Github-Mirror/seaweedfs
1
0 Fork 0
mirror of https://github.com/seaweedfs/seaweedfs.git synced 2026-11-12 06:48:45 +01:00
Code Issues Packages Projects Releases Wiki Activity

feat: Phase 8 CP8-1/2/3/4 -- ops control plane, multi-replica, CSI snapshots, observability

Browse Source 
CP8-1: HTTP REST API (create/delete/lookup/list/assign/servers), blockapi Go
client with multi-master failover, 5 shell commands, HTML dashboard at /block/.

CP8-2: RF=2/RF=3 multi-replica support -- ShipperGroup fan-out, distributed
sync, health scoring, segment-based scrub, gated promotion (heartbeat
freshness + WAL LSN + role checks), failover/rebuild for N>2 replicas.

CP8-3: CSI snapshot + expansion -- CreateSnapshot/DeleteSnapshot/ListSnapshots
RPCs, NodeExpandVolume with iSCSI rescan, snapshot ID helpers, 20 adversarial
tests covering concurrent ops, edge cases, and error injection.

CP8-4: Observability -- EngineMetrics atomic counters for flusher/group-commit/
WAL-shipper/scrub, 10 new Prometheus metrics, barrier_lag_lsn SLO gauge,
failover/promotion/rebuild counters, request ID correlation in master gRPC
logs, baseline regression framework with 7 hard-fail conditions.

Total: 63 files, ~11.2K LOC, 160+ new tests.

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
Ping QiuandClaude Opus 4.6 committed 2026-03-06 00:05:17 -08:00
1 parent 8b2b5f6f66
commit 979a9b496c
63 files changed
+11207 -659
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

weed/command/master.go
+7 -4
View File
@@ -71,8 +71,9 @@ type MasterOptions struct {
raftBootstrap *bool
telemetryUrl *string
telemetryEnabled *bool
debug *bool
debugPort *int
debug *bool
debugPort *int
blockPromotionLSNTolerance *int
}
func init() {
@@ -104,6 +105,7 @@ func init() {
m.telemetryEnabled = cmdMaster.Flag.Bool("telemetry", false, "enable telemetry reporting")
m.debug = cmdMaster.Flag.Bool("debug", false, "serves runtime profiling data via pprof on the port specified by -debug.port")
m.debugPort = cmdMaster.Flag.Int("debug.port", 6060, "http port for debugging")
m.blockPromotionLSNTolerance = cmdMaster.Flag.Int("block.promotion.lsnTolerance", 100, "max WAL LSN lag for block volume replica promotion eligibility")
}
var cmdMaster = &Command{
@@ -410,7 +412,8 @@ func (m *MasterOptions) toMasterOption(whiteList []string) *weed_server.MasterOp
DisableHttp: *m.disableHttp,
MetricsAddress: *m.metricsAddress,
MetricsIntervalSec: *m.metricsIntervalSec,
TelemetryUrl: *m.telemetryUrl,
TelemetryEnabled: *m.telemetryEnabled,
TelemetryUrl: *m.telemetryUrl,
TelemetryEnabled: *m.telemetryEnabled,
BlockPromotionLSNTolerance: *m.blockPromotionLSNTolerance,
}
}
weed/command/volume.go
+3
View File
@@ -310,6 +310,9 @@ func (v VolumeServerOptions) startVolumeServer(volumeFolders, maxVolumeCounts, v
// Start block volume iSCSI service (disabled if block.dir is empty).
blockService := weed_server.StartBlockService(*v.blockListen, *v.blockDir, *v.blockIQNPrefix)
if blockService != nil {
volumeServer.SetBlockService(blockService)
}
grace.OnReload(volumeServer.LoadNewVolumes)
grace.OnReload(volumeServer.Reload)
weed/pb/master.proto
+59
View File
@@ -63,6 +63,14 @@ service Seaweed {
}
rpc LookupBlockVolume (LookupBlockVolumeRequest) returns (LookupBlockVolumeResponse) {
}
rpc CreateBlockSnapshot (CreateBlockSnapshotRequest) returns (CreateBlockSnapshotResponse) {
}
rpc DeleteBlockSnapshot (DeleteBlockSnapshotRequest) returns (DeleteBlockSnapshotResponse) {
}
rpc ListBlockSnapshots (ListBlockSnapshotsRequest) returns (ListBlockSnapshotsResponse) {
}
rpc ExpandBlockVolume (ExpandBlockVolumeRequest) returns (ExpandBlockVolumeResponse) {
}
}
//////////////////////////////////////////////////
@@ -493,6 +501,10 @@ message BlockVolumeInfoMessage {
string disk_type = 9;
string replica_data_addr = 10;
string replica_ctrl_addr = 11;
double health_score = 12;
int64 scrub_errors = 13;
int64 last_scrub_time = 14;
bool replica_degraded = 15;
}
message BlockVolumeShortInfoMessage {
@@ -510,12 +522,19 @@ message BlockVolumeAssignment {
string replica_data_addr = 5;
string replica_ctrl_addr = 6;
string rebuild_addr = 7;
repeated ReplicaAddrMessage replica_addrs = 8;
}
message ReplicaAddrMessage {
string data_addr = 1;
string ctrl_addr = 2;
}
message CreateBlockVolumeRequest {
string name = 1;
uint64 size_bytes = 2;
string disk_type = 3;
uint32 replica_factor = 4;
}
message CreateBlockVolumeResponse {
string volume_id = 1;
@@ -524,6 +543,7 @@ message CreateBlockVolumeResponse {
string iqn = 4;
uint64 capacity_bytes = 5;
string replica_server = 6;
repeated string replica_servers = 7;
}
message DeleteBlockVolumeRequest {
@@ -541,4 +561,43 @@ message LookupBlockVolumeResponse {
string iqn = 3;
uint64 capacity_bytes = 4;
string replica_server = 5;
uint32 replica_factor = 6;
repeated string replica_servers = 7;
}
message CreateBlockSnapshotRequest {
string volume_name = 1;
uint32 snapshot_id = 2;
}
message CreateBlockSnapshotResponse {
uint32 snapshot_id = 1;
int64 created_at = 2; // Unix timestamp in seconds
uint64 size_bytes = 3;
}
message DeleteBlockSnapshotRequest {
string volume_name = 1;
uint32 snapshot_id = 2;
}
message DeleteBlockSnapshotResponse {
}
message ListBlockSnapshotsRequest {
string volume_name = 1;
}
message ListBlockSnapshotsResponse {
repeated BlockSnapshotInfo snapshots = 1;
}
message BlockSnapshotInfo {
uint32 snapshot_id = 1;
int64 created_at = 2; // Unix timestamp in seconds
uint64 volume_size_bytes = 3;
}
message ExpandBlockVolumeRequest {
string name = 1;
uint64 new_size_bytes = 2;
}
message ExpandBlockVolumeResponse {
uint64 capacity_bytes = 1;
}
weed/pb/master_pb/master.pb.go
+728 -121
View File
File diff suppressed because it is too large. Load diff
weed/pb/master_pb/master_grpc.pb.go
+152
View File
@@ -46,6 +46,10 @@ const (
Seaweed_CreateBlockVolume_FullMethodName = "/master_pb.Seaweed/CreateBlockVolume"
Seaweed_DeleteBlockVolume_FullMethodName = "/master_pb.Seaweed/DeleteBlockVolume"
Seaweed_LookupBlockVolume_FullMethodName = "/master_pb.Seaweed/LookupBlockVolume"
Seaweed_CreateBlockSnapshot_FullMethodName = "/master_pb.Seaweed/CreateBlockSnapshot"
Seaweed_DeleteBlockSnapshot_FullMethodName = "/master_pb.Seaweed/DeleteBlockSnapshot"
Seaweed_ListBlockSnapshots_FullMethodName = "/master_pb.Seaweed/ListBlockSnapshots"
Seaweed_ExpandBlockVolume_FullMethodName = "/master_pb.Seaweed/ExpandBlockVolume"
)
// SeaweedClient is the client API for Seaweed service.
@@ -79,6 +83,10 @@ type SeaweedClient interface {
CreateBlockVolume(ctx context.Context, in *CreateBlockVolumeRequest, opts ...grpc.CallOption) (*CreateBlockVolumeResponse, error)
DeleteBlockVolume(ctx context.Context, in *DeleteBlockVolumeRequest, opts ...grpc.CallOption) (*DeleteBlockVolumeResponse, error)
LookupBlockVolume(ctx context.Context, in *LookupBlockVolumeRequest, opts ...grpc.CallOption) (*LookupBlockVolumeResponse, error)
CreateBlockSnapshot(ctx context.Context, in *CreateBlockSnapshotRequest, opts ...grpc.CallOption) (*CreateBlockSnapshotResponse, error)
DeleteBlockSnapshot(ctx context.Context, in *DeleteBlockSnapshotRequest, opts ...grpc.CallOption) (*DeleteBlockSnapshotResponse, error)
ListBlockSnapshots(ctx context.Context, in *ListBlockSnapshotsRequest, opts ...grpc.CallOption) (*ListBlockSnapshotsResponse, error)
ExpandBlockVolume(ctx context.Context, in *ExpandBlockVolumeRequest, opts ...grpc.CallOption) (*ExpandBlockVolumeResponse, error)
}
type seaweedClient struct {
@@ -368,6 +376,46 @@ func (c *seaweedClient) LookupBlockVolume(ctx context.Context, in *LookupBlockVo
return out, nil
}
func (c *seaweedClient) CreateBlockSnapshot(ctx context.Context, in *CreateBlockSnapshotRequest, opts ...grpc.CallOption) (*CreateBlockSnapshotResponse, error) {
cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...)
out := new(CreateBlockSnapshotResponse)
err := c.cc.Invoke(ctx, Seaweed_CreateBlockSnapshot_FullMethodName, in, out, cOpts...)
if err != nil {
return nil, err
}
return out, nil
}
func (c *seaweedClient) DeleteBlockSnapshot(ctx context.Context, in *DeleteBlockSnapshotRequest, opts ...grpc.CallOption) (*DeleteBlockSnapshotResponse, error) {
cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...)
out := new(DeleteBlockSnapshotResponse)
err := c.cc.Invoke(ctx, Seaweed_DeleteBlockSnapshot_FullMethodName, in, out, cOpts...)
if err != nil {
return nil, err
}
return out, nil
}
func (c *seaweedClient) ListBlockSnapshots(ctx context.Context, in *ListBlockSnapshotsRequest, opts ...grpc.CallOption) (*ListBlockSnapshotsResponse, error) {
cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...)
out := new(ListBlockSnapshotsResponse)
err := c.cc.Invoke(ctx, Seaweed_ListBlockSnapshots_FullMethodName, in, out, cOpts...)
if err != nil {
return nil, err
}
return out, nil
}
func (c *seaweedClient) ExpandBlockVolume(ctx context.Context, in *ExpandBlockVolumeRequest, opts ...grpc.CallOption) (*ExpandBlockVolumeResponse, error) {
cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...)
out := new(ExpandBlockVolumeResponse)
err := c.cc.Invoke(ctx, Seaweed_ExpandBlockVolume_FullMethodName, in, out, cOpts...)
if err != nil {
return nil, err
}
return out, nil
}
// SeaweedServer is the server API for Seaweed service.
// All implementations must embed UnimplementedSeaweedServer
// for forward compatibility.
@@ -399,6 +447,10 @@ type SeaweedServer interface {
CreateBlockVolume(context.Context, *CreateBlockVolumeRequest) (*CreateBlockVolumeResponse, error)
DeleteBlockVolume(context.Context, *DeleteBlockVolumeRequest) (*DeleteBlockVolumeResponse, error)
LookupBlockVolume(context.Context, *LookupBlockVolumeRequest) (*LookupBlockVolumeResponse, error)
CreateBlockSnapshot(context.Context, *CreateBlockSnapshotRequest) (*CreateBlockSnapshotResponse, error)
DeleteBlockSnapshot(context.Context, *DeleteBlockSnapshotRequest) (*DeleteBlockSnapshotResponse, error)
ListBlockSnapshots(context.Context, *ListBlockSnapshotsRequest) (*ListBlockSnapshotsResponse, error)
ExpandBlockVolume(context.Context, *ExpandBlockVolumeRequest) (*ExpandBlockVolumeResponse, error)
mustEmbedUnimplementedSeaweedServer()
}
@@ -490,6 +542,18 @@ func (UnimplementedSeaweedServer) DeleteBlockVolume(context.Context, *DeleteBloc
func (UnimplementedSeaweedServer) LookupBlockVolume(context.Context, *LookupBlockVolumeRequest) (*LookupBlockVolumeResponse, error) {
return nil, status.Error(codes.Unimplemented, "method LookupBlockVolume not implemented")
}
func (UnimplementedSeaweedServer) CreateBlockSnapshot(context.Context, *CreateBlockSnapshotRequest) (*CreateBlockSnapshotResponse, error) {
return nil, status.Error(codes.Unimplemented, "method CreateBlockSnapshot not implemented")
}
func (UnimplementedSeaweedServer) DeleteBlockSnapshot(context.Context, *DeleteBlockSnapshotRequest) (*DeleteBlockSnapshotResponse, error) {
return nil, status.Error(codes.Unimplemented, "method DeleteBlockSnapshot not implemented")
}
func (UnimplementedSeaweedServer) ListBlockSnapshots(context.Context, *ListBlockSnapshotsRequest) (*ListBlockSnapshotsResponse, error) {
return nil, status.Error(codes.Unimplemented, "method ListBlockSnapshots not implemented")
}
func (UnimplementedSeaweedServer) ExpandBlockVolume(context.Context, *ExpandBlockVolumeRequest) (*ExpandBlockVolumeResponse, error) {
return nil, status.Error(codes.Unimplemented, "method ExpandBlockVolume not implemented")
}
func (UnimplementedSeaweedServer) mustEmbedUnimplementedSeaweedServer() {}
func (UnimplementedSeaweedServer) testEmbeddedByValue() {}
@@ -964,6 +1028,78 @@ func _Seaweed_LookupBlockVolume_Handler(srv interface{}, ctx context.Context, de
return interceptor(ctx, in, info, handler)
}
func _Seaweed_CreateBlockSnapshot_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
in := new(CreateBlockSnapshotRequest)
if err := dec(in); err != nil {
return nil, err
}
if interceptor == nil {
return srv.(SeaweedServer).CreateBlockSnapshot(ctx, in)
}
info := &grpc.UnaryServerInfo{
Server: srv,
FullMethod: Seaweed_CreateBlockSnapshot_FullMethodName,
}
handler := func(ctx context.Context, req interface{}) (interface{}, error) {
return srv.(SeaweedServer).CreateBlockSnapshot(ctx, req.(*CreateBlockSnapshotRequest))
}
return interceptor(ctx, in, info, handler)
}
func _Seaweed_DeleteBlockSnapshot_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
in := new(DeleteBlockSnapshotRequest)
if err := dec(in); err != nil {
return nil, err
}
if interceptor == nil {
return srv.(SeaweedServer).DeleteBlockSnapshot(ctx, in)
}
info := &grpc.UnaryServerInfo{
Server: srv,
FullMethod: Seaweed_DeleteBlockSnapshot_FullMethodName,
}
handler := func(ctx context.Context, req interface{}) (interface{}, error) {
return srv.(SeaweedServer).DeleteBlockSnapshot(ctx, req.(*DeleteBlockSnapshotRequest))
}
return interceptor(ctx, in, info, handler)
}
func _Seaweed_ListBlockSnapshots_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
in := new(ListBlockSnapshotsRequest)
if err := dec(in); err != nil {
return nil, err
}
if interceptor == nil {
return srv.(SeaweedServer).ListBlockSnapshots(ctx, in)
}
info := &grpc.UnaryServerInfo{
Server: srv,
FullMethod: Seaweed_ListBlockSnapshots_FullMethodName,
}
handler := func(ctx context.Context, req interface{}) (interface{}, error) {
return srv.(SeaweedServer).ListBlockSnapshots(ctx, req.(*ListBlockSnapshotsRequest))
}
return interceptor(ctx, in, info, handler)
}
func _Seaweed_ExpandBlockVolume_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
in := new(ExpandBlockVolumeRequest)
if err := dec(in); err != nil {
return nil, err
}
if interceptor == nil {
return srv.(SeaweedServer).ExpandBlockVolume(ctx, in)
}
info := &grpc.UnaryServerInfo{
Server: srv,
FullMethod: Seaweed_ExpandBlockVolume_FullMethodName,
}
handler := func(ctx context.Context, req interface{}) (interface{}, error) {
return srv.(SeaweedServer).ExpandBlockVolume(ctx, req.(*ExpandBlockVolumeRequest))
}
return interceptor(ctx, in, info, handler)
}
// Seaweed_ServiceDesc is the grpc.ServiceDesc for Seaweed service.
// It's only intended for direct use with grpc.RegisterService,
// and not to be introspected or modified (even as a copy)
@@ -1067,6 +1203,22 @@ var Seaweed_ServiceDesc = grpc.ServiceDesc{
MethodName: "LookupBlockVolume",
Handler: _Seaweed_LookupBlockVolume_Handler,
},
{
MethodName: "CreateBlockSnapshot",
Handler: _Seaweed_CreateBlockSnapshot_Handler,
},
{
MethodName: "DeleteBlockSnapshot",
Handler: _Seaweed_DeleteBlockSnapshot_Handler,
},
{
MethodName: "ListBlockSnapshots",
Handler: _Seaweed_ListBlockSnapshots_Handler,
},
{
MethodName: "ExpandBlockVolume",
Handler: _Seaweed_ExpandBlockVolume_Handler,
},
},
Streams: []grpc.StreamDesc{
{
weed/pb/volume_server.proto
+45
View File
@@ -143,6 +143,14 @@ service VolumeServer {
}
rpc VolumeServerDeleteBlockVolume (VolumeServerDeleteBlockVolumeRequest) returns (VolumeServerDeleteBlockVolumeResponse) {
}
rpc SnapshotBlockVolume (SnapshotBlockVolumeRequest) returns (SnapshotBlockVolumeResponse) {
}
rpc DeleteBlockSnapshot (DeleteBlockSnapshotRequest) returns (DeleteBlockSnapshotResponse) {
}
rpc ListBlockSnapshots (ListBlockSnapshotsRequest) returns (ListBlockSnapshotsResponse) {
}
rpc ExpandBlockVolume (ExpandBlockVolumeRequest) returns (ExpandBlockVolumeResponse) {
}
}
@@ -786,3 +794,40 @@ message VolumeServerDeleteBlockVolumeRequest {
}
message VolumeServerDeleteBlockVolumeResponse {
}
message SnapshotBlockVolumeRequest {
string name = 1;
uint32 snapshot_id = 2;
}
message SnapshotBlockVolumeResponse {
uint32 snapshot_id = 1;
int64 created_at = 2; // Unix timestamp in seconds
uint64 size_bytes = 3;
}
message DeleteBlockSnapshotRequest {
string name = 1;
uint32 snapshot_id = 2;
}
message DeleteBlockSnapshotResponse {
}
message ListBlockSnapshotsRequest {
string name = 1;
}
message ListBlockSnapshotsResponse {
repeated BlockSnapshotInfo snapshots = 1;
}
message BlockSnapshotInfo {
uint32 snapshot_id = 1;
int64 created_at = 2; // Unix timestamp in seconds
uint64 volume_size_bytes = 3;
}
message ExpandBlockVolumeRequest {
string name = 1;
uint64 new_size_bytes = 2;
}
message ExpandBlockVolumeResponse {
uint64 capacity_bytes = 1;
}
weed/pb/volume_server_pb/volume_server.pb.go
+661 -161
View File
@@ -6246,15 +6246,15 @@ func (x *AllocateBlockVolumeRequest) GetDiskType() string {
}
type AllocateBlockVolumeResponse struct {
state protoimpl.MessageState `protogen:"open.v1"`
Path string `protobuf:"bytes,1,opt,name=path,proto3" json:"path,omitempty"`
Iqn string `protobuf:"bytes,2,opt,name=iqn,proto3" json:"iqn,omitempty"`
IscsiAddr string `protobuf:"bytes,3,opt,name=iscsi_addr,json=iscsiAddr,proto3" json:"iscsi_addr,omitempty"`
ReplicaDataAddr string `protobuf:"bytes,4,opt,name=replica_data_addr,json=replicaDataAddr,proto3" json:"replica_data_addr,omitempty"`
ReplicaCtrlAddr string `protobuf:"bytes,5,opt,name=replica_ctrl_addr,json=replicaCtrlAddr,proto3" json:"replica_ctrl_addr,omitempty"`
RebuildListenAddr string `protobuf:"bytes,6,opt,name=rebuild_listen_addr,json=rebuildListenAddr,proto3" json:"rebuild_listen_addr,omitempty"`
unknownFields protoimpl.UnknownFields
sizeCache protoimpl.SizeCache
state protoimpl.MessageState `protogen:"open.v1"`
Path string `protobuf:"bytes,1,opt,name=path,proto3" json:"path,omitempty"`
Iqn string `protobuf:"bytes,2,opt,name=iqn,proto3" json:"iqn,omitempty"`
IscsiAddr string `protobuf:"bytes,3,opt,name=iscsi_addr,json=iscsiAddr,proto3" json:"iscsi_addr,omitempty"`
ReplicaDataAddr string `protobuf:"bytes,4,opt,name=replica_data_addr,json=replicaDataAddr,proto3" json:"replica_data_addr,omitempty"`
ReplicaCtrlAddr string `protobuf:"bytes,5,opt,name=replica_ctrl_addr,json=replicaCtrlAddr,proto3" json:"replica_ctrl_addr,omitempty"`
RebuildListenAddr string `protobuf:"bytes,6,opt,name=rebuild_listen_addr,json=rebuildListenAddr,proto3" json:"rebuild_listen_addr,omitempty"`
unknownFields protoimpl.UnknownFields
sizeCache protoimpl.SizeCache
}
func (x *AllocateBlockVolumeResponse) Reset() {
@@ -6409,6 +6409,450 @@ func (*VolumeServerDeleteBlockVolumeResponse) Descriptor() ([]byte, []int) {
return file_volume_server_proto_rawDescGZIP(), []int{110}
}
type SnapshotBlockVolumeRequest struct {
state protoimpl.MessageState `protogen:"open.v1"`
Name string `protobuf:"bytes,1,opt,name=name,proto3" json:"name,omitempty"`
SnapshotId uint32 `protobuf:"varint,2,opt,name=snapshot_id,json=snapshotId,proto3" json:"snapshot_id,omitempty"`
unknownFields protoimpl.UnknownFields
sizeCache protoimpl.SizeCache
}
func (x *SnapshotBlockVolumeRequest) Reset() {
*x = SnapshotBlockVolumeRequest{}
mi := &file_volume_server_proto_msgTypes[111]
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
ms.StoreMessageInfo(mi)
}
func (x *SnapshotBlockVolumeRequest) String() string {
return protoimpl.X.MessageStringOf(x)
}
func (*SnapshotBlockVolumeRequest) ProtoMessage() {}
func (x *SnapshotBlockVolumeRequest) ProtoReflect() protoreflect.Message {
mi := &file_volume_server_proto_msgTypes[111]
if x != nil {
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
if ms.LoadMessageInfo() == nil {
ms.StoreMessageInfo(mi)
}
return ms
}
return mi.MessageOf(x)
}
// Deprecated: Use SnapshotBlockVolumeRequest.ProtoReflect.Descriptor instead.
func (*SnapshotBlockVolumeRequest) Descriptor() ([]byte, []int) {
return file_volume_server_proto_rawDescGZIP(), []int{111}
}
func (x *SnapshotBlockVolumeRequest) GetName() string {
if x != nil {
return x.Name
}
return ""
}
func (x *SnapshotBlockVolumeRequest) GetSnapshotId() uint32 {
if x != nil {
return x.SnapshotId
}
return 0
}
type SnapshotBlockVolumeResponse struct {
state protoimpl.MessageState `protogen:"open.v1"`
SnapshotId uint32 `protobuf:"varint,1,opt,name=snapshot_id,json=snapshotId,proto3" json:"snapshot_id,omitempty"`
CreatedAt int64 `protobuf:"varint,2,opt,name=created_at,json=createdAt,proto3" json:"created_at,omitempty"` // Unix timestamp in seconds
SizeBytes uint64 `protobuf:"varint,3,opt,name=size_bytes,json=sizeBytes,proto3" json:"size_bytes,omitempty"`
unknownFields protoimpl.UnknownFields
sizeCache protoimpl.SizeCache
}
func (x *SnapshotBlockVolumeResponse) Reset() {
*x = SnapshotBlockVolumeResponse{}
mi := &file_volume_server_proto_msgTypes[112]
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
ms.StoreMessageInfo(mi)
}
func (x *SnapshotBlockVolumeResponse) String() string {
return protoimpl.X.MessageStringOf(x)
}
func (*SnapshotBlockVolumeResponse) ProtoMessage() {}
func (x *SnapshotBlockVolumeResponse) ProtoReflect() protoreflect.Message {
mi := &file_volume_server_proto_msgTypes[112]
if x != nil {
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
if ms.LoadMessageInfo() == nil {
ms.StoreMessageInfo(mi)
}
return ms
}
return mi.MessageOf(x)
}
// Deprecated: Use SnapshotBlockVolumeResponse.ProtoReflect.Descriptor instead.
func (*SnapshotBlockVolumeResponse) Descriptor() ([]byte, []int) {
return file_volume_server_proto_rawDescGZIP(), []int{112}
}
func (x *SnapshotBlockVolumeResponse) GetSnapshotId() uint32 {
if x != nil {
return x.SnapshotId
}
return 0
}
func (x *SnapshotBlockVolumeResponse) GetCreatedAt() int64 {
if x != nil {
return x.CreatedAt
}
return 0
}
func (x *SnapshotBlockVolumeResponse) GetSizeBytes() uint64 {
if x != nil {
return x.SizeBytes
}
return 0
}
type DeleteBlockSnapshotRequest struct {
state protoimpl.MessageState `protogen:"open.v1"`
Name string `protobuf:"bytes,1,opt,name=name,proto3" json:"name,omitempty"`
SnapshotId uint32 `protobuf:"varint,2,opt,name=snapshot_id,json=snapshotId,proto3" json:"snapshot_id,omitempty"`
unknownFields protoimpl.UnknownFields
sizeCache protoimpl.SizeCache
}
func (x *DeleteBlockSnapshotRequest) Reset() {
*x = DeleteBlockSnapshotRequest{}
mi := &file_volume_server_proto_msgTypes[113]
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
ms.StoreMessageInfo(mi)
}
func (x *DeleteBlockSnapshotRequest) String() string {
return protoimpl.X.MessageStringOf(x)
}
func (*DeleteBlockSnapshotRequest) ProtoMessage() {}
func (x *DeleteBlockSnapshotRequest) ProtoReflect() protoreflect.Message {
mi := &file_volume_server_proto_msgTypes[113]
if x != nil {
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
if ms.LoadMessageInfo() == nil {
ms.StoreMessageInfo(mi)
}
return ms
}
return mi.MessageOf(x)
}
// Deprecated: Use DeleteBlockSnapshotRequest.ProtoReflect.Descriptor instead.
func (*DeleteBlockSnapshotRequest) Descriptor() ([]byte, []int) {
return file_volume_server_proto_rawDescGZIP(), []int{113}
}
func (x *DeleteBlockSnapshotRequest) GetName() string {
if x != nil {
return x.Name
}
return ""
}
func (x *DeleteBlockSnapshotRequest) GetSnapshotId() uint32 {
if x != nil {
return x.SnapshotId
}
return 0
}
type DeleteBlockSnapshotResponse struct {
state protoimpl.MessageState `protogen:"open.v1"`
unknownFields protoimpl.UnknownFields
sizeCache protoimpl.SizeCache
}
func (x *DeleteBlockSnapshotResponse) Reset() {
*x = DeleteBlockSnapshotResponse{}
mi := &file_volume_server_proto_msgTypes[114]
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
ms.StoreMessageInfo(mi)
}
func (x *DeleteBlockSnapshotResponse) String() string {
return protoimpl.X.MessageStringOf(x)
}
func (*DeleteBlockSnapshotResponse) ProtoMessage() {}
func (x *DeleteBlockSnapshotResponse) ProtoReflect() protoreflect.Message {
mi := &file_volume_server_proto_msgTypes[114]
if x != nil {
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
if ms.LoadMessageInfo() == nil {
ms.StoreMessageInfo(mi)
}
return ms
}
return mi.MessageOf(x)
}
// Deprecated: Use DeleteBlockSnapshotResponse.ProtoReflect.Descriptor instead.
func (*DeleteBlockSnapshotResponse) Descriptor() ([]byte, []int) {
return file_volume_server_proto_rawDescGZIP(), []int{114}
}
type ListBlockSnapshotsRequest struct {
state protoimpl.MessageState `protogen:"open.v1"`
Name string `protobuf:"bytes,1,opt,name=name,proto3" json:"name,omitempty"`
unknownFields protoimpl.UnknownFields
sizeCache protoimpl.SizeCache
}
func (x *ListBlockSnapshotsRequest) Reset() {
*x = ListBlockSnapshotsRequest{}
mi := &file_volume_server_proto_msgTypes[115]
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
ms.StoreMessageInfo(mi)
}
func (x *ListBlockSnapshotsRequest) String() string {
return protoimpl.X.MessageStringOf(x)
}
func (*ListBlockSnapshotsRequest) ProtoMessage() {}
func (x *ListBlockSnapshotsRequest) ProtoReflect() protoreflect.Message {
mi := &file_volume_server_proto_msgTypes[115]
if x != nil {
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
if ms.LoadMessageInfo() == nil {
ms.StoreMessageInfo(mi)
}
return ms
}
return mi.MessageOf(x)
}
// Deprecated: Use ListBlockSnapshotsRequest.ProtoReflect.Descriptor instead.
func (*ListBlockSnapshotsRequest) Descriptor() ([]byte, []int) {
return file_volume_server_proto_rawDescGZIP(), []int{115}
}
func (x *ListBlockSnapshotsRequest) GetName() string {
if x != nil {
return x.Name
}
return ""
}
type ListBlockSnapshotsResponse struct {
state protoimpl.MessageState `protogen:"open.v1"`
Snapshots []*BlockSnapshotInfo `protobuf:"bytes,1,rep,name=snapshots,proto3" json:"snapshots,omitempty"`
unknownFields protoimpl.UnknownFields
sizeCache protoimpl.SizeCache
}
func (x *ListBlockSnapshotsResponse) Reset() {
*x = ListBlockSnapshotsResponse{}
mi := &file_volume_server_proto_msgTypes[116]
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
ms.StoreMessageInfo(mi)
}
func (x *ListBlockSnapshotsResponse) String() string {
return protoimpl.X.MessageStringOf(x)
}
func (*ListBlockSnapshotsResponse) ProtoMessage() {}
func (x *ListBlockSnapshotsResponse) ProtoReflect() protoreflect.Message {
mi := &file_volume_server_proto_msgTypes[116]
if x != nil {
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
if ms.LoadMessageInfo() == nil {
ms.StoreMessageInfo(mi)
}
return ms
}
return mi.MessageOf(x)
}
// Deprecated: Use ListBlockSnapshotsResponse.ProtoReflect.Descriptor instead.
func (*ListBlockSnapshotsResponse) Descriptor() ([]byte, []int) {
return file_volume_server_proto_rawDescGZIP(), []int{116}
}
func (x *ListBlockSnapshotsResponse) GetSnapshots() []*BlockSnapshotInfo {
if x != nil {
return x.Snapshots
}
return nil
}
type BlockSnapshotInfo struct {
state protoimpl.MessageState `protogen:"open.v1"`
SnapshotId uint32 `protobuf:"varint,1,opt,name=snapshot_id,json=snapshotId,proto3" json:"snapshot_id,omitempty"`
CreatedAt int64 `protobuf:"varint,2,opt,name=created_at,json=createdAt,proto3" json:"created_at,omitempty"` // Unix timestamp in seconds
VolumeSizeBytes uint64 `protobuf:"varint,3,opt,name=volume_size_bytes,json=volumeSizeBytes,proto3" json:"volume_size_bytes,omitempty"`
unknownFields protoimpl.UnknownFields
sizeCache protoimpl.SizeCache
}
func (x *BlockSnapshotInfo) Reset() {
*x = BlockSnapshotInfo{}
mi := &file_volume_server_proto_msgTypes[117]
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
ms.StoreMessageInfo(mi)
}
func (x *BlockSnapshotInfo) String() string {
return protoimpl.X.MessageStringOf(x)
}
func (*BlockSnapshotInfo) ProtoMessage() {}
func (x *BlockSnapshotInfo) ProtoReflect() protoreflect.Message {
mi := &file_volume_server_proto_msgTypes[117]
if x != nil {
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
if ms.LoadMessageInfo() == nil {
ms.StoreMessageInfo(mi)
}
return ms
}
return mi.MessageOf(x)
}
// Deprecated: Use BlockSnapshotInfo.ProtoReflect.Descriptor instead.
func (*BlockSnapshotInfo) Descriptor() ([]byte, []int) {
return file_volume_server_proto_rawDescGZIP(), []int{117}
}
func (x *BlockSnapshotInfo) GetSnapshotId() uint32 {
if x != nil {
return x.SnapshotId
}
return 0
}
func (x *BlockSnapshotInfo) GetCreatedAt() int64 {
if x != nil {
return x.CreatedAt
}
return 0
}
func (x *BlockSnapshotInfo) GetVolumeSizeBytes() uint64 {
if x != nil {
return x.VolumeSizeBytes
}
return 0
}
type ExpandBlockVolumeRequest struct {
state protoimpl.MessageState `protogen:"open.v1"`
Name string `protobuf:"bytes,1,opt,name=name,proto3" json:"name,omitempty"`
NewSizeBytes uint64 `protobuf:"varint,2,opt,name=new_size_bytes,json=newSizeBytes,proto3" json:"new_size_bytes,omitempty"`
unknownFields protoimpl.UnknownFields
sizeCache protoimpl.SizeCache
}
func (x *ExpandBlockVolumeRequest) Reset() {
*x = ExpandBlockVolumeRequest{}
mi := &file_volume_server_proto_msgTypes[118]
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
ms.StoreMessageInfo(mi)
}
func (x *ExpandBlockVolumeRequest) String() string {
return protoimpl.X.MessageStringOf(x)
}
func (*ExpandBlockVolumeRequest) ProtoMessage() {}
func (x *ExpandBlockVolumeRequest) ProtoReflect() protoreflect.Message {
mi := &file_volume_server_proto_msgTypes[118]
if x != nil {
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
if ms.LoadMessageInfo() == nil {
ms.StoreMessageInfo(mi)
}
return ms
}
return mi.MessageOf(x)
}
// Deprecated: Use ExpandBlockVolumeRequest.ProtoReflect.Descriptor instead.
func (*ExpandBlockVolumeRequest) Descriptor() ([]byte, []int) {
return file_volume_server_proto_rawDescGZIP(), []int{118}
}
func (x *ExpandBlockVolumeRequest) GetName() string {
if x != nil {
return x.Name
}
return ""
}
func (x *ExpandBlockVolumeRequest) GetNewSizeBytes() uint64 {
if x != nil {
return x.NewSizeBytes
}
return 0
}
type ExpandBlockVolumeResponse struct {
state protoimpl.MessageState `protogen:"open.v1"`
CapacityBytes uint64 `protobuf:"varint,1,opt,name=capacity_bytes,json=capacityBytes,proto3" json:"capacity_bytes,omitempty"`
unknownFields protoimpl.UnknownFields
sizeCache protoimpl.SizeCache
}
func (x *ExpandBlockVolumeResponse) Reset() {
*x = ExpandBlockVolumeResponse{}
mi := &file_volume_server_proto_msgTypes[119]
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
ms.StoreMessageInfo(mi)
}
func (x *ExpandBlockVolumeResponse) String() string {
return protoimpl.X.MessageStringOf(x)
}
func (*ExpandBlockVolumeResponse) ProtoMessage() {}
func (x *ExpandBlockVolumeResponse) ProtoReflect() protoreflect.Message {
mi := &file_volume_server_proto_msgTypes[119]
if x != nil {
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
if ms.LoadMessageInfo() == nil {
ms.StoreMessageInfo(mi)
}
return ms
}
return mi.MessageOf(x)
}
// Deprecated: Use ExpandBlockVolumeResponse.ProtoReflect.Descriptor instead.
func (*ExpandBlockVolumeResponse) Descriptor() ([]byte, []int) {
return file_volume_server_proto_rawDescGZIP(), []int{119}
}
func (x *ExpandBlockVolumeResponse) GetCapacityBytes() uint64 {
if x != nil {
return x.CapacityBytes
}
return 0
}
type FetchAndWriteNeedleRequest_Replica struct {
state protoimpl.MessageState `protogen:"open.v1"`
Url string `protobuf:"bytes,1,opt,name=url,proto3" json:"url,omitempty"`
@@ -6420,7 +6864,7 @@ type FetchAndWriteNeedleRequest_Replica struct {
func (x *FetchAndWriteNeedleRequest_Replica) Reset() {
*x = FetchAndWriteNeedleRequest_Replica{}
mi := &file_volume_server_proto_msgTypes[111]
mi := &file_volume_server_proto_msgTypes[120]
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
ms.StoreMessageInfo(mi)
}
@@ -6432,7 +6876,7 @@ func (x *FetchAndWriteNeedleRequest_Replica) String() string {
func (*FetchAndWriteNeedleRequest_Replica) ProtoMessage() {}
func (x *FetchAndWriteNeedleRequest_Replica) ProtoReflect() protoreflect.Message {
mi := &file_volume_server_proto_msgTypes[111]
mi := &file_volume_server_proto_msgTypes[120]
if x != nil {
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
if ms.LoadMessageInfo() == nil {
@@ -6480,7 +6924,7 @@ type QueryRequest_Filter struct {
func (x *QueryRequest_Filter) Reset() {
*x = QueryRequest_Filter{}
mi := &file_volume_server_proto_msgTypes[112]
mi := &file_volume_server_proto_msgTypes[121]
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
ms.StoreMessageInfo(mi)
}
@@ -6492,7 +6936,7 @@ func (x *QueryRequest_Filter) String() string {
func (*QueryRequest_Filter) ProtoMessage() {}
func (x *QueryRequest_Filter) ProtoReflect() protoreflect.Message {
mi := &file_volume_server_proto_msgTypes[112]
mi := &file_volume_server_proto_msgTypes[121]
if x != nil {
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
if ms.LoadMessageInfo() == nil {
@@ -6542,7 +6986,7 @@ type QueryRequest_InputSerialization struct {
func (x *QueryRequest_InputSerialization) Reset() {
*x = QueryRequest_InputSerialization{}
mi := &file_volume_server_proto_msgTypes[113]
mi := &file_volume_server_proto_msgTypes[122]
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
ms.StoreMessageInfo(mi)
}
@@ -6554,7 +6998,7 @@ func (x *QueryRequest_InputSerialization) String() string {
func (*QueryRequest_InputSerialization) ProtoMessage() {}
func (x *QueryRequest_InputSerialization) ProtoReflect() protoreflect.Message {
mi := &file_volume_server_proto_msgTypes[113]
mi := &file_volume_server_proto_msgTypes[122]
if x != nil {
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
if ms.LoadMessageInfo() == nil {
@@ -6608,7 +7052,7 @@ type QueryRequest_OutputSerialization struct {
func (x *QueryRequest_OutputSerialization) Reset() {
*x = QueryRequest_OutputSerialization{}
mi := &file_volume_server_proto_msgTypes[114]
mi := &file_volume_server_proto_msgTypes[123]
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
ms.StoreMessageInfo(mi)
}
@@ -6620,7 +7064,7 @@ func (x *QueryRequest_OutputSerialization) String() string {
func (*QueryRequest_OutputSerialization) ProtoMessage() {}
func (x *QueryRequest_OutputSerialization) ProtoReflect() protoreflect.Message {
mi := &file_volume_server_proto_msgTypes[114]
mi := &file_volume_server_proto_msgTypes[123]
if x != nil {
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
if ms.LoadMessageInfo() == nil {
@@ -6666,7 +7110,7 @@ type QueryRequest_InputSerialization_CSVInput struct {
func (x *QueryRequest_InputSerialization_CSVInput) Reset() {
*x = QueryRequest_InputSerialization_CSVInput{}
mi := &file_volume_server_proto_msgTypes[115]
mi := &file_volume_server_proto_msgTypes[124]
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
ms.StoreMessageInfo(mi)
}
@@ -6678,7 +7122,7 @@ func (x *QueryRequest_InputSerialization_CSVInput) String() string {
func (*QueryRequest_InputSerialization_CSVInput) ProtoMessage() {}
func (x *QueryRequest_InputSerialization_CSVInput) ProtoReflect() protoreflect.Message {
mi := &file_volume_server_proto_msgTypes[115]
mi := &file_volume_server_proto_msgTypes[124]
if x != nil {
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
if ms.LoadMessageInfo() == nil {
@@ -6752,7 +7196,7 @@ type QueryRequest_InputSerialization_JSONInput struct {
func (x *QueryRequest_InputSerialization_JSONInput) Reset() {
*x = QueryRequest_InputSerialization_JSONInput{}
mi := &file_volume_server_proto_msgTypes[116]
mi := &file_volume_server_proto_msgTypes[125]
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
ms.StoreMessageInfo(mi)
}
@@ -6764,7 +7208,7 @@ func (x *QueryRequest_InputSerialization_JSONInput) String() string {
func (*QueryRequest_InputSerialization_JSONInput) ProtoMessage() {}
func (x *QueryRequest_InputSerialization_JSONInput) ProtoReflect() protoreflect.Message {
mi := &file_volume_server_proto_msgTypes[116]
mi := &file_volume_server_proto_msgTypes[125]
if x != nil {
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
if ms.LoadMessageInfo() == nil {
@@ -6795,7 +7239,7 @@ type QueryRequest_InputSerialization_ParquetInput struct {
func (x *QueryRequest_InputSerialization_ParquetInput) Reset() {
*x = QueryRequest_InputSerialization_ParquetInput{}
mi := &file_volume_server_proto_msgTypes[117]
mi := &file_volume_server_proto_msgTypes[126]
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
ms.StoreMessageInfo(mi)
}
@@ -6807,7 +7251,7 @@ func (x *QueryRequest_InputSerialization_ParquetInput) String() string {
func (*QueryRequest_InputSerialization_ParquetInput) ProtoMessage() {}
func (x *QueryRequest_InputSerialization_ParquetInput) ProtoReflect() protoreflect.Message {
mi := &file_volume_server_proto_msgTypes[117]
mi := &file_volume_server_proto_msgTypes[126]
if x != nil {
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
if ms.LoadMessageInfo() == nil {
@@ -6836,7 +7280,7 @@ type QueryRequest_OutputSerialization_CSVOutput struct {
func (x *QueryRequest_OutputSerialization_CSVOutput) Reset() {
*x = QueryRequest_OutputSerialization_CSVOutput{}
mi := &file_volume_server_proto_msgTypes[118]
mi := &file_volume_server_proto_msgTypes[127]
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
ms.StoreMessageInfo(mi)
}
@@ -6848,7 +7292,7 @@ func (x *QueryRequest_OutputSerialization_CSVOutput) String() string {
func (*QueryRequest_OutputSerialization_CSVOutput) ProtoMessage() {}
func (x *QueryRequest_OutputSerialization_CSVOutput) ProtoReflect() protoreflect.Message {
mi := &file_volume_server_proto_msgTypes[118]
mi := &file_volume_server_proto_msgTypes[127]
if x != nil {
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
if ms.LoadMessageInfo() == nil {
@@ -6908,7 +7352,7 @@ type QueryRequest_OutputSerialization_JSONOutput struct {
func (x *QueryRequest_OutputSerialization_JSONOutput) Reset() {
*x = QueryRequest_OutputSerialization_JSONOutput{}
mi := &file_volume_server_proto_msgTypes[119]
mi := &file_volume_server_proto_msgTypes[128]
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
ms.StoreMessageInfo(mi)
}
@@ -6920,7 +7364,7 @@ func (x *QueryRequest_OutputSerialization_JSONOutput) String() string {
func (*QueryRequest_OutputSerialization_JSONOutput) ProtoMessage() {}
func (x *QueryRequest_OutputSerialization_JSONOutput) ProtoReflect() protoreflect.Message {
mi := &file_volume_server_proto_msgTypes[119]
mi := &file_volume_server_proto_msgTypes[128]
if x != nil {
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
if ms.LoadMessageInfo() == nil {
@@ -7451,20 +7895,54 @@ const file_volume_server_proto_rawDesc = "" +
"\x04name\x18\x01 \x01(\tR\x04name\x12\x1d\n" +
"\n" +
"size_bytes\x18\x02 \x01(\x04R\tsizeBytes\x12\x1b\n" +
"\tdisk_type\x18\x03 \x01(\tR\bdiskType\"b\n" +
"\tdisk_type\x18\x03 \x01(\tR\bdiskType\"\xea\x01\n" +
"\x1bAllocateBlockVolumeResponse\x12\x12\n" +
"\x04path\x18\x01 \x01(\tR\x04path\x12\x10\n" +
"\x03iqn\x18\x02 \x01(\tR\x03iqn\x12\x1d\n" +
"\n" +
"iscsi_addr\x18\x03 \x01(\tR\tiscsiAddr\":\n" +
"iscsi_addr\x18\x03 \x01(\tR\tiscsiAddr\x12*\n" +
"\x11replica_data_addr\x18\x04 \x01(\tR\x0freplicaDataAddr\x12*\n" +
"\x11replica_ctrl_addr\x18\x05 \x01(\tR\x0freplicaCtrlAddr\x12.\n" +
"\x13rebuild_listen_addr\x18\x06 \x01(\tR\x11rebuildListenAddr\":\n" +
"$VolumeServerDeleteBlockVolumeRequest\x12\x12\n" +
"\x04name\x18\x01 \x01(\tR\x04name\"'\n" +
"%VolumeServerDeleteBlockVolumeResponse*>\n" +
"%VolumeServerDeleteBlockVolumeResponse\"Q\n" +
"\x1aSnapshotBlockVolumeRequest\x12\x12\n" +
"\x04name\x18\x01 \x01(\tR\x04name\x12\x1f\n" +
"\vsnapshot_id\x18\x02 \x01(\rR\n" +
"snapshotId\"|\n" +
"\x1bSnapshotBlockVolumeResponse\x12\x1f\n" +
"\vsnapshot_id\x18\x01 \x01(\rR\n" +
"snapshotId\x12\x1d\n" +
"\n" +
"created_at\x18\x02 \x01(\x03R\tcreatedAt\x12\x1d\n" +
"\n" +
"size_bytes\x18\x03 \x01(\x04R\tsizeBytes\"Q\n" +
"\x1aDeleteBlockSnapshotRequest\x12\x12\n" +
"\x04name\x18\x01 \x01(\tR\x04name\x12\x1f\n" +
"\vsnapshot_id\x18\x02 \x01(\rR\n" +
"snapshotId\"\x1d\n" +
"\x1bDeleteBlockSnapshotResponse\"/\n" +
"\x19ListBlockSnapshotsRequest\x12\x12\n" +
"\x04name\x18\x01 \x01(\tR\x04name\"_\n" +
"\x1aListBlockSnapshotsResponse\x12A\n" +
"\tsnapshots\x18\x01 \x03(\v2#.volume_server_pb.BlockSnapshotInfoR\tsnapshots\"\x7f\n" +
"\x11BlockSnapshotInfo\x12\x1f\n" +
"\vsnapshot_id\x18\x01 \x01(\rR\n" +
"snapshotId\x12\x1d\n" +
"\n" +
"created_at\x18\x02 \x01(\x03R\tcreatedAt\x12*\n" +
"\x11volume_size_bytes\x18\x03 \x01(\x04R\x0fvolumeSizeBytes\"T\n" +
"\x18ExpandBlockVolumeRequest\x12\x12\n" +
"\x04name\x18\x01 \x01(\tR\x04name\x12$\n" +
"\x0enew_size_bytes\x18\x02 \x01(\x04R\fnewSizeBytes\"B\n" +
"\x19ExpandBlockVolumeResponse\x12%\n" +
"\x0ecapacity_bytes\x18\x01 \x01(\x04R\rcapacityBytes*>\n" +
"\x0fVolumeScrubMode\x12\v\n" +
"\aUNKNOWN\x10\x00\x12\t\n" +
"\x05INDEX\x10\x01\x12\b\n" +
"\x04FULL\x10\x02\x12\t\n" +
"\x05LOCAL\x10\x032\x86+\n" +
"\x05LOCAL\x10\x032\xd5.\n" +
"\fVolumeServer\x12\\\n" +
"\vBatchDelete\x12$.volume_server_pb.BatchDeleteRequest\x1a%.volume_server_pb.BatchDeleteResponse\"\x00\x12n\n" +
"\x11VacuumVolumeCheck\x12*.volume_server_pb.VacuumVolumeCheckRequest\x1a+.volume_server_pb.VacuumVolumeCheckResponse\"\x00\x12v\n" +
@@ -7516,7 +7994,11 @@ const file_volume_server_proto_rawDesc = "" +
"\x12VolumeNeedleStatus\x12+.volume_server_pb.VolumeNeedleStatusRequest\x1a,.volume_server_pb.VolumeNeedleStatusResponse\"\x00\x12G\n" +
"\x04Ping\x12\x1d.volume_server_pb.PingRequest\x1a\x1e.volume_server_pb.PingResponse\"\x00\x12t\n" +
"\x13AllocateBlockVolume\x12,.volume_server_pb.AllocateBlockVolumeRequest\x1a-.volume_server_pb.AllocateBlockVolumeResponse\"\x00\x12\x92\x01\n" +
"\x1dVolumeServerDeleteBlockVolume\x126.volume_server_pb.VolumeServerDeleteBlockVolumeRequest\x1a7.volume_server_pb.VolumeServerDeleteBlockVolumeResponse\"\x00B9Z7github.com/seaweedfs/seaweedfs/weed/pb/volume_server_pbb\x06proto3"
"\x1dVolumeServerDeleteBlockVolume\x126.volume_server_pb.VolumeServerDeleteBlockVolumeRequest\x1a7.volume_server_pb.VolumeServerDeleteBlockVolumeResponse\"\x00\x12t\n" +
"\x13SnapshotBlockVolume\x12,.volume_server_pb.SnapshotBlockVolumeRequest\x1a-.volume_server_pb.SnapshotBlockVolumeResponse\"\x00\x12t\n" +
"\x13DeleteBlockSnapshot\x12,.volume_server_pb.DeleteBlockSnapshotRequest\x1a-.volume_server_pb.DeleteBlockSnapshotResponse\"\x00\x12q\n" +
"\x12ListBlockSnapshots\x12+.volume_server_pb.ListBlockSnapshotsRequest\x1a,.volume_server_pb.ListBlockSnapshotsResponse\"\x00\x12n\n" +
"\x11ExpandBlockVolume\x12*.volume_server_pb.ExpandBlockVolumeRequest\x1a+.volume_server_pb.ExpandBlockVolumeResponse\"\x00B9Z7github.com/seaweedfs/seaweedfs/weed/pb/volume_server_pbb\x06proto3"
var (
file_volume_server_proto_rawDescOnce sync.Once
@@ -7531,7 +8013,7 @@ func file_volume_server_proto_rawDescGZIP() []byte {
}
var file_volume_server_proto_enumTypes = make([]protoimpl.EnumInfo, 1)
var file_volume_server_proto_msgTypes = make([]protoimpl.MessageInfo, 120)
var file_volume_server_proto_msgTypes = make([]protoimpl.MessageInfo, 129)
var file_volume_server_proto_goTypes = []any{
(VolumeScrubMode)(0), // 0: volume_server_pb.VolumeScrubMode
(*VolumeServerState)(nil), // 1: volume_server_pb.VolumeServerState
@@ -7645,17 +8127,26 @@ var file_volume_server_proto_goTypes = []any{
(*AllocateBlockVolumeResponse)(nil), // 109: volume_server_pb.AllocateBlockVolumeResponse
(*VolumeServerDeleteBlockVolumeRequest)(nil), // 110: volume_server_pb.VolumeServerDeleteBlockVolumeRequest
(*VolumeServerDeleteBlockVolumeResponse)(nil), // 111: volume_server_pb.VolumeServerDeleteBlockVolumeResponse
(*FetchAndWriteNeedleRequest_Replica)(nil), // 112: volume_server_pb.FetchAndWriteNeedleRequest.Replica
(*QueryRequest_Filter)(nil), // 113: volume_server_pb.QueryRequest.Filter
(*QueryRequest_InputSerialization)(nil), // 114: volume_server_pb.QueryRequest.InputSerialization
(*QueryRequest_OutputSerialization)(nil), // 115: volume_server_pb.QueryRequest.OutputSerialization
(*QueryRequest_InputSerialization_CSVInput)(nil), // 116: volume_server_pb.QueryRequest.InputSerialization.CSVInput
(*QueryRequest_InputSerialization_JSONInput)(nil), // 117: volume_server_pb.QueryRequest.InputSerialization.JSONInput
(*QueryRequest_InputSerialization_ParquetInput)(nil), // 118: volume_server_pb.QueryRequest.InputSerialization.ParquetInput
(*QueryRequest_OutputSerialization_CSVOutput)(nil), // 119: volume_server_pb.QueryRequest.OutputSerialization.CSVOutput
(*QueryRequest_OutputSerialization_JSONOutput)(nil), // 120: volume_server_pb.QueryRequest.OutputSerialization.JSONOutput
(*remote_pb.RemoteConf)(nil), // 121: remote_pb.RemoteConf
(*remote_pb.RemoteStorageLocation)(nil), // 122: remote_pb.RemoteStorageLocation
(*SnapshotBlockVolumeRequest)(nil), // 112: volume_server_pb.SnapshotBlockVolumeRequest
(*SnapshotBlockVolumeResponse)(nil), // 113: volume_server_pb.SnapshotBlockVolumeResponse
(*DeleteBlockSnapshotRequest)(nil), // 114: volume_server_pb.DeleteBlockSnapshotRequest
(*DeleteBlockSnapshotResponse)(nil), // 115: volume_server_pb.DeleteBlockSnapshotResponse
(*ListBlockSnapshotsRequest)(nil), // 116: volume_server_pb.ListBlockSnapshotsRequest
(*ListBlockSnapshotsResponse)(nil), // 117: volume_server_pb.ListBlockSnapshotsResponse
(*BlockSnapshotInfo)(nil), // 118: volume_server_pb.BlockSnapshotInfo
(*ExpandBlockVolumeRequest)(nil), // 119: volume_server_pb.ExpandBlockVolumeRequest
(*ExpandBlockVolumeResponse)(nil), // 120: volume_server_pb.ExpandBlockVolumeResponse
(*FetchAndWriteNeedleRequest_Replica)(nil), // 121: volume_server_pb.FetchAndWriteNeedleRequest.Replica
(*QueryRequest_Filter)(nil), // 122: volume_server_pb.QueryRequest.Filter
(*QueryRequest_InputSerialization)(nil), // 123: volume_server_pb.QueryRequest.InputSerialization
(*QueryRequest_OutputSerialization)(nil), // 124: volume_server_pb.QueryRequest.OutputSerialization
(*QueryRequest_InputSerialization_CSVInput)(nil), // 125: volume_server_pb.QueryRequest.InputSerialization.CSVInput
(*QueryRequest_InputSerialization_JSONInput)(nil), // 126: volume_server_pb.QueryRequest.InputSerialization.JSONInput
(*QueryRequest_InputSerialization_ParquetInput)(nil), // 127: volume_server_pb.QueryRequest.InputSerialization.ParquetInput
(*QueryRequest_OutputSerialization_CSVOutput)(nil), // 128: volume_server_pb.QueryRequest.OutputSerialization.CSVOutput
(*QueryRequest_OutputSerialization_JSONOutput)(nil), // 129: volume_server_pb.QueryRequest.OutputSerialization.JSONOutput
(*remote_pb.RemoteConf)(nil), // 130: remote_pb.RemoteConf
(*remote_pb.RemoteStorageLocation)(nil), // 131: remote_pb.RemoteStorageLocation
}
var file_volume_server_proto_depIdxs = []int32{
4, // 0: volume_server_pb.BatchDeleteResponse.results:type_name -> volume_server_pb.DeleteResult
@@ -7671,125 +8162,134 @@ var file_volume_server_proto_depIdxs = []int32{
82, // 10: volume_server_pb.VolumeServerStatusResponse.disk_statuses:type_name -> volume_server_pb.DiskStatus
83, // 11: volume_server_pb.VolumeServerStatusResponse.memory_status:type_name -> volume_server_pb.MemStatus
1, // 12: volume_server_pb.VolumeServerStatusResponse.state:type_name -> volume_server_pb.VolumeServerState
112, // 13: volume_server_pb.FetchAndWriteNeedleRequest.replicas:type_name -> volume_server_pb.FetchAndWriteNeedleRequest.Replica
121, // 14: volume_server_pb.FetchAndWriteNeedleRequest.remote_conf:type_name -> remote_pb.RemoteConf
122, // 15: volume_server_pb.FetchAndWriteNeedleRequest.remote_location:type_name -> remote_pb.RemoteStorageLocation
121, // 13: volume_server_pb.FetchAndWriteNeedleRequest.replicas:type_name -> volume_server_pb.FetchAndWriteNeedleRequest.Replica
130, // 14: volume_server_pb.FetchAndWriteNeedleRequest.remote_conf:type_name -> remote_pb.RemoteConf
131, // 15: volume_server_pb.FetchAndWriteNeedleRequest.remote_location:type_name -> remote_pb.RemoteStorageLocation
0, // 16: volume_server_pb.ScrubVolumeRequest.mode:type_name -> volume_server_pb.VolumeScrubMode
0, // 17: volume_server_pb.ScrubEcVolumeRequest.mode:type_name -> volume_server_pb.VolumeScrubMode
79, // 18: volume_server_pb.ScrubEcVolumeResponse.broken_shard_infos:type_name -> volume_server_pb.EcShardInfo
113, // 19: volume_server_pb.QueryRequest.filter:type_name -> volume_server_pb.QueryRequest.Filter
114, // 20: volume_server_pb.QueryRequest.input_serialization:type_name -> volume_server_pb.QueryRequest.InputSerialization
115, // 21: volume_server_pb.QueryRequest.output_serialization:type_name -> volume_server_pb.QueryRequest.OutputSerialization
116, // 22: volume_server_pb.QueryRequest.InputSerialization.csv_input:type_name -> volume_server_pb.QueryRequest.InputSerialization.CSVInput
117, // 23: volume_server_pb.QueryRequest.InputSerialization.json_input:type_name -> volume_server_pb.QueryRequest.InputSerialization.JSONInput
118, // 24: volume_server_pb.QueryRequest.InputSerialization.parquet_input:type_name -> volume_server_pb.QueryRequest.InputSerialization.ParquetInput
119, // 25: volume_server_pb.QueryRequest.OutputSerialization.csv_output:type_name -> volume_server_pb.QueryRequest.OutputSerialization.CSVOutput
120, // 26: volume_server_pb.QueryRequest.OutputSerialization.json_output:type_name -> volume_server_pb.QueryRequest.OutputSerialization.JSONOutput
2, // 27: volume_server_pb.VolumeServer.BatchDelete:input_type -> volume_server_pb.BatchDeleteRequest
6, // 28: volume_server_pb.VolumeServer.VacuumVolumeCheck:input_type -> volume_server_pb.VacuumVolumeCheckRequest
8, // 29: volume_server_pb.VolumeServer.VacuumVolumeCompact:input_type -> volume_server_pb.VacuumVolumeCompactRequest
10, // 30: volume_server_pb.VolumeServer.VacuumVolumeCommit:input_type -> volume_server_pb.VacuumVolumeCommitRequest
12, // 31: volume_server_pb.VolumeServer.VacuumVolumeCleanup:input_type -> volume_server_pb.VacuumVolumeCleanupRequest
14, // 32: volume_server_pb.VolumeServer.DeleteCollection:input_type -> volume_server_pb.DeleteCollectionRequest
16, // 33: volume_server_pb.VolumeServer.AllocateVolume:input_type -> volume_server_pb.AllocateVolumeRequest
18, // 34: volume_server_pb.VolumeServer.VolumeSyncStatus:input_type -> volume_server_pb.VolumeSyncStatusRequest
20, // 35: volume_server_pb.VolumeServer.VolumeIncrementalCopy:input_type -> volume_server_pb.VolumeIncrementalCopyRequest
22, // 36: volume_server_pb.VolumeServer.VolumeMount:input_type -> volume_server_pb.VolumeMountRequest
24, // 37: volume_server_pb.VolumeServer.VolumeUnmount:input_type -> volume_server_pb.VolumeUnmountRequest
26, // 38: volume_server_pb.VolumeServer.VolumeDelete:input_type -> volume_server_pb.VolumeDeleteRequest
28, // 39: volume_server_pb.VolumeServer.VolumeMarkReadonly:input_type -> volume_server_pb.VolumeMarkReadonlyRequest
30, // 40: volume_server_pb.VolumeServer.VolumeMarkWritable:input_type -> volume_server_pb.VolumeMarkWritableRequest
32, // 41: volume_server_pb.VolumeServer.VolumeConfigure:input_type -> volume_server_pb.VolumeConfigureRequest
34, // 42: volume_server_pb.VolumeServer.VolumeStatus:input_type -> volume_server_pb.VolumeStatusRequest
36, // 43: volume_server_pb.VolumeServer.GetState:input_type -> volume_server_pb.GetStateRequest
38, // 44: volume_server_pb.VolumeServer.SetState:input_type -> volume_server_pb.SetStateRequest
40, // 45: volume_server_pb.VolumeServer.VolumeCopy:input_type -> volume_server_pb.VolumeCopyRequest
80, // 46: volume_server_pb.VolumeServer.ReadVolumeFileStatus:input_type -> volume_server_pb.ReadVolumeFileStatusRequest
42, // 47: volume_server_pb.VolumeServer.CopyFile:input_type -> volume_server_pb.CopyFileRequest
44, // 48: volume_server_pb.VolumeServer.ReceiveFile:input_type -> volume_server_pb.ReceiveFileRequest
47, // 49: volume_server_pb.VolumeServer.ReadNeedleBlob:input_type -> volume_server_pb.ReadNeedleBlobRequest
49, // 50: volume_server_pb.VolumeServer.ReadNeedleMeta:input_type -> volume_server_pb.ReadNeedleMetaRequest
51, // 51: volume_server_pb.VolumeServer.WriteNeedleBlob:input_type -> volume_server_pb.WriteNeedleBlobRequest
53, // 52: volume_server_pb.VolumeServer.ReadAllNeedles:input_type -> volume_server_pb.ReadAllNeedlesRequest
55, // 53: volume_server_pb.VolumeServer.VolumeTailSender:input_type -> volume_server_pb.VolumeTailSenderRequest
57, // 54: volume_server_pb.VolumeServer.VolumeTailReceiver:input_type -> volume_server_pb.VolumeTailReceiverRequest
59, // 55: volume_server_pb.VolumeServer.VolumeEcShardsGenerate:input_type -> volume_server_pb.VolumeEcShardsGenerateRequest
61, // 56: volume_server_pb.VolumeServer.VolumeEcShardsRebuild:input_type -> volume_server_pb.VolumeEcShardsRebuildRequest
63, // 57: volume_server_pb.VolumeServer.VolumeEcShardsCopy:input_type -> volume_server_pb.VolumeEcShardsCopyRequest
65, // 58: volume_server_pb.VolumeServer.VolumeEcShardsDelete:input_type -> volume_server_pb.VolumeEcShardsDeleteRequest
67, // 59: volume_server_pb.VolumeServer.VolumeEcShardsMount:input_type -> volume_server_pb.VolumeEcShardsMountRequest
69, // 60: volume_server_pb.VolumeServer.VolumeEcShardsUnmount:input_type -> volume_server_pb.VolumeEcShardsUnmountRequest
71, // 61: volume_server_pb.VolumeServer.VolumeEcShardRead:input_type -> volume_server_pb.VolumeEcShardReadRequest
73, // 62: volume_server_pb.VolumeServer.VolumeEcBlobDelete:input_type -> volume_server_pb.VolumeEcBlobDeleteRequest
75, // 63: volume_server_pb.VolumeServer.VolumeEcShardsToVolume:input_type -> volume_server_pb.VolumeEcShardsToVolumeRequest
77, // 64: volume_server_pb.VolumeServer.VolumeEcShardsInfo:input_type -> volume_server_pb.VolumeEcShardsInfoRequest
88, // 65: volume_server_pb.VolumeServer.VolumeTierMoveDatToRemote:input_type -> volume_server_pb.VolumeTierMoveDatToRemoteRequest
90, // 66: volume_server_pb.VolumeServer.VolumeTierMoveDatFromRemote:input_type -> volume_server_pb.VolumeTierMoveDatFromRemoteRequest
92, // 67: volume_server_pb.VolumeServer.VolumeServerStatus:input_type -> volume_server_pb.VolumeServerStatusRequest
94, // 68: volume_server_pb.VolumeServer.VolumeServerLeave:input_type -> volume_server_pb.VolumeServerLeaveRequest
96, // 69: volume_server_pb.VolumeServer.FetchAndWriteNeedle:input_type -> volume_server_pb.FetchAndWriteNeedleRequest
98, // 70: volume_server_pb.VolumeServer.ScrubVolume:input_type -> volume_server_pb.ScrubVolumeRequest
100, // 71: volume_server_pb.VolumeServer.ScrubEcVolume:input_type -> volume_server_pb.ScrubEcVolumeRequest
102, // 72: volume_server_pb.VolumeServer.Query:input_type -> volume_server_pb.QueryRequest
104, // 73: volume_server_pb.VolumeServer.VolumeNeedleStatus:input_type -> volume_server_pb.VolumeNeedleStatusRequest
106, // 74: volume_server_pb.VolumeServer.Ping:input_type -> volume_server_pb.PingRequest
108, // 75: volume_server_pb.VolumeServer.AllocateBlockVolume:input_type -> volume_server_pb.AllocateBlockVolumeRequest
110, // 76: volume_server_pb.VolumeServer.VolumeServerDeleteBlockVolume:input_type -> volume_server_pb.VolumeServerDeleteBlockVolumeRequest
3, // 77: volume_server_pb.VolumeServer.BatchDelete:output_type -> volume_server_pb.BatchDeleteResponse
7, // 78: volume_server_pb.VolumeServer.VacuumVolumeCheck:output_type -> volume_server_pb.VacuumVolumeCheckResponse
9, // 79: volume_server_pb.VolumeServer.VacuumVolumeCompact:output_type -> volume_server_pb.VacuumVolumeCompactResponse
11, // 80: volume_server_pb.VolumeServer.VacuumVolumeCommit:output_type -> volume_server_pb.VacuumVolumeCommitResponse
13, // 81: volume_server_pb.VolumeServer.VacuumVolumeCleanup:output_type -> volume_server_pb.VacuumVolumeCleanupResponse
15, // 82: volume_server_pb.VolumeServer.DeleteCollection:output_type -> volume_server_pb.DeleteCollectionResponse
17, // 83: volume_server_pb.VolumeServer.AllocateVolume:output_type -> volume_server_pb.AllocateVolumeResponse
19, // 84: volume_server_pb.VolumeServer.VolumeSyncStatus:output_type -> volume_server_pb.VolumeSyncStatusResponse
21, // 85: volume_server_pb.VolumeServer.VolumeIncrementalCopy:output_type -> volume_server_pb.VolumeIncrementalCopyResponse
23, // 86: volume_server_pb.VolumeServer.VolumeMount:output_type -> volume_server_pb.VolumeMountResponse
25, // 87: volume_server_pb.VolumeServer.VolumeUnmount:output_type -> volume_server_pb.VolumeUnmountResponse
27, // 88: volume_server_pb.VolumeServer.VolumeDelete:output_type -> volume_server_pb.VolumeDeleteResponse
29, // 89: volume_server_pb.VolumeServer.VolumeMarkReadonly:output_type -> volume_server_pb.VolumeMarkReadonlyResponse
31, // 90: volume_server_pb.VolumeServer.VolumeMarkWritable:output_type -> volume_server_pb.VolumeMarkWritableResponse
33, // 91: volume_server_pb.VolumeServer.VolumeConfigure:output_type -> volume_server_pb.VolumeConfigureResponse
35, // 92: volume_server_pb.VolumeServer.VolumeStatus:output_type -> volume_server_pb.VolumeStatusResponse
37, // 93: volume_server_pb.VolumeServer.GetState:output_type -> volume_server_pb.GetStateResponse
39, // 94: volume_server_pb.VolumeServer.SetState:output_type -> volume_server_pb.SetStateResponse
41, // 95: volume_server_pb.VolumeServer.VolumeCopy:output_type -> volume_server_pb.VolumeCopyResponse
81, // 96: volume_server_pb.VolumeServer.ReadVolumeFileStatus:output_type -> volume_server_pb.ReadVolumeFileStatusResponse
43, // 97: volume_server_pb.VolumeServer.CopyFile:output_type -> volume_server_pb.CopyFileResponse
46, // 98: volume_server_pb.VolumeServer.ReceiveFile:output_type -> volume_server_pb.ReceiveFileResponse
48, // 99: volume_server_pb.VolumeServer.ReadNeedleBlob:output_type -> volume_server_pb.ReadNeedleBlobResponse
50, // 100: volume_server_pb.VolumeServer.ReadNeedleMeta:output_type -> volume_server_pb.ReadNeedleMetaResponse
52, // 101: volume_server_pb.VolumeServer.WriteNeedleBlob:output_type -> volume_server_pb.WriteNeedleBlobResponse
54, // 102: volume_server_pb.VolumeServer.ReadAllNeedles:output_type -> volume_server_pb.ReadAllNeedlesResponse
56, // 103: volume_server_pb.VolumeServer.VolumeTailSender:output_type -> volume_server_pb.VolumeTailSenderResponse
58, // 104: volume_server_pb.VolumeServer.VolumeTailReceiver:output_type -> volume_server_pb.VolumeTailReceiverResponse
60, // 105: volume_server_pb.VolumeServer.VolumeEcShardsGenerate:output_type -> volume_server_pb.VolumeEcShardsGenerateResponse
62, // 106: volume_server_pb.VolumeServer.VolumeEcShardsRebuild:output_type -> volume_server_pb.VolumeEcShardsRebuildResponse
64, // 107: volume_server_pb.VolumeServer.VolumeEcShardsCopy:output_type -> volume_server_pb.VolumeEcShardsCopyResponse
66, // 108: volume_server_pb.VolumeServer.VolumeEcShardsDelete:output_type -> volume_server_pb.VolumeEcShardsDeleteResponse
68, // 109: volume_server_pb.VolumeServer.VolumeEcShardsMount:output_type -> volume_server_pb.VolumeEcShardsMountResponse
70, // 110: volume_server_pb.VolumeServer.VolumeEcShardsUnmount:output_type -> volume_server_pb.VolumeEcShardsUnmountResponse
72, // 111: volume_server_pb.VolumeServer.VolumeEcShardRead:output_type -> volume_server_pb.VolumeEcShardReadResponse
74, // 112: volume_server_pb.VolumeServer.VolumeEcBlobDelete:output_type -> volume_server_pb.VolumeEcBlobDeleteResponse
76, // 113: volume_server_pb.VolumeServer.VolumeEcShardsToVolume:output_type -> volume_server_pb.VolumeEcShardsToVolumeResponse
78, // 114: volume_server_pb.VolumeServer.VolumeEcShardsInfo:output_type -> volume_server_pb.VolumeEcShardsInfoResponse
89, // 115: volume_server_pb.VolumeServer.VolumeTierMoveDatToRemote:output_type -> volume_server_pb.VolumeTierMoveDatToRemoteResponse
91, // 116: volume_server_pb.VolumeServer.VolumeTierMoveDatFromRemote:output_type -> volume_server_pb.VolumeTierMoveDatFromRemoteResponse
93, // 117: volume_server_pb.VolumeServer.VolumeServerStatus:output_type -> volume_server_pb.VolumeServerStatusResponse
95, // 118: volume_server_pb.VolumeServer.VolumeServerLeave:output_type -> volume_server_pb.VolumeServerLeaveResponse
97, // 119: volume_server_pb.VolumeServer.FetchAndWriteNeedle:output_type -> volume_server_pb.FetchAndWriteNeedleResponse
99, // 120: volume_server_pb.VolumeServer.ScrubVolume:output_type -> volume_server_pb.ScrubVolumeResponse
101, // 121: volume_server_pb.VolumeServer.ScrubEcVolume:output_type -> volume_server_pb.ScrubEcVolumeResponse
103, // 122: volume_server_pb.VolumeServer.Query:output_type -> volume_server_pb.QueriedStripe
105, // 123: volume_server_pb.VolumeServer.VolumeNeedleStatus:output_type -> volume_server_pb.VolumeNeedleStatusResponse
107, // 124: volume_server_pb.VolumeServer.Ping:output_type -> volume_server_pb.PingResponse
109, // 125: volume_server_pb.VolumeServer.AllocateBlockVolume:output_type -> volume_server_pb.AllocateBlockVolumeResponse
111, // 126: volume_server_pb.VolumeServer.VolumeServerDeleteBlockVolume:output_type -> volume_server_pb.VolumeServerDeleteBlockVolumeResponse
77, // [77:127] is the sub-list for method output_type
27, // [27:77] is the sub-list for method input_type
27, // [27:27] is the sub-list for extension type_name
27, // [27:27] is the sub-list for extension extendee
0, // [0:27] is the sub-list for field type_name
122, // 19: volume_server_pb.QueryRequest.filter:type_name -> volume_server_pb.QueryRequest.Filter
123, // 20: volume_server_pb.QueryRequest.input_serialization:type_name -> volume_server_pb.QueryRequest.InputSerialization
124, // 21: volume_server_pb.QueryRequest.output_serialization:type_name -> volume_server_pb.QueryRequest.OutputSerialization
118, // 22: volume_server_pb.ListBlockSnapshotsResponse.snapshots:type_name -> volume_server_pb.BlockSnapshotInfo
125, // 23: volume_server_pb.QueryRequest.InputSerialization.csv_input:type_name -> volume_server_pb.QueryRequest.InputSerialization.CSVInput
126, // 24: volume_server_pb.QueryRequest.InputSerialization.json_input:type_name -> volume_server_pb.QueryRequest.InputSerialization.JSONInput
127, // 25: volume_server_pb.QueryRequest.InputSerialization.parquet_input:type_name -> volume_server_pb.QueryRequest.InputSerialization.ParquetInput
128, // 26: volume_server_pb.QueryRequest.OutputSerialization.csv_output:type_name -> volume_server_pb.QueryRequest.OutputSerialization.CSVOutput
129, // 27: volume_server_pb.QueryRequest.OutputSerialization.json_output:type_name -> volume_server_pb.QueryRequest.OutputSerialization.JSONOutput
2, // 28: volume_server_pb.VolumeServer.BatchDelete:input_type -> volume_server_pb.BatchDeleteRequest
6, // 29: volume_server_pb.VolumeServer.VacuumVolumeCheck:input_type -> volume_server_pb.VacuumVolumeCheckRequest
8, // 30: volume_server_pb.VolumeServer.VacuumVolumeCompact:input_type -> volume_server_pb.VacuumVolumeCompactRequest
10, // 31: volume_server_pb.VolumeServer.VacuumVolumeCommit:input_type -> volume_server_pb.VacuumVolumeCommitRequest
12, // 32: volume_server_pb.VolumeServer.VacuumVolumeCleanup:input_type -> volume_server_pb.VacuumVolumeCleanupRequest
14, // 33: volume_server_pb.VolumeServer.DeleteCollection:input_type -> volume_server_pb.DeleteCollectionRequest
16, // 34: volume_server_pb.VolumeServer.AllocateVolume:input_type -> volume_server_pb.AllocateVolumeRequest
18, // 35: volume_server_pb.VolumeServer.VolumeSyncStatus:input_type -> volume_server_pb.VolumeSyncStatusRequest
20, // 36: volume_server_pb.VolumeServer.VolumeIncrementalCopy:input_type -> volume_server_pb.VolumeIncrementalCopyRequest
22, // 37: volume_server_pb.VolumeServer.VolumeMount:input_type -> volume_server_pb.VolumeMountRequest
24, // 38: volume_server_pb.VolumeServer.VolumeUnmount:input_type -> volume_server_pb.VolumeUnmountRequest
26, // 39: volume_server_pb.VolumeServer.VolumeDelete:input_type -> volume_server_pb.VolumeDeleteRequest
28, // 40: volume_server_pb.VolumeServer.VolumeMarkReadonly:input_type -> volume_server_pb.VolumeMarkReadonlyRequest
30, // 41: volume_server_pb.VolumeServer.VolumeMarkWritable:input_type -> volume_server_pb.VolumeMarkWritableRequest
32, // 42: volume_server_pb.VolumeServer.VolumeConfigure:input_type -> volume_server_pb.VolumeConfigureRequest
34, // 43: volume_server_pb.VolumeServer.VolumeStatus:input_type -> volume_server_pb.VolumeStatusRequest
36, // 44: volume_server_pb.VolumeServer.GetState:input_type -> volume_server_pb.GetStateRequest
38, // 45: volume_server_pb.VolumeServer.SetState:input_type -> volume_server_pb.SetStateRequest
40, // 46: volume_server_pb.VolumeServer.VolumeCopy:input_type -> volume_server_pb.VolumeCopyRequest
80, // 47: volume_server_pb.VolumeServer.ReadVolumeFileStatus:input_type -> volume_server_pb.ReadVolumeFileStatusRequest
42, // 48: volume_server_pb.VolumeServer.CopyFile:input_type -> volume_server_pb.CopyFileRequest
44, // 49: volume_server_pb.VolumeServer.ReceiveFile:input_type -> volume_server_pb.ReceiveFileRequest
47, // 50: volume_server_pb.VolumeServer.ReadNeedleBlob:input_type -> volume_server_pb.ReadNeedleBlobRequest
49, // 51: volume_server_pb.VolumeServer.ReadNeedleMeta:input_type -> volume_server_pb.ReadNeedleMetaRequest
51, // 52: volume_server_pb.VolumeServer.WriteNeedleBlob:input_type -> volume_server_pb.WriteNeedleBlobRequest
53, // 53: volume_server_pb.VolumeServer.ReadAllNeedles:input_type -> volume_server_pb.ReadAllNeedlesRequest
55, // 54: volume_server_pb.VolumeServer.VolumeTailSender:input_type -> volume_server_pb.VolumeTailSenderRequest
57, // 55: volume_server_pb.VolumeServer.VolumeTailReceiver:input_type -> volume_server_pb.VolumeTailReceiverRequest
59, // 56: volume_server_pb.VolumeServer.VolumeEcShardsGenerate:input_type -> volume_server_pb.VolumeEcShardsGenerateRequest
61, // 57: volume_server_pb.VolumeServer.VolumeEcShardsRebuild:input_type -> volume_server_pb.VolumeEcShardsRebuildRequest
63, // 58: volume_server_pb.VolumeServer.VolumeEcShardsCopy:input_type -> volume_server_pb.VolumeEcShardsCopyRequest
65, // 59: volume_server_pb.VolumeServer.VolumeEcShardsDelete:input_type -> volume_server_pb.VolumeEcShardsDeleteRequest
67, // 60: volume_server_pb.VolumeServer.VolumeEcShardsMount:input_type -> volume_server_pb.VolumeEcShardsMountRequest
69, // 61: volume_server_pb.VolumeServer.VolumeEcShardsUnmount:input_type -> volume_server_pb.VolumeEcShardsUnmountRequest
71, // 62: volume_server_pb.VolumeServer.VolumeEcShardRead:input_type -> volume_server_pb.VolumeEcShardReadRequest
73, // 63: volume_server_pb.VolumeServer.VolumeEcBlobDelete:input_type -> volume_server_pb.VolumeEcBlobDeleteRequest
75, // 64: volume_server_pb.VolumeServer.VolumeEcShardsToVolume:input_type -> volume_server_pb.VolumeEcShardsToVolumeRequest
77, // 65: volume_server_pb.VolumeServer.VolumeEcShardsInfo:input_type -> volume_server_pb.VolumeEcShardsInfoRequest
88, // 66: volume_server_pb.VolumeServer.VolumeTierMoveDatToRemote:input_type -> volume_server_pb.VolumeTierMoveDatToRemoteRequest
90, // 67: volume_server_pb.VolumeServer.VolumeTierMoveDatFromRemote:input_type -> volume_server_pb.VolumeTierMoveDatFromRemoteRequest
92, // 68: volume_server_pb.VolumeServer.VolumeServerStatus:input_type -> volume_server_pb.VolumeServerStatusRequest
94, // 69: volume_server_pb.VolumeServer.VolumeServerLeave:input_type -> volume_server_pb.VolumeServerLeaveRequest
96, // 70: volume_server_pb.VolumeServer.FetchAndWriteNeedle:input_type -> volume_server_pb.FetchAndWriteNeedleRequest
98, // 71: volume_server_pb.VolumeServer.ScrubVolume:input_type -> volume_server_pb.ScrubVolumeRequest
100, // 72: volume_server_pb.VolumeServer.ScrubEcVolume:input_type -> volume_server_pb.ScrubEcVolumeRequest
102, // 73: volume_server_pb.VolumeServer.Query:input_type -> volume_server_pb.QueryRequest
104, // 74: volume_server_pb.VolumeServer.VolumeNeedleStatus:input_type -> volume_server_pb.VolumeNeedleStatusRequest
106, // 75: volume_server_pb.VolumeServer.Ping:input_type -> volume_server_pb.PingRequest
108, // 76: volume_server_pb.VolumeServer.AllocateBlockVolume:input_type -> volume_server_pb.AllocateBlockVolumeRequest
110, // 77: volume_server_pb.VolumeServer.VolumeServerDeleteBlockVolume:input_type -> volume_server_pb.VolumeServerDeleteBlockVolumeRequest
112, // 78: volume_server_pb.VolumeServer.SnapshotBlockVolume:input_type -> volume_server_pb.SnapshotBlockVolumeRequest
114, // 79: volume_server_pb.VolumeServer.DeleteBlockSnapshot:input_type -> volume_server_pb.DeleteBlockSnapshotRequest
116, // 80: volume_server_pb.VolumeServer.ListBlockSnapshots:input_type -> volume_server_pb.ListBlockSnapshotsRequest
119, // 81: volume_server_pb.VolumeServer.ExpandBlockVolume:input_type -> volume_server_pb.ExpandBlockVolumeRequest
3, // 82: volume_server_pb.VolumeServer.BatchDelete:output_type -> volume_server_pb.BatchDeleteResponse
7, // 83: volume_server_pb.VolumeServer.VacuumVolumeCheck:output_type -> volume_server_pb.VacuumVolumeCheckResponse
9, // 84: volume_server_pb.VolumeServer.VacuumVolumeCompact:output_type -> volume_server_pb.VacuumVolumeCompactResponse
11, // 85: volume_server_pb.VolumeServer.VacuumVolumeCommit:output_type -> volume_server_pb.VacuumVolumeCommitResponse
13, // 86: volume_server_pb.VolumeServer.VacuumVolumeCleanup:output_type -> volume_server_pb.VacuumVolumeCleanupResponse
15, // 87: volume_server_pb.VolumeServer.DeleteCollection:output_type -> volume_server_pb.DeleteCollectionResponse
17, // 88: volume_server_pb.VolumeServer.AllocateVolume:output_type -> volume_server_pb.AllocateVolumeResponse
19, // 89: volume_server_pb.VolumeServer.VolumeSyncStatus:output_type -> volume_server_pb.VolumeSyncStatusResponse
21, // 90: volume_server_pb.VolumeServer.VolumeIncrementalCopy:output_type -> volume_server_pb.VolumeIncrementalCopyResponse
23, // 91: volume_server_pb.VolumeServer.VolumeMount:output_type -> volume_server_pb.VolumeMountResponse
25, // 92: volume_server_pb.VolumeServer.VolumeUnmount:output_type -> volume_server_pb.VolumeUnmountResponse
27, // 93: volume_server_pb.VolumeServer.VolumeDelete:output_type -> volume_server_pb.VolumeDeleteResponse
29, // 94: volume_server_pb.VolumeServer.VolumeMarkReadonly:output_type -> volume_server_pb.VolumeMarkReadonlyResponse
31, // 95: volume_server_pb.VolumeServer.VolumeMarkWritable:output_type -> volume_server_pb.VolumeMarkWritableResponse
33, // 96: volume_server_pb.VolumeServer.VolumeConfigure:output_type -> volume_server_pb.VolumeConfigureResponse
35, // 97: volume_server_pb.VolumeServer.VolumeStatus:output_type -> volume_server_pb.VolumeStatusResponse
37, // 98: volume_server_pb.VolumeServer.GetState:output_type -> volume_server_pb.GetStateResponse
39, // 99: volume_server_pb.VolumeServer.SetState:output_type -> volume_server_pb.SetStateResponse
41, // 100: volume_server_pb.VolumeServer.VolumeCopy:output_type -> volume_server_pb.VolumeCopyResponse
81, // 101: volume_server_pb.VolumeServer.ReadVolumeFileStatus:output_type -> volume_server_pb.ReadVolumeFileStatusResponse
43, // 102: volume_server_pb.VolumeServer.CopyFile:output_type -> volume_server_pb.CopyFileResponse
46, // 103: volume_server_pb.VolumeServer.ReceiveFile:output_type -> volume_server_pb.ReceiveFileResponse
48, // 104: volume_server_pb.VolumeServer.ReadNeedleBlob:output_type -> volume_server_pb.ReadNeedleBlobResponse
50, // 105: volume_server_pb.VolumeServer.ReadNeedleMeta:output_type -> volume_server_pb.ReadNeedleMetaResponse
52, // 106: volume_server_pb.VolumeServer.WriteNeedleBlob:output_type -> volume_server_pb.WriteNeedleBlobResponse
54, // 107: volume_server_pb.VolumeServer.ReadAllNeedles:output_type -> volume_server_pb.ReadAllNeedlesResponse
56, // 108: volume_server_pb.VolumeServer.VolumeTailSender:output_type -> volume_server_pb.VolumeTailSenderResponse
58, // 109: volume_server_pb.VolumeServer.VolumeTailReceiver:output_type -> volume_server_pb.VolumeTailReceiverResponse
60, // 110: volume_server_pb.VolumeServer.VolumeEcShardsGenerate:output_type -> volume_server_pb.VolumeEcShardsGenerateResponse
62, // 111: volume_server_pb.VolumeServer.VolumeEcShardsRebuild:output_type -> volume_server_pb.VolumeEcShardsRebuildResponse
64, // 112: volume_server_pb.VolumeServer.VolumeEcShardsCopy:output_type -> volume_server_pb.VolumeEcShardsCopyResponse
66, // 113: volume_server_pb.VolumeServer.VolumeEcShardsDelete:output_type -> volume_server_pb.VolumeEcShardsDeleteResponse
68, // 114: volume_server_pb.VolumeServer.VolumeEcShardsMount:output_type -> volume_server_pb.VolumeEcShardsMountResponse
70, // 115: volume_server_pb.VolumeServer.VolumeEcShardsUnmount:output_type -> volume_server_pb.VolumeEcShardsUnmountResponse
72, // 116: volume_server_pb.VolumeServer.VolumeEcShardRead:output_type -> volume_server_pb.VolumeEcShardReadResponse
74, // 117: volume_server_pb.VolumeServer.VolumeEcBlobDelete:output_type -> volume_server_pb.VolumeEcBlobDeleteResponse
76, // 118: volume_server_pb.VolumeServer.VolumeEcShardsToVolume:output_type -> volume_server_pb.VolumeEcShardsToVolumeResponse
78, // 119: volume_server_pb.VolumeServer.VolumeEcShardsInfo:output_type -> volume_server_pb.VolumeEcShardsInfoResponse
89, // 120: volume_server_pb.VolumeServer.VolumeTierMoveDatToRemote:output_type -> volume_server_pb.VolumeTierMoveDatToRemoteResponse
91, // 121: volume_server_pb.VolumeServer.VolumeTierMoveDatFromRemote:output_type -> volume_server_pb.VolumeTierMoveDatFromRemoteResponse
93, // 122: volume_server_pb.VolumeServer.VolumeServerStatus:output_type -> volume_server_pb.VolumeServerStatusResponse
95, // 123: volume_server_pb.VolumeServer.VolumeServerLeave:output_type -> volume_server_pb.VolumeServerLeaveResponse
97, // 124: volume_server_pb.VolumeServer.FetchAndWriteNeedle:output_type -> volume_server_pb.FetchAndWriteNeedleResponse
99, // 125: volume_server_pb.VolumeServer.ScrubVolume:output_type -> volume_server_pb.ScrubVolumeResponse
101, // 126: volume_server_pb.VolumeServer.ScrubEcVolume:output_type -> volume_server_pb.ScrubEcVolumeResponse
103, // 127: volume_server_pb.VolumeServer.Query:output_type -> volume_server_pb.QueriedStripe
105, // 128: volume_server_pb.VolumeServer.VolumeNeedleStatus:output_type -> volume_server_pb.VolumeNeedleStatusResponse
107, // 129: volume_server_pb.VolumeServer.Ping:output_type -> volume_server_pb.PingResponse
109, // 130: volume_server_pb.VolumeServer.AllocateBlockVolume:output_type -> volume_server_pb.AllocateBlockVolumeResponse
111, // 131: volume_server_pb.VolumeServer.VolumeServerDeleteBlockVolume:output_type -> volume_server_pb.VolumeServerDeleteBlockVolumeResponse
113, // 132: volume_server_pb.VolumeServer.SnapshotBlockVolume:output_type -> volume_server_pb.SnapshotBlockVolumeResponse
115, // 133: volume_server_pb.VolumeServer.DeleteBlockSnapshot:output_type -> volume_server_pb.DeleteBlockSnapshotResponse
117, // 134: volume_server_pb.VolumeServer.ListBlockSnapshots:output_type -> volume_server_pb.ListBlockSnapshotsResponse
120, // 135: volume_server_pb.VolumeServer.ExpandBlockVolume:output_type -> volume_server_pb.ExpandBlockVolumeResponse
82, // [82:136] is the sub-list for method output_type
28, // [28:82] is the sub-list for method input_type
28, // [28:28] is the sub-list for extension type_name
28, // [28:28] is the sub-list for extension extendee
0, // [0:28] is the sub-list for field type_name
}
func init() { file_volume_server_proto_init() }
@@ -7807,7 +8307,7 @@ func file_volume_server_proto_init() {
GoPackagePath: reflect.TypeOf(x{}).PkgPath(),
RawDescriptor: unsafe.Slice(unsafe.StringData(file_volume_server_proto_rawDesc), len(file_volume_server_proto_rawDesc)),
NumEnums: 1,
NumMessages: 120,
NumMessages: 129,
NumExtensions: 0,
NumServices: 1,
},
weed/pb/volume_server_pb/volume_server_grpc.pb.go
+152
View File
@@ -69,6 +69,10 @@ const (
VolumeServer_Ping_FullMethodName = "/volume_server_pb.VolumeServer/Ping"
VolumeServer_AllocateBlockVolume_FullMethodName = "/volume_server_pb.VolumeServer/AllocateBlockVolume"
VolumeServer_VolumeServerDeleteBlockVolume_FullMethodName = "/volume_server_pb.VolumeServer/VolumeServerDeleteBlockVolume"
VolumeServer_SnapshotBlockVolume_FullMethodName = "/volume_server_pb.VolumeServer/SnapshotBlockVolume"
VolumeServer_DeleteBlockSnapshot_FullMethodName = "/volume_server_pb.VolumeServer/DeleteBlockSnapshot"
VolumeServer_ListBlockSnapshots_FullMethodName = "/volume_server_pb.VolumeServer/ListBlockSnapshots"
VolumeServer_ExpandBlockVolume_FullMethodName = "/volume_server_pb.VolumeServer/ExpandBlockVolume"
)
// VolumeServerClient is the client API for VolumeServer service.
@@ -133,6 +137,10 @@ type VolumeServerClient interface {
// block volumes
AllocateBlockVolume(ctx context.Context, in *AllocateBlockVolumeRequest, opts ...grpc.CallOption) (*AllocateBlockVolumeResponse, error)
VolumeServerDeleteBlockVolume(ctx context.Context, in *VolumeServerDeleteBlockVolumeRequest, opts ...grpc.CallOption) (*VolumeServerDeleteBlockVolumeResponse, error)
SnapshotBlockVolume(ctx context.Context, in *SnapshotBlockVolumeRequest, opts ...grpc.CallOption) (*SnapshotBlockVolumeResponse, error)
DeleteBlockSnapshot(ctx context.Context, in *DeleteBlockSnapshotRequest, opts ...grpc.CallOption) (*DeleteBlockSnapshotResponse, error)
ListBlockSnapshots(ctx context.Context, in *ListBlockSnapshotsRequest, opts ...grpc.CallOption) (*ListBlockSnapshotsResponse, error)
ExpandBlockVolume(ctx context.Context, in *ExpandBlockVolumeRequest, opts ...grpc.CallOption) (*ExpandBlockVolumeResponse, error)
}
type volumeServerClient struct {
@@ -736,6 +744,46 @@ func (c *volumeServerClient) VolumeServerDeleteBlockVolume(ctx context.Context,
return out, nil
}
func (c *volumeServerClient) SnapshotBlockVolume(ctx context.Context, in *SnapshotBlockVolumeRequest, opts ...grpc.CallOption) (*SnapshotBlockVolumeResponse, error) {
cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...)
out := new(SnapshotBlockVolumeResponse)
err := c.cc.Invoke(ctx, VolumeServer_SnapshotBlockVolume_FullMethodName, in, out, cOpts...)
if err != nil {
return nil, err
}
return out, nil
}
func (c *volumeServerClient) DeleteBlockSnapshot(ctx context.Context, in *DeleteBlockSnapshotRequest, opts ...grpc.CallOption) (*DeleteBlockSnapshotResponse, error) {
cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...)
out := new(DeleteBlockSnapshotResponse)
err := c.cc.Invoke(ctx, VolumeServer_DeleteBlockSnapshot_FullMethodName, in, out, cOpts...)
if err != nil {
return nil, err
}
return out, nil
}
func (c *volumeServerClient) ListBlockSnapshots(ctx context.Context, in *ListBlockSnapshotsRequest, opts ...grpc.CallOption) (*ListBlockSnapshotsResponse, error) {
cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...)
out := new(ListBlockSnapshotsResponse)
err := c.cc.Invoke(ctx, VolumeServer_ListBlockSnapshots_FullMethodName, in, out, cOpts...)
if err != nil {
return nil, err
}
return out, nil
}
func (c *volumeServerClient) ExpandBlockVolume(ctx context.Context, in *ExpandBlockVolumeRequest, opts ...grpc.CallOption) (*ExpandBlockVolumeResponse, error) {
cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...)
out := new(ExpandBlockVolumeResponse)
err := c.cc.Invoke(ctx, VolumeServer_ExpandBlockVolume_FullMethodName, in, out, cOpts...)
if err != nil {
return nil, err
}
return out, nil
}
// VolumeServerServer is the server API for VolumeServer service.
// All implementations must embed UnimplementedVolumeServerServer
// for forward compatibility.
@@ -798,6 +846,10 @@ type VolumeServerServer interface {
// block volumes
AllocateBlockVolume(context.Context, *AllocateBlockVolumeRequest) (*AllocateBlockVolumeResponse, error)
VolumeServerDeleteBlockVolume(context.Context, *VolumeServerDeleteBlockVolumeRequest) (*VolumeServerDeleteBlockVolumeResponse, error)
SnapshotBlockVolume(context.Context, *SnapshotBlockVolumeRequest) (*SnapshotBlockVolumeResponse, error)
DeleteBlockSnapshot(context.Context, *DeleteBlockSnapshotRequest) (*DeleteBlockSnapshotResponse, error)
ListBlockSnapshots(context.Context, *ListBlockSnapshotsRequest) (*ListBlockSnapshotsResponse, error)
ExpandBlockVolume(context.Context, *ExpandBlockVolumeRequest) (*ExpandBlockVolumeResponse, error)
mustEmbedUnimplementedVolumeServerServer()
}
@@ -958,6 +1010,18 @@ func (UnimplementedVolumeServerServer) AllocateBlockVolume(context.Context, *All
func (UnimplementedVolumeServerServer) VolumeServerDeleteBlockVolume(context.Context, *VolumeServerDeleteBlockVolumeRequest) (*VolumeServerDeleteBlockVolumeResponse, error) {
return nil, status.Error(codes.Unimplemented, "method VolumeServerDeleteBlockVolume not implemented")
}
func (UnimplementedVolumeServerServer) SnapshotBlockVolume(context.Context, *SnapshotBlockVolumeRequest) (*SnapshotBlockVolumeResponse, error) {
return nil, status.Error(codes.Unimplemented, "method SnapshotBlockVolume not implemented")
}
func (UnimplementedVolumeServerServer) DeleteBlockSnapshot(context.Context, *DeleteBlockSnapshotRequest) (*DeleteBlockSnapshotResponse, error) {
return nil, status.Error(codes.Unimplemented, "method DeleteBlockSnapshot not implemented")
}
func (UnimplementedVolumeServerServer) ListBlockSnapshots(context.Context, *ListBlockSnapshotsRequest) (*ListBlockSnapshotsResponse, error) {
return nil, status.Error(codes.Unimplemented, "method ListBlockSnapshots not implemented")
}
func (UnimplementedVolumeServerServer) ExpandBlockVolume(context.Context, *ExpandBlockVolumeRequest) (*ExpandBlockVolumeResponse, error) {
return nil, status.Error(codes.Unimplemented, "method ExpandBlockVolume not implemented")
}
func (UnimplementedVolumeServerServer) mustEmbedUnimplementedVolumeServerServer() {}
func (UnimplementedVolumeServerServer) testEmbeddedByValue() {}
@@ -1798,6 +1862,78 @@ func _VolumeServer_VolumeServerDeleteBlockVolume_Handler(srv interface{}, ctx co
return interceptor(ctx, in, info, handler)
}
func _VolumeServer_SnapshotBlockVolume_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
in := new(SnapshotBlockVolumeRequest)
if err := dec(in); err != nil {
return nil, err
}
if interceptor == nil {
return srv.(VolumeServerServer).SnapshotBlockVolume(ctx, in)
}
info := &grpc.UnaryServerInfo{
Server: srv,
FullMethod: VolumeServer_SnapshotBlockVolume_FullMethodName,
}
handler := func(ctx context.Context, req interface{}) (interface{}, error) {
return srv.(VolumeServerServer).SnapshotBlockVolume(ctx, req.(*SnapshotBlockVolumeRequest))
}
return interceptor(ctx, in, info, handler)
}
func _VolumeServer_DeleteBlockSnapshot_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
in := new(DeleteBlockSnapshotRequest)
if err := dec(in); err != nil {
return nil, err
}
if interceptor == nil {
return srv.(VolumeServerServer).DeleteBlockSnapshot(ctx, in)
}
info := &grpc.UnaryServerInfo{
Server: srv,
FullMethod: VolumeServer_DeleteBlockSnapshot_FullMethodName,
}
handler := func(ctx context.Context, req interface{}) (interface{}, error) {
return srv.(VolumeServerServer).DeleteBlockSnapshot(ctx, req.(*DeleteBlockSnapshotRequest))
}
return interceptor(ctx, in, info, handler)
}
func _VolumeServer_ListBlockSnapshots_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
in := new(ListBlockSnapshotsRequest)
if err := dec(in); err != nil {
return nil, err
}
if interceptor == nil {
return srv.(VolumeServerServer).ListBlockSnapshots(ctx, in)
}
info := &grpc.UnaryServerInfo{
Server: srv,
FullMethod: VolumeServer_ListBlockSnapshots_FullMethodName,
}
handler := func(ctx context.Context, req interface{}) (interface{}, error) {
return srv.(VolumeServerServer).ListBlockSnapshots(ctx, req.(*ListBlockSnapshotsRequest))
}
return interceptor(ctx, in, info, handler)
}
func _VolumeServer_ExpandBlockVolume_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
in := new(ExpandBlockVolumeRequest)
if err := dec(in); err != nil {
return nil, err
}
if interceptor == nil {
return srv.(VolumeServerServer).ExpandBlockVolume(ctx, in)
}
info := &grpc.UnaryServerInfo{
Server: srv,
FullMethod: VolumeServer_ExpandBlockVolume_FullMethodName,
}
handler := func(ctx context.Context, req interface{}) (interface{}, error) {
return srv.(VolumeServerServer).ExpandBlockVolume(ctx, req.(*ExpandBlockVolumeRequest))
}
return interceptor(ctx, in, info, handler)
}
// VolumeServer_ServiceDesc is the grpc.ServiceDesc for VolumeServer service.
// It's only intended for direct use with grpc.RegisterService,
// and not to be introspected or modified (even as a copy)
@@ -1961,6 +2097,22 @@ var VolumeServer_ServiceDesc = grpc.ServiceDesc{
MethodName: "VolumeServerDeleteBlockVolume",
Handler: _VolumeServer_VolumeServerDeleteBlockVolume_Handler,
},
{
MethodName: "SnapshotBlockVolume",
Handler: _VolumeServer_SnapshotBlockVolume_Handler,
},
{
MethodName: "DeleteBlockSnapshot",
Handler: _VolumeServer_DeleteBlockSnapshot_Handler,
},
{
MethodName: "ListBlockSnapshots",
Handler: _VolumeServer_ListBlockSnapshots_Handler,
},
{
MethodName: "ExpandBlockVolume",
Handler: _VolumeServer_ExpandBlockVolume_Handler,
},
},
Streams: []grpc.StreamDesc{
{
weed/server/integration_block_test.go
+21 -5
View File
@@ -641,7 +641,22 @@ func TestIntegration_DoubleFailover(t *testing.T) {
t.Fatalf("first failover epoch: got %d, want 2", e1.Epoch)
}
// Second failover: vs2 dies → vs1 promoted (it's now the replica).
// CP8-2: PromoteBestReplica does NOT add old primary back as replica.
// Reconnect vs1 first so it becomes a replica (via recoverBlockVolumes).
ms.recoverBlockVolumes(vs1)
// Simulate heartbeat from vs1 that restores iSCSI addr and health score
// (in production this happens when the VS re-registers after reconnect).
e1, _ = ms.blockRegistry.Lookup("pvc-double-1")
for i := range e1.Replicas {
if e1.Replicas[i].Server == vs1 {
e1.Replicas[i].ISCSIAddr = vs1 + ":3260"
e1.Replicas[i].HealthScore = 1.0
}
}
// Now vs1 is back as replica. Second failover: vs2 dies → vs1 promoted.
e1, _ = ms.blockRegistry.Lookup("pvc-double-1")
e1.LastLeaseGrant = time.Now().Add(-1 * time.Minute)
ms.failoverBlockVolumes(vs2)
@@ -724,9 +739,10 @@ func TestIntegration_MultiVolumeFailoverRebuild(t *testing.T) {
}
}
_ = otherServer
// rebuildCount should equal the number of volumes that were primary on deadServer.
if rebuildCount != primaryCounts[deadServer] {
t.Fatalf("expected %d rebuild assignments for %s, got %d",
primaryCounts[deadServer], deadServer, rebuildCount)
// CP8-2: With 2 servers and 3 volumes, deadServer hosts all 3 volumes
// (as primary for some, replica for others). All need rebuild on reconnect.
if rebuildCount != 3 {
t.Fatalf("expected 3 rebuild assignments for %s (all volumes), got %d",
deadServer, rebuildCount)
}
}
weed/server/master_block_assignment_queue.go
+11
View File
@@ -123,3 +123,14 @@ func (q *BlockAssignmentQueue) Pending(server string) int {
defer q.mu.Unlock()
return len(q.queues[server])
}
// TotalPending returns the total number of pending assignments across all servers.
func (q *BlockAssignmentQueue) TotalPending() int {
q.mu.Lock()
defer q.mu.Unlock()
total := 0
for _, queue := range q.queues {
total += len(queue)
}
return total
}
weed/server/master_block_failover.go
+77 -38
View File
@@ -32,82 +32,116 @@ func newBlockFailoverState() *blockFailoverState {
}
// failoverBlockVolumes is called when a volume server disconnects.
// It checks each block volume on that server and promotes the replica
// if the lease has expired (F2).
// It checks each block volume on that server and:
// - If dead server is primary: promote best replica (if lease expired).
// - If dead server hosts a replica: remove from replica list, record pending rebuild.
func (ms *MasterServer) failoverBlockVolumes(deadServer string) {
if ms.blockRegistry == nil {
return
}
ms.blockRegistry.FailoversTotal.Add(1)
entries := ms.blockRegistry.ListByServer(deadServer)
now := time.Now()
for _, entry := range entries {
if blockvol.RoleFromWire(entry.Role) != blockvol.RolePrimary {
// Case 1: Dead server is the primary.
if entry.VolumeServer == deadServer &&
blockvol.RoleFromWire(entry.Role) == blockvol.RolePrimary {
if !entry.HasReplica() {
glog.Warningf("failover: %q has no replica, cannot promote", entry.Name)
continue
}
// F2: Wait for lease expiry before promoting.
leaseExpiry := entry.LastLeaseGrant.Add(entry.LeaseTTL)
if now.Before(leaseExpiry) {
delay := leaseExpiry.Sub(now)
glog.V(0).Infof("failover: %q lease expires in %v, deferring promotion", entry.Name, delay)
volumeName := entry.Name
timer := time.AfterFunc(delay, func() {
ms.promoteReplica(volumeName)
})
ms.blockFailover.mu.Lock()
ms.blockFailover.deferredTimers[deadServer] = append(
ms.blockFailover.deferredTimers[deadServer], timer)
ms.blockFailover.mu.Unlock()
continue
}
// Lease already expired — promote immediately.
ms.promoteReplica(entry.Name)
continue
}
// Only failover volumes whose primary is the dead server.
// Case 2: Dead server hosts a replica (not the primary).
if entry.VolumeServer != deadServer {
continue
ri := entry.ReplicaByServer(deadServer)
if ri != nil {
replicaPath := ri.Path
// Remove dead replica from registry.
if err := ms.blockRegistry.RemoveReplica(entry.Name, deadServer); err != nil {
glog.Warningf("failover: RemoveReplica %q on %s: %v", entry.Name, deadServer, err)
continue
}
// Record pending rebuild for when dead server reconnects.
ms.recordPendingRebuild(deadServer, pendingRebuild{
VolumeName: entry.Name,
OldPath: replicaPath,
NewPrimary: entry.VolumeServer, // current primary (unchanged)
Epoch: entry.Epoch,
})
glog.V(0).Infof("failover: removed dead replica %s for %q, pending rebuild",
deadServer, entry.Name)
}
}
if entry.ReplicaServer == "" {
glog.Warningf("failover: %q has no replica, cannot promote", entry.Name)
continue
}
// F2: Wait for lease expiry before promoting.
leaseExpiry := entry.LastLeaseGrant.Add(entry.LeaseTTL)
if now.Before(leaseExpiry) {
delay := leaseExpiry.Sub(now)
glog.V(0).Infof("failover: %q lease expires in %v, deferring promotion", entry.Name, delay)
volumeName := entry.Name
timer := time.AfterFunc(delay, func() {
ms.promoteReplica(volumeName)
})
// R2-F2: Store timer so it can be cancelled if the server reconnects.
ms.blockFailover.mu.Lock()
ms.blockFailover.deferredTimers[deadServer] = append(
ms.blockFailover.deferredTimers[deadServer], timer)
ms.blockFailover.mu.Unlock()
continue
}
// Lease already expired — promote immediately.
ms.promoteReplica(entry.Name)
}
}
// promoteReplica swaps primary and replica for the named volume,
// promoteReplica promotes the best replica to primary for the named volume,
// enqueues an assignment for the new primary, and records a pending rebuild.
func (ms *MasterServer) promoteReplica(volumeName string) {
entry, ok := ms.blockRegistry.Lookup(volumeName)
if !ok {
return
}
if entry.ReplicaServer == "" {
if !entry.HasReplica() {
return
}
oldPrimary := entry.VolumeServer
oldPath := entry.Path
// R2-F5: Epoch computed atomically inside SwapPrimaryReplica (under lock).
newEpoch, err := ms.blockRegistry.SwapPrimaryReplica(volumeName)
// CP8-2: Use PromoteBestReplica (picks by health score, tie-break by WALHeadLSN).
newEpoch, err := ms.blockRegistry.PromoteBestReplica(volumeName)
if err != nil {
glog.Warningf("failover: SwapPrimaryReplica %q: %v", volumeName, err)
glog.Warningf("failover: PromoteBestReplica %q: %v", volumeName, err)
return
}
// Re-read entry after swap.
// Re-read entry after promotion.
entry, ok = ms.blockRegistry.Lookup(volumeName)
if !ok {
return
}
// Enqueue assignment for new primary.
// Build assignment for new primary. Include ReplicaAddrs for remaining replicas.
leaseTTLMs := blockvol.LeaseTTLToWire(30 * time.Second)
ms.blockAssignmentQueue.Enqueue(entry.VolumeServer, blockvol.BlockVolumeAssignment{
assignment := blockvol.BlockVolumeAssignment{
Path: entry.Path,
Epoch: newEpoch,
Role: blockvol.RoleToWire(blockvol.RolePrimary),
LeaseTtlMs: leaseTTLMs,
})
}
for _, ri := range entry.Replicas {
assignment.ReplicaAddrs = append(assignment.ReplicaAddrs, blockvol.ReplicaAddr{
DataAddr: ri.DataAddr,
CtrlAddr: ri.CtrlAddr,
})
}
// Backward compat: also set scalar fields if exactly 1 replica.
if len(entry.Replicas) == 1 {
assignment.ReplicaDataAddr = entry.Replicas[0].DataAddr
assignment.ReplicaCtrlAddr = entry.Replicas[0].CtrlAddr
}
ms.blockAssignmentQueue.Enqueue(entry.VolumeServer, assignment)
// Record pending rebuild for when dead server reconnects.
ms.recordPendingRebuild(oldPrimary, pendingRebuild{
@@ -117,6 +151,7 @@ func (ms *MasterServer) promoteReplica(volumeName string) {
Epoch: newEpoch,
})
ms.blockRegistry.PromotionsTotal.Add(1)
glog.V(0).Infof("failover: promoted replica for %q: new primary=%s epoch=%d (old primary=%s)",
volumeName, entry.VolumeServer, newEpoch, oldPrimary)
}
@@ -180,8 +215,11 @@ func (ms *MasterServer) recoverBlockVolumes(reconnectedServer string) {
continue
}
// Update registry: reconnected server becomes the new replica.
ms.blockRegistry.SetReplica(rb.VolumeName, reconnectedServer, rb.OldPath, "", "")
// Update registry: reconnected server becomes a replica (via AddReplica for RF≥2 support).
ms.blockRegistry.AddReplica(rb.VolumeName, ReplicaInfo{
Server: reconnectedServer,
Path: rb.OldPath,
})
// Enqueue rebuild assignment for the reconnected server.
ms.blockAssignmentQueue.Enqueue(reconnectedServer, blockvol.BlockVolumeAssignment{
@@ -191,6 +229,7 @@ func (ms *MasterServer) recoverBlockVolumes(reconnectedServer string) {
RebuildAddr: entry.RebuildListenAddr,
})
ms.blockRegistry.RebuildsTotal.Add(1)
glog.V(0).Infof("rebuild: enqueued rebuild for %q on %s (epoch=%d, rebuildAddr=%s)",
rb.VolumeName, reconnectedServer, entry.Epoch, entry.RebuildListenAddr)
}
weed/server/master_block_failover_test.go
+368 -6
View File
@@ -50,6 +50,16 @@ func registerVolumeWithReplica(t *testing.T, ms *MasterServer, name, primary, re
ReplicaISCSIAddr: replica + ":3260",
LeaseTTL: leaseTTL,
LastLeaseGrant: time.Now().Add(-2 * leaseTTL), // expired
// CP8-2: also populate Replicas[] for PromoteBestReplica.
Replicas: []ReplicaInfo{
{
Server: replica,
Path: fmt.Sprintf("/data/%s.blk", name),
IQN: fmt.Sprintf("iqn.2024.test:%s-replica", name),
ISCSIAddr: replica + ":3260",
HealthScore: 1.0,
},
},
}
if err := ms.blockRegistry.Register(entry); err != nil {
t.Fatalf("register %s: %v", name, err)
@@ -128,12 +138,14 @@ func TestFailover_RegistryUpdated(t *testing.T) {
ms.failoverBlockVolumes("vs1")
entry, _ := ms.blockRegistry.Lookup("vol1")
// After swap: new primary = vs2, old primary (vs1) becomes replica.
// After PromoteBestReplica: new primary = vs2.
// Old primary (vs1) is NOT added back to Replicas (needs rebuild).
if entry.VolumeServer != "vs2" {
t.Fatalf("VolumeServer: got %q, want vs2", entry.VolumeServer)
}
if entry.ReplicaServer != "vs1" {
t.Fatalf("ReplicaServer: got %q, want vs1 (old primary)", entry.ReplicaServer)
// Replicas should be empty (only had 1 replica which was promoted).
if len(entry.Replicas) != 0 {
t.Fatalf("Replicas should be empty after promotion, got %d", len(entry.Replicas))
}
}
@@ -196,6 +208,9 @@ func TestFailover_LeaseNotExpired_DeferredPromotion(t *testing.T) {
ReplicaISCSIAddr: "vs2:3260",
LeaseTTL: 200 * time.Millisecond,
LastLeaseGrant: time.Now(), // just granted, NOT expired yet
Replicas: []ReplicaInfo{
{Server: "vs2", Path: "/data/vol1.blk", IQN: "iqn:vol1-r", ISCSIAddr: "vs2:3260", HealthScore: 1.0},
},
}
ms.blockRegistry.Register(entry)
@@ -369,13 +384,14 @@ func TestRebuild_RegistryUpdatedWithNewReplica(t *testing.T) {
ms.failoverBlockVolumes("vs1")
ms.recoverBlockVolumes("vs1")
// After recovery, vs1 should be the new replica for vol1.
// After recovery, vs1 should be a replica for vol1 (added back via AddReplica).
entry, _ := ms.blockRegistry.Lookup("vol1")
if entry.VolumeServer != "vs2" {
t.Fatalf("primary should be vs2, got %q", entry.VolumeServer)
}
if entry.ReplicaServer != "vs1" {
t.Fatalf("replica should be vs1 (reconnected), got %q", entry.ReplicaServer)
// Replicas[] should contain vs1 (added by recoverBlockVolumes).
if len(entry.Replicas) != 1 || entry.Replicas[0].Server != "vs1" {
t.Fatalf("replica should be vs1 (reconnected), got %+v", entry.Replicas)
}
}
@@ -396,6 +412,9 @@ func TestRebuild_AssignmentContainsRebuildAddr(t *testing.T) {
RebuildListenAddr: "vs1:15000",
LeaseTTL: 5 * time.Second,
LastLeaseGrant: time.Now().Add(-10 * time.Second),
Replicas: []ReplicaInfo{
{Server: "vs2", Path: "/data/vol1.blk", IQN: "iqn:vol1-r", ISCSIAddr: "vs2:3260", HealthScore: 1.0},
},
}
ms.blockRegistry.Register(entry)
@@ -437,6 +456,9 @@ func TestFailover_TransientDisconnect_NoPromotion(t *testing.T) {
ReplicaISCSIAddr: "vs2:3260",
LeaseTTL: 30 * time.Second,
LastLeaseGrant: time.Now(), // just granted
Replicas: []ReplicaInfo{
{Server: "vs2", Path: "/data/vol1.blk", IQN: "iqn:vol1-r", ISCSIAddr: "vs2:3260", HealthScore: 1.0},
},
}
ms.blockRegistry.Register(entry)
@@ -526,3 +548,343 @@ func TestLifecycle_CreateFailoverRebuild(t *testing.T) {
t.Fatal("expected rebuild assignment for reconnected server")
}
}
// ============================================================
// CP8-2 T8: RF=3 (N>2) Failover Tests
// ============================================================
// registerVolumeRF3 creates a volume entry with primary + 2 replicas for RF=3 tests.
func registerVolumeRF3(t *testing.T, ms *MasterServer, name, primary, replica1, replica2 string, epoch uint64, leaseTTL time.Duration) {
t.Helper()
entry := &BlockVolumeEntry{
Name: name,
VolumeServer: primary,
Path: fmt.Sprintf("/data/%s.blk", name),
IQN: fmt.Sprintf("iqn.2024.test:%s", name),
ISCSIAddr: primary + ":3260",
SizeBytes: 1 << 30,
Epoch: epoch,
Role: blockvol.RoleToWire(blockvol.RolePrimary),
Status: StatusActive,
ReplicaFactor: 3,
LeaseTTL: leaseTTL,
LastLeaseGrant: time.Now().Add(-2 * leaseTTL), // expired
// Deprecated scalar fields (backward compat): first replica only.
ReplicaServer: replica1,
ReplicaPath: fmt.Sprintf("/data/%s.blk", name),
ReplicaIQN: fmt.Sprintf("iqn.2024.test:%s-r1", name),
ReplicaISCSIAddr: replica1 + ":3260",
Replicas: []ReplicaInfo{
{
Server: replica1,
Path: fmt.Sprintf("/data/%s.blk", name),
IQN: fmt.Sprintf("iqn.2024.test:%s-r1", name),
ISCSIAddr: replica1 + ":3260",
HealthScore: 1.0,
WALHeadLSN: 100,
},
{
Server: replica2,
Path: fmt.Sprintf("/data/%s.blk", name),
IQN: fmt.Sprintf("iqn.2024.test:%s-r2", name),
ISCSIAddr: replica2 + ":3260",
HealthScore: 1.0,
WALHeadLSN: 100,
},
},
}
if err := ms.blockRegistry.Register(entry); err != nil {
t.Fatalf("register %s: %v", name, err)
}
}
// RF3: Primary dies → best replica promoted, other replica survives.
func TestRF3_PrimaryDies_BestReplicaPromoted(t *testing.T) {
ms := testMasterServerForFailover(t)
registerVolumeRF3(t, ms, "vol1", "vs1", "vs2", "vs3", 1, 5*time.Second)
// Give vs3 a higher health score so it should be promoted.
entry, _ := ms.blockRegistry.Lookup("vol1")
entry.Replicas[0].HealthScore = 0.8 // vs2
entry.Replicas[1].HealthScore = 1.0 // vs3
ms.failoverBlockVolumes("vs1")
entry, _ = ms.blockRegistry.Lookup("vol1")
if entry.VolumeServer != "vs3" {
t.Fatalf("primary should be vs3 (highest health), got %q", entry.VolumeServer)
}
if entry.Epoch != 2 {
t.Fatalf("epoch: got %d, want 2", entry.Epoch)
}
// vs2 should remain as the only replica.
if len(entry.Replicas) != 1 {
t.Fatalf("expected 1 remaining replica, got %d", len(entry.Replicas))
}
if entry.Replicas[0].Server != "vs2" {
t.Fatalf("remaining replica should be vs2, got %q", entry.Replicas[0].Server)
}
}
// RF3: One replica dies → primary stays, dead replica removed, pending rebuild.
func TestRF3_OneReplicaDies_PrimaryUnchanged(t *testing.T) {
ms := testMasterServerForFailover(t)
registerVolumeRF3(t, ms, "vol1", "vs1", "vs2", "vs3", 1, 5*time.Second)
// vs3 dies (a replica, not the primary).
ms.failoverBlockVolumes("vs3")
entry, _ := ms.blockRegistry.Lookup("vol1")
// Primary should remain vs1.
if entry.VolumeServer != "vs1" {
t.Fatalf("primary should remain vs1, got %q", entry.VolumeServer)
}
// Epoch should NOT change (no promotion).
if entry.Epoch != 1 {
t.Fatalf("epoch should remain 1, got %d", entry.Epoch)
}
// vs3 should be removed from replicas, leaving only vs2.
if len(entry.Replicas) != 1 {
t.Fatalf("expected 1 replica after vs3 death, got %d", len(entry.Replicas))
}
if entry.Replicas[0].Server != "vs2" {
t.Fatalf("remaining replica should be vs2, got %q", entry.Replicas[0].Server)
}
// Pending rebuild should be recorded for vs3.
ms.blockFailover.mu.Lock()
rebuilds := ms.blockFailover.pendingRebuilds["vs3"]
ms.blockFailover.mu.Unlock()
if len(rebuilds) != 1 || rebuilds[0].VolumeName != "vol1" {
t.Fatalf("expected 1 pending rebuild for vs3/vol1, got %+v", rebuilds)
}
}
// RF3: Dead replica reconnects → rebuilds, rejoins replica list.
func TestRF3_RecoverRebuildsDeadReplica(t *testing.T) {
ms := testMasterServerForFailover(t)
registerVolumeRF3(t, ms, "vol1", "vs1", "vs2", "vs3", 1, 5*time.Second)
// vs3 dies.
ms.failoverBlockVolumes("vs3")
// vs3 reconnects.
ms.recoverBlockVolumes("vs3")
entry, _ := ms.blockRegistry.Lookup("vol1")
// Primary still vs1.
if entry.VolumeServer != "vs1" {
t.Fatalf("primary should remain vs1, got %q", entry.VolumeServer)
}
// vs3 should be back in replicas (added by recoverBlockVolumes via AddReplica).
if len(entry.Replicas) != 2 {
t.Fatalf("expected 2 replicas after recovery, got %d", len(entry.Replicas))
}
// Verify rebuild assignment was queued for vs3.
assignments := ms.blockAssignmentQueue.Peek("vs3")
foundRebuild := false
for _, a := range assignments {
if blockvol.RoleFromWire(a.Role) == blockvol.RoleRebuilding {
foundRebuild = true
}
}
if !foundRebuild {
t.Fatal("expected rebuild assignment for vs3")
}
}
// RF3: Other replicas survive promotion.
func TestRF3_OtherReplicasSurvivePromotion(t *testing.T) {
ms := testMasterServerForFailover(t)
registerVolumeRF3(t, ms, "vol1", "vs1", "vs2", "vs3", 1, 5*time.Second)
// Primary (vs1) dies → one replica promoted.
ms.failoverBlockVolumes("vs1")
entry, _ := ms.blockRegistry.Lookup("vol1")
newPrimary := entry.VolumeServer
// New primary should be vs2 or vs3 (both have equal health/LSN, so first wins).
if newPrimary != "vs2" && newPrimary != "vs3" {
t.Fatalf("new primary should be vs2 or vs3, got %q", newPrimary)
}
// The other replica should still be in the list.
if len(entry.Replicas) != 1 {
t.Fatalf("expected 1 surviving replica, got %d", len(entry.Replicas))
}
// The surviving replica is whichever wasn't promoted.
expectedSurvivor := "vs3"
if newPrimary == "vs3" {
expectedSurvivor = "vs2"
}
if entry.Replicas[0].Server != expectedSurvivor {
t.Fatalf("surviving replica should be %q, got %q", expectedSurvivor, entry.Replicas[0].Server)
}
// Assignment for new primary should include ReplicaAddrs for the surviving replica.
assignments := ms.blockAssignmentQueue.Peek(newPrimary)
foundPrimary := false
for _, a := range assignments {
if blockvol.RoleFromWire(a.Role) == blockvol.RolePrimary {
foundPrimary = true
if len(a.ReplicaAddrs) != 1 {
t.Fatalf("expected 1 ReplicaAddr in assignment, got %d", len(a.ReplicaAddrs))
}
}
}
if !foundPrimary {
t.Fatal("expected primary assignment for new primary")
}
}
// RF2 unchanged: verify RF=2 still works identically.
func TestRF2_Unchanged_AfterCP82(t *testing.T) {
ms := testMasterServerForFailover(t)
registerVolumeWithReplica(t, ms, "vol1", "vs1", "vs2", 1, 5*time.Second)
ms.failoverBlockVolumes("vs1")
entry, _ := ms.blockRegistry.Lookup("vol1")
if entry.VolumeServer != "vs2" {
t.Fatalf("RF2: primary should be vs2, got %q", entry.VolumeServer)
}
if entry.Epoch != 2 {
t.Fatalf("RF2: epoch should be 2, got %d", entry.Epoch)
}
// No replicas left (RF=2, one promoted).
if len(entry.Replicas) != 0 {
t.Fatalf("RF2: expected 0 replicas after promotion, got %d", len(entry.Replicas))
}
// Rebuild works the same.
ms.recoverBlockVolumes("vs1")
entry, _ = ms.blockRegistry.Lookup("vol1")
if len(entry.Replicas) != 1 || entry.Replicas[0].Server != "vs1" {
t.Fatalf("RF2: vs1 should be back as replica after recovery, got %+v", entry.Replicas)
}
}
// RF3: All replicas dead → no promotion possible.
func TestRF3_AllReplicasDead_NoPromotion(t *testing.T) {
ms := testMasterServerForFailover(t)
registerVolumeRF3(t, ms, "vol1", "vs1", "vs2", "vs3", 1, 5*time.Second)
// Both replicas die first (not primary).
ms.failoverBlockVolumes("vs2")
ms.failoverBlockVolumes("vs3")
entry, _ := ms.blockRegistry.Lookup("vol1")
// Primary still vs1, replicas removed.
if entry.VolumeServer != "vs1" {
t.Fatalf("primary should remain vs1, got %q", entry.VolumeServer)
}
if len(entry.Replicas) != 0 {
t.Fatalf("expected 0 replicas, got %d", len(entry.Replicas))
}
// Now primary dies → no promotion possible (no replicas left).
ms.failoverBlockVolumes("vs1")
entry, _ = ms.blockRegistry.Lookup("vol1")
if entry.VolumeServer != "vs1" {
t.Fatalf("should remain vs1 (no replicas to promote), got %q", entry.VolumeServer)
}
}
// RF3: Lease deferred promotion with RF=3.
func TestRF3_LeaseDeferred_Promotion(t *testing.T) {
ms := testMasterServerForFailover(t)
entry := &BlockVolumeEntry{
Name: "vol1",
VolumeServer: "vs1",
Path: "/data/vol1.blk",
IQN: "iqn:vol1",
ISCSIAddr: "vs1:3260",
SizeBytes: 1 << 30,
Epoch: 1,
Role: blockvol.RoleToWire(blockvol.RolePrimary),
Status: StatusActive,
ReplicaFactor: 3,
LeaseTTL: 200 * time.Millisecond,
LastLeaseGrant: time.Now(), // just granted → NOT expired
Replicas: []ReplicaInfo{
{Server: "vs2", Path: "/data/vol1.blk", ISCSIAddr: "vs2:3260", HealthScore: 1.0, WALHeadLSN: 50},
{Server: "vs3", Path: "/data/vol1.blk", ISCSIAddr: "vs3:3260", HealthScore: 0.9, WALHeadLSN: 50},
},
// Deprecated scalar fields.
ReplicaServer: "vs2", ReplicaPath: "/data/vol1.blk", ReplicaISCSIAddr: "vs2:3260",
}
ms.blockRegistry.Register(entry)
ms.failoverBlockVolumes("vs1")
// Immediately: no promotion (lease not expired).
e, _ := ms.blockRegistry.Lookup("vol1")
if e.VolumeServer != "vs1" {
t.Fatalf("should NOT promote yet (lease active), got %q", e.VolumeServer)
}
// Wait for lease to expire + deferred timer.
time.Sleep(350 * time.Millisecond)
e, _ = ms.blockRegistry.Lookup("vol1")
if e.VolumeServer != "vs2" {
t.Fatalf("should promote vs2 (highest health) after lease expires, got %q", e.VolumeServer)
}
// vs3 should survive as the remaining replica.
if len(e.Replicas) != 1 || e.Replicas[0].Server != "vs3" {
t.Fatalf("vs3 should remain as replica, got %+v", e.Replicas)
}
}
// RF3: Cancel deferred timers on reconnect.
func TestRF3_CancelDeferredOnReconnect(t *testing.T) {
ms := testMasterServerForFailover(t)
entry := &BlockVolumeEntry{
Name: "vol1",
VolumeServer: "vs1",
Path: "/data/vol1.blk",
IQN: "iqn:vol1",
ISCSIAddr: "vs1:3260",
SizeBytes: 1 << 30,
Epoch: 1,
Role: blockvol.RoleToWire(blockvol.RolePrimary),
Status: StatusActive,
ReplicaFactor: 3,
LeaseTTL: 5 * time.Second,
LastLeaseGrant: time.Now(), // just granted → long lease
Replicas: []ReplicaInfo{
{Server: "vs2", Path: "/data/vol1.blk", ISCSIAddr: "vs2:3260", HealthScore: 1.0},
{Server: "vs3", Path: "/data/vol1.blk", ISCSIAddr: "vs3:3260", HealthScore: 1.0},
},
ReplicaServer: "vs2", ReplicaPath: "/data/vol1.blk", ReplicaISCSIAddr: "vs2:3260",
}
ms.blockRegistry.Register(entry)
// vs1 disconnects → deferred timer created (lease not expired).
ms.failoverBlockVolumes("vs1")
ms.blockFailover.mu.Lock()
timerCount := len(ms.blockFailover.deferredTimers["vs1"])
ms.blockFailover.mu.Unlock()
if timerCount == 0 {
t.Fatal("expected deferred timer for vs1")
}
// vs1 reconnects before lease expires → timers cancelled.
ms.recoverBlockVolumes("vs1")
ms.blockFailover.mu.Lock()
timerCount = len(ms.blockFailover.deferredTimers["vs1"])
ms.blockFailover.mu.Unlock()
if timerCount != 0 {
t.Fatalf("expected 0 deferred timers after reconnect, got %d", timerCount)
}
// Wait past original lease time — no promotion should happen.
time.Sleep(200 * time.Millisecond)
e, _ := ms.blockRegistry.Lookup("vol1")
if e.VolumeServer != "vs1" {
t.Fatalf("vs1 should remain primary (timer cancelled), got %q", e.VolumeServer)
}
}
weed/server/master_block_registry.go
+591 -43
View File
@@ -2,9 +2,14 @@ package weed_server
import (
"fmt"
"path/filepath"
"sort"
"strings"
"sync"
"sync/atomic"
"time"
"github.com/seaweedfs/seaweedfs/weed/glog"
"github.com/seaweedfs/seaweedfs/weed/pb/master_pb"
"github.com/seaweedfs/seaweedfs/weed/storage/blockvol"
)
@@ -17,32 +22,102 @@ const (
StatusActive // Confirmed by heartbeat from volume server
)
// ReplicaInfo tracks one replica of a block volume (CP8-2).
type ReplicaInfo struct {
Server string // replica VS address
Path string // file path on replica VS
ISCSIAddr string // iSCSI target address
IQN string // iSCSI qualified name
DataAddr string // WAL receiver data listen addr
CtrlAddr string // WAL receiver ctrl listen addr
HealthScore float64 // from heartbeat (0.0-1.0)
WALHeadLSN uint64 // from heartbeat
WALLag uint64 // computed: primary.WALHeadLSN - replica.WALHeadLSN
LastHeartbeat time.Time // last heartbeat received from this replica
Role uint32 // replica role (RoleReplica, RoleRebuilding, etc.)
}
const (
// DefaultPromotionLSNTolerance is the max WAL LSN lag allowed for promotion eligibility.
// Configurable per-registry via SetPromotionLSNTolerance.
DefaultPromotionLSNTolerance uint64 = 100
)
// BlockVolumeEntry tracks one block volume across the cluster.
type BlockVolumeEntry struct {
Name string
VolumeServer string // volume server address (ip:port or grpc addr)
Path string // file path on volume server
IQN string
ISCSIAddr string
SizeBytes uint64
Epoch uint64
Role uint32
Status VolumeStatus
Name string
VolumeServer string // volume server address (ip:port or grpc addr)
Path string // file path on volume server
IQN string
ISCSIAddr string
SizeBytes uint64
ReplicaPlacement string // SeaweedFS placement string: "000", "001", "010", "100"
Epoch uint64
Role uint32
Status VolumeStatus
// Replica tracking (CP6-3).
ReplicaServer string // replica VS address
ReplicaPath string // file path on replica VS
// Deprecated scalar replica fields (CP6-3). Use Replicas[] for new code.
ReplicaServer string
ReplicaPath string
ReplicaISCSIAddr string
ReplicaIQN string
ReplicaDataAddr string // replica receiver data listen addr
ReplicaCtrlAddr string // replica receiver ctrl listen addr
ReplicaDataAddr string
ReplicaCtrlAddr string
RebuildListenAddr string // rebuild server listen addr on primary
// CP8-2: Multi-replica support.
ReplicaFactor int // 2 or 3 (default 2)
Replicas []ReplicaInfo // one per replica (RF-1 entries)
HealthScore float64 // primary health score from heartbeat
ReplicaDegraded bool // primary reports degraded replicas
WALHeadLSN uint64 // primary WAL head LSN from heartbeat
// Lease tracking for failover (CP6-3 F2).
LastLeaseGrant time.Time
LeaseTTL time.Duration
}
// HasReplica returns true if this volume has any replica (checks both new and deprecated fields).
func (e *BlockVolumeEntry) HasReplica() bool {
return len(e.Replicas) > 0 || e.ReplicaServer != ""
}
// FirstReplica returns the first replica info, or nil if none.
func (e *BlockVolumeEntry) FirstReplica() *ReplicaInfo {
if len(e.Replicas) > 0 {
return &e.Replicas[0]
}
return nil
}
// ReplicaByServer returns the replica hosted on the given server, or nil.
func (e *BlockVolumeEntry) ReplicaByServer(server string) *ReplicaInfo {
for i := range e.Replicas {
if e.Replicas[i].Server == server {
return &e.Replicas[i]
}
}
return nil
}
// BestReplicaForPromotion returns the best replica for promotion, or nil if none eligible.
// Criteria: highest HealthScore, tie-break by highest WALHeadLSN, then first in list.
func (e *BlockVolumeEntry) BestReplicaForPromotion() *ReplicaInfo {
if len(e.Replicas) == 0 {
return nil
}
best := 0
for i := 1; i < len(e.Replicas); i++ {
if e.Replicas[i].HealthScore > e.Replicas[best].HealthScore {
best = i
} else if e.Replicas[i].HealthScore == e.Replicas[best].HealthScore &&
e.Replicas[i].WALHeadLSN > e.Replicas[best].WALHeadLSN {
best = i
}
}
return &e.Replicas[best]
}
// BlockVolumeRegistry is the in-memory registry of block volumes.
// Rebuilt from heartbeats on master restart (no persistence).
type BlockVolumeRegistry struct {
@@ -51,8 +126,16 @@ type BlockVolumeRegistry struct {
byServer map[string]map[string]bool // server -> set of volume names
blockServers map[string]bool // servers known to support block volumes
// Promotion eligibility: max WAL LSN lag for replica to be promotable.
promotionLSNTolerance uint64
// inflight guards concurrent CreateBlockVolume for the same name.
inflight sync.Map // name -> *inflightEntry
// Metrics (CP8-4).
PromotionsTotal atomic.Uint64
FailoversTotal atomic.Uint64
RebuildsTotal atomic.Uint64
}
type inflightEntry struct{}
@@ -60,9 +143,10 @@ type inflightEntry struct{}
// NewBlockVolumeRegistry creates an empty registry.
func NewBlockVolumeRegistry() *BlockVolumeRegistry {
return &BlockVolumeRegistry{
volumes: make(map[string]*BlockVolumeEntry),
byServer: make(map[string]map[string]bool),
blockServers: make(map[string]bool),
volumes: make(map[string]*BlockVolumeEntry),
byServer: make(map[string]map[string]bool),
blockServers: make(map[string]bool),
promotionLSNTolerance: DefaultPromotionLSNTolerance,
}
}
@@ -76,6 +160,10 @@ func (r *BlockVolumeRegistry) Register(entry *BlockVolumeEntry) error {
}
r.volumes[entry.Name] = entry
r.addToServer(entry.VolumeServer, entry.Name)
// Also index replica servers so ListByServer finds them.
for _, ri := range entry.Replicas {
r.addToServer(ri.Server, entry.Name)
}
return nil
}
@@ -89,9 +177,25 @@ func (r *BlockVolumeRegistry) Unregister(name string) *BlockVolumeEntry {
}
delete(r.volumes, name)
r.removeFromServer(entry.VolumeServer, name)
for _, ri := range entry.Replicas {
r.removeFromServer(ri.Server, name)
}
return entry
}
// UpdateSize updates the size of a registered volume.
// Called only after a successful VS expand to keep registry in sync.
func (r *BlockVolumeRegistry) UpdateSize(name string, newSizeBytes uint64) error {
r.mu.Lock()
defer r.mu.Unlock()
entry, ok := r.volumes[name]
if !ok {
return fmt.Errorf("block volume %q not found in registry", name)
}
entry.SizeBytes = newSizeBytes
return nil
}
// Lookup returns the entry for the given name.
func (r *BlockVolumeRegistry) Lookup(name string) (*BlockVolumeEntry, bool) {
r.mu.RLock()
@@ -134,51 +238,172 @@ func (r *BlockVolumeRegistry) UpdateFullHeartbeat(server string, infos []*master
reported[info.Path] = info
}
// Find entries for this server that are NOT reported -> remove them.
// Find entries for this server that are NOT reported -> reconcile.
if names, ok := r.byServer[server]; ok {
for name := range names {
entry := r.volumes[name]
if entry == nil {
continue
}
if _, found := reported[entry.Path]; !found {
delete(r.volumes, name)
delete(names, name)
if entry.VolumeServer == server {
// Server is the primary: check if primary path is reported.
if _, found := reported[entry.Path]; !found {
delete(r.volumes, name)
delete(names, name)
// Also clean up replica entries from byServer.
for _, ri := range entry.Replicas {
r.removeFromServer(ri.Server, name)
}
}
} else {
// Server is a replica: check if replica path is reported.
ri := entry.ReplicaByServer(server)
if ri == nil {
// No replica record — stale byServer index, just clean up.
delete(names, name)
continue
}
if _, found := reported[ri.Path]; !found {
// Replica path not reported — remove this replica, NOT the whole volume.
r.removeReplicaLocked(entry, server, name)
delete(names, name)
glog.V(0).Infof("block registry: removed stale replica %s for %q (path %s not in heartbeat)",
server, name, ri.Path)
}
}
}
}
// Update or add entries for reported volumes.
for _, info := range infos {
// Find existing entry by path on this server.
// Find existing entry: search byServer index for matching path (primary or replica).
var existing *BlockVolumeEntry
var existingName string
if names, ok := r.byServer[server]; ok {
for name := range names {
if e := r.volumes[name]; e != nil && e.Path == info.Path {
existing = e
break
for vname := range names {
if e := r.volumes[vname]; e != nil {
if e.VolumeServer == server && e.Path == info.Path {
existing = e
existingName = vname
break
}
if ri := e.ReplicaByServer(server); ri != nil && ri.Path == info.Path {
existing = e
existingName = vname
break
}
}
}
}
// Also try lookup by name derived from path (handles post-restart).
if existing == nil {
name := nameFromPath(info.Path)
if name != "" {
if e, ok := r.volumes[name]; ok {
existing = e
existingName = name
}
}
}
if existing != nil {
// Update fields from heartbeat.
existing.SizeBytes = info.VolumeSize
existing.Epoch = info.Epoch
existing.Role = info.Role
existing.Status = StatusActive
// R1-5: Refresh lease on heartbeat — VS is alive and running this volume.
existing.LastLeaseGrant = time.Now()
// F5: update replica addresses from heartbeat info.
if info.ReplicaDataAddr != "" {
existing.ReplicaDataAddr = info.ReplicaDataAddr
isPrimary := existing.VolumeServer == server
isReplica := existing.ReplicaByServer(server) != nil
if isPrimary {
// Primary heartbeat: update primary fields.
existing.SizeBytes = info.VolumeSize
existing.Epoch = info.Epoch
existing.Role = info.Role
existing.Status = StatusActive
existing.LastLeaseGrant = time.Now()
existing.HealthScore = info.HealthScore
existing.ReplicaDegraded = info.ReplicaDegraded
existing.WALHeadLSN = info.WalHeadLsn
// F5: update replica addresses from heartbeat info.
if info.ReplicaDataAddr != "" {
existing.ReplicaDataAddr = info.ReplicaDataAddr
}
if info.ReplicaCtrlAddr != "" {
existing.ReplicaCtrlAddr = info.ReplicaCtrlAddr
}
// Sync first replica's data addrs to Replicas[].
if info.ReplicaDataAddr != "" && len(existing.Replicas) > 0 {
existing.Replicas[0].DataAddr = info.ReplicaDataAddr
existing.Replicas[0].CtrlAddr = info.ReplicaCtrlAddr
}
} else if isReplica {
// Replica heartbeat: update ReplicaInfo fields.
for i := range existing.Replicas {
if existing.Replicas[i].Server == server {
existing.Replicas[i].Path = info.Path
existing.Replicas[i].WALHeadLSN = info.WalHeadLsn
existing.Replicas[i].HealthScore = info.HealthScore
existing.Replicas[i].LastHeartbeat = time.Now()
existing.Replicas[i].Role = info.Role
if existing.WALHeadLSN > info.WalHeadLsn {
existing.Replicas[i].WALLag = existing.WALHeadLSN - info.WalHeadLsn
} else {
existing.Replicas[i].WALLag = 0
}
break
}
}
} else {
// Fix #3: Server reports a volume that exists but has no record of this server.
// This happens after master restart: primary heartbeat re-created the entry,
// but replica heartbeat arrives and isn't linked. Add as replica.
ri := ReplicaInfo{
Server: server,
Path: info.Path,
HealthScore: info.HealthScore,
WALHeadLSN: info.WalHeadLsn,
LastHeartbeat: time.Now(),
Role: info.Role,
}
existing.Replicas = append(existing.Replicas, ri)
r.addToServer(server, existingName)
// Sync deprecated scalar fields.
if len(existing.Replicas) == 1 {
existing.ReplicaServer = ri.Server
existing.ReplicaPath = ri.Path
}
glog.V(0).Infof("block registry: attached replica %s for %q from heartbeat (path=%s)",
server, existingName, info.Path)
}
} else {
// Auto-register volumes reported by heartbeat but not in registry.
// This recovers state after master restart.
name := nameFromPath(info.Path)
if name == "" {
continue
}
if info.ReplicaCtrlAddr != "" {
existing.ReplicaCtrlAddr = info.ReplicaCtrlAddr
if _, dup := r.volumes[name]; !dup {
entry := &BlockVolumeEntry{
Name: name,
VolumeServer: server,
Path: info.Path,
SizeBytes: info.VolumeSize,
Epoch: info.Epoch,
Role: info.Role,
Status: StatusActive,
LastLeaseGrant: time.Now(),
LeaseTTL: 30 * time.Second,
HealthScore: info.HealthScore,
WALHeadLSN: info.WalHeadLsn,
}
if info.ReplicaDataAddr != "" {
entry.ReplicaDataAddr = info.ReplicaDataAddr
}
if info.ReplicaCtrlAddr != "" {
entry.ReplicaCtrlAddr = info.ReplicaCtrlAddr
}
r.volumes[name] = entry
r.addToServer(server, name)
glog.V(0).Infof("block registry: auto-registered %q from heartbeat (server=%s, path=%s, size=%d)",
name, server, info.Path, info.VolumeSize)
}
}
// If no existing entry found by path, it was created outside master
// (e.g., manually). We don't auto-register unknown volumes — they
// must be created via CreateBlockVolume RPC.
}
}
@@ -274,7 +499,38 @@ func (r *BlockVolumeRegistry) removeFromServer(server, name string) {
}
}
// removeReplicaLocked removes a replica from an entry by server address.
// Caller must hold r.mu. Also syncs deprecated scalar fields.
func (r *BlockVolumeRegistry) removeReplicaLocked(entry *BlockVolumeEntry, server, name string) {
newReplicas := make([]ReplicaInfo, 0, len(entry.Replicas))
for _, ri := range entry.Replicas {
if ri.Server == server {
continue
}
newReplicas = append(newReplicas, ri)
}
entry.Replicas = newReplicas
// Sync deprecated scalar fields.
if len(entry.Replicas) > 0 {
r0 := &entry.Replicas[0]
entry.ReplicaServer = r0.Server
entry.ReplicaPath = r0.Path
entry.ReplicaISCSIAddr = r0.ISCSIAddr
entry.ReplicaIQN = r0.IQN
entry.ReplicaDataAddr = r0.DataAddr
entry.ReplicaCtrlAddr = r0.CtrlAddr
} else {
entry.ReplicaServer = ""
entry.ReplicaPath = ""
entry.ReplicaISCSIAddr = ""
entry.ReplicaIQN = ""
entry.ReplicaDataAddr = ""
entry.ReplicaCtrlAddr = ""
}
}
// SetReplica sets replica info for a registered volume.
// Deprecated: use AddReplica for new code. This method syncs both scalar and Replicas[].
func (r *BlockVolumeRegistry) SetReplica(name, server, path, iscsiAddr, iqn string) error {
r.mu.Lock()
defer r.mu.Unlock()
@@ -290,12 +546,31 @@ func (r *BlockVolumeRegistry) SetReplica(name, server, path, iscsiAddr, iqn stri
entry.ReplicaPath = path
entry.ReplicaISCSIAddr = iscsiAddr
entry.ReplicaIQN = iqn
// Also add to byServer index for the replica server.
r.addToServer(server, name)
// CP8-2: also sync to Replicas[].
info := ReplicaInfo{Server: server, Path: path, ISCSIAddr: iscsiAddr, IQN: iqn}
replaced := false
for i := range entry.Replicas {
if entry.Replicas[i].Server == server {
// Preserve existing health/LSN data.
info.HealthScore = entry.Replicas[i].HealthScore
info.WALHeadLSN = entry.Replicas[i].WALHeadLSN
info.DataAddr = entry.Replicas[i].DataAddr
info.CtrlAddr = entry.Replicas[i].CtrlAddr
entry.Replicas[i] = info
replaced = true
break
}
}
if !replaced {
entry.Replicas = append(entry.Replicas, info)
}
return nil
}
// ClearReplica removes replica info for a registered volume.
// ClearReplica removes all replica info for a registered volume.
// Deprecated: use RemoveReplica for new code. This method clears both scalar and Replicas[].
func (r *BlockVolumeRegistry) ClearReplica(name string) error {
r.mu.Lock()
defer r.mu.Unlock()
@@ -306,12 +581,19 @@ func (r *BlockVolumeRegistry) ClearReplica(name string) error {
if entry.ReplicaServer != "" {
r.removeFromServer(entry.ReplicaServer, name)
}
// Remove all replicas from byServer index.
for _, ri := range entry.Replicas {
if ri.Server != entry.ReplicaServer {
r.removeFromServer(ri.Server, name)
}
}
entry.ReplicaServer = ""
entry.ReplicaPath = ""
entry.ReplicaISCSIAddr = ""
entry.ReplicaIQN = ""
entry.ReplicaDataAddr = ""
entry.ReplicaCtrlAddr = ""
entry.Replicas = nil
return nil
}
@@ -363,6 +645,198 @@ func (r *BlockVolumeRegistry) SwapPrimaryReplica(name string) (uint64, error) {
return newEpoch, nil
}
// AddReplica adds or replaces a replica in the Replicas slice (by server).
// Also updates the byServer index and deprecated scalar fields for backward compat.
func (r *BlockVolumeRegistry) AddReplica(name string, info ReplicaInfo) error {
r.mu.Lock()
defer r.mu.Unlock()
entry, ok := r.volumes[name]
if !ok {
return fmt.Errorf("block volume %q not found", name)
}
// Replace if same server already exists.
replaced := false
for i := range entry.Replicas {
if entry.Replicas[i].Server == info.Server {
entry.Replicas[i] = info
replaced = true
break
}
}
if !replaced {
entry.Replicas = append(entry.Replicas, info)
}
r.addToServer(info.Server, name)
// Sync deprecated scalar fields (first replica → scalar).
if len(entry.Replicas) > 0 {
r0 := &entry.Replicas[0]
entry.ReplicaServer = r0.Server
entry.ReplicaPath = r0.Path
entry.ReplicaISCSIAddr = r0.ISCSIAddr
entry.ReplicaIQN = r0.IQN
entry.ReplicaDataAddr = r0.DataAddr
entry.ReplicaCtrlAddr = r0.CtrlAddr
}
return nil
}
// RemoveReplica removes a replica by server address.
func (r *BlockVolumeRegistry) RemoveReplica(name, server string) error {
r.mu.Lock()
defer r.mu.Unlock()
entry, ok := r.volumes[name]
if !ok {
return fmt.Errorf("block volume %q not found", name)
}
found := false
newReplicas := make([]ReplicaInfo, 0, len(entry.Replicas))
for _, ri := range entry.Replicas {
if ri.Server == server {
found = true
r.removeFromServer(server, name)
continue
}
newReplicas = append(newReplicas, ri)
}
if !found {
return fmt.Errorf("replica on %q not found for volume %q", server, name)
}
entry.Replicas = newReplicas
// Sync deprecated scalar fields.
if len(entry.Replicas) > 0 {
r0 := &entry.Replicas[0]
entry.ReplicaServer = r0.Server
entry.ReplicaPath = r0.Path
entry.ReplicaISCSIAddr = r0.ISCSIAddr
entry.ReplicaIQN = r0.IQN
entry.ReplicaDataAddr = r0.DataAddr
entry.ReplicaCtrlAddr = r0.CtrlAddr
} else {
entry.ReplicaServer = ""
entry.ReplicaPath = ""
entry.ReplicaISCSIAddr = ""
entry.ReplicaIQN = ""
entry.ReplicaDataAddr = ""
entry.ReplicaCtrlAddr = ""
}
return nil
}
// SetPromotionLSNTolerance configures the max WAL LSN lag for promotion eligibility.
func (r *BlockVolumeRegistry) SetPromotionLSNTolerance(tolerance uint64) {
r.mu.Lock()
defer r.mu.Unlock()
r.promotionLSNTolerance = tolerance
}
// PromotionLSNTolerance returns the current promotion LSN tolerance.
func (r *BlockVolumeRegistry) PromotionLSNTolerance() uint64 {
r.mu.RLock()
defer r.mu.RUnlock()
return r.promotionLSNTolerance
}
// PromoteBestReplica promotes the best eligible replica to primary.
// Eligibility: heartbeat fresh (within 2×LeaseTTL), WALHeadLSN within tolerance of primary,
// and role must be RoleReplica (not RoleRebuilding).
// The promoted replica is removed from Replicas[]. Other replicas stay.
// Old primary is NOT added to Replicas (needs rebuild).
// Returns the new epoch.
func (r *BlockVolumeRegistry) PromoteBestReplica(name string) (uint64, error) {
r.mu.Lock()
defer r.mu.Unlock()
entry, ok := r.volumes[name]
if !ok {
return 0, fmt.Errorf("block volume %q not found", name)
}
if len(entry.Replicas) == 0 {
return 0, fmt.Errorf("block volume %q has no replicas", name)
}
// Filter eligible replicas.
now := time.Now()
freshnessCutoff := 2 * entry.LeaseTTL
if freshnessCutoff == 0 {
freshnessCutoff = 60 * time.Second // default if LeaseTTL not set
}
primaryLSN := entry.WALHeadLSN
bestIdx := -1
for i := range entry.Replicas {
ri := &entry.Replicas[i]
// Gate 1: heartbeat freshness.
if !ri.LastHeartbeat.IsZero() && now.Sub(ri.LastHeartbeat) > freshnessCutoff {
continue
}
// Gate 2: WAL LSN recency (skip if primary LSN is 0 — no data yet, all eligible).
if primaryLSN > 0 && ri.WALHeadLSN+r.promotionLSNTolerance < primaryLSN {
continue
}
// Gate 3: role must be RoleReplica (not rebuilding/stale).
if ri.Role != 0 && blockvol.RoleFromWire(ri.Role) != blockvol.RoleReplica {
continue
}
// Eligible — pick best by health score, tie-break by WALHeadLSN.
if bestIdx == -1 {
bestIdx = i
} else if ri.HealthScore > entry.Replicas[bestIdx].HealthScore {
bestIdx = i
} else if ri.HealthScore == entry.Replicas[bestIdx].HealthScore &&
ri.WALHeadLSN > entry.Replicas[bestIdx].WALHeadLSN {
bestIdx = i
}
}
if bestIdx == -1 {
return 0, fmt.Errorf("block volume %q: no eligible replicas for promotion", name)
}
promoted := entry.Replicas[bestIdx]
// Remove old primary from byServer index.
r.removeFromServer(entry.VolumeServer, name)
// Bump epoch atomically.
newEpoch := entry.Epoch + 1
// Promote replica to primary.
entry.VolumeServer = promoted.Server
entry.Path = promoted.Path
entry.IQN = promoted.IQN
entry.ISCSIAddr = promoted.ISCSIAddr
entry.Epoch = newEpoch
entry.Role = blockvol.RoleToWire(blockvol.RolePrimary)
entry.LastLeaseGrant = time.Now()
// Remove promoted from Replicas. Others stay.
entry.Replicas = append(entry.Replicas[:bestIdx], entry.Replicas[bestIdx+1:]...)
// Sync deprecated scalar fields.
if len(entry.Replicas) > 0 {
r0 := &entry.Replicas[0]
entry.ReplicaServer = r0.Server
entry.ReplicaPath = r0.Path
entry.ReplicaISCSIAddr = r0.ISCSIAddr
entry.ReplicaIQN = r0.IQN
entry.ReplicaDataAddr = r0.DataAddr
entry.ReplicaCtrlAddr = r0.CtrlAddr
} else {
entry.ReplicaServer = ""
entry.ReplicaPath = ""
entry.ReplicaISCSIAddr = ""
entry.ReplicaIQN = ""
entry.ReplicaDataAddr = ""
entry.ReplicaCtrlAddr = ""
}
// Update byServer index: new primary server now hosts this volume.
r.addToServer(entry.VolumeServer, name)
return newEpoch, nil
}
// MarkBlockCapable records that the given server supports block volumes.
func (r *BlockVolumeRegistry) MarkBlockCapable(server string) {
r.mu.Lock()
@@ -377,6 +851,45 @@ func (r *BlockVolumeRegistry) UnmarkBlockCapable(server string) {
r.mu.Unlock()
}
// ListAll returns all registered block volume entries, sorted by name.
func (r *BlockVolumeRegistry) ListAll() []*BlockVolumeEntry {
r.mu.RLock()
defer r.mu.RUnlock()
entries := make([]*BlockVolumeEntry, 0, len(r.volumes))
for _, e := range r.volumes {
entries = append(entries, e)
}
sort.Slice(entries, func(i, j int) bool { return entries[i].Name < entries[j].Name })
return entries
}
// BlockServerSummary summarizes a block-capable volume server.
type BlockServerSummary struct {
Address string
VolumeCount int
BlockCapable bool
}
// ServerSummaries returns a summary for each block-capable server.
func (r *BlockVolumeRegistry) ServerSummaries() []BlockServerSummary {
r.mu.RLock()
defer r.mu.RUnlock()
summaries := make([]BlockServerSummary, 0, len(r.blockServers))
for addr := range r.blockServers {
count := 0
if names, ok := r.byServer[addr]; ok {
count = len(names)
}
summaries = append(summaries, BlockServerSummary{
Address: addr,
VolumeCount: count,
BlockCapable: true,
})
}
sort.Slice(summaries, func(i, j int) bool { return summaries[i].Address < summaries[j].Address })
return summaries
}
// BlockCapableServers returns the list of servers known to support block volumes.
func (r *BlockVolumeRegistry) BlockCapableServers() []string {
r.mu.RLock()
@@ -387,3 +900,38 @@ func (r *BlockVolumeRegistry) BlockCapableServers() []string {
}
return servers
}
// MaxBarrierLagLSN returns the maximum WAL lag across all volumes and replicas.
// This is the primary durability-risk metric: primary WALHeadLSN minus
// replica's WALHeadLSN. SLO threshold: < 100 under normal load.
func (r *BlockVolumeRegistry) MaxBarrierLagLSN() uint64 {
r.mu.RLock()
defer r.mu.RUnlock()
var maxLag uint64
for _, entry := range r.volumes {
for _, ri := range entry.Replicas {
if ri.WALLag > maxLag {
maxLag = ri.WALLag
}
}
}
return maxLag
}
// AssignmentQueueDepth returns the total number of pending assignments across all servers.
func (r *BlockVolumeRegistry) AssignmentQueueDepth() int {
// Delegated to the assignment queue, not the registry.
// Placeholder: the queue tracks its own depth.
return 0
}
// nameFromPath extracts the volume name from a .blk file path.
// e.g. "/opt/data/block/my-volume.blk" -> "my-volume"
func nameFromPath(path string) string {
base := filepath.Base(path)
if strings.HasSuffix(base, ".blk") {
return strings.TrimSuffix(base, ".blk")
}
return base
}
weed/server/master_block_registry_test.go
+557
View File
@@ -4,8 +4,10 @@ import (
"fmt"
"sync"
"testing"
"time"
"github.com/seaweedfs/seaweedfs/weed/pb/master_pb"
"github.com/seaweedfs/seaweedfs/weed/storage/blockvol"
)
func TestRegistry_RegisterLookup(t *testing.T) {
@@ -434,3 +436,558 @@ func TestFullHeartbeat_UpdatesReplicaAddrs(t *testing.T) {
t.Fatalf("ReplicaCtrlAddr: got %q, want 10.0.0.2:14261", entry.ReplicaCtrlAddr)
}
}
// --- CP8-2 new tests ---
func TestRegistry_AddReplica(t *testing.T) {
r := NewBlockVolumeRegistry()
r.Register(&BlockVolumeEntry{Name: "vol1", VolumeServer: "s1", Path: "/v1.blk"})
err := r.AddReplica("vol1", ReplicaInfo{
Server: "s2",
Path: "/replica/v1.blk",
ISCSIAddr: "10.0.0.2:3260",
IQN: "iqn:vol1-r1",
DataAddr: "s2:14260",
CtrlAddr: "s2:14261",
})
if err != nil {
t.Fatalf("AddReplica: %v", err)
}
e, _ := r.Lookup("vol1")
if len(e.Replicas) != 1 {
t.Fatalf("Replicas len: got %d, want 1", len(e.Replicas))
}
if e.Replicas[0].Server != "s2" {
t.Fatalf("Replicas[0].Server: got %q", e.Replicas[0].Server)
}
// Deprecated scalar should be synced.
if e.ReplicaServer != "s2" {
t.Fatalf("ReplicaServer (deprecated): got %q", e.ReplicaServer)
}
// byServer index should include replica.
if len(r.ListByServer("s2")) != 1 {
t.Fatalf("ListByServer(s2): got %d, want 1", len(r.ListByServer("s2")))
}
}
func TestRegistry_AddReplica_TwoRF3(t *testing.T) {
r := NewBlockVolumeRegistry()
r.Register(&BlockVolumeEntry{Name: "vol1", VolumeServer: "s1", Path: "/v1.blk", ReplicaFactor: 3})
r.AddReplica("vol1", ReplicaInfo{Server: "s2", Path: "/r1.blk", IQN: "iqn:r1"})
r.AddReplica("vol1", ReplicaInfo{Server: "s3", Path: "/r2.blk", IQN: "iqn:r2"})
e, _ := r.Lookup("vol1")
if len(e.Replicas) != 2 {
t.Fatalf("Replicas len: got %d, want 2", len(e.Replicas))
}
if e.Replicas[0].Server != "s2" || e.Replicas[1].Server != "s3" {
t.Fatalf("Replicas: got %+v", e.Replicas)
}
// byServer index should include both.
if len(r.ListByServer("s2")) != 1 || len(r.ListByServer("s3")) != 1 {
t.Fatal("byServer should include both replica servers")
}
}
func TestRegistry_AddReplica_Upsert(t *testing.T) {
r := NewBlockVolumeRegistry()
r.Register(&BlockVolumeEntry{Name: "vol1", VolumeServer: "s1", Path: "/v1.blk"})
r.AddReplica("vol1", ReplicaInfo{Server: "s2", Path: "/r1.blk"})
r.AddReplica("vol1", ReplicaInfo{Server: "s2", Path: "/r1-new.blk"})
e, _ := r.Lookup("vol1")
if len(e.Replicas) != 1 {
t.Fatalf("Replicas len: got %d, want 1 (upsert, not duplicate)", len(e.Replicas))
}
if e.Replicas[0].Path != "/r1-new.blk" {
t.Fatalf("Replicas[0].Path: got %q, want /r1-new.blk", e.Replicas[0].Path)
}
}
func TestRegistry_RemoveReplica(t *testing.T) {
r := NewBlockVolumeRegistry()
r.Register(&BlockVolumeEntry{Name: "vol1", VolumeServer: "s1", Path: "/v1.blk"})
r.AddReplica("vol1", ReplicaInfo{Server: "s2", Path: "/r1.blk"})
r.AddReplica("vol1", ReplicaInfo{Server: "s3", Path: "/r2.blk"})
err := r.RemoveReplica("vol1", "s2")
if err != nil {
t.Fatalf("RemoveReplica: %v", err)
}
e, _ := r.Lookup("vol1")
if len(e.Replicas) != 1 {
t.Fatalf("Replicas len: got %d, want 1", len(e.Replicas))
}
if e.Replicas[0].Server != "s3" {
t.Fatalf("remaining replica should be s3, got %q", e.Replicas[0].Server)
}
// Deprecated scalar should sync to first remaining replica.
if e.ReplicaServer != "s3" {
t.Fatalf("ReplicaServer (deprecated): got %q, want s3", e.ReplicaServer)
}
// s2 should be removed from byServer.
if len(r.ListByServer("s2")) != 0 {
t.Fatalf("ListByServer(s2): got %d, want 0", len(r.ListByServer("s2")))
}
}
func TestRegistry_PromoteBestReplica_PicksHighest(t *testing.T) {
r := NewBlockVolumeRegistry()
r.Register(&BlockVolumeEntry{
Name: "vol1",
VolumeServer: "s1",
Path: "/v1.blk",
Epoch: 5,
Role: 1,
Replicas: []ReplicaInfo{
{Server: "s2", Path: "/r1.blk", IQN: "iqn:r1", ISCSIAddr: "s2:3260", HealthScore: 0.8, WALHeadLSN: 100},
{Server: "s3", Path: "/r2.blk", IQN: "iqn:r2", ISCSIAddr: "s3:3260", HealthScore: 0.95, WALHeadLSN: 90},
},
})
// Add to byServer for s2 and s3.
r.mu.Lock()
r.addToServer("s2", "vol1")
r.addToServer("s3", "vol1")
r.mu.Unlock()
newEpoch, err := r.PromoteBestReplica("vol1")
if err != nil {
t.Fatalf("PromoteBestReplica: %v", err)
}
if newEpoch != 6 {
t.Fatalf("newEpoch: got %d, want 6", newEpoch)
}
e, _ := r.Lookup("vol1")
// s3 had higher health score → promoted.
if e.VolumeServer != "s3" {
t.Fatalf("VolumeServer: got %q, want s3 (higher health)", e.VolumeServer)
}
if e.Path != "/r2.blk" {
t.Fatalf("Path: got %q", e.Path)
}
// s2 should remain in Replicas.
if len(e.Replicas) != 1 {
t.Fatalf("Replicas len: got %d, want 1 (s2 stays)", len(e.Replicas))
}
if e.Replicas[0].Server != "s2" {
t.Fatalf("remaining replica: got %q, want s2", e.Replicas[0].Server)
}
}
func TestRegistry_PromoteBestReplica_NoReplica(t *testing.T) {
r := NewBlockVolumeRegistry()
r.Register(&BlockVolumeEntry{Name: "vol1", VolumeServer: "s1", Path: "/v1.blk"})
_, err := r.PromoteBestReplica("vol1")
if err == nil {
t.Fatal("PromoteBestReplica with no replicas should return error")
}
}
func TestRegistry_PromoteBestReplica_TiebreakByLSN(t *testing.T) {
r := NewBlockVolumeRegistry()
r.Register(&BlockVolumeEntry{
Name: "vol1",
VolumeServer: "s1",
Path: "/v1.blk",
Epoch: 3,
Replicas: []ReplicaInfo{
{Server: "s2", Path: "/r1.blk", IQN: "iqn:r1", ISCSIAddr: "s2:3260", HealthScore: 0.9, WALHeadLSN: 50},
{Server: "s3", Path: "/r2.blk", IQN: "iqn:r2", ISCSIAddr: "s3:3260", HealthScore: 0.9, WALHeadLSN: 100},
},
})
r.mu.Lock()
r.addToServer("s2", "vol1")
r.addToServer("s3", "vol1")
r.mu.Unlock()
newEpoch, err := r.PromoteBestReplica("vol1")
if err != nil {
t.Fatalf("PromoteBestReplica: %v", err)
}
if newEpoch != 4 {
t.Fatalf("newEpoch: got %d, want 4", newEpoch)
}
e, _ := r.Lookup("vol1")
// Same health → tie-break by WALHeadLSN → s3 wins.
if e.VolumeServer != "s3" {
t.Fatalf("VolumeServer: got %q, want s3 (higher LSN)", e.VolumeServer)
}
if len(e.Replicas) != 1 || e.Replicas[0].Server != "s2" {
t.Fatalf("remaining replica: got %+v, want [s2]", e.Replicas)
}
}
func TestRegistry_PromoteBestReplica_KeepsOthers(t *testing.T) {
r := NewBlockVolumeRegistry()
r.Register(&BlockVolumeEntry{
Name: "vol1",
VolumeServer: "s1",
Path: "/v1.blk",
Epoch: 1,
Replicas: []ReplicaInfo{
{Server: "s2", Path: "/r1.blk", IQN: "iqn:r1", ISCSIAddr: "s2:3260", HealthScore: 1.0, WALHeadLSN: 100},
{Server: "s3", Path: "/r2.blk", IQN: "iqn:r2", ISCSIAddr: "s3:3260", HealthScore: 0.5, WALHeadLSN: 100},
},
})
r.mu.Lock()
r.addToServer("s2", "vol1")
r.addToServer("s3", "vol1")
r.mu.Unlock()
r.PromoteBestReplica("vol1")
e, _ := r.Lookup("vol1")
// s2 promoted, s3 stays.
if e.VolumeServer != "s2" {
t.Fatalf("VolumeServer: got %q, want s2", e.VolumeServer)
}
if len(e.Replicas) != 1 || e.Replicas[0].Server != "s3" {
t.Fatalf("remaining replicas: got %+v, want [s3]", e.Replicas)
}
}
func TestRegistry_BackwardCompatAccessors(t *testing.T) {
r := NewBlockVolumeRegistry()
r.Register(&BlockVolumeEntry{Name: "vol1", VolumeServer: "s1", Path: "/v1.blk"})
e, _ := r.Lookup("vol1")
if e.HasReplica() {
t.Fatal("HasReplica should be false with no replicas")
}
if e.FirstReplica() != nil {
t.Fatal("FirstReplica should be nil")
}
if e.BestReplicaForPromotion() != nil {
t.Fatal("BestReplicaForPromotion should be nil")
}
r.AddReplica("vol1", ReplicaInfo{Server: "s2", Path: "/r.blk", HealthScore: 0.9})
e, _ = r.Lookup("vol1")
if !e.HasReplica() {
t.Fatal("HasReplica should be true after AddReplica")
}
if e.FirstReplica() == nil || e.FirstReplica().Server != "s2" {
t.Fatal("FirstReplica should return s2")
}
if e.ReplicaByServer("s2") == nil {
t.Fatal("ReplicaByServer(s2) should not be nil")
}
if e.ReplicaByServer("s3") != nil {
t.Fatal("ReplicaByServer(s3) should be nil")
}
}
func TestRegistry_ReplicaFactorDefault(t *testing.T) {
r := NewBlockVolumeRegistry()
r.Register(&BlockVolumeEntry{Name: "vol1", VolumeServer: "s1", Path: "/v1.blk"})
e, _ := r.Lookup("vol1")
// ReplicaFactor defaults to 0 (zero value). API handler defaults to 2.
if e.ReplicaFactor != 0 {
t.Fatalf("default ReplicaFactor: got %d, want 0", e.ReplicaFactor)
}
// Explicit RF=3.
r.Register(&BlockVolumeEntry{Name: "vol2", VolumeServer: "s1", Path: "/v2.blk", ReplicaFactor: 3})
e2, _ := r.Lookup("vol2")
if e2.ReplicaFactor != 3 {
t.Fatalf("ReplicaFactor: got %d, want 3", e2.ReplicaFactor)
}
}
func TestRegistry_FullHeartbeat_UpdatesHealthScore(t *testing.T) {
r := NewBlockVolumeRegistry()
r.Register(&BlockVolumeEntry{
Name: "vol1",
VolumeServer: "s1",
Path: "/v1.blk",
Status: StatusPending,
})
r.UpdateFullHeartbeat("s1", []*master_pb.BlockVolumeInfoMessage{
{
Path: "/v1.blk",
VolumeSize: 1 << 30,
Epoch: 1,
Role: 1,
HealthScore: 0.85,
ScrubErrors: 2,
WalHeadLsn: 500,
},
})
e, _ := r.Lookup("vol1")
if e.HealthScore != 0.85 {
t.Fatalf("HealthScore: got %f, want 0.85", e.HealthScore)
}
if e.WALHeadLSN != 500 {
t.Fatalf("WALHeadLSN: got %d, want 500", e.WALHeadLSN)
}
}
// Fix #1: Replica heartbeat must NOT delete the volume.
func TestRegistry_ReplicaHeartbeat_DoesNotDeleteVolume(t *testing.T) {
r := NewBlockVolumeRegistry()
r.Register(&BlockVolumeEntry{
Name: "vol1",
VolumeServer: "primary",
Path: "/data/vol1.blk",
Status: StatusActive,
Replicas: []ReplicaInfo{
{Server: "replica1", Path: "/data/vol1.blk"},
},
})
// Replica sends heartbeat reporting its path.
r.UpdateFullHeartbeat("replica1", []*master_pb.BlockVolumeInfoMessage{
{Path: "/data/vol1.blk", Epoch: 1, Role: 2},
})
// Volume must still exist with primary intact.
e, ok := r.Lookup("vol1")
if !ok {
t.Fatal("vol1 should not be deleted when replica sends heartbeat")
}
if e.VolumeServer != "primary" {
t.Fatalf("primary should remain 'primary', got %q", e.VolumeServer)
}
}
// Fix #1: Replica path NOT reported → replica removed, volume preserved.
func TestRegistry_ReplicaHeartbeat_StaleReplicaRemoved(t *testing.T) {
r := NewBlockVolumeRegistry()
r.Register(&BlockVolumeEntry{
Name: "vol1",
VolumeServer: "primary",
Path: "/data/vol1.blk",
Status: StatusActive,
Replicas: []ReplicaInfo{
{Server: "replica1", Path: "/data/vol1.blk"},
{Server: "replica2", Path: "/data/vol1.blk"},
},
})
// replica1 heartbeat does NOT report vol1 path → stale replica.
r.UpdateFullHeartbeat("replica1", []*master_pb.BlockVolumeInfoMessage{})
// Volume still exists, but replica1 removed.
e, ok := r.Lookup("vol1")
if !ok {
t.Fatal("vol1 should exist (only replica removed)")
}
if len(e.Replicas) != 1 {
t.Fatalf("expected 1 replica after stale removal, got %d", len(e.Replicas))
}
if e.Replicas[0].Server != "replica2" {
t.Fatalf("remaining replica should be replica2, got %q", e.Replicas[0].Server)
}
}
// Fix #3: Replica heartbeat after master restart reconstructs ReplicaInfo.
func TestRegistry_ReplicaHeartbeat_ReconstructsAfterRestart(t *testing.T) {
r := NewBlockVolumeRegistry()
// Simulate master restart: primary heartbeat re-created entry without replicas.
r.Register(&BlockVolumeEntry{
Name: "vol1",
VolumeServer: "primary",
Path: "/data/vol1.blk",
Status: StatusActive,
})
// Replica heartbeat arrives — vol1 exists but has no record of this server.
r.UpdateFullHeartbeat("replica1", []*master_pb.BlockVolumeInfoMessage{
{Path: "/data/vol1.blk", Epoch: 1, Role: 2, HealthScore: 0.95, WalHeadLsn: 42},
})
// vol1 should now have replica1 in Replicas[].
e, ok := r.Lookup("vol1")
if !ok {
t.Fatal("vol1 should exist")
}
if len(e.Replicas) != 1 {
t.Fatalf("expected 1 replica after reconstruction, got %d", len(e.Replicas))
}
ri := e.Replicas[0]
if ri.Server != "replica1" {
t.Fatalf("replica server: got %q, want replica1", ri.Server)
}
if ri.HealthScore != 0.95 {
t.Fatalf("replica health: got %f, want 0.95", ri.HealthScore)
}
if ri.WALHeadLSN != 42 {
t.Fatalf("replica WALHeadLSN: got %d, want 42", ri.WALHeadLSN)
}
// byServer index should include replica1.
entries := r.ListByServer("replica1")
if len(entries) != 1 || entries[0].Name != "vol1" {
t.Fatalf("ListByServer(replica1) should return vol1, got %+v", entries)
}
}
// Fix #2: Stale replica (old heartbeat) not eligible for promotion.
func TestRegistry_PromoteBestReplica_StaleHeartbeatIneligible(t *testing.T) {
r := NewBlockVolumeRegistry()
r.Register(&BlockVolumeEntry{
Name: "vol1",
VolumeServer: "primary",
Path: "/data/vol1.blk",
Epoch: 1,
LeaseTTL: 5 * time.Second,
WALHeadLSN: 100,
Replicas: []ReplicaInfo{
{
Server: "stale-replica",
Path: "/data/vol1.blk",
HealthScore: 1.0,
WALHeadLSN: 100,
LastHeartbeat: time.Now().Add(-30 * time.Second), // stale (>2×5s)
},
},
})
_, err := r.PromoteBestReplica("vol1")
if err == nil {
t.Fatal("expected error: stale replica should not be eligible")
}
}
// Fix #2: Replica with WAL lag too large is not eligible.
func TestRegistry_PromoteBestReplica_WALLagIneligible(t *testing.T) {
r := NewBlockVolumeRegistry()
r.Register(&BlockVolumeEntry{
Name: "vol1",
VolumeServer: "primary",
Path: "/data/vol1.blk",
Epoch: 1,
LeaseTTL: 30 * time.Second,
WALHeadLSN: 1000,
Replicas: []ReplicaInfo{
{
Server: "lagging",
Path: "/data/vol1.blk",
HealthScore: 1.0,
WALHeadLSN: 800, // lag=200, tolerance=100
LastHeartbeat: time.Now(),
},
},
})
_, err := r.PromoteBestReplica("vol1")
if err == nil {
t.Fatal("expected error: lagging replica should not be eligible")
}
}
// Fix #2: Rebuilding replica is not eligible for promotion.
func TestRegistry_PromoteBestReplica_RebuildingIneligible(t *testing.T) {
r := NewBlockVolumeRegistry()
r.Register(&BlockVolumeEntry{
Name: "vol1",
VolumeServer: "primary",
Path: "/data/vol1.blk",
Epoch: 1,
LeaseTTL: 30 * time.Second,
WALHeadLSN: 100,
Replicas: []ReplicaInfo{
{
Server: "rebuilding",
Path: "/data/vol1.blk",
HealthScore: 1.0,
WALHeadLSN: 100,
LastHeartbeat: time.Now(),
Role: blockvol.RoleToWire(blockvol.RoleRebuilding),
},
},
})
_, err := r.PromoteBestReplica("vol1")
if err == nil {
t.Fatal("expected error: rebuilding replica should not be eligible")
}
}
// Fix #2: Among eligible replicas, best (health+LSN) wins.
func TestRegistry_PromoteBestReplica_EligibilityFiltersCorrectly(t *testing.T) {
r := NewBlockVolumeRegistry()
r.Register(&BlockVolumeEntry{
Name: "vol1",
VolumeServer: "primary",
Path: "/data/vol1.blk",
Epoch: 1,
LeaseTTL: 30 * time.Second,
WALHeadLSN: 100,
Replicas: []ReplicaInfo{
{
Server: "stale", // ineligible: old heartbeat
Path: "/data/vol1.blk",
HealthScore: 1.0,
WALHeadLSN: 100,
LastHeartbeat: time.Now().Add(-2 * time.Minute),
},
{
Server: "good", // eligible
Path: "/data/vol1.blk",
HealthScore: 0.8,
WALHeadLSN: 95,
LastHeartbeat: time.Now(),
},
},
})
_, err := r.PromoteBestReplica("vol1")
if err != nil {
t.Fatalf("expected promotion to succeed: %v", err)
}
e, _ := r.Lookup("vol1")
if e.VolumeServer != "good" {
t.Fatalf("expected 'good' promoted (only eligible), got %q", e.VolumeServer)
}
}
// Configurable tolerance: widen tolerance to allow lagging replicas.
func TestRegistry_PromoteBestReplica_ConfigurableTolerance(t *testing.T) {
r := NewBlockVolumeRegistry()
r.Register(&BlockVolumeEntry{
Name: "vol1",
VolumeServer: "primary",
Path: "/data/vol1.blk",
Epoch: 1,
LeaseTTL: 30 * time.Second,
WALHeadLSN: 1000,
Replicas: []ReplicaInfo{
{
Server: "lagging",
Path: "/data/vol1.blk",
HealthScore: 1.0,
WALHeadLSN: 800, // lag=200
LastHeartbeat: time.Now(),
},
},
})
// Default tolerance (100): lag 200 > tolerance → ineligible.
_, err := r.PromoteBestReplica("vol1")
if err == nil {
t.Fatal("expected error with default tolerance")
}
// Widen tolerance to 250: lag 200 < tolerance → eligible.
r.SetPromotionLSNTolerance(250)
_, err = r.PromoteBestReplica("vol1")
if err != nil {
t.Fatalf("expected success with widened tolerance: %v", err)
}
e, _ := r.Lookup("vol1")
if e.VolumeServer != "lagging" {
t.Fatalf("expected 'lagging' promoted, got %q", e.VolumeServer)
}
}
weed/server/master_grpc_server_block.go
+244 -75
View File
@@ -8,8 +8,21 @@ import (
"github.com/seaweedfs/seaweedfs/weed/glog"
"github.com/seaweedfs/seaweedfs/weed/pb/master_pb"
"github.com/seaweedfs/seaweedfs/weed/storage/blockvol"
"github.com/seaweedfs/seaweedfs/weed/util/request_id"
)
// blockReqID extracts a short request ID from context for log correlation.
func blockReqID(ctx context.Context) string {
id := request_id.Get(ctx)
if len(id) > 8 {
return id[:8]
}
if id == "" {
return "no-id"
}
return id
}
// CreateBlockVolume picks a volume server, delegates creation, and records
// the mapping in the block volume registry.
func (ms *MasterServer) CreateBlockVolume(ctx context.Context, req *master_pb.CreateBlockVolumeRequest) (*master_pb.CreateBlockVolumeResponse, error) {
@@ -25,13 +38,7 @@ func (ms *MasterServer) CreateBlockVolume(ctx context.Context, req *master_pb.Cr
if entry.SizeBytes < req.SizeBytes {
return nil, fmt.Errorf("block volume %q exists with size %d (requested %d)", req.Name, entry.SizeBytes, req.SizeBytes)
}
return &master_pb.CreateBlockVolumeResponse{
VolumeId: entry.Name,
VolumeServer: entry.VolumeServer,
IscsiAddr: entry.ISCSIAddr,
Iqn: entry.IQN,
CapacityBytes: entry.SizeBytes,
}, nil
return ms.createBlockVolumeResponseFromEntry(entry), nil
}
// Per-name inflight lock prevents concurrent creates for the same name.
@@ -42,13 +49,7 @@ func (ms *MasterServer) CreateBlockVolume(ctx context.Context, req *master_pb.Cr
// Double-check after acquiring lock (another goroutine may have finished).
if entry, ok := ms.blockRegistry.Lookup(req.Name); ok {
return &master_pb.CreateBlockVolumeResponse{
VolumeId: entry.Name,
VolumeServer: entry.VolumeServer,
IscsiAddr: entry.ISCSIAddr,
Iqn: entry.IQN,
CapacityBytes: entry.SizeBytes,
}, nil
return ms.createBlockVolumeResponseFromEntry(entry), nil
}
// Get candidate servers.
@@ -73,11 +74,17 @@ func (ms *MasterServer) CreateBlockVolume(ctx context.Context, req *master_pb.Cr
result, err := ms.blockVSAllocate(ctx, server, req.Name, req.SizeBytes, req.DiskType)
if err != nil {
lastErr = fmt.Errorf("server %s: %w", server, err)
glog.V(0).Infof("CreateBlockVolume %q: attempt %d on %s failed: %v", req.Name, attempt+1, server, err)
glog.V(0).Infof("[reqID=%s] CreateBlockVolume %q: attempt %d on %s failed: %v", blockReqID(ctx), req.Name, attempt+1, server, err)
servers = removeServer(servers, server)
continue
}
// CP8-2: determine replica factor from request (default 2).
replicaFactor := 2
if req.ReplicaFactor > 0 && req.ReplicaFactor <= 3 {
replicaFactor = int(req.ReplicaFactor)
}
entry := &BlockVolumeEntry{
Name: req.Name,
VolumeServer: server,
@@ -88,66 +95,85 @@ func (ms *MasterServer) CreateBlockVolume(ctx context.Context, req *master_pb.Cr
Epoch: 1,
Role: blockvol.RoleToWire(blockvol.RolePrimary),
Status: StatusActive,
ReplicaFactor: replicaFactor,
LeaseTTL: 30 * time.Second,
LastLeaseGrant: time.Now(), // R2-F1: set BEFORE Register to avoid stale-lease race
}
// Try to create replica on a different server (F4: partial create OK).
var replicaServer string
remainingServers := removeServer(servers, server)
if len(remainingServers) > 0 {
replicaServer = ms.tryCreateReplica(ctx, req, entry, result, remainingServers)
} else {
glog.V(0).Infof("CreateBlockVolume %q: single-copy mode (only 1 server)", req.Name)
// Create replicaFactor-1 replicas on different servers (F4: partial create OK).
remaining := removeServer(servers, server)
for i := 0; i < replicaFactor-1 && len(remaining) > 0; i++ {
replicaServer := ms.tryCreateOneReplica(ctx, req, entry, result, remaining)
if replicaServer != "" {
remaining = removeServer(remaining, replicaServer)
}
}
if len(entry.Replicas) == 0 && replicaFactor > 1 {
glog.V(0).Infof("[reqID=%s] CreateBlockVolume %q: single-copy mode (replica allocation failed)", blockReqID(ctx), req.Name)
}
// Sync deprecated scalar fields from first replica.
if len(entry.Replicas) > 0 {
r0 := &entry.Replicas[0]
entry.ReplicaServer = r0.Server
entry.ReplicaPath = r0.Path
entry.ReplicaIQN = r0.IQN
entry.ReplicaISCSIAddr = r0.ISCSIAddr
entry.ReplicaDataAddr = r0.DataAddr
entry.ReplicaCtrlAddr = r0.CtrlAddr
}
// Register in registry as Active (VS confirmed creation).
if err := ms.blockRegistry.Register(entry); err != nil {
// Already registered (race condition) — return the existing entry.
if existing, ok := ms.blockRegistry.Lookup(req.Name); ok {
return &master_pb.CreateBlockVolumeResponse{
VolumeId: existing.Name,
VolumeServer: existing.VolumeServer,
IscsiAddr: existing.ISCSIAddr,
Iqn: existing.IQN,
CapacityBytes: existing.SizeBytes,
ReplicaServer: existing.ReplicaServer,
}, nil
return ms.createBlockVolumeResponseFromEntry(existing), nil
}
return nil, fmt.Errorf("register block volume: %w", err)
}
// Enqueue assignments for primary (and replica if available).
// Enqueue assignments for primary (and replicas if available).
leaseTTLMs := blockvol.LeaseTTLToWire(30 * time.Second)
ms.blockAssignmentQueue.Enqueue(server, blockvol.BlockVolumeAssignment{
Path: result.Path,
Epoch: 1,
Role: blockvol.RoleToWire(blockvol.RolePrimary),
LeaseTtlMs: leaseTTLMs,
ReplicaDataAddr: entry.ReplicaDataAddr,
ReplicaCtrlAddr: entry.ReplicaCtrlAddr,
})
if entry.ReplicaServer != "" {
ms.blockAssignmentQueue.Enqueue(entry.ReplicaServer, blockvol.BlockVolumeAssignment{
Path: entry.ReplicaPath,
primaryAssignment := blockvol.BlockVolumeAssignment{
Path: result.Path,
Epoch: 1,
Role: blockvol.RoleToWire(blockvol.RolePrimary),
LeaseTtlMs: leaseTTLMs,
}
// CP8-2: populate ReplicaAddrs for multi-replica.
for _, ri := range entry.Replicas {
primaryAssignment.ReplicaAddrs = append(primaryAssignment.ReplicaAddrs, blockvol.ReplicaAddr{
DataAddr: ri.DataAddr,
CtrlAddr: ri.CtrlAddr,
})
}
// Backward compat: also set scalar fields if exactly 1 replica.
if len(entry.Replicas) == 1 {
primaryAssignment.ReplicaDataAddr = entry.Replicas[0].DataAddr
primaryAssignment.ReplicaCtrlAddr = entry.Replicas[0].CtrlAddr
}
ms.blockAssignmentQueue.Enqueue(server, primaryAssignment)
// Enqueue assignments for each replica.
for _, ri := range entry.Replicas {
ms.blockAssignmentQueue.Enqueue(ri.Server, blockvol.BlockVolumeAssignment{
Path: ri.Path,
Epoch: 1,
Role: blockvol.RoleToWire(blockvol.RoleReplica),
LeaseTtlMs: leaseTTLMs,
ReplicaDataAddr: entry.ReplicaDataAddr,
ReplicaCtrlAddr: entry.ReplicaCtrlAddr,
ReplicaDataAddr: ri.DataAddr,
ReplicaCtrlAddr: ri.CtrlAddr,
})
}
glog.V(0).Infof("CreateBlockVolume %q: created on %s (path=%s, iqn=%s, replica=%s)",
req.Name, server, result.Path, result.IQN, replicaServer)
return &master_pb.CreateBlockVolumeResponse{
VolumeId: req.Name,
VolumeServer: server,
IscsiAddr: result.ISCSIAddr,
Iqn: result.IQN,
CapacityBytes: req.SizeBytes,
ReplicaServer: replicaServer,
}, nil
// Collect replica server addresses for response.
var replicaServers []string
for _, ri := range entry.Replicas {
replicaServers = append(replicaServers, ri.Server)
}
glog.V(0).Infof("[reqID=%s] CreateBlockVolume %q: created on %s (path=%s, iqn=%s, replicas=%v)",
blockReqID(ctx), req.Name, server, result.Path, result.IQN, replicaServers)
return ms.createBlockVolumeResponseFromEntry(entry), nil
}
return nil, fmt.Errorf("all volume servers failed for %q: %v", req.Name, lastErr)
@@ -170,16 +196,16 @@ func (ms *MasterServer) DeleteBlockVolume(ctx context.Context, req *master_pb.De
return nil, fmt.Errorf("delete block volume %q on %s: %w", req.Name, entry.VolumeServer, err)
}
// R2-F4: Also delete replica (best-effort, don't fail if replica is down).
if entry.ReplicaServer != "" {
if err := ms.blockVSDelete(ctx, entry.ReplicaServer, req.Name); err != nil {
glog.Warningf("DeleteBlockVolume %q: replica delete on %s failed (best-effort): %v",
req.Name, entry.ReplicaServer, err)
// CP8-2: Delete ALL replicas (best-effort, don't fail if replica is down).
for _, ri := range entry.Replicas {
if err := ms.blockVSDelete(ctx, ri.Server, req.Name); err != nil {
glog.Warningf("[reqID=%s] DeleteBlockVolume %q: replica delete on %s failed (best-effort): %v",
blockReqID(ctx), req.Name, ri.Server, err)
}
}
ms.blockRegistry.Unregister(req.Name)
glog.V(0).Infof("DeleteBlockVolume %q: removed from %s (replica=%s)", req.Name, entry.VolumeServer, entry.ReplicaServer)
glog.V(0).Infof("[reqID=%s] DeleteBlockVolume %q: removed from %s (replicas=%d)", blockReqID(ctx), req.Name, entry.VolumeServer, len(entry.Replicas))
return &master_pb.DeleteBlockVolumeResponse{}, nil
}
@@ -194,37 +220,180 @@ func (ms *MasterServer) LookupBlockVolume(ctx context.Context, req *master_pb.Lo
return nil, fmt.Errorf("block volume %q not found", req.Name)
}
replicaServers := replicaServerList(entry)
rf := entry.ReplicaFactor
if rf == 0 {
rf = 2 // default for pre-CP8-2 entries
}
return &master_pb.LookupBlockVolumeResponse{
VolumeServer: entry.VolumeServer,
IscsiAddr: entry.ISCSIAddr,
Iqn: entry.IQN,
CapacityBytes: entry.SizeBytes,
ReplicaServer: entry.ReplicaServer,
VolumeServer: entry.VolumeServer,
IscsiAddr: entry.ISCSIAddr,
Iqn: entry.IQN,
CapacityBytes: entry.SizeBytes,
ReplicaServer: entry.ReplicaServer, // backward compat
ReplicaFactor: uint32(rf),
ReplicaServers: replicaServers,
}, nil
}
// tryCreateReplica attempts to create a replica volume on a different server.
// tryCreateOneReplica attempts to create one replica volume on a different server.
// Returns the replica server address on success, or empty string on failure (F4).
func (ms *MasterServer) tryCreateReplica(ctx context.Context, req *master_pb.CreateBlockVolumeRequest, entry *BlockVolumeEntry, primaryResult *blockAllocResult, candidates []string) string {
func (ms *MasterServer) tryCreateOneReplica(ctx context.Context, req *master_pb.CreateBlockVolumeRequest, entry *BlockVolumeEntry, primaryResult *blockAllocResult, candidates []string) string {
for _, replicaServerStr := range candidates {
replicaResult, err := ms.blockVSAllocate(ctx, replicaServerStr, req.Name, req.SizeBytes, req.DiskType)
if err != nil {
glog.V(0).Infof("CreateBlockVolume %q: replica on %s failed: %v", req.Name, replicaServerStr, err)
glog.V(0).Infof("[reqID=%s] CreateBlockVolume %q: replica on %s failed: %v", blockReqID(ctx), req.Name, replicaServerStr, err)
continue
}
entry.ReplicaServer = replicaServerStr
entry.ReplicaPath = replicaResult.Path
entry.ReplicaIQN = replicaResult.IQN
entry.ReplicaISCSIAddr = replicaResult.ISCSIAddr
entry.ReplicaDataAddr = replicaResult.ReplicaDataAddr
entry.ReplicaCtrlAddr = replicaResult.ReplicaCtrlAddr
entry.RebuildListenAddr = primaryResult.RebuildListenAddr
// CP8-2: populate Replicas[].
entry.Replicas = append(entry.Replicas, ReplicaInfo{
Server: replicaServerStr,
Path: replicaResult.Path,
ISCSIAddr: replicaResult.ISCSIAddr,
IQN: replicaResult.IQN,
DataAddr: replicaResult.ReplicaDataAddr,
CtrlAddr: replicaResult.ReplicaCtrlAddr,
})
return replicaServerStr
}
glog.Warningf("CreateBlockVolume %q: created without replica (replica allocation failed)", req.Name)
glog.Warningf("[reqID=%s] CreateBlockVolume %q: created without replica (replica allocation failed)", blockReqID(ctx), req.Name)
return ""
}
// CreateBlockSnapshot creates a snapshot on a block volume via the volume server.
func (ms *MasterServer) CreateBlockSnapshot(ctx context.Context, req *master_pb.CreateBlockSnapshotRequest) (*master_pb.CreateBlockSnapshotResponse, error) {
if req.VolumeName == "" {
return nil, fmt.Errorf("volume_name is required")
}
entry, ok := ms.blockRegistry.Lookup(req.VolumeName)
if !ok {
return nil, fmt.Errorf("block volume %q not found", req.VolumeName)
}
createdAt, sizeBytes, err := ms.blockVSSnapshot(ctx, entry.VolumeServer, req.VolumeName, req.SnapshotId)
if err != nil {
return nil, fmt.Errorf("create snapshot on %s: %w", entry.VolumeServer, err)
}
return &master_pb.CreateBlockSnapshotResponse{
SnapshotId: req.SnapshotId,
CreatedAt: createdAt,
SizeBytes: sizeBytes,
}, nil
}
// DeleteBlockSnapshot deletes a snapshot from a block volume via the volume server.
func (ms *MasterServer) DeleteBlockSnapshot(ctx context.Context, req *master_pb.DeleteBlockSnapshotRequest) (*master_pb.DeleteBlockSnapshotResponse, error) {
if req.VolumeName == "" {
return nil, fmt.Errorf("volume_name is required")
}
entry, ok := ms.blockRegistry.Lookup(req.VolumeName)
if !ok {
// Idempotent: volume not found → snapshot doesn't exist either.
return &master_pb.DeleteBlockSnapshotResponse{}, nil
}
if err := ms.blockVSDeleteSnap(ctx, entry.VolumeServer, req.VolumeName, req.SnapshotId); err != nil {
return nil, fmt.Errorf("delete snapshot on %s: %w", entry.VolumeServer, err)
}
return &master_pb.DeleteBlockSnapshotResponse{}, nil
}
// ListBlockSnapshots lists all snapshots on a block volume via the volume server.
func (ms *MasterServer) ListBlockSnapshots(ctx context.Context, req *master_pb.ListBlockSnapshotsRequest) (*master_pb.ListBlockSnapshotsResponse, error) {
if req.VolumeName == "" {
return nil, fmt.Errorf("volume_name is required")
}
entry, ok := ms.blockRegistry.Lookup(req.VolumeName)
if !ok {
return nil, fmt.Errorf("block volume %q not found", req.VolumeName)
}
vsInfos, err := ms.blockVSListSnaps(ctx, entry.VolumeServer, req.VolumeName)
if err != nil {
return nil, fmt.Errorf("list snapshots on %s: %w", entry.VolumeServer, err)
}
resp := &master_pb.ListBlockSnapshotsResponse{}
for _, si := range vsInfos {
resp.Snapshots = append(resp.Snapshots, &master_pb.BlockSnapshotInfo{
SnapshotId: si.SnapshotId,
CreatedAt: si.CreatedAt,
VolumeSizeBytes: si.VolumeSizeBytes,
})
}
return resp, nil
}
// ExpandBlockVolume expands a block volume via the volume server, then updates registry.
func (ms *MasterServer) ExpandBlockVolume(ctx context.Context, req *master_pb.ExpandBlockVolumeRequest) (*master_pb.ExpandBlockVolumeResponse, error) {
if req.Name == "" {
return nil, fmt.Errorf("name is required")
}
if req.NewSizeBytes == 0 {
return nil, fmt.Errorf("new_size_bytes must be > 0")
}
entry, ok := ms.blockRegistry.Lookup(req.Name)
if !ok {
return nil, fmt.Errorf("block volume %q not found", req.Name)
}
// Expand primary first; only update registry on success.
capacity, err := ms.blockVSExpand(ctx, entry.VolumeServer, req.Name, req.NewSizeBytes)
if err != nil {
return nil, fmt.Errorf("expand on %s: %w", entry.VolumeServer, err)
}
// CP8-2: Expand ALL replicas (best-effort, log warning on failure).
for _, ri := range entry.Replicas {
if _, err := ms.blockVSExpand(ctx, ri.Server, req.Name, req.NewSizeBytes); err != nil {
glog.Warningf("[reqID=%s] ExpandBlockVolume %q: replica expand on %s failed (best-effort): %v",
blockReqID(ctx), req.Name, ri.Server, err)
}
}
// Update registry with actual new size.
if uerr := ms.blockRegistry.UpdateSize(req.Name, capacity); uerr != nil {
glog.Warningf("[reqID=%s] ExpandBlockVolume %q: registry update failed (VS succeeded): %v", blockReqID(ctx), req.Name, uerr)
}
return &master_pb.ExpandBlockVolumeResponse{
CapacityBytes: capacity,
}, nil
}
// createBlockVolumeResponseFromEntry builds a CreateBlockVolumeResponse from a registry entry.
func (ms *MasterServer) createBlockVolumeResponseFromEntry(entry *BlockVolumeEntry) *master_pb.CreateBlockVolumeResponse {
return &master_pb.CreateBlockVolumeResponse{
VolumeId: entry.Name,
VolumeServer: entry.VolumeServer,
IscsiAddr: entry.ISCSIAddr,
Iqn: entry.IQN,
CapacityBytes: entry.SizeBytes,
ReplicaServer: entry.ReplicaServer, // backward compat
ReplicaServers: replicaServerList(entry),
}
}
// replicaServerList returns the list of replica server addresses.
// Order matches Replicas[] (append-order), ensuring ReplicaServers[0] == ReplicaServer (legacy).
func replicaServerList(entry *BlockVolumeEntry) []string {
if len(entry.Replicas) == 0 {
return nil
}
servers := make([]string, len(entry.Replicas))
for i, ri := range entry.Replicas {
servers[i] = ri.Server
}
return servers
}
// removeServer returns a new slice without the specified server.
func removeServer(servers []string, server string) []string {
result := make([]string, 0, len(servers)-1)
weed/server/master_grpc_server_block_test.go
+488
View File
@@ -8,6 +8,7 @@ import (
"testing"
"github.com/seaweedfs/seaweedfs/weed/pb/master_pb"
"github.com/seaweedfs/seaweedfs/weed/pb/volume_server_pb"
)
// testMasterServer creates a minimal MasterServer with mock VS calls for testing.
@@ -492,6 +493,142 @@ func TestMaster_LookupReturnsReplicaServer(t *testing.T) {
}
}
func TestMaster_CreateBlockSnapshot(t *testing.T) {
ms := testMasterServer(t)
ms.blockRegistry.MarkBlockCapable("vs1:9333")
ms.blockVSSnapshot = func(ctx context.Context, server string, name string, snapID uint32) (int64, uint64, error) {
return 1709654400, 1 << 30, nil
}
// Create volume first.
ms.CreateBlockVolume(context.Background(), &master_pb.CreateBlockVolumeRequest{
Name: "snap-vol", SizeBytes: 1 << 30,
})
resp, err := ms.CreateBlockSnapshot(context.Background(), &master_pb.CreateBlockSnapshotRequest{
VolumeName: "snap-vol", SnapshotId: 42,
})
if err != nil {
t.Fatalf("CreateBlockSnapshot: %v", err)
}
if resp.SnapshotId != 42 {
t.Fatalf("SnapshotId: got %d, want 42", resp.SnapshotId)
}
if resp.CreatedAt != 1709654400 {
t.Fatalf("CreatedAt: got %d, want 1709654400", resp.CreatedAt)
}
}
func TestMaster_CreateBlockSnapshot_VolumeNotFound(t *testing.T) {
ms := testMasterServer(t)
_, err := ms.CreateBlockSnapshot(context.Background(), &master_pb.CreateBlockSnapshotRequest{
VolumeName: "nonexistent", SnapshotId: 1,
})
if err == nil {
t.Fatal("expected error for nonexistent volume")
}
}
func TestMaster_DeleteBlockSnapshot(t *testing.T) {
ms := testMasterServer(t)
ms.blockRegistry.MarkBlockCapable("vs1:9333")
ms.blockVSDeleteSnap = func(ctx context.Context, server string, name string, snapID uint32) error {
return nil
}
ms.CreateBlockVolume(context.Background(), &master_pb.CreateBlockVolumeRequest{
Name: "snap-vol", SizeBytes: 1 << 30,
})
_, err := ms.DeleteBlockSnapshot(context.Background(), &master_pb.DeleteBlockSnapshotRequest{
VolumeName: "snap-vol", SnapshotId: 42,
})
if err != nil {
t.Fatalf("DeleteBlockSnapshot: %v", err)
}
}
func TestMaster_ListBlockSnapshots(t *testing.T) {
ms := testMasterServer(t)
ms.blockRegistry.MarkBlockCapable("vs1:9333")
ms.blockVSListSnaps = func(ctx context.Context, server string, name string) ([]*volume_server_pb.BlockSnapshotInfo, error) {
return []*volume_server_pb.BlockSnapshotInfo{
{SnapshotId: 1, CreatedAt: 100},
{SnapshotId: 2, CreatedAt: 200},
}, nil
}
ms.CreateBlockVolume(context.Background(), &master_pb.CreateBlockVolumeRequest{
Name: "snap-vol", SizeBytes: 1 << 30,
})
resp, err := ms.ListBlockSnapshots(context.Background(), &master_pb.ListBlockSnapshotsRequest{
VolumeName: "snap-vol",
})
if err != nil {
t.Fatalf("ListBlockSnapshots: %v", err)
}
if len(resp.Snapshots) != 2 {
t.Fatalf("expected 2 snapshots, got %d", len(resp.Snapshots))
}
}
func TestMaster_ExpandBlockVolume(t *testing.T) {
ms := testMasterServer(t)
ms.blockRegistry.MarkBlockCapable("vs1:9333")
ms.blockVSExpand = func(ctx context.Context, server string, name string, newSize uint64) (uint64, error) {
return newSize, nil
}
ms.CreateBlockVolume(context.Background(), &master_pb.CreateBlockVolumeRequest{
Name: "expand-vol", SizeBytes: 1 << 30,
})
resp, err := ms.ExpandBlockVolume(context.Background(), &master_pb.ExpandBlockVolumeRequest{
Name: "expand-vol", NewSizeBytes: 2 << 30,
})
if err != nil {
t.Fatalf("ExpandBlockVolume: %v", err)
}
if resp.CapacityBytes != 2<<30 {
t.Fatalf("CapacityBytes: got %d, want %d", resp.CapacityBytes, 2<<30)
}
// Verify registry was updated.
entry, ok := ms.blockRegistry.Lookup("expand-vol")
if !ok {
t.Fatal("volume not found in registry")
}
if entry.SizeBytes != 2<<30 {
t.Fatalf("registry size: got %d, want %d", entry.SizeBytes, 2<<30)
}
}
func TestMaster_ExpandBlockVolume_VSFailure(t *testing.T) {
ms := testMasterServer(t)
ms.blockRegistry.MarkBlockCapable("vs1:9333")
ms.blockVSExpand = func(ctx context.Context, server string, name string, newSize uint64) (uint64, error) {
return 0, fmt.Errorf("disk full")
}
ms.CreateBlockVolume(context.Background(), &master_pb.CreateBlockVolumeRequest{
Name: "expand-vol", SizeBytes: 1 << 30,
})
_, err := ms.ExpandBlockVolume(context.Background(), &master_pb.ExpandBlockVolumeRequest{
Name: "expand-vol", NewSizeBytes: 2 << 30,
})
if err == nil {
t.Fatal("expected error when VS expand fails")
}
// Registry should NOT have been updated.
entry, _ := ms.blockRegistry.Lookup("expand-vol")
if entry.SizeBytes != 1<<30 {
t.Fatalf("registry size should be unchanged: got %d, want %d", entry.SizeBytes, 1<<30)
}
}
func TestMaster_LookupBlockVolume(t *testing.T) {
ms := testMasterServer(t)
ms.blockRegistry.MarkBlockCapable("vs1:9333")
@@ -525,3 +662,354 @@ func TestMaster_LookupBlockVolume(t *testing.T) {
t.Fatal("lookup nonexistent should return error")
}
}
// ============================================================
// CP8-2 T9: Multi-Replica Create/Delete/Assign Tests
// ============================================================
func testMasterServerRF3(t *testing.T) *MasterServer {
t.Helper()
ms := testMasterServer(t)
ms.blockVSAllocate = func(ctx context.Context, server string, name string, sizeBytes uint64, diskType string) (*blockAllocResult, error) {
return &blockAllocResult{
Path: fmt.Sprintf("/data/%s.blk", name),
IQN: fmt.Sprintf("iqn.2024.test:%s", name),
ISCSIAddr: server + ":3260",
ReplicaDataAddr: server + ":14260",
ReplicaCtrlAddr: server + ":14261",
}, nil
}
ms.blockRegistry.MarkBlockCapable("vs1:9333")
ms.blockRegistry.MarkBlockCapable("vs2:9333")
ms.blockRegistry.MarkBlockCapable("vs3:9333")
return ms
}
// RF=3 with 3 servers: should create 2 replicas.
func TestMaster_CreateRF3_ThreeServers(t *testing.T) {
ms := testMasterServerRF3(t)
resp, err := ms.CreateBlockVolume(context.Background(), &master_pb.CreateBlockVolumeRequest{
Name: "vol1",
SizeBytes: 1 << 30,
ReplicaFactor: 3,
})
if err != nil {
t.Fatalf("CreateBlockVolume: %v", err)
}
entry, ok := ms.blockRegistry.Lookup("vol1")
if !ok {
t.Fatal("vol1 not in registry")
}
if entry.ReplicaFactor != 3 {
t.Fatalf("ReplicaFactor: got %d, want 3", entry.ReplicaFactor)
}
if len(entry.Replicas) != 2 {
t.Fatalf("expected 2 replicas, got %d", len(entry.Replicas))
}
// All 3 servers should be different.
servers := map[string]bool{resp.VolumeServer: true}
for _, ri := range entry.Replicas {
if servers[ri.Server] {
t.Fatalf("duplicate server: %q", ri.Server)
}
servers[ri.Server] = true
}
if len(servers) != 3 {
t.Fatalf("expected 3 distinct servers, got %d", len(servers))
}
}
// RF=3 with only 2 servers: should create 1 replica (partial).
func TestMaster_CreateRF3_TwoServers(t *testing.T) {
ms := testMasterServer(t)
ms.blockVSAllocate = func(ctx context.Context, server string, name string, sizeBytes uint64, diskType string) (*blockAllocResult, error) {
return &blockAllocResult{
Path: fmt.Sprintf("/data/%s.blk", name),
IQN: fmt.Sprintf("iqn.2024.test:%s", name),
ISCSIAddr: server + ":3260",
ReplicaDataAddr: server + ":14260",
ReplicaCtrlAddr: server + ":14261",
}, nil
}
ms.blockRegistry.MarkBlockCapable("vs1:9333")
ms.blockRegistry.MarkBlockCapable("vs2:9333")
_, err := ms.CreateBlockVolume(context.Background(), &master_pb.CreateBlockVolumeRequest{
Name: "vol1",
SizeBytes: 1 << 30,
ReplicaFactor: 3,
})
if err != nil {
t.Fatalf("CreateBlockVolume: %v", err)
}
entry, _ := ms.blockRegistry.Lookup("vol1")
// Only 2 servers available, so only 1 replica (not 2).
if len(entry.Replicas) != 1 {
t.Fatalf("expected 1 replica (only 2 servers), got %d", len(entry.Replicas))
}
}
// RF=2 unchanged after CP8-2 changes.
func TestMaster_CreateRF2_Unchanged(t *testing.T) {
ms := testMasterServerRF3(t)
_, err := ms.CreateBlockVolume(context.Background(), &master_pb.CreateBlockVolumeRequest{
Name: "vol1",
SizeBytes: 1 << 30,
// No ReplicaFactor → defaults to 2.
})
if err != nil {
t.Fatalf("CreateBlockVolume: %v", err)
}
entry, _ := ms.blockRegistry.Lookup("vol1")
if entry.ReplicaFactor != 2 {
t.Fatalf("ReplicaFactor: got %d, want 2 (default)", entry.ReplicaFactor)
}
if len(entry.Replicas) != 1 {
t.Fatalf("expected 1 replica for RF=2, got %d", len(entry.Replicas))
}
// Backward compat scalar fields should be set.
if entry.ReplicaServer == "" {
t.Fatal("ReplicaServer (deprecated) should still be set for backward compat")
}
}
// DeleteBlockVolume RF=3 deletes all replicas.
func TestMaster_DeleteRF3_DeletesAllReplicas(t *testing.T) {
ms := testMasterServerRF3(t)
var deletedServers []string
ms.blockVSDelete = func(ctx context.Context, server string, name string) error {
deletedServers = append(deletedServers, server)
return nil
}
_, err := ms.CreateBlockVolume(context.Background(), &master_pb.CreateBlockVolumeRequest{
Name: "vol1",
SizeBytes: 1 << 30,
ReplicaFactor: 3,
})
if err != nil {
t.Fatalf("create: %v", err)
}
_, err = ms.DeleteBlockVolume(context.Background(), &master_pb.DeleteBlockVolumeRequest{
Name: "vol1",
})
if err != nil {
t.Fatalf("delete: %v", err)
}
// Should have deleted on primary + 2 replicas = 3 servers.
if len(deletedServers) != 3 {
t.Fatalf("expected 3 delete calls (primary + 2 replicas), got %d: %v",
len(deletedServers), deletedServers)
}
}
// ExpandBlockVolume RF=3 expands all replicas.
func TestMaster_ExpandRF3_ExpandsAllReplicas(t *testing.T) {
ms := testMasterServerRF3(t)
var expandedServers []string
ms.blockVSExpand = func(ctx context.Context, server string, name string, newSize uint64) (uint64, error) {
expandedServers = append(expandedServers, server)
return newSize, nil
}
_, err := ms.CreateBlockVolume(context.Background(), &master_pb.CreateBlockVolumeRequest{
Name: "vol1",
SizeBytes: 1 << 30,
ReplicaFactor: 3,
})
if err != nil {
t.Fatalf("create: %v", err)
}
_, err = ms.ExpandBlockVolume(context.Background(), &master_pb.ExpandBlockVolumeRequest{
Name: "vol1", NewSizeBytes: 2 << 30,
})
if err != nil {
t.Fatalf("expand: %v", err)
}
// Should have expanded on primary + 2 replicas = 3 servers.
if len(expandedServers) != 3 {
t.Fatalf("expected 3 expand calls (primary + 2 replicas), got %d: %v",
len(expandedServers), expandedServers)
}
}
// ProcessAssignments with multi-replica addrs.
func TestMaster_CreateRF3_AssignmentsIncludeReplicaAddrs(t *testing.T) {
ms := testMasterServerRF3(t)
resp, err := ms.CreateBlockVolume(context.Background(), &master_pb.CreateBlockVolumeRequest{
Name: "vol1",
SizeBytes: 1 << 30,
ReplicaFactor: 3,
})
if err != nil {
t.Fatalf("create: %v", err)
}
// Primary assignment should have ReplicaAddrs with 2 entries.
assignments := ms.blockAssignmentQueue.Peek(resp.VolumeServer)
if len(assignments) != 1 {
t.Fatalf("expected 1 primary assignment, got %d", len(assignments))
}
pa := assignments[0]
if len(pa.ReplicaAddrs) != 2 {
t.Fatalf("primary assignment ReplicaAddrs: got %d, want 2", len(pa.ReplicaAddrs))
}
// Each replica addr should have non-empty data/ctrl.
for i, ra := range pa.ReplicaAddrs {
if ra.DataAddr == "" || ra.CtrlAddr == "" {
t.Fatalf("ReplicaAddrs[%d] has empty addr: %+v", i, ra)
}
}
}
// Fix #4: CreateBlockVolumeResponse includes ReplicaServers.
func TestMaster_CreateResponse_IncludesReplicaServers(t *testing.T) {
ms := testMasterServerRF3(t)
resp, err := ms.CreateBlockVolume(context.Background(), &master_pb.CreateBlockVolumeRequest{
Name: "vol1",
SizeBytes: 1 << 30,
ReplicaFactor: 3,
})
if err != nil {
t.Fatalf("CreateBlockVolume: %v", err)
}
if len(resp.ReplicaServers) != 2 {
t.Fatalf("ReplicaServers: got %d, want 2", len(resp.ReplicaServers))
}
// ReplicaServers should match registry entry Replicas.
entry, _ := ms.blockRegistry.Lookup("vol1")
for i, ri := range entry.Replicas {
if resp.ReplicaServers[i] != ri.Server {
t.Errorf("ReplicaServers[%d]: got %q, want %q", i, resp.ReplicaServers[i], ri.Server)
}
}
// Backward compat: ReplicaServer (scalar) should be first replica.
if resp.ReplicaServer != entry.ReplicaServer {
t.Errorf("ReplicaServer: got %q, want %q", resp.ReplicaServer, entry.ReplicaServer)
}
}
// Fix #4: LookupBlockVolumeResponse includes ReplicaFactor and ReplicaServers.
func TestMaster_LookupResponse_IncludesReplicaFields(t *testing.T) {
ms := testMasterServerRF3(t)
_, err := ms.CreateBlockVolume(context.Background(), &master_pb.CreateBlockVolumeRequest{
Name: "vol1",
SizeBytes: 1 << 30,
ReplicaFactor: 3,
})
if err != nil {
t.Fatalf("CreateBlockVolume: %v", err)
}
resp, err := ms.LookupBlockVolume(context.Background(), &master_pb.LookupBlockVolumeRequest{
Name: "vol1",
})
if err != nil {
t.Fatalf("LookupBlockVolume: %v", err)
}
if resp.ReplicaFactor != 3 {
t.Fatalf("ReplicaFactor: got %d, want 3", resp.ReplicaFactor)
}
if len(resp.ReplicaServers) != 2 {
t.Fatalf("ReplicaServers: got %d, want 2", len(resp.ReplicaServers))
}
// Backward compat: ReplicaServer (scalar).
if resp.ReplicaServer == "" {
t.Error("ReplicaServer (scalar) should be non-empty for backward compat")
}
}
// Fix #4: Idempotent create returns ReplicaServers.
func TestMaster_CreateIdempotent_IncludesReplicaServers(t *testing.T) {
ms := testMasterServerRF3(t)
_, err := ms.CreateBlockVolume(context.Background(), &master_pb.CreateBlockVolumeRequest{
Name: "vol1",
SizeBytes: 1 << 30,
ReplicaFactor: 3,
})
if err != nil {
t.Fatalf("first create: %v", err)
}
// Second create (idempotent) should also include ReplicaServers.
resp, err := ms.CreateBlockVolume(context.Background(), &master_pb.CreateBlockVolumeRequest{
Name: "vol1",
SizeBytes: 1 << 30,
ReplicaFactor: 3,
})
if err != nil {
t.Fatalf("idempotent create: %v", err)
}
if len(resp.ReplicaServers) != 2 {
t.Fatalf("idempotent ReplicaServers: got %d, want 2", len(resp.ReplicaServers))
}
}
// Lookup returns ReplicaFactor=2 for pre-CP8-2 entries where ReplicaFactor is 0.
func TestMaster_LookupResponse_ReplicaFactorDefault(t *testing.T) {
ms := testMasterServer(t)
ms.blockRegistry.MarkBlockCapable("vs1:9333")
// Manually register entry with ReplicaFactor=0 (pre-CP8-2 legacy).
ms.blockRegistry.Register(&BlockVolumeEntry{
Name: "legacy-vol",
VolumeServer: "vs1:9333",
Path: "/data/legacy-vol.blk",
SizeBytes: 1 << 30,
Status: StatusActive,
// ReplicaFactor intentionally 0 (zero value).
})
resp, err := ms.LookupBlockVolume(context.Background(), &master_pb.LookupBlockVolumeRequest{
Name: "legacy-vol",
})
if err != nil {
t.Fatalf("LookupBlockVolume: %v", err)
}
if resp.ReplicaFactor != 2 {
t.Fatalf("ReplicaFactor: got %d, want 2 (default for pre-CP8-2)", resp.ReplicaFactor)
}
}
// ReplicaServers[0] matches ReplicaServer (legacy scalar).
func TestMaster_ResponseConsistency_ReplicaServerVsReplicaServers(t *testing.T) {
ms := testMasterServerRF3(t)
resp, err := ms.CreateBlockVolume(context.Background(), &master_pb.CreateBlockVolumeRequest{
Name: "vol1",
SizeBytes: 1 << 30,
ReplicaFactor: 3,
})
if err != nil {
t.Fatalf("CreateBlockVolume: %v", err)
}
if len(resp.ReplicaServers) < 1 {
t.Fatal("expected at least 1 replica server")
}
if resp.ReplicaServer != resp.ReplicaServers[0] {
t.Fatalf("ReplicaServer=%q != ReplicaServers[0]=%q",
resp.ReplicaServer, resp.ReplicaServers[0])
}
// Same for Lookup.
lresp, err := ms.LookupBlockVolume(context.Background(), &master_pb.LookupBlockVolumeRequest{
Name: "vol1",
})
if err != nil {
t.Fatalf("LookupBlockVolume: %v", err)
}
if lresp.ReplicaServer != lresp.ReplicaServers[0] {
t.Fatalf("Lookup: ReplicaServer=%q != ReplicaServers[0]=%q",
lresp.ReplicaServer, lresp.ReplicaServers[0])
}
}
weed/server/master_server.go
+88 -5
View File
@@ -60,9 +60,10 @@ type MasterOption struct {
MetricsAddress string
MetricsIntervalSec int
IsFollower bool
TelemetryUrl string
TelemetryEnabled bool
VolumeGrowthDisabled bool
TelemetryUrl string
TelemetryEnabled bool
VolumeGrowthDisabled bool
BlockPromotionLSNTolerance int
}
type MasterServer struct {
@@ -97,8 +98,12 @@ type MasterServer struct {
blockRegistry *BlockVolumeRegistry
blockAssignmentQueue *BlockAssignmentQueue
blockFailover *blockFailoverState
blockVSAllocate func(ctx context.Context, server string, name string, sizeBytes uint64, diskType string) (*blockAllocResult, error)
blockVSDelete func(ctx context.Context, server string, name string) error
blockVSAllocate func(ctx context.Context, server string, name string, sizeBytes uint64, diskType string) (*blockAllocResult, error)
blockVSDelete func(ctx context.Context, server string, name string) error
blockVSSnapshot func(ctx context.Context, server string, name string, snapID uint32) (int64, uint64, error)
blockVSDeleteSnap func(ctx context.Context, server string, name string, snapID uint32) error
blockVSListSnaps func(ctx context.Context, server string, name string) ([]*volume_server_pb.BlockSnapshotInfo, error)
blockVSExpand func(ctx context.Context, server string, name string, newSize uint64) (uint64, error)
}
func NewMasterServer(r *mux.Router, option *MasterOption, peers map[string]pb.ServerAddress) *MasterServer {
@@ -148,10 +153,17 @@ func NewMasterServer(r *mux.Router, option *MasterOption, peers map[string]pb.Se
}
ms.blockRegistry = NewBlockVolumeRegistry()
if option.BlockPromotionLSNTolerance > 0 {
ms.blockRegistry.SetPromotionLSNTolerance(uint64(option.BlockPromotionLSNTolerance))
}
ms.blockAssignmentQueue = NewBlockAssignmentQueue()
ms.blockFailover = newBlockFailoverState()
ms.blockVSAllocate = ms.defaultBlockVSAllocate
ms.blockVSDelete = ms.defaultBlockVSDelete
ms.blockVSSnapshot = ms.defaultBlockVSSnapshot
ms.blockVSDeleteSnap = ms.defaultBlockVSDeleteSnap
ms.blockVSListSnaps = ms.defaultBlockVSListSnaps
ms.blockVSExpand = ms.defaultBlockVSExpand
ms.MasterClient.SetOnPeerUpdateFn(ms.OnPeerUpdate)
@@ -197,6 +209,18 @@ func NewMasterServer(r *mux.Router, option *MasterOption, peers map[string]pb.Se
r.HandleFunc("/stats/counter", ms.guard.WhiteList(statsCounterHandler))
r.HandleFunc("/stats/memory", ms.guard.WhiteList(statsMemoryHandler))
*/
// Block volume management routes.
r.HandleFunc("/block/volume", ms.proxyToLeader(ms.guard.WhiteList(requestIDMiddleware(ms.blockVolumeCreateHandler)))).Methods("POST")
r.HandleFunc("/block/volume/{name}", ms.proxyToLeader(ms.guard.WhiteList(requestIDMiddleware(ms.blockVolumeDeleteHandler)))).Methods("DELETE")
r.HandleFunc("/block/volume/{name}", ms.guard.WhiteList(requestIDMiddleware(ms.blockVolumeLookupHandler))).Methods("GET")
r.HandleFunc("/block/volumes", ms.guard.WhiteList(requestIDMiddleware(ms.blockVolumeListHandler))).Methods("GET")
r.HandleFunc("/block/assign", ms.proxyToLeader(ms.guard.WhiteList(requestIDMiddleware(ms.blockAssignHandler)))).Methods("POST")
r.HandleFunc("/block/servers", ms.guard.WhiteList(requestIDMiddleware(ms.blockServersHandler))).Methods("GET")
r.HandleFunc("/block/status", ms.guard.WhiteList(requestIDMiddleware(ms.blockStatusHandler))).Methods("GET")
r.HandleFunc("/block/ops", requestIDMiddleware(ms.blockOpsHandler))
r.HandleFunc("/block/", requestIDMiddleware(ms.blockUIHandler))
r.HandleFunc("/{fileId}", requestIDMiddleware(ms.redirectHandler))
}
@@ -560,3 +584,62 @@ func (ms *MasterServer) defaultBlockVSDelete(ctx context.Context, server string,
return err
})
}
func (ms *MasterServer) defaultBlockVSSnapshot(ctx context.Context, server string, name string, snapID uint32) (int64, uint64, error) {
var createdAt int64
var sizeBytes uint64
err := operation.WithVolumeServerClient(false, pb.ServerAddress(server), ms.grpcDialOption, func(client volume_server_pb.VolumeServerClient) error {
resp, rerr := client.SnapshotBlockVolume(ctx, &volume_server_pb.SnapshotBlockVolumeRequest{
Name: name,
SnapshotId: snapID,
})
if rerr != nil {
return rerr
}
createdAt = resp.CreatedAt
sizeBytes = resp.SizeBytes
return nil
})
return createdAt, sizeBytes, err
}
func (ms *MasterServer) defaultBlockVSDeleteSnap(ctx context.Context, server string, name string, snapID uint32) error {
return operation.WithVolumeServerClient(false, pb.ServerAddress(server), ms.grpcDialOption, func(client volume_server_pb.VolumeServerClient) error {
_, err := client.DeleteBlockSnapshot(ctx, &volume_server_pb.DeleteBlockSnapshotRequest{
Name: name,
SnapshotId: snapID,
})
return err
})
}
func (ms *MasterServer) defaultBlockVSListSnaps(ctx context.Context, server string, name string) ([]*volume_server_pb.BlockSnapshotInfo, error) {
var infos []*volume_server_pb.BlockSnapshotInfo
err := operation.WithVolumeServerClient(false, pb.ServerAddress(server), ms.grpcDialOption, func(client volume_server_pb.VolumeServerClient) error {
resp, rerr := client.ListBlockSnapshots(ctx, &volume_server_pb.ListBlockSnapshotsRequest{
Name: name,
})
if rerr != nil {
return rerr
}
infos = resp.Snapshots
return nil
})
return infos, err
}
func (ms *MasterServer) defaultBlockVSExpand(ctx context.Context, server string, name string, newSize uint64) (uint64, error) {
var capacity uint64
err := operation.WithVolumeServerClient(false, pb.ServerAddress(server), ms.grpcDialOption, func(client volume_server_pb.VolumeServerClient) error {
resp, rerr := client.ExpandBlockVolume(ctx, &volume_server_pb.ExpandBlockVolumeRequest{
Name: name,
NewSizeBytes: newSize,
})
if rerr != nil {
return rerr
}
capacity = resp.CapacityBytes
return nil
})
return capacity, err
}
weed/server/master_server_handlers_block.go
+209
View File
@@ -0,0 +1,209 @@
package weed_server
import (
"encoding/json"
"fmt"
"net/http"
"github.com/gorilla/mux"
"github.com/seaweedfs/seaweedfs/weed/pb/master_pb"
"github.com/seaweedfs/seaweedfs/weed/storage/blockvol"
"github.com/seaweedfs/seaweedfs/weed/storage/blockvol/blockapi"
)
// blockVolumeCreateHandler handles POST /block/volume.
func (ms *MasterServer) blockVolumeCreateHandler(w http.ResponseWriter, r *http.Request) {
var req blockapi.CreateVolumeRequest
if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
writeJsonError(w, r, http.StatusBadRequest, fmt.Errorf("invalid request body: %w", err))
return
}
// Store replica_placement in registry after creation.
replicaPlacement := req.ReplicaPlacement
if replicaPlacement == "" {
replicaPlacement = "000"
}
resp, err := ms.CreateBlockVolume(r.Context(), &master_pb.CreateBlockVolumeRequest{
Name: req.Name,
SizeBytes: req.SizeBytes,
DiskType: req.DiskType,
})
if err != nil {
writeJsonError(w, r, http.StatusInternalServerError, err)
return
}
// Store replica_placement on the registry entry.
if entry, ok := ms.blockRegistry.Lookup(resp.VolumeId); ok {
entry.ReplicaPlacement = replicaPlacement
}
// Look up the full entry to populate all fields.
info := blockapi.VolumeInfo{
Name: resp.VolumeId,
VolumeServer: resp.VolumeServer,
SizeBytes: resp.CapacityBytes,
ReplicaPlacement: replicaPlacement,
ISCSIAddr: resp.IscsiAddr,
IQN: resp.Iqn,
}
if entry, ok := ms.blockRegistry.Lookup(resp.VolumeId); ok {
info = entryToVolumeInfo(entry)
}
writeJsonQuiet(w, r, http.StatusOK, info)
}
// blockVolumeDeleteHandler handles DELETE /block/volume/{name}.
func (ms *MasterServer) blockVolumeDeleteHandler(w http.ResponseWriter, r *http.Request) {
name := mux.Vars(r)["name"]
if name == "" {
writeJsonError(w, r, http.StatusBadRequest, fmt.Errorf("name is required"))
return
}
_, err := ms.DeleteBlockVolume(r.Context(), &master_pb.DeleteBlockVolumeRequest{
Name: name,
})
if err != nil {
writeJsonError(w, r, http.StatusInternalServerError, err)
return
}
writeJsonQuiet(w, r, http.StatusOK, map[string]string{"status": "deleted"})
}
// blockVolumeLookupHandler handles GET /block/volume/{name}.
func (ms *MasterServer) blockVolumeLookupHandler(w http.ResponseWriter, r *http.Request) {
name := mux.Vars(r)["name"]
if name == "" {
writeJsonError(w, r, http.StatusBadRequest, fmt.Errorf("name is required"))
return
}
entry, ok := ms.blockRegistry.Lookup(name)
if !ok {
writeJsonError(w, r, http.StatusNotFound, fmt.Errorf("block volume %q not found", name))
return
}
writeJsonQuiet(w, r, http.StatusOK, entryToVolumeInfo(entry))
}
// blockVolumeListHandler handles GET /block/volumes.
func (ms *MasterServer) blockVolumeListHandler(w http.ResponseWriter, r *http.Request) {
entries := ms.blockRegistry.ListAll()
infos := make([]blockapi.VolumeInfo, len(entries))
for i, e := range entries {
infos[i] = entryToVolumeInfo(e)
}
writeJsonQuiet(w, r, http.StatusOK, infos)
}
// blockAssignHandler handles POST /block/assign.
func (ms *MasterServer) blockAssignHandler(w http.ResponseWriter, r *http.Request) {
var req blockapi.AssignRequest
if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
writeJsonError(w, r, http.StatusBadRequest, fmt.Errorf("invalid request body: %w", err))
return
}
if req.Name == "" {
writeJsonError(w, r, http.StatusBadRequest, fmt.Errorf("name is required"))
return
}
// Resolve name → registry entry.
entry, ok := ms.blockRegistry.Lookup(req.Name)
if !ok {
writeJsonError(w, r, http.StatusNotFound, fmt.Errorf("block volume %q not found", req.Name))
return
}
// Determine target server + path based on role.
server, path := entry.VolumeServer, entry.Path
if req.Role == "replica" {
if entry.ReplicaServer == "" {
writeJsonError(w, r, http.StatusBadRequest, fmt.Errorf("block volume %q has no replica", req.Name))
return
}
server, path = entry.ReplicaServer, entry.ReplicaPath
}
ms.blockAssignmentQueue.Enqueue(server, blockvol.BlockVolumeAssignment{
Path: path,
Epoch: req.Epoch,
Role: blockapi.RoleFromString(req.Role),
LeaseTtlMs: uint32(req.LeaseTTLMs),
})
writeJsonQuiet(w, r, http.StatusOK, map[string]string{"status": "queued"})
}
// blockServersHandler handles GET /block/servers.
func (ms *MasterServer) blockServersHandler(w http.ResponseWriter, r *http.Request) {
summaries := ms.blockRegistry.ServerSummaries()
infos := make([]blockapi.ServerInfo, len(summaries))
for i, s := range summaries {
infos[i] = blockapi.ServerInfo{
Address: s.Address,
VolumeCount: s.VolumeCount,
BlockCapable: s.BlockCapable,
}
}
writeJsonQuiet(w, r, http.StatusOK, infos)
}
// blockStatusHandler handles GET /block/status — returns registry configuration for debugging.
func (ms *MasterServer) blockStatusHandler(w http.ResponseWriter, r *http.Request) {
status := map[string]interface{}{
"promotion_lsn_tolerance": ms.blockRegistry.PromotionLSNTolerance(),
"volume_count": len(ms.blockRegistry.ListAll()),
"server_count": len(ms.blockRegistry.BlockCapableServers()),
"barrier_lag_lsn": ms.blockRegistry.MaxBarrierLagLSN(),
"promotions_total": ms.blockRegistry.PromotionsTotal.Load(),
"failovers_total": ms.blockRegistry.FailoversTotal.Load(),
"rebuilds_total": ms.blockRegistry.RebuildsTotal.Load(),
"assignment_queue_depth": ms.blockAssignmentQueue.TotalPending(),
}
writeJsonQuiet(w, r, http.StatusOK, status)
}
// entryToVolumeInfo converts a BlockVolumeEntry to a blockapi.VolumeInfo.
func entryToVolumeInfo(e *BlockVolumeEntry) blockapi.VolumeInfo {
status := "pending"
if e.Status == StatusActive {
status = "active"
}
rf := e.ReplicaFactor
if rf == 0 {
rf = 2 // default
}
info := blockapi.VolumeInfo{
Name: e.Name,
VolumeServer: e.VolumeServer,
SizeBytes: e.SizeBytes,
ReplicaPlacement: e.ReplicaPlacement,
Epoch: e.Epoch,
Role: blockvol.RoleFromWire(e.Role).String(),
Status: status,
ISCSIAddr: e.ISCSIAddr,
IQN: e.IQN,
ReplicaServer: e.ReplicaServer,
ReplicaISCSIAddr: e.ReplicaISCSIAddr,
ReplicaIQN: e.ReplicaIQN,
ReplicaDataAddr: e.ReplicaDataAddr,
ReplicaCtrlAddr: e.ReplicaCtrlAddr,
ReplicaFactor: rf,
HealthScore: e.HealthScore,
ReplicaDegraded: e.ReplicaDegraded,
}
for _, ri := range e.Replicas {
info.Replicas = append(info.Replicas, blockapi.ReplicaDetail{
Server: ri.Server,
ISCSIAddr: ri.ISCSIAddr,
IQN: ri.IQN,
HealthScore: ri.HealthScore,
WALLag: ri.WALLag,
})
}
return info
}
weed/server/master_server_handlers_block_test.go
+288
View File
@@ -0,0 +1,288 @@
package weed_server
import (
"bytes"
"context"
"encoding/json"
"fmt"
"net/http"
"net/http/httptest"
"testing"
"github.com/gorilla/mux"
"github.com/seaweedfs/seaweedfs/weed/storage/blockvol/blockapi"
)
// blockTestServer creates a MasterServer + httptest.Server with block routes registered.
func blockTestServer(t *testing.T) (*MasterServer, *httptest.Server) {
t.Helper()
ms := &MasterServer{
blockRegistry: NewBlockVolumeRegistry(),
blockAssignmentQueue: NewBlockAssignmentQueue(),
}
ms.blockVSAllocate = func(ctx context.Context, server string, name string, sizeBytes uint64, diskType string) (*blockAllocResult, error) {
return &blockAllocResult{
Path: fmt.Sprintf("/data/%s.blk", name),
IQN: fmt.Sprintf("iqn.2024.test:%s", name),
ISCSIAddr: server + ":3260",
}, nil
}
ms.blockVSDelete = func(ctx context.Context, server string, name string) error {
return nil
}
ms.blockRegistry.MarkBlockCapable("vs1:9333")
r := mux.NewRouter()
r.HandleFunc("/block/volume", ms.blockVolumeCreateHandler).Methods("POST")
r.HandleFunc("/block/volume/{name}", ms.blockVolumeDeleteHandler).Methods("DELETE")
r.HandleFunc("/block/volume/{name}", ms.blockVolumeLookupHandler).Methods("GET")
r.HandleFunc("/block/volumes", ms.blockVolumeListHandler).Methods("GET")
r.HandleFunc("/block/assign", ms.blockAssignHandler).Methods("POST")
r.HandleFunc("/block/servers", ms.blockServersHandler).Methods("GET")
ts := httptest.NewServer(r)
t.Cleanup(ts.Close)
return ms, ts
}
func TestBlockVolumeCreateHandler(t *testing.T) {
_, ts := blockTestServer(t)
body, _ := json.Marshal(blockapi.CreateVolumeRequest{
Name: "vol1",
SizeBytes: 1 << 30,
})
resp, err := http.Post(ts.URL+"/block/volume", "application/json", bytes.NewReader(body))
if err != nil {
t.Fatal(err)
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusOK {
t.Fatalf("expected 200, got %d", resp.StatusCode)
}
var info blockapi.VolumeInfo
if err := json.NewDecoder(resp.Body).Decode(&info); err != nil {
t.Fatal(err)
}
if info.Name != "vol1" {
t.Errorf("expected name vol1, got %s", info.Name)
}
if info.VolumeServer == "" {
t.Error("expected non-empty volume server")
}
}
func TestBlockVolumeListHandler(t *testing.T) {
ms, ts := blockTestServer(t)
// Create two volumes via gRPC.
for _, name := range []string{"alpha", "beta"} {
ms.blockRegistry.Register(&BlockVolumeEntry{
Name: name,
VolumeServer: "vs1:9333",
SizeBytes: 1 << 20,
Status: StatusActive,
})
}
resp, err := http.Get(ts.URL + "/block/volumes")
if err != nil {
t.Fatal(err)
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusOK {
t.Fatalf("expected 200, got %d", resp.StatusCode)
}
var infos []blockapi.VolumeInfo
if err := json.NewDecoder(resp.Body).Decode(&infos); err != nil {
t.Fatal(err)
}
if len(infos) != 2 {
t.Fatalf("expected 2 volumes, got %d", len(infos))
}
if infos[0].Name != "alpha" || infos[1].Name != "beta" {
t.Errorf("expected sorted [alpha, beta], got [%s, %s]", infos[0].Name, infos[1].Name)
}
}
func TestBlockVolumeLookupHandler(t *testing.T) {
ms, ts := blockTestServer(t)
ms.blockRegistry.Register(&BlockVolumeEntry{
Name: "vol1",
VolumeServer: "vs1:9333",
SizeBytes: 1 << 30,
Epoch: 3,
Role: 1, // RolePrimary
Status: StatusActive,
ReplicaServer: "vs2:9333",
})
resp, err := http.Get(ts.URL + "/block/volume/vol1")
if err != nil {
t.Fatal(err)
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusOK {
t.Fatalf("expected 200, got %d", resp.StatusCode)
}
var info blockapi.VolumeInfo
if err := json.NewDecoder(resp.Body).Decode(&info); err != nil {
t.Fatal(err)
}
if info.Epoch != 3 {
t.Errorf("expected epoch 3, got %d", info.Epoch)
}
if info.Role != "primary" {
t.Errorf("expected role primary, got %s", info.Role)
}
if info.ReplicaServer != "vs2:9333" {
t.Errorf("expected replica vs2:9333, got %s", info.ReplicaServer)
}
// Not found case.
resp2, err := http.Get(ts.URL + "/block/volume/nonexistent")
if err != nil {
t.Fatal(err)
}
defer resp2.Body.Close()
if resp2.StatusCode != http.StatusNotFound {
t.Fatalf("expected 404, got %d", resp2.StatusCode)
}
}
func TestBlockVolumeDeleteHandler(t *testing.T) {
ms, ts := blockTestServer(t)
ms.blockRegistry.Register(&BlockVolumeEntry{
Name: "vol1",
VolumeServer: "vs1:9333",
SizeBytes: 1 << 30,
Status: StatusActive,
})
req, _ := http.NewRequest("DELETE", ts.URL+"/block/volume/vol1", nil)
resp, err := http.DefaultClient.Do(req)
if err != nil {
t.Fatal(err)
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusOK {
t.Fatalf("expected 200, got %d", resp.StatusCode)
}
// Verify it's gone.
if _, ok := ms.blockRegistry.Lookup("vol1"); ok {
t.Error("expected vol1 to be unregistered")
}
}
func TestBlockAssignHandler(t *testing.T) {
ms, ts := blockTestServer(t)
ms.blockRegistry.Register(&BlockVolumeEntry{
Name: "vol1",
VolumeServer: "vs1:9333",
Path: "/data/vol1.blk",
SizeBytes: 1 << 30,
Status: StatusActive,
})
body, _ := json.Marshal(blockapi.AssignRequest{
Name: "vol1",
Epoch: 5,
Role: "primary",
LeaseTTLMs: 30000,
})
resp, err := http.Post(ts.URL+"/block/assign", "application/json", bytes.NewReader(body))
if err != nil {
t.Fatal(err)
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusOK {
t.Fatalf("expected 200, got %d", resp.StatusCode)
}
// Verify assignment was queued.
pending := ms.blockAssignmentQueue.Peek("vs1:9333")
if len(pending) != 1 {
t.Fatalf("expected 1 pending assignment, got %d", len(pending))
}
if pending[0].Epoch != 5 {
t.Errorf("expected epoch 5, got %d", pending[0].Epoch)
}
// Not-found case.
body2, _ := json.Marshal(blockapi.AssignRequest{Name: "missing", Epoch: 1, Role: "primary"})
resp2, err := http.Post(ts.URL+"/block/assign", "application/json", bytes.NewReader(body2))
if err != nil {
t.Fatal(err)
}
defer resp2.Body.Close()
if resp2.StatusCode != http.StatusNotFound {
t.Fatalf("expected 404, got %d", resp2.StatusCode)
}
}
func TestBlockServersHandler(t *testing.T) {
_, ts := blockTestServer(t)
resp, err := http.Get(ts.URL + "/block/servers")
if err != nil {
t.Fatal(err)
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusOK {
t.Fatalf("expected 200, got %d", resp.StatusCode)
}
var servers []blockapi.ServerInfo
if err := json.NewDecoder(resp.Body).Decode(&servers); err != nil {
t.Fatal(err)
}
if len(servers) != 1 {
t.Fatalf("expected 1 server, got %d", len(servers))
}
if servers[0].Address != "vs1:9333" {
t.Errorf("expected vs1:9333, got %s", servers[0].Address)
}
}
func TestListAll(t *testing.T) {
r := NewBlockVolumeRegistry()
r.Register(&BlockVolumeEntry{Name: "charlie", VolumeServer: "vs1:9333"})
r.Register(&BlockVolumeEntry{Name: "alpha", VolumeServer: "vs1:9333"})
r.Register(&BlockVolumeEntry{Name: "bravo", VolumeServer: "vs1:9333"})
entries := r.ListAll()
if len(entries) != 3 {
t.Fatalf("expected 3 entries, got %d", len(entries))
}
if entries[0].Name != "alpha" || entries[1].Name != "bravo" || entries[2].Name != "charlie" {
t.Errorf("expected sorted order, got %s %s %s", entries[0].Name, entries[1].Name, entries[2].Name)
}
}
func TestServerSummaries(t *testing.T) {
r := NewBlockVolumeRegistry()
r.MarkBlockCapable("vs1:9333")
r.MarkBlockCapable("vs2:9333")
r.Register(&BlockVolumeEntry{Name: "vol1", VolumeServer: "vs1:9333"})
r.Register(&BlockVolumeEntry{Name: "vol2", VolumeServer: "vs1:9333"})
r.Register(&BlockVolumeEntry{Name: "vol3", VolumeServer: "vs2:9333"})
summaries := r.ServerSummaries()
if len(summaries) != 2 {
t.Fatalf("expected 2 summaries, got %d", len(summaries))
}
// Sorted by address.
if summaries[0].Address != "vs1:9333" || summaries[0].VolumeCount != 2 {
t.Errorf("vs1: got %+v", summaries[0])
}
if summaries[1].Address != "vs2:9333" || summaries[1].VolumeCount != 1 {
t.Errorf("vs2: got %+v", summaries[1])
}
}
weed/server/master_server_handlers_block_ui.go
+304
View File
@@ -0,0 +1,304 @@
package weed_server
import (
"html/template"
"net/http"
"github.com/seaweedfs/seaweedfs/weed/glog"
"github.com/seaweedfs/seaweedfs/weed/storage/blockvol/blockapi"
)
var blockDashTemplate = template.Must(template.New("blockDash").Parse(blockLayoutHTML + blockDashContentHTML))
var blockOpsTemplate = template.Must(template.New("blockOps").Parse(blockLayoutHTML + blockOpsContentHTML))
type blockUIVolume struct {
blockapi.VolumeInfo
SizeMB uint64
}
type blockUIData struct {
Tab string // "dashboard" or "ops"
Volumes []blockUIVolume
Servers []blockapi.ServerInfo
// Dashboard stats
TotalVolumes int
ActiveCount int
PendingCount int
TotalSizeMB uint64
}
func (ms *MasterServer) buildBlockUIData(tab string) blockUIData {
entries := ms.blockRegistry.ListAll()
volumes := make([]blockUIVolume, len(entries))
var totalSizeMB uint64
var activeCount, pendingCount int
for i, e := range entries {
info := entryToVolumeInfo(e)
mb := info.SizeBytes / (1024 * 1024)
volumes[i] = blockUIVolume{
VolumeInfo: info,
SizeMB: mb,
}
totalSizeMB += mb
if e.Status == StatusActive {
activeCount++
} else {
pendingCount++
}
}
summaries := ms.blockRegistry.ServerSummaries()
servers := make([]blockapi.ServerInfo, len(summaries))
for i, s := range summaries {
servers[i] = blockapi.ServerInfo{
Address: s.Address,
VolumeCount: s.VolumeCount,
BlockCapable: s.BlockCapable,
}
}
return blockUIData{
Tab: tab,
Volumes: volumes,
Servers: servers,
TotalVolumes: len(entries),
ActiveCount: activeCount,
PendingCount: pendingCount,
TotalSizeMB: totalSizeMB,
}
}
// blockUIHandler serves the /block/ dashboard page.
func (ms *MasterServer) blockUIHandler(w http.ResponseWriter, r *http.Request) {
data := ms.buildBlockUIData("dashboard")
w.Header().Set("Content-Type", "text/html; charset=utf-8")
if err := blockDashTemplate.Execute(w, data); err != nil {
glog.V(0).Infof("block UI template error: %v", err)
}
}
// blockOpsHandler serves the /block/ops operations page.
func (ms *MasterServer) blockOpsHandler(w http.ResponseWriter, r *http.Request) {
data := ms.buildBlockUIData("ops")
w.Header().Set("Content-Type", "text/html; charset=utf-8")
if err := blockOpsTemplate.Execute(w, data); err != nil {
glog.V(0).Infof("block UI template error: %v", err)
}
}
// blockLayoutHTML is the shared HTML layout with tab navigation.
const blockLayoutHTML = `<!DOCTYPE html>
<html>
<head>
<title>SeaweedFS Block Storage</title>
<style>
* { box-sizing: border-box; }
body { font-family: -apple-system, BlinkMacSystemFont, "Segoe UI", Roboto, sans-serif; margin: 0; padding: 0; background: #f5f6fa; color: #333; }
.header { background: #2d3436; color: white; padding: 16px 24px; }
.header h1 { margin: 0; font-size: 20px; font-weight: 500; }
.tabs { display: flex; background: #fff; border-bottom: 2px solid #dfe6e9; padding: 0 24px; }
.tab { padding: 12px 20px; text-decoration: none; color: #636e72; font-size: 14px; font-weight: 500; border-bottom: 2px solid transparent; margin-bottom: -2px; }
.tab:hover { color: #2d3436; }
.tab.active { color: #0984e3; border-bottom-color: #0984e3; }
.content { padding: 24px; max-width: 1400px; }
.cards { display: flex; gap: 16px; margin-bottom: 24px; flex-wrap: wrap; }
.card { background: #fff; border-radius: 8px; padding: 20px; min-width: 180px; flex: 1; box-shadow: 0 1px 3px rgba(0,0,0,0.08); }
.card .label { font-size: 12px; color: #636e72; text-transform: uppercase; letter-spacing: 0.5px; }
.card .value { font-size: 28px; font-weight: 600; margin-top: 4px; }
.card .value.green { color: #00b894; }
.card .value.orange { color: #fdcb6e; }
.card .value.blue { color: #0984e3; }
table { border-collapse: collapse; width: 100%; background: #fff; border-radius: 8px; overflow: hidden; box-shadow: 0 1px 3px rgba(0,0,0,0.08); margin-bottom: 24px; }
th { background: #dfe6e9; color: #2d3436; font-size: 12px; text-transform: uppercase; letter-spacing: 0.5px; padding: 10px 12px; text-align: left; }
td { padding: 10px 12px; border-top: 1px solid #f1f2f6; font-size: 13px; }
tr:hover td { background: #f8f9fa; }
h2 { font-size: 16px; color: #2d3436; margin: 24px 0 12px; }
.form-section { background: #fff; padding: 20px; border-radius: 8px; box-shadow: 0 1px 3px rgba(0,0,0,0.08); margin-bottom: 24px; }
.form-section h3 { margin-top: 0; font-size: 15px; }
.form-row { display: flex; align-items: center; gap: 12px; margin-bottom: 10px; flex-wrap: wrap; }
.form-row label { width: 90px; font-size: 13px; color: #636e72; }
.form-row input, .form-row select { padding: 6px 10px; border: 1px solid #dfe6e9; border-radius: 4px; font-size: 13px; }
.form-row input[type="text"] { width: 200px; }
.form-row input[type="number"] { width: 120px; }
.btn-create { background: #0984e3; color: white; border: none; border-radius: 4px; padding: 8px 20px; cursor: pointer; font-size: 13px; }
.btn-create:hover { background: #0874c9; }
.btn-delete { background: #d63031; color: white; border: none; border-radius: 4px; font-size: 12px; padding: 4px 12px; cursor: pointer; }
.btn-delete:hover { background: #c0392b; }
.badge { display: inline-block; padding: 2px 8px; border-radius: 10px; font-size: 11px; font-weight: 500; }
.badge-active { background: #dff9ec; color: #00b894; }
.badge-pending { background: #ffeaa7; color: #d68910; }
.badge-primary { background: #dfe6e9; color: #2d3436; }
.badge-replica { background: #e8daef; color: #6c3483; }
.empty { color: #b2bec3; font-style: italic; padding: 20px; text-align: center; }
</style>
</head>
<body>
<div class="header"><h1>SeaweedFS Block Storage</h1></div>
<div class="tabs">
<a class="tab {{if eq .Tab "dashboard"}}active{{end}}" href="/block/">Dashboard</a>
<a class="tab {{if eq .Tab "ops"}}active{{end}}" href="/block/ops">Operations</a>
</div>
<div class="content">
`
// blockDashContentHTML is the Dashboard tab content.
const blockDashContentHTML = `
<div class="cards">
<div class="card">
<div class="label">Block Servers</div>
<div class="value blue">{{len .Servers}}</div>
</div>
<div class="card">
<div class="label">Total Volumes</div>
<div class="value">{{.TotalVolumes}}</div>
</div>
<div class="card">
<div class="label">Active</div>
<div class="value green">{{.ActiveCount}}</div>
</div>
<div class="card">
<div class="label">Pending</div>
<div class="value orange">{{.PendingCount}}</div>
</div>
<div class="card">
<div class="label">Total Capacity</div>
<div class="value">{{.TotalSizeMB}} MB</div>
</div>
</div>
<h2>Servers</h2>
{{if .Servers}}
<table>
<tr><th>Address</th><th>Volume Count</th><th>Block Capable</th></tr>
{{range .Servers}}
<tr>
<td>{{.Address}}</td>
<td>{{.VolumeCount}}</td>
<td>{{if .BlockCapable}}Yes{{else}}No{{end}}</td>
</tr>
{{end}}
</table>
{{else}}
<div class="empty">No block servers connected</div>
{{end}}
<h2>Volumes</h2>
{{if .Volumes}}
<table>
<tr>
<th>Name</th><th>Server</th><th>Size</th><th>Placement</th>
<th>Epoch</th><th>Role</th><th>Status</th><th>iSCSI</th><th>Replica</th>
</tr>
{{range .Volumes}}
<tr>
<td>{{.Name}}</td>
<td>{{.VolumeServer}}</td>
<td>{{.SizeMB}} MB</td>
<td>{{.ReplicaPlacement}}</td>
<td>{{.Epoch}}</td>
<td>{{if eq .Role "primary"}}<span class="badge badge-primary">primary</span>{{else if eq .Role "replica"}}<span class="badge badge-replica">replica</span>{{else}}{{.Role}}{{end}}</td>
<td>{{if eq .Status "active"}}<span class="badge badge-active">active</span>{{else}}<span class="badge badge-pending">{{.Status}}</span>{{end}}</td>
<td>{{.ISCSIAddr}}</td>
<td>{{.ReplicaServer}}</td>
</tr>
{{end}}
</table>
{{else}}
<div class="empty">No block volumes</div>
{{end}}
</div></body></html>`
// blockOpsContentHTML is the Operations tab content.
const blockOpsContentHTML = `
<div class="form-section">
<h3>Create Volume</h3>
<form id="createForm">
<div class="form-row">
<label>Name:</label>
<input type="text" id="cName" required placeholder="my-volume">
</div>
<div class="form-row">
<label>Size (MB):</label>
<input type="number" id="cSize" required min="1" placeholder="100">
</div>
<div class="form-row">
<label>Placement:</label>
<select id="cPlacement">
<option value="000">000 - No replica</option>
<option value="001" selected>001 - Different server</option>
<option value="010">010 - Different rack</option>
<option value="100">100 - Different DC</option>
</select>
</div>
<div class="form-row">
<label>Disk type:</label>
<input type="text" id="cDisk" placeholder="ssd">
</div>
<div class="form-row">
<label></label>
<button type="submit" class="btn-create">Create Volume</button>
</div>
</form>
</div>
<h2>Volumes</h2>
{{if .Volumes}}
<table>
<tr>
<th>Name</th><th>Server</th><th>Size</th><th>Placement</th>
<th>Epoch</th><th>Role</th><th>Status</th><th>iSCSI</th><th>Replica</th><th>Action</th>
</tr>
{{range .Volumes}}
<tr>
<td>{{.Name}}</td>
<td>{{.VolumeServer}}</td>
<td>{{.SizeMB}} MB</td>
<td>{{.ReplicaPlacement}}</td>
<td>{{.Epoch}}</td>
<td>{{if eq .Role "primary"}}<span class="badge badge-primary">primary</span>{{else if eq .Role "replica"}}<span class="badge badge-replica">replica</span>{{else}}{{.Role}}{{end}}</td>
<td>{{if eq .Status "active"}}<span class="badge badge-active">active</span>{{else}}<span class="badge badge-pending">{{.Status}}</span>{{end}}</td>
<td>{{.ISCSIAddr}}</td>
<td>{{.ReplicaServer}}</td>
<td><button class="btn-delete" onclick="deleteVol('{{.Name}}')">Delete</button></td>
</tr>
{{end}}
</table>
{{else}}
<div class="empty">No block volumes</div>
{{end}}
<script>
document.getElementById('createForm').addEventListener('submit', function(e) {
e.preventDefault();
var btn = e.target.querySelector('button[type="submit"]');
btn.disabled = true;
btn.textContent = 'Creating...';
fetch('/block/volume', {
method: 'POST',
headers: {'Content-Type': 'application/json'},
body: JSON.stringify({
name: document.getElementById('cName').value,
size_bytes: parseInt(document.getElementById('cSize').value) * 1024 * 1024,
replica_placement: document.getElementById('cPlacement').value,
disk_type: document.getElementById('cDisk').value
})
}).then(function(r) {
if (!r.ok) return r.json().then(function(j) { throw new Error(j.error || r.statusText); });
window.location.reload();
}).catch(function(err) {
alert('Create failed: ' + err.message);
btn.disabled = false;
btn.textContent = 'Create Volume';
});
});
function deleteVol(name) {
if (!confirm('Delete block volume "' + name + '"?')) return;
fetch('/block/volume/' + encodeURIComponent(name), {method: 'DELETE'})
.then(function(r) {
if (!r.ok) return r.json().then(function(j) { throw new Error(j.error || r.statusText); });
window.location.reload();
}).catch(function(err) { alert('Delete failed: ' + err.message); });
}
</script>
</div></body></html>`
weed/server/qa_block_cp63_test.go
+23 -9
View File
@@ -62,6 +62,16 @@ func registerQAVolume(t *testing.T, ms *MasterServer, name, primary, replica str
entry.ReplicaPath = fmt.Sprintf("/data/%s.blk", name)
entry.ReplicaIQN = fmt.Sprintf("iqn.2024.test:%s-r", name)
entry.ReplicaISCSIAddr = replica + ":3260"
// CP8-2: also populate Replicas[].
entry.Replicas = []ReplicaInfo{
{
Server: replica,
Path: fmt.Sprintf("/data/%s.blk", name),
IQN: fmt.Sprintf("iqn.2024.test:%s-r", name),
ISCSIAddr: replica + ":3260",
HealthScore: 1.0,
},
}
}
if err := ms.blockRegistry.Register(entry); err != nil {
t.Fatalf("register %s: %v", name, err)
@@ -382,11 +392,12 @@ func TestQA_Failover_PromoteIdempotent_NoReplicaAfterFirstSwap(t *testing.T) {
ms := testMSForQA(t)
registerQAVolume(t, ms, "vol1", "vs1", "vs2", 1, 5*time.Second, true)
ms.failoverBlockVolumes("vs1") // promotes vs2, vs1 becomes replica
ms.failoverBlockVolumes("vs1") // promotes vs2
// CP8-2: PromoteBestReplica does NOT add old primary back as replica.
// Reconnect vs1 first so it becomes a replica.
ms.recoverBlockVolumes("vs1")
// Now if vs2 also disconnects, it should try to failover.
// After first failover: primary=vs2, replica=vs1.
// vs2 disconnects: primary IS vs2, replica=vs1 — should swap back.
e, _ := ms.blockRegistry.Lookup("vol1")
e.LastLeaseGrant = time.Now().Add(-1 * time.Minute) // expire the new lease
ms.failoverBlockVolumes("vs2")
@@ -555,17 +566,20 @@ func TestQA_Create_ReplicaDeleteFailure_PrimaryStillDeleted(t *testing.T) {
ms.blockRegistry.MarkBlockCapable("vs1")
ms.blockRegistry.MarkBlockCapable("vs2")
ms.CreateBlockVolume(context.Background(), &master_pb.CreateBlockVolumeRequest{
Name: "vol1", SizeBytes: 1 << 30,
})
// Find the replica server and make its delete fail.
entry, _ := ms.blockRegistry.Lookup("vol1")
replicaServer := entry.ReplicaServer
ms.blockVSDelete = func(ctx context.Context, server string, name string) error {
if server == "vs2" {
if server == replicaServer {
return fmt.Errorf("replica down")
}
return nil
}
ms.CreateBlockVolume(context.Background(), &master_pb.CreateBlockVolumeRequest{
Name: "vol1", SizeBytes: 1 << 30,
})
// Delete should succeed even if replica delete fails (best-effort).
_, err := ms.DeleteBlockVolume(context.Background(), &master_pb.DeleteBlockVolumeRequest{Name: "vol1"})
if err != nil {
weed/server/qa_block_cp82_adversarial_test.go
+849
View File
@@ -0,0 +1,849 @@
package weed_server
import (
"context"
"fmt"
"sync"
"testing"
"time"
"github.com/seaweedfs/seaweedfs/weed/pb/master_pb"
"github.com/seaweedfs/seaweedfs/weed/storage/blockvol"
)
// ============================================================
// CP8-2 Adversarial Test Suite
//
// 12 scenarios covering multi-replica, scrub, promotion gating,
// heartbeat reconciliation, and partial-failure edge cases.
// ============================================================
// qaCP82Master creates a MasterServer with 3 block-capable servers
// and configurable mocks for adversarial testing.
func qaCP82Master(t *testing.T) *MasterServer {
t.Helper()
ms := &MasterServer{
blockRegistry: NewBlockVolumeRegistry(),
blockAssignmentQueue: NewBlockAssignmentQueue(),
blockFailover: newBlockFailoverState(),
}
ms.blockVSAllocate = func(ctx context.Context, server string, name string, sizeBytes uint64, diskType string) (*blockAllocResult, error) {
return &blockAllocResult{
Path: fmt.Sprintf("/data/%s.blk", name),
IQN: fmt.Sprintf("iqn.2024.test:%s", name),
ISCSIAddr: server + ":3260",
ReplicaDataAddr: server + ":14260",
ReplicaCtrlAddr: server + ":14261",
RebuildListenAddr: server + ":15000",
}, nil
}
ms.blockVSDelete = func(ctx context.Context, server string, name string) error {
return nil
}
ms.blockVSExpand = func(ctx context.Context, server string, name string, newSize uint64) (uint64, error) {
return newSize, nil
}
ms.blockRegistry.MarkBlockCapable("vs1:9333")
ms.blockRegistry.MarkBlockCapable("vs2:9333")
ms.blockRegistry.MarkBlockCapable("vs3:9333")
return ms
}
// qaRegisterRF2WithLSN creates an RF=2 volume with explicit WAL LSN state.
func qaRegisterRF2WithLSN(t *testing.T, ms *MasterServer, name, primary, replica string, epoch, primaryLSN, replicaLSN uint64) {
t.Helper()
entry := &BlockVolumeEntry{
Name: name,
VolumeServer: primary,
Path: fmt.Sprintf("/data/%s.blk", name),
IQN: fmt.Sprintf("iqn.2024.test:%s", name),
ISCSIAddr: primary + ":3260",
SizeBytes: 1 << 30,
Epoch: epoch,
Role: blockvol.RoleToWire(blockvol.RolePrimary),
Status: StatusActive,
WALHeadLSN: primaryLSN,
LeaseTTL: 5 * time.Second,
LastLeaseGrant: time.Now().Add(-10 * time.Second), // expired
ReplicaServer: replica,
ReplicaPath: fmt.Sprintf("/data/%s.blk", name),
ReplicaIQN: fmt.Sprintf("iqn.2024.test:%s-r", name),
ReplicaISCSIAddr: replica + ":3260",
Replicas: []ReplicaInfo{
{
Server: replica,
Path: fmt.Sprintf("/data/%s.blk", name),
IQN: fmt.Sprintf("iqn.2024.test:%s-r", name),
ISCSIAddr: replica + ":3260",
HealthScore: 1.0,
WALHeadLSN: replicaLSN,
LastHeartbeat: time.Now(), // fresh
Role: blockvol.RoleToWire(blockvol.RoleReplica),
},
},
}
if err := ms.blockRegistry.Register(entry); err != nil {
t.Fatalf("register %s: %v", name, err)
}
}
// qaRegisterRF3WithState creates an RF=3 volume with per-replica state control.
func qaRegisterRF3WithState(t *testing.T, ms *MasterServer, name, primary string,
replicas []struct {
Server string
HealthScore float64
WALHeadLSN uint64
Role blockvol.Role
Heartbeat time.Time
},
primaryLSN uint64,
) {
t.Helper()
entry := &BlockVolumeEntry{
Name: name,
VolumeServer: primary,
Path: fmt.Sprintf("/data/%s.blk", name),
IQN: fmt.Sprintf("iqn.2024.test:%s", name),
ISCSIAddr: primary + ":3260",
SizeBytes: 1 << 30,
Epoch: 1,
Role: blockvol.RoleToWire(blockvol.RolePrimary),
Status: StatusActive,
WALHeadLSN: primaryLSN,
ReplicaFactor: len(replicas) + 1,
LeaseTTL: 5 * time.Second,
LastLeaseGrant: time.Now().Add(-10 * time.Second), // expired
}
for i, r := range replicas {
ri := ReplicaInfo{
Server: r.Server,
Path: fmt.Sprintf("/data/%s.blk", name),
IQN: fmt.Sprintf("iqn.2024.test:%s-r%d", name, i+1),
ISCSIAddr: r.Server + ":3260",
HealthScore: r.HealthScore,
WALHeadLSN: r.WALHeadLSN,
Role: blockvol.RoleToWire(r.Role),
LastHeartbeat: r.Heartbeat,
}
entry.Replicas = append(entry.Replicas, ri)
}
// Sync scalar fields from first replica.
if len(entry.Replicas) > 0 {
entry.ReplicaServer = entry.Replicas[0].Server
entry.ReplicaPath = entry.Replicas[0].Path
entry.ReplicaIQN = entry.Replicas[0].IQN
entry.ReplicaISCSIAddr = entry.Replicas[0].ISCSIAddr
}
if err := ms.blockRegistry.Register(entry); err != nil {
t.Fatalf("register %s: %v", name, err)
}
}
// ────────────────────────────────────────────────────────────
// QA-1: PrimaryCrash_AfterAck_BeforeReplicaBarrier
//
// Verify failover data loss window is bounded by barrier_lag_lsn.
// Simulates primary at LSN 200, replica at barrier LSN 190.
// After failover, new primary should be at replica's LSN (190).
// The gap (10 LSN entries) is the bounded loss window.
// ────────────────────────────────────────────────────────────
func TestQA_CP82_PrimaryCrash_AfterAck_BeforeReplicaBarrier(t *testing.T) {
ms := qaCP82Master(t)
qaRegisterRF2WithLSN(t, ms, "vol-lag", "vs1:9333", "vs2:9333", 1, 200, 190)
// Kill primary.
ms.failoverBlockVolumes("vs1:9333")
entry, ok := ms.blockRegistry.Lookup("vol-lag")
if !ok {
t.Fatal("volume should still exist after failover")
}
if entry.VolumeServer != "vs2:9333" {
t.Fatalf("replica should be promoted, got %q", entry.VolumeServer)
}
// The promoted replica was at LSN 190. The 10-LSN gap represents
// acknowledged-but-unbarriered writes (bounded data loss).
// New epoch should be 2.
if entry.Epoch != 2 {
t.Fatalf("epoch: got %d, want 2", entry.Epoch)
}
}
// ────────────────────────────────────────────────────────────
// QA-2: ReplicaHeartbeatSpoof_DoesNotDeletePrimary
//
// Regression test for Fix #1. A replica heartbeat that does NOT
// include the primary path must NOT delete the volume entry.
// ────────────────────────────────────────────────────────────
func TestQA_CP82_ReplicaHeartbeatSpoof_DoesNotDeletePrimary(t *testing.T) {
ms := qaCP82Master(t)
qaRegisterRF2WithLSN(t, ms, "vol-spoof", "vs1:9333", "vs2:9333", 1, 100, 100)
// Simulate replica heartbeat from vs2 with ONLY its replica path.
// This must NOT remove the volume (vs1 is the primary, not vs2).
ms.blockRegistry.UpdateFullHeartbeat("vs2:9333", []*master_pb.BlockVolumeInfoMessage{
{
Path: "/data/vol-spoof.blk",
VolumeSize: 1 << 30,
Epoch: 1,
Role: blockvol.RoleToWire(blockvol.RoleReplica),
},
})
entry, ok := ms.blockRegistry.Lookup("vol-spoof")
if !ok {
t.Fatal("volume must NOT be deleted by replica heartbeat (Fix #1 regression)")
}
if entry.VolumeServer != "vs1:9333" {
t.Fatalf("primary must remain vs1:9333, got %q", entry.VolumeServer)
}
// Now simulate a full heartbeat from vs2 with NO volumes at all.
// This should remove vs2 as replica but NOT delete the volume.
ms.blockRegistry.UpdateFullHeartbeat("vs2:9333", []*master_pb.BlockVolumeInfoMessage{})
entry, ok = ms.blockRegistry.Lookup("vol-spoof")
if !ok {
t.Fatal("volume must still exist after empty replica heartbeat")
}
if entry.VolumeServer != "vs1:9333" {
t.Fatal("primary must still be vs1:9333")
}
}
// ────────────────────────────────────────────────────────────
// QA-3: PromotionRejects_StaleButHealthyReplica
//
// Replica has perfect health (1.0) but stale heartbeat.
// Must NOT be promoted (Gate 1: heartbeat freshness).
// ────────────────────────────────────────────────────────────
func TestQA_CP82_PromotionRejects_StaleButHealthyReplica(t *testing.T) {
ms := qaCP82Master(t)
staleTime := time.Now().Add(-5 * time.Minute) // way beyond 2×LeaseTTL
qaRegisterRF3WithState(t, ms, "vol-stale", "vs1:9333",
[]struct {
Server string
HealthScore float64
WALHeadLSN uint64
Role blockvol.Role
Heartbeat time.Time
}{
{"vs2:9333", 1.0, 100, blockvol.RoleReplica, staleTime}, // healthy but stale
{"vs3:9333", 0.5, 100, blockvol.RoleReplica, staleTime}, // also stale
},
100,
)
_, err := ms.blockRegistry.PromoteBestReplica("vol-stale")
if err == nil {
t.Fatal("expected promotion to fail — all replicas have stale heartbeats")
}
// Volume should remain unchanged.
entry, _ := ms.blockRegistry.Lookup("vol-stale")
if entry.VolumeServer != "vs1:9333" {
t.Fatalf("primary must not change, got %q", entry.VolumeServer)
}
}
// ────────────────────────────────────────────────────────────
// QA-4: PromotionRejects_RebuildingReplica
//
// Replica is fresh and healthy but in RoleRebuilding.
// Must NOT be promoted (Gate 3: role check).
// ────────────────────────────────────────────────────────────
func TestQA_CP82_PromotionRejects_RebuildingReplica(t *testing.T) {
ms := qaCP82Master(t)
now := time.Now()
qaRegisterRF3WithState(t, ms, "vol-rebuild", "vs1:9333",
[]struct {
Server string
HealthScore float64
WALHeadLSN uint64
Role blockvol.Role
Heartbeat time.Time
}{
{"vs2:9333", 1.0, 100, blockvol.RoleRebuilding, now}, // rebuilding
{"vs3:9333", 1.0, 100, blockvol.RoleRebuilding, now}, // also rebuilding
},
100,
)
_, err := ms.blockRegistry.PromoteBestReplica("vol-rebuild")
if err == nil {
t.Fatal("expected promotion to fail — all replicas are rebuilding")
}
// One rebuilding, one ready: only the ready one should be promoted.
ms2 := qaCP82Master(t)
qaRegisterRF3WithState(t, ms2, "vol-mixed", "vs1:9333",
[]struct {
Server string
HealthScore float64
WALHeadLSN uint64
Role blockvol.Role
Heartbeat time.Time
}{
{"vs2:9333", 1.0, 100, blockvol.RoleRebuilding, now},
{"vs3:9333", 0.7, 100, blockvol.RoleReplica, now}, // eligible despite lower health
},
100,
)
_, err = ms2.blockRegistry.PromoteBestReplica("vol-mixed")
if err != nil {
t.Fatalf("expected promotion to succeed: %v", err)
}
entry, _ := ms2.blockRegistry.Lookup("vol-mixed")
if entry.VolumeServer != "vs3:9333" {
t.Fatalf("vs3 should be promoted (only eligible), got %q", entry.VolumeServer)
}
}
// ────────────────────────────────────────────────────────────
// QA-5: PromotionToleranceBoundary_ExactLSN
//
// Tests promotion_lsn_tolerance boundary: tolerance-1, exact, tolerance+1.
// Catches off-by-one in the eligibility gate.
// ────────────────────────────────────────────────────────────
func TestQA_CP82_PromotionToleranceBoundary_ExactLSN(t *testing.T) {
const tolerance uint64 = 50
const primaryLSN uint64 = 200
cases := []struct {
name string
replicaLSN uint64
wantErr bool
}{
// replicaLSN + tolerance < primaryLSN → ineligible
// 149 + 50 = 199 < 200 → INELIGIBLE
{"below_tolerance", primaryLSN - tolerance - 1, true},
// 150 + 50 = 200 == 200 → ELIGIBLE (not strict less-than)
{"at_tolerance", primaryLSN - tolerance, false},
// 151 + 50 = 201 > 200 → ELIGIBLE
{"above_tolerance", primaryLSN - tolerance + 1, false},
}
for _, tc := range cases {
t.Run(tc.name, func(t *testing.T) {
ms := qaCP82Master(t)
ms.blockRegistry.SetPromotionLSNTolerance(tolerance)
now := time.Now()
qaRegisterRF3WithState(t, ms, "vol-tol", "vs1:9333",
[]struct {
Server string
HealthScore float64
WALHeadLSN uint64
Role blockvol.Role
Heartbeat time.Time
}{
{"vs2:9333", 1.0, tc.replicaLSN, blockvol.RoleReplica, now},
},
primaryLSN,
)
_, err := ms.blockRegistry.PromoteBestReplica("vol-tol")
if tc.wantErr && err == nil {
t.Fatalf("replicaLSN=%d: expected promotion to fail", tc.replicaLSN)
}
if !tc.wantErr && err != nil {
t.Fatalf("replicaLSN=%d: expected promotion to succeed: %v", tc.replicaLSN, err)
}
})
}
}
// ────────────────────────────────────────────────────────────
// QA-6: MasterRestart_ReconstructReplicas_ThenFailover
//
// Simulates master restart: empty registry, full heartbeats
// from primary and replica servers reconstruct state, then
// failover correctly promotes the replica.
// ────────────────────────────────────────────────────────────
func TestQA_CP82_MasterRestart_ReconstructReplicas_ThenFailover(t *testing.T) {
ms := qaCP82Master(t)
// Phase 1: Primary heartbeat — auto-registers volume from heartbeat.
ms.blockRegistry.UpdateFullHeartbeat("vs1:9333", []*master_pb.BlockVolumeInfoMessage{
{
Path: "/data/vol-restart.blk",
VolumeSize: 1 << 30,
Epoch: 5,
Role: blockvol.RoleToWire(blockvol.RolePrimary),
WalHeadLsn: 500,
},
})
entry, ok := ms.blockRegistry.Lookup("vol-restart")
if !ok {
t.Fatal("volume should be auto-registered from primary heartbeat")
}
if entry.VolumeServer != "vs1:9333" {
t.Fatalf("primary: got %q, want vs1:9333", entry.VolumeServer)
}
// Phase 2: Replica heartbeat — should reconstruct ReplicaInfo.
ms.blockRegistry.UpdateFullHeartbeat("vs2:9333", []*master_pb.BlockVolumeInfoMessage{
{
Path: "/data/vol-restart.blk",
VolumeSize: 1 << 30,
Epoch: 5,
Role: blockvol.RoleToWire(blockvol.RoleReplica),
WalHeadLsn: 498,
},
})
entry, _ = ms.blockRegistry.Lookup("vol-restart")
if len(entry.Replicas) == 0 {
t.Fatal("replica should be reconstructed from heartbeat (Fix #3)")
}
if entry.Replicas[0].Server != "vs2:9333" {
t.Fatalf("replica server: got %q, want vs2:9333", entry.Replicas[0].Server)
}
// Phase 3: Set lease expired and trigger failover.
entry.LeaseTTL = 5 * time.Second
entry.LastLeaseGrant = time.Now().Add(-1 * time.Minute)
// Ensure replica is eligible for promotion.
entry.Replicas[0].LastHeartbeat = time.Now()
entry.Replicas[0].Role = blockvol.RoleToWire(blockvol.RoleReplica)
ms.failoverBlockVolumes("vs1:9333")
entry, ok = ms.blockRegistry.Lookup("vol-restart")
if !ok {
t.Fatal("volume should survive failover")
}
if entry.VolumeServer != "vs2:9333" {
t.Fatalf("after failover: primary should be vs2:9333, got %q", entry.VolumeServer)
}
if entry.Epoch <= 5 {
t.Fatalf("epoch should be bumped, got %d", entry.Epoch)
}
}
// ────────────────────────────────────────────────────────────
// QA-7: RF3_OneReplicaFlaps_UnderWriteLoad
//
// One replica rapidly disconnects/reconnects while concurrent
// failover checks run. Must not cause split-brain or panic.
// Simulates via concurrent UpdateFullHeartbeat + failoverBlockVolumes.
// ────────────────────────────────────────────────────────────
func TestQA_CP82_RF3_OneReplicaFlaps_UnderWriteLoad(t *testing.T) {
ms := qaCP82Master(t)
now := time.Now()
qaRegisterRF3WithState(t, ms, "vol-flap", "vs1:9333",
[]struct {
Server string
HealthScore float64
WALHeadLSN uint64
Role blockvol.Role
Heartbeat time.Time
}{
{"vs2:9333", 1.0, 100, blockvol.RoleReplica, now},
{"vs3:9333", 1.0, 100, blockvol.RoleReplica, now},
},
100,
)
var wg sync.WaitGroup
const rounds = 50
// Goroutine 1: flapping replica vs3 (heartbeat with/without volume).
wg.Add(1)
go func() {
defer wg.Done()
for i := 0; i < rounds; i++ {
if i%2 == 0 {
// Replica reports volume.
ms.blockRegistry.UpdateFullHeartbeat("vs3:9333", []*master_pb.BlockVolumeInfoMessage{
{
Path: "/data/vol-flap.blk",
VolumeSize: 1 << 30,
Epoch: 1,
Role: blockvol.RoleToWire(blockvol.RoleReplica),
WalHeadLsn: 100,
},
})
} else {
// Replica reports no volumes (simulates disconnect).
ms.blockRegistry.UpdateFullHeartbeat("vs3:9333", []*master_pb.BlockVolumeInfoMessage{})
}
}
}()
// Goroutine 2: concurrent heartbeats from primary.
wg.Add(1)
go func() {
defer wg.Done()
for i := 0; i < rounds; i++ {
ms.blockRegistry.UpdateFullHeartbeat("vs1:9333", []*master_pb.BlockVolumeInfoMessage{
{
Path: "/data/vol-flap.blk",
VolumeSize: 1 << 30,
Epoch: 1,
Role: blockvol.RoleToWire(blockvol.RolePrimary),
WalHeadLsn: uint64(100 + i),
},
})
}
}()
// Goroutine 3: concurrent heartbeats from stable replica vs2.
wg.Add(1)
go func() {
defer wg.Done()
for i := 0; i < rounds; i++ {
ms.blockRegistry.UpdateFullHeartbeat("vs2:9333", []*master_pb.BlockVolumeInfoMessage{
{
Path: "/data/vol-flap.blk",
VolumeSize: 1 << 30,
Epoch: 1,
Role: blockvol.RoleToWire(blockvol.RoleReplica),
WalHeadLsn: uint64(100 + i),
},
})
}
}()
wg.Wait()
// Invariant: volume must still exist, primary must be vs1.
entry, ok := ms.blockRegistry.Lookup("vol-flap")
if !ok {
t.Fatal("volume must survive flapping replica")
}
if entry.VolumeServer != "vs1:9333" {
t.Fatalf("primary must remain vs1:9333, got %q (split-brain!)", entry.VolumeServer)
}
}
// ────────────────────────────────────────────────────────────
// QA-8: AssignmentPrecedence_ReplicaAddrsVsScalar
//
// Creates RF=3 volume and verifies primary assignment includes
// ReplicaAddrs (not just scalar fields). VS should use
// ReplicaAddrs when non-empty.
// ────────────────────────────────────────────────────────────
func TestQA_CP82_AssignmentPrecedence_ReplicaAddrsVsScalar(t *testing.T) {
ms := qaCP82Master(t)
ctx := context.Background()
resp, err := ms.CreateBlockVolume(ctx, &master_pb.CreateBlockVolumeRequest{
Name: "vol-addrs",
SizeBytes: 1 << 30,
ReplicaFactor: 3,
})
if err != nil {
t.Fatalf("create: %v", err)
}
primary := resp.VolumeServer
assignments := ms.blockAssignmentQueue.Peek(primary)
// Find the primary assignment.
var primaryAssignment *blockvol.BlockVolumeAssignment
for i := range assignments {
if blockvol.RoleFromWire(assignments[i].Role) == blockvol.RolePrimary {
primaryAssignment = &assignments[i]
break
}
}
if primaryAssignment == nil {
t.Fatal("no primary assignment found")
}
// RF=3 → 2 replicas → ReplicaAddrs should have 2 entries.
if len(primaryAssignment.ReplicaAddrs) != 2 {
t.Fatalf("primary assignment ReplicaAddrs: got %d, want 2", len(primaryAssignment.ReplicaAddrs))
}
// Verify each ReplicaAddr has data+ctrl.
for i, ra := range primaryAssignment.ReplicaAddrs {
if ra.DataAddr == "" || ra.CtrlAddr == "" {
t.Errorf("ReplicaAddrs[%d]: data=%q ctrl=%q — both must be non-empty", i, ra.DataAddr, ra.CtrlAddr)
}
}
// For RF=3, legacy scalar ReplicaDataAddr is NOT set on the primary
// assignment (only ReplicaAddrs is used). Verify precedence: ReplicaAddrs
// is the authoritative source when non-empty.
if len(primaryAssignment.ReplicaAddrs) > 0 && primaryAssignment.ReplicaDataAddr != "" {
t.Error("legacy ReplicaDataAddr should be empty when ReplicaAddrs is populated (RF=3)")
}
}
// ────────────────────────────────────────────────────────────
// QA-9: ScrubConcurrentWrites_NoFalseCorruption
//
// Heavy writes while scrub runs. Expect zero false positives.
// Tests at the HealthScore level (engine scrub tested separately).
// ────────────────────────────────────────────────────────────
func TestQA_CP82_ScrubConcurrentWrites_NoFalseCorruption(t *testing.T) {
ms := qaCP82Master(t)
now := time.Now()
qaRegisterRF3WithState(t, ms, "vol-scrub", "vs1:9333",
[]struct {
Server string
HealthScore float64
WALHeadLSN uint64
Role blockvol.Role
Heartbeat time.Time
}{
{"vs2:9333", 1.0, 500, blockvol.RoleReplica, now},
{"vs3:9333", 1.0, 500, blockvol.RoleReplica, now},
},
500,
)
var wg sync.WaitGroup
const rounds = 100
// Simulate heartbeats with varying health scores
// (as would happen during scrub with concurrent writes).
wg.Add(1)
go func() {
defer wg.Done()
for i := 0; i < rounds; i++ {
// Health stays 1.0 — no corruption detected (correct behavior
// when writes are excluded from scrub checks).
ms.blockRegistry.UpdateFullHeartbeat("vs1:9333", []*master_pb.BlockVolumeInfoMessage{
{
Path: "/data/vol-scrub.blk",
VolumeSize: 1 << 30,
Epoch: 1,
Role: blockvol.RoleToWire(blockvol.RolePrimary),
WalHeadLsn: uint64(500 + i),
HealthScore: 1.0, // Clean — no false positives
},
})
}
}()
// Concurrent replica heartbeats.
wg.Add(1)
go func() {
defer wg.Done()
for i := 0; i < rounds; i++ {
ms.blockRegistry.UpdateFullHeartbeat("vs2:9333", []*master_pb.BlockVolumeInfoMessage{
{
Path: "/data/vol-scrub.blk",
VolumeSize: 1 << 30,
Epoch: 1,
Role: blockvol.RoleToWire(blockvol.RoleReplica),
WalHeadLsn: uint64(500 + i),
HealthScore: 1.0,
},
})
}
}()
wg.Wait()
entry, _ := ms.blockRegistry.Lookup("vol-scrub")
if entry.HealthScore < 1.0 {
t.Fatalf("health score should be 1.0 (no corruption), got %f", entry.HealthScore)
}
}
// ────────────────────────────────────────────────────────────
// QA-10: ScrubDetectsCorruption_HealthDrops_PromotionAvoids
//
// One replica has corruption (low health), the other is clean.
// Promotion must prefer the healthy replica.
// ────────────────────────────────────────────────────────────
func TestQA_CP82_ScrubDetectsCorruption_HealthDrops_PromotionAvoids(t *testing.T) {
ms := qaCP82Master(t)
now := time.Now()
qaRegisterRF3WithState(t, ms, "vol-corrupt", "vs1:9333",
[]struct {
Server string
HealthScore float64
WALHeadLSN uint64
Role blockvol.Role
Heartbeat time.Time
}{
{"vs2:9333", 0.3, 100, blockvol.RoleReplica, now}, // corrupted (low health)
{"vs3:9333", 1.0, 100, blockvol.RoleReplica, now}, // clean
},
100,
)
// Simulate heartbeat reporting low health on vs2 (scrub found errors).
ms.blockRegistry.UpdateFullHeartbeat("vs2:9333", []*master_pb.BlockVolumeInfoMessage{
{
Path: "/data/vol-corrupt.blk",
VolumeSize: 1 << 30,
Epoch: 1,
Role: blockvol.RoleToWire(blockvol.RoleReplica),
WalHeadLsn: 100,
HealthScore: 0.3,
ScrubErrors: 5,
},
})
// Trigger failover — vs3 (healthy) should be promoted, not vs2.
newEpoch, err := ms.blockRegistry.PromoteBestReplica("vol-corrupt")
if err != nil {
t.Fatalf("promotion should succeed: %v", err)
}
if newEpoch != 2 {
t.Fatalf("epoch: got %d, want 2", newEpoch)
}
entry, _ := ms.blockRegistry.Lookup("vol-corrupt")
if entry.VolumeServer != "vs3:9333" {
t.Fatalf("vs3 (healthy) should be promoted, got %q", entry.VolumeServer)
}
// vs2 (corrupted) should remain as replica.
if len(entry.Replicas) != 1 || entry.Replicas[0].Server != "vs2:9333" {
t.Fatalf("vs2 should remain as replica, got %+v", entry.Replicas)
}
}
// ────────────────────────────────────────────────────────────
// QA-11: ExpandRF3_PartialReplicaFailure
//
// Primary expand succeeds, one replica expand fails.
// Verify: operation succeeds (best-effort), registry updated,
// failed replica state is degraded.
// ────────────────────────────────────────────────────────────
func TestQA_CP82_ExpandRF3_PartialReplicaFailure(t *testing.T) {
ms := qaCP82Master(t)
ctx := context.Background()
// Create RF=3 volume first.
_, err := ms.CreateBlockVolume(ctx, &master_pb.CreateBlockVolumeRequest{
Name: "vol-expand",
SizeBytes: 1 << 30,
ReplicaFactor: 3,
})
if err != nil {
t.Fatalf("create: %v", err)
}
entry, _ := ms.blockRegistry.Lookup("vol-expand")
failServer := entry.Replicas[1].Server
// Override expand mock: one replica fails.
ms.blockVSExpand = func(ctx context.Context, server string, name string, newSize uint64) (uint64, error) {
if server == failServer {
return 0, fmt.Errorf("disk full on %s", server)
}
return newSize, nil
}
// Expand should succeed (primary + one replica succeed, one fails best-effort).
resp, err := ms.ExpandBlockVolume(ctx, &master_pb.ExpandBlockVolumeRequest{
Name: "vol-expand",
NewSizeBytes: 2 << 30,
})
if err != nil {
t.Fatalf("expand should succeed despite partial replica failure: %v", err)
}
if resp.CapacityBytes != 2<<30 {
t.Fatalf("capacity: got %d, want %d", resp.CapacityBytes, uint64(2<<30))
}
// Registry should reflect new size.
entry, _ = ms.blockRegistry.Lookup("vol-expand")
if entry.SizeBytes != 2<<30 {
t.Fatalf("registry size: got %d, want %d", entry.SizeBytes, uint64(2<<30))
}
}
// ────────────────────────────────────────────────────────────
// QA-12: ByServerIndexConsistency_AfterReplicaMove
//
// Moves replica between servers repeatedly. ListByServer and
// byServer counts must stay exact — no dangling references.
// ────────────────────────────────────────────────────────────
func TestQA_CP82_ByServerIndexConsistency_AfterReplicaMove(t *testing.T) {
ms := qaCP82Master(t)
now := time.Now()
qaRegisterRF3WithState(t, ms, "vol-move", "vs1:9333",
[]struct {
Server string
HealthScore float64
WALHeadLSN uint64
Role blockvol.Role
Heartbeat time.Time
}{
{"vs2:9333", 1.0, 100, blockvol.RoleReplica, now},
},
100,
)
// Verify initial state: vs1 and vs2 both have vol-move.
assertServerHasVolume(t, ms, "vs1:9333", "vol-move")
assertServerHasVolume(t, ms, "vs2:9333", "vol-move")
// Move replica from vs2 to vs3 repeatedly.
servers := []string{"vs3:9333", "vs2:9333", "vs3:9333", "vs2:9333", "vs3:9333"}
for _, newServer := range servers {
// Remove old replica.
entry, _ := ms.blockRegistry.Lookup("vol-move")
oldServer := ""
if len(entry.Replicas) > 0 {
oldServer = entry.Replicas[0].Server
}
if oldServer != "" {
if err := ms.blockRegistry.RemoveReplica("vol-move", oldServer); err != nil {
t.Fatalf("remove replica from %s: %v", oldServer, err)
}
}
// Add new replica.
if err := ms.blockRegistry.AddReplica("vol-move", ReplicaInfo{
Server: newServer,
Path: "/data/vol-move.blk",
IQN: fmt.Sprintf("iqn.2024.test:vol-move-r"),
ISCSIAddr: newServer + ":3260",
HealthScore: 1.0,
}); err != nil {
t.Fatalf("add replica on %s: %v", newServer, err)
}
// Invariant: primary always indexed.
assertServerHasVolume(t, ms, "vs1:9333", "vol-move")
// New replica indexed.
assertServerHasVolume(t, ms, newServer, "vol-move")
// Old server should NOT have it (unless it's the primary).
if oldServer != "" && oldServer != "vs1:9333" && oldServer != newServer {
assertServerDoesNotHaveVolume(t, ms, oldServer, "vol-move")
}
}
// Final check: ListAll should have exactly 1 volume.
all := ms.blockRegistry.ListAll()
if len(all) != 1 {
t.Fatalf("expected 1 volume, got %d", len(all))
}
}
// ────────────────────────────────────────────────────────────
// Helpers
// ────────────────────────────────────────────────────────────
func assertServerHasVolume(t *testing.T, ms *MasterServer, server, volName string) {
t.Helper()
entries := ms.blockRegistry.ListByServer(server)
for _, e := range entries {
if e.Name == volName {
return
}
}
t.Errorf("server %q should have volume %q in byServer index", server, volName)
}
func assertServerDoesNotHaveVolume(t *testing.T, ms *MasterServer, server, volName string) {
t.Helper()
entries := ms.blockRegistry.ListByServer(server)
for _, e := range entries {
if e.Name == volName {
t.Errorf("server %q should NOT have volume %q in byServer index", server, volName)
return
}
}
}
weed/server/qa_block_rf3_test.go
+430
View File
@@ -0,0 +1,430 @@
package weed_server
import (
"context"
"fmt"
"testing"
"time"
"github.com/seaweedfs/seaweedfs/weed/pb/master_pb"
"github.com/seaweedfs/seaweedfs/weed/storage/blockvol"
)
// ============================================================
// CP8-2 T10: RF=3 QA + Integration Tests
//
// Adversarial and integration tests for multi-replica (N>2)
// support introduced in CP8-2.
// ============================================================
// Helper: create master with 3 block-capable servers and RF=3-aware alloc mock.
func qaRF3Master(t *testing.T) *MasterServer {
t.Helper()
ms := &MasterServer{
blockRegistry: NewBlockVolumeRegistry(),
blockAssignmentQueue: NewBlockAssignmentQueue(),
blockFailover: newBlockFailoverState(),
}
ms.blockVSAllocate = func(ctx context.Context, server string, name string, sizeBytes uint64, diskType string) (*blockAllocResult, error) {
return &blockAllocResult{
Path: fmt.Sprintf("/data/%s.blk", name),
IQN: fmt.Sprintf("iqn.2024.test:%s", name),
ISCSIAddr: server + ":3260",
ReplicaDataAddr: server + ":14260",
ReplicaCtrlAddr: server + ":14261",
RebuildListenAddr: server + ":15000",
}, nil
}
ms.blockVSDelete = func(ctx context.Context, server string, name string) error {
return nil
}
ms.blockVSExpand = func(ctx context.Context, server string, name string, newSize uint64) (uint64, error) {
return newSize, nil
}
ms.blockRegistry.MarkBlockCapable("vs1:9333")
ms.blockRegistry.MarkBlockCapable("vs2:9333")
ms.blockRegistry.MarkBlockCapable("vs3:9333")
return ms
}
// 1. Create RF=3, verify 2 replicas in registry with correct data.
func TestQA_RF3_CreateAndVerifyReplicas(t *testing.T) {
ms := qaRF3Master(t)
ctx := context.Background()
resp, err := ms.CreateBlockVolume(ctx, &master_pb.CreateBlockVolumeRequest{
Name: "pvc-rf3-1",
SizeBytes: 1 << 30,
ReplicaFactor: 3,
})
if err != nil {
t.Fatalf("create: %v", err)
}
entry, ok := ms.blockRegistry.Lookup("pvc-rf3-1")
if !ok {
t.Fatal("volume not in registry")
}
// Verify ReplicaFactor.
if entry.ReplicaFactor != 3 {
t.Fatalf("ReplicaFactor: got %d, want 3", entry.ReplicaFactor)
}
// Verify 2 replicas.
if len(entry.Replicas) != 2 {
t.Fatalf("expected 2 replicas, got %d", len(entry.Replicas))
}
// All 3 servers are distinct.
servers := map[string]bool{resp.VolumeServer: true}
for _, ri := range entry.Replicas {
if servers[ri.Server] {
t.Fatalf("duplicate server in RF=3 volume: %q", ri.Server)
}
servers[ri.Server] = true
// Each replica should have path, IQN, ISCSIAddr set.
if ri.Path == "" || ri.IQN == "" || ri.ISCSIAddr == "" {
t.Fatalf("replica %q missing fields: path=%q iqn=%q iscsi=%q",
ri.Server, ri.Path, ri.IQN, ri.ISCSIAddr)
}
}
// Backward compat: ReplicaServer (deprecated) should point to first replica.
if entry.ReplicaServer == "" {
t.Fatal("deprecated ReplicaServer should be set for backward compat")
}
if entry.ReplicaServer != entry.Replicas[0].Server {
t.Fatalf("deprecated ReplicaServer %q != Replicas[0].Server %q",
entry.ReplicaServer, entry.Replicas[0].Server)
}
// Assignments: primary should have 2 ReplicaAddrs.
primaryAssignments := ms.blockAssignmentQueue.Peek(resp.VolumeServer)
if len(primaryAssignments) != 1 {
t.Fatalf("expected 1 primary assignment, got %d", len(primaryAssignments))
}
if len(primaryAssignments[0].ReplicaAddrs) != 2 {
t.Fatalf("primary assignment ReplicaAddrs: got %d, want 2",
len(primaryAssignments[0].ReplicaAddrs))
}
// Each replica server should have 1 assignment.
for _, ri := range entry.Replicas {
ra := ms.blockAssignmentQueue.Peek(ri.Server)
if len(ra) != 1 {
t.Fatalf("replica %q: expected 1 assignment, got %d", ri.Server, len(ra))
}
if blockvol.RoleFromWire(ra[0].Role) != blockvol.RoleReplica {
t.Fatalf("replica %q: expected RoleReplica, got %d", ri.Server, ra[0].Role)
}
}
}
// 2. Kill primary → best replica (highest health score) promoted.
func TestQA_RF3_PrimaryDies_BestReplicaPromoted(t *testing.T) {
ms := qaRF3Master(t)
ctx := context.Background()
resp, err := ms.CreateBlockVolume(ctx, &master_pb.CreateBlockVolumeRequest{
Name: "pvc-rf3-promo",
SizeBytes: 1 << 30,
ReplicaFactor: 3,
})
if err != nil {
t.Fatalf("create: %v", err)
}
entry, _ := ms.blockRegistry.Lookup("pvc-rf3-promo")
primary := resp.VolumeServer
// Set different health scores on replicas.
for i := range entry.Replicas {
if i == 0 {
entry.Replicas[i].HealthScore = 0.7
} else {
entry.Replicas[i].HealthScore = 1.0
}
}
expectedPromoted := entry.Replicas[1].Server // higher health score
entry.LastLeaseGrant = time.Now().Add(-1 * time.Minute) // expired
// Kill primary.
ms.failoverBlockVolumes(primary)
entry, _ = ms.blockRegistry.Lookup("pvc-rf3-promo")
if entry.VolumeServer != expectedPromoted {
t.Fatalf("expected %q promoted (highest health), got %q", expectedPromoted, entry.VolumeServer)
}
// 1 replica should remain (the one not promoted).
if len(entry.Replicas) != 1 {
t.Fatalf("expected 1 remaining replica, got %d", len(entry.Replicas))
}
// Pending rebuild for old primary.
ms.blockFailover.mu.Lock()
rebuilds := ms.blockFailover.pendingRebuilds[primary]
ms.blockFailover.mu.Unlock()
if len(rebuilds) != 1 || rebuilds[0].VolumeName != "pvc-rf3-promo" {
t.Fatalf("expected 1 pending rebuild for %q, got %+v", primary, rebuilds)
}
}
// 3. Kill one replica → writes unaffected (primary unchanged, dead replica removed).
func TestQA_RF3_OneReplicaDies_WritesUnaffected(t *testing.T) {
ms := qaRF3Master(t)
ctx := context.Background()
_, err := ms.CreateBlockVolume(ctx, &master_pb.CreateBlockVolumeRequest{
Name: "pvc-rf3-repdeath",
SizeBytes: 1 << 30,
ReplicaFactor: 3,
})
if err != nil {
t.Fatalf("create: %v", err)
}
entry, _ := ms.blockRegistry.Lookup("pvc-rf3-repdeath")
originalPrimary := entry.VolumeServer
originalEpoch := entry.Epoch
deadReplica := entry.Replicas[1].Server // kill second replica
ms.failoverBlockVolumes(deadReplica)
entry, _ = ms.blockRegistry.Lookup("pvc-rf3-repdeath")
// Primary unchanged.
if entry.VolumeServer != originalPrimary {
t.Fatalf("primary should remain %q, got %q", originalPrimary, entry.VolumeServer)
}
// Epoch unchanged (no promotion).
if entry.Epoch != originalEpoch {
t.Fatalf("epoch should remain %d, got %d", originalEpoch, entry.Epoch)
}
// Dead replica removed, 1 replica remains.
if len(entry.Replicas) != 1 {
t.Fatalf("expected 1 replica after death, got %d", len(entry.Replicas))
}
if entry.Replicas[0].Server == deadReplica {
t.Fatalf("dead replica %q should be removed", deadReplica)
}
}
// 4. Dead replica reconnects → rebuilds and rejoins.
func TestQA_RF3_Rebuild_DeadReplicaCatchesUp(t *testing.T) {
ms := qaRF3Master(t)
ctx := context.Background()
_, err := ms.CreateBlockVolume(ctx, &master_pb.CreateBlockVolumeRequest{
Name: "pvc-rf3-rebuild",
SizeBytes: 1 << 30,
ReplicaFactor: 3,
})
if err != nil {
t.Fatalf("create: %v", err)
}
entry, _ := ms.blockRegistry.Lookup("pvc-rf3-rebuild")
deadReplica := entry.Replicas[0].Server
// Kill one replica.
ms.failoverBlockVolumes(deadReplica)
// Verify it's removed.
entry, _ = ms.blockRegistry.Lookup("pvc-rf3-rebuild")
if len(entry.Replicas) != 1 {
t.Fatalf("after kill: expected 1 replica, got %d", len(entry.Replicas))
}
// Dead replica reconnects.
ms.recoverBlockVolumes(deadReplica)
// Verify it's back.
entry, _ = ms.blockRegistry.Lookup("pvc-rf3-rebuild")
if len(entry.Replicas) != 2 {
t.Fatalf("after reconnect: expected 2 replicas, got %d", len(entry.Replicas))
}
// Rebuild assignment should be queued.
assignments := ms.blockAssignmentQueue.Peek(deadReplica)
foundRebuild := false
for _, a := range assignments {
if blockvol.RoleFromWire(a.Role) == blockvol.RoleRebuilding {
foundRebuild = true
if a.RebuildAddr == "" {
t.Fatal("rebuild assignment missing RebuildAddr")
}
}
}
if !foundRebuild {
t.Fatalf("expected rebuild assignment for %q", deadReplica)
}
}
// 5. Health score influences failover preference.
func TestQA_RF3_HealthScore_FailoverPreference(t *testing.T) {
ms := qaRF3Master(t)
ctx := context.Background()
_, err := ms.CreateBlockVolume(ctx, &master_pb.CreateBlockVolumeRequest{
Name: "pvc-rf3-health",
SizeBytes: 1 << 30,
ReplicaFactor: 3,
})
if err != nil {
t.Fatalf("create: %v", err)
}
entry, _ := ms.blockRegistry.Lookup("pvc-rf3-health")
primary := entry.VolumeServer
// Set health scores: replica[0] = 0.3 (low), replica[1] = 0.9 (high).
entry.Replicas[0].HealthScore = 0.3
entry.Replicas[0].WALHeadLSN = 100
entry.Replicas[1].HealthScore = 0.9
entry.Replicas[1].WALHeadLSN = 100
expectedWinner := entry.Replicas[1].Server
entry.LastLeaseGrant = time.Now().Add(-1 * time.Minute) // expired
ms.failoverBlockVolumes(primary)
entry, _ = ms.blockRegistry.Lookup("pvc-rf3-health")
if entry.VolumeServer != expectedWinner {
t.Fatalf("expected %q (health=0.9) promoted, got %q", expectedWinner, entry.VolumeServer)
}
// The low-health replica should remain in the list.
if len(entry.Replicas) != 1 {
t.Fatalf("expected 1 remaining replica, got %d", len(entry.Replicas))
}
if entry.Replicas[0].HealthScore != 0.3 {
t.Fatalf("remaining replica health: got %.1f, want 0.3", entry.Replicas[0].HealthScore)
}
}
// 6. RF=2 backward compat: identical behavior to pre-CP8-2.
func TestQA_RF3_BackwardCompat_RF2_Unchanged(t *testing.T) {
ms := qaRF3Master(t)
ctx := context.Background()
// Create with default RF (should be 2).
resp, err := ms.CreateBlockVolume(ctx, &master_pb.CreateBlockVolumeRequest{
Name: "pvc-rf2-compat",
SizeBytes: 1 << 30,
})
if err != nil {
t.Fatalf("create: %v", err)
}
entry, _ := ms.blockRegistry.Lookup("pvc-rf2-compat")
// RF should default to 2.
if entry.ReplicaFactor != 2 {
t.Fatalf("RF default: got %d, want 2", entry.ReplicaFactor)
}
// Exactly 1 replica.
if len(entry.Replicas) != 1 {
t.Fatalf("expected 1 replica for RF=2, got %d", len(entry.Replicas))
}
// Failover should work identically.
primary := resp.VolumeServer
entry.LastLeaseGrant = time.Now().Add(-1 * time.Minute)
ms.failoverBlockVolumes(primary)
entry, _ = ms.blockRegistry.Lookup("pvc-rf2-compat")
if entry.VolumeServer == primary {
t.Fatalf("failover failed: primary should have changed from %q", primary)
}
// No replicas left after RF=2 promotion.
if len(entry.Replicas) != 0 {
t.Fatalf("expected 0 replicas after RF=2 promotion, got %d", len(entry.Replicas))
}
// Recover → rebuild.
ms.recoverBlockVolumes(primary)
entry, _ = ms.blockRegistry.Lookup("pvc-rf2-compat")
if len(entry.Replicas) != 1 || entry.Replicas[0].Server != primary {
t.Fatalf("after recovery: expected %q as replica, got %+v", primary, entry.Replicas)
}
}
// 7. Full lifecycle: create RF=3 → kill primary → promote → kill another → promote → recover all.
func TestQA_RF3_FullLifecycle(t *testing.T) {
ms := qaRF3Master(t)
ctx := context.Background()
resp, err := ms.CreateBlockVolume(ctx, &master_pb.CreateBlockVolumeRequest{
Name: "pvc-rf3-lifecycle",
SizeBytes: 1 << 30,
ReplicaFactor: 3,
})
if err != nil {
t.Fatalf("create: %v", err)
}
entry, _ := ms.blockRegistry.Lookup("pvc-rf3-lifecycle")
vs1 := resp.VolumeServer
vs2 := entry.Replicas[0].Server
vs3 := entry.Replicas[1].Server
entry.LastLeaseGrant = time.Now().Add(-1 * time.Minute)
// Step 1: Kill vs1 (primary). One of vs2/vs3 promoted.
ms.failoverBlockVolumes(vs1)
entry, _ = ms.blockRegistry.Lookup("pvc-rf3-lifecycle")
if entry.VolumeServer == vs1 {
t.Fatal("step1: primary should not be vs1 after failover")
}
step1Primary := entry.VolumeServer
if len(entry.Replicas) != 1 {
t.Fatalf("step1: expected 1 replica, got %d", len(entry.Replicas))
}
step1Replica := entry.Replicas[0].Server
// Step 2: Recover vs1 → becomes replica (rebuilding).
ms.recoverBlockVolumes(vs1)
entry, _ = ms.blockRegistry.Lookup("pvc-rf3-lifecycle")
if len(entry.Replicas) != 2 {
t.Fatalf("step2: expected 2 replicas after vs1 recovery, got %d", len(entry.Replicas))
}
// Step 3: Kill step1Primary. The surviving replica (step1Replica or vs1) should be promoted.
entry.LastLeaseGrant = time.Now().Add(-1 * time.Minute)
// Set health scores: vs1 low (just rebuilt), step1Replica high.
for i := range entry.Replicas {
if entry.Replicas[i].Server == vs1 {
entry.Replicas[i].HealthScore = 0.5
} else {
entry.Replicas[i].HealthScore = 1.0
}
}
ms.failoverBlockVolumes(step1Primary)
entry, _ = ms.blockRegistry.Lookup("pvc-rf3-lifecycle")
if entry.VolumeServer == step1Primary {
t.Fatal("step3: primary should not be step1Primary after second failover")
}
// The higher health replica should win.
if entry.VolumeServer != step1Replica {
t.Fatalf("step3: expected %q promoted (health=1.0), got %q", step1Replica, entry.VolumeServer)
}
// Step 4: Recover step1Primary → becomes replica.
ms.recoverBlockVolumes(step1Primary)
entry, _ = ms.blockRegistry.Lookup("pvc-rf3-lifecycle")
// Should have 2 replicas now (vs1 from step2 still there + step1Primary recovered).
// Actually vs1's health was 0.5 and was NOT promoted, so vs1 stays as replica.
if len(entry.Replicas) < 1 {
t.Fatalf("step4: expected at least 1 replica after recovery, got %d", len(entry.Replicas))
}
// Step 5: Delete.
_, err = ms.DeleteBlockVolume(ctx, &master_pb.DeleteBlockVolumeRequest{Name: "pvc-rf3-lifecycle"})
if err != nil {
t.Fatalf("delete: %v", err)
}
if _, ok := ms.blockRegistry.Lookup("pvc-rf3-lifecycle"); ok {
t.Fatal("volume should be deleted")
}
_ = vs2
_ = vs3
}
weed/server/volume_grpc_block.go
+84
View File
@@ -57,3 +57,87 @@ func (vs *VolumeServer) VolumeServerDeleteBlockVolume(_ context.Context, req *vo
return &volume_server_pb.VolumeServerDeleteBlockVolumeResponse{}, nil
}
// SnapshotBlockVolume creates a snapshot on a block volume.
func (vs *VolumeServer) SnapshotBlockVolume(_ context.Context, req *volume_server_pb.SnapshotBlockVolumeRequest) (*volume_server_pb.SnapshotBlockVolumeResponse, error) {
if vs.blockService == nil {
return nil, fmt.Errorf("block service not enabled on this volume server")
}
if req.Name == "" {
return nil, fmt.Errorf("name is required")
}
createdAt, sizeBytes, err := vs.blockService.SnapshotBlockVol(req.Name, req.SnapshotId)
if err != nil {
return nil, fmt.Errorf("snapshot block volume %q: %w", req.Name, err)
}
return &volume_server_pb.SnapshotBlockVolumeResponse{
SnapshotId: req.SnapshotId,
CreatedAt: createdAt,
SizeBytes: sizeBytes,
}, nil
}
// DeleteBlockSnapshot deletes a snapshot from a block volume.
func (vs *VolumeServer) DeleteBlockSnapshot(_ context.Context, req *volume_server_pb.DeleteBlockSnapshotRequest) (*volume_server_pb.DeleteBlockSnapshotResponse, error) {
if vs.blockService == nil {
return nil, fmt.Errorf("block service not enabled on this volume server")
}
if req.Name == "" {
return nil, fmt.Errorf("name is required")
}
if err := vs.blockService.DeleteBlockSnapshot(req.Name, req.SnapshotId); err != nil {
return nil, fmt.Errorf("delete block snapshot %q/%d: %w", req.Name, req.SnapshotId, err)
}
return &volume_server_pb.DeleteBlockSnapshotResponse{}, nil
}
// ListBlockSnapshots lists all snapshots on a block volume.
func (vs *VolumeServer) ListBlockSnapshots(_ context.Context, req *volume_server_pb.ListBlockSnapshotsRequest) (*volume_server_pb.ListBlockSnapshotsResponse, error) {
if vs.blockService == nil {
return nil, fmt.Errorf("block service not enabled on this volume server")
}
if req.Name == "" {
return nil, fmt.Errorf("name is required")
}
infos, volSize, err := vs.blockService.ListBlockSnapshots(req.Name)
if err != nil {
return nil, fmt.Errorf("list block snapshots %q: %w", req.Name, err)
}
resp := &volume_server_pb.ListBlockSnapshotsResponse{}
for _, s := range infos {
resp.Snapshots = append(resp.Snapshots, &volume_server_pb.BlockSnapshotInfo{
SnapshotId: s.ID,
CreatedAt: s.CreatedAt.Unix(),
VolumeSizeBytes: volSize,
})
}
return resp, nil
}
// ExpandBlockVolume expands a block volume to a new size.
func (vs *VolumeServer) ExpandBlockVolume(_ context.Context, req *volume_server_pb.ExpandBlockVolumeRequest) (*volume_server_pb.ExpandBlockVolumeResponse, error) {
if vs.blockService == nil {
return nil, fmt.Errorf("block service not enabled on this volume server")
}
if req.Name == "" {
return nil, fmt.Errorf("name is required")
}
if req.NewSizeBytes == 0 {
return nil, fmt.Errorf("new_size_bytes must be > 0")
}
actualSize, err := vs.blockService.ExpandBlockVol(req.Name, req.NewSizeBytes)
if err != nil {
return nil, fmt.Errorf("expand block volume %q: %w", req.Name, err)
}
return &volume_server_pb.ExpandBlockVolumeResponse{
CapacityBytes: actualSize,
}, nil
}
weed/server/volume_grpc_block_test.go
+93
View File
@@ -108,3 +108,96 @@ func TestVS_DeleteNotFound(t *testing.T) {
t.Fatalf("delete non-existent should not error: %v", err)
}
}
func TestVS_SnapshotBlockVol(t *testing.T) {
bs, _ := newTestBlockServiceWithDir(t)
bs.CreateBlockVol("snap-vol", 4*1024*1024, "")
createdAt, sizeBytes, err := bs.SnapshotBlockVol("snap-vol", 1)
if err != nil {
t.Fatalf("SnapshotBlockVol: %v", err)
}
if createdAt == 0 {
t.Fatal("createdAt should not be zero")
}
if sizeBytes == 0 {
t.Fatal("sizeBytes should not be zero")
}
}
func TestVS_SnapshotVolumeNotFound(t *testing.T) {
bs, _ := newTestBlockServiceWithDir(t)
_, _, err := bs.SnapshotBlockVol("nonexistent", 1)
if err == nil {
t.Fatal("expected error for nonexistent volume")
}
}
func TestVS_DeleteBlockSnapshot(t *testing.T) {
bs, _ := newTestBlockServiceWithDir(t)
bs.CreateBlockVol("snap-vol", 4*1024*1024, "")
bs.SnapshotBlockVol("snap-vol", 1)
if err := bs.DeleteBlockSnapshot("snap-vol", 1); err != nil {
t.Fatalf("DeleteBlockSnapshot: %v", err)
}
// Delete again should be idempotent.
if err := bs.DeleteBlockSnapshot("snap-vol", 1); err != nil {
t.Fatalf("idempotent delete: %v", err)
}
}
func TestVS_ListBlockSnapshots(t *testing.T) {
bs, _ := newTestBlockServiceWithDir(t)
bs.CreateBlockVol("snap-vol", 4*1024*1024, "")
bs.SnapshotBlockVol("snap-vol", 1)
bs.SnapshotBlockVol("snap-vol", 2)
infos, volSize, err := bs.ListBlockSnapshots("snap-vol")
if err != nil {
t.Fatalf("ListBlockSnapshots: %v", err)
}
if len(infos) != 2 {
t.Fatalf("expected 2 snapshots, got %d", len(infos))
}
if volSize != 4*1024*1024 {
t.Fatalf("volSize: got %d, want %d", volSize, 4*1024*1024)
}
}
func TestVS_ListSnapshotsVolumeNotFound(t *testing.T) {
bs, _ := newTestBlockServiceWithDir(t)
_, _, err := bs.ListBlockSnapshots("nonexistent")
if err == nil {
t.Fatal("expected error for nonexistent volume")
}
}
func TestVS_ExpandBlockVol(t *testing.T) {
bs, _ := newTestBlockServiceWithDir(t)
bs.CreateBlockVol("expand-vol", 4*1024*1024, "")
actualSize, err := bs.ExpandBlockVol("expand-vol", 8*1024*1024)
if err != nil {
t.Fatalf("ExpandBlockVol: %v", err)
}
if actualSize != 8*1024*1024 {
t.Fatalf("expected 8MiB, got %d", actualSize)
}
}
func TestVS_ExpandVolumeNotFound(t *testing.T) {
bs, _ := newTestBlockServiceWithDir(t)
_, err := bs.ExpandBlockVol("nonexistent", 8*1024*1024)
if err == nil {
t.Fatal("expected error for nonexistent volume")
}
}
weed/server/volume_server_block.go
+113 -1
View File
@@ -228,7 +228,10 @@ func (bs *BlockService) ProcessAssignments(assignments []blockvol.BlockVolumeAss
// 2. Replication setup based on role + addresses.
switch role {
case blockvol.RolePrimary:
if a.ReplicaDataAddr != "" && a.ReplicaCtrlAddr != "" {
// CP8-2: ReplicaAddrs (multi-replica) takes precedence over scalar fields.
if len(a.ReplicaAddrs) > 0 {
bs.setupPrimaryReplicationMulti(a.Path, a.ReplicaAddrs)
} else if a.ReplicaDataAddr != "" && a.ReplicaCtrlAddr != "" {
bs.setupPrimaryReplication(a.Path, a.ReplicaDataAddr, a.ReplicaCtrlAddr)
}
case blockvol.RoleReplica:
@@ -279,6 +282,43 @@ func (bs *BlockService) setupPrimaryReplication(path, replicaDataAddr, replicaCt
glog.V(0).Infof("block service: primary %s shipping WAL to %s/%s (rebuild=%s)", path, replicaDataAddr, replicaCtrlAddr, rebuildAddr)
}
// setupPrimaryReplicationMulti configures WAL shipping from primary to N replicas
// using SetReplicaAddrs (CP8-2: multi-replica support).
func (bs *BlockService) setupPrimaryReplicationMulti(path string, addrs []blockvol.ReplicaAddr) {
// Compute deterministic rebuild listen address.
_, _, rebuildPort := bs.ReplicationPorts(path)
host := bs.listenAddr
if idx := strings.LastIndex(host, ":"); idx >= 0 {
host = host[:idx]
}
rebuildAddr := fmt.Sprintf("%s:%d", host, rebuildPort)
if err := bs.blockStore.WithVolume(path, func(vol *blockvol.BlockVol) error {
vol.SetReplicaAddrs(addrs)
if err := vol.StartRebuildServer(rebuildAddr); err != nil {
glog.Warningf("block service: start rebuild server %s on %s: %v", path, rebuildAddr, err)
}
return nil
}); err != nil {
glog.Warningf("block service: setup primary replication (multi) %s: %v", path, err)
return
}
// Track replication state for heartbeat reporting.
bs.replMu.Lock()
if bs.replStates == nil {
bs.replStates = make(map[string]*volReplState)
}
// Store first replica in replState for backward compat heartbeat reporting.
if len(addrs) > 0 {
bs.replStates[path] = &volReplState{
replicaDataAddr: addrs[0].DataAddr,
replicaCtrlAddr: addrs[0].CtrlAddr,
}
}
bs.replMu.Unlock()
glog.V(0).Infof("block service: primary %s shipping WAL to %d replicas (rebuild=%s)", path, len(addrs), rebuildAddr)
}
// setupReplicaReceiver starts the replica WAL receiver.
func (bs *BlockService) setupReplicaReceiver(path, dataAddr, ctrlAddr string) {
if err := bs.blockStore.WithVolume(path, func(vol *blockvol.BlockVol) error {
@@ -318,6 +358,72 @@ func (bs *BlockService) startRebuild(path, rebuildAddr string, epoch uint64) {
}()
}
// SnapshotBlockVol creates a snapshot on the named volume.
func (bs *BlockService) SnapshotBlockVol(name string, snapID uint32) (createdAt int64, sizeBytes uint64, err error) {
path := bs.volumePath(name)
err = bs.blockStore.WithVolume(path, func(vol *blockvol.BlockVol) error {
if serr := vol.CreateSnapshot(snapID); serr != nil {
return serr
}
// Find the snapshot we just created for its metadata.
for _, s := range vol.ListSnapshots() {
if s.ID == snapID {
createdAt = s.CreatedAt.Unix()
sizeBytes = vol.Info().VolumeSize
return nil
}
}
return fmt.Errorf("snapshot %d created but not found in list", snapID)
})
return
}
// DeleteBlockSnapshot deletes a snapshot on the named volume.
// Idempotent: returns nil if snapshot does not exist.
func (bs *BlockService) DeleteBlockSnapshot(name string, snapID uint32) error {
path := bs.volumePath(name)
return bs.blockStore.WithVolume(path, func(vol *blockvol.BlockVol) error {
err := vol.DeleteSnapshot(snapID)
if err != nil && err.Error() == "blockvol: snapshot not found" {
return nil // idempotent
}
return err
})
}
// ListBlockSnapshots lists all snapshots on the named volume.
func (bs *BlockService) ListBlockSnapshots(name string) ([]blockvol.SnapshotInfo, uint64, error) {
path := bs.volumePath(name)
var infos []blockvol.SnapshotInfo
var volSize uint64
err := bs.blockStore.WithVolume(path, func(vol *blockvol.BlockVol) error {
infos = vol.ListSnapshots()
volSize = vol.Info().VolumeSize
return nil
})
return infos, volSize, err
}
// ExpandBlockVol expands the named volume to newSize bytes.
func (bs *BlockService) ExpandBlockVol(name string, newSize uint64) (uint64, error) {
path := bs.volumePath(name)
var actualSize uint64
err := bs.blockStore.WithVolume(path, func(vol *blockvol.BlockVol) error {
if eerr := vol.Expand(newSize); eerr != nil {
return eerr
}
actualSize = vol.Info().VolumeSize
return nil
})
return actualSize, err
}
// volumePath converts a volume name to its .blk file path.
func (bs *BlockService) volumePath(name string) string {
sanitized := blockvol.SanitizeFilename(name)
return filepath.Join(bs.blockDir, sanitized+".blk")
}
// GetReplState returns the replication state for a volume path.
func (bs *BlockService) GetReplState(path string) (dataAddr, ctrlAddr string) {
bs.replMu.RLock()
@@ -374,3 +480,9 @@ func (bs *BlockService) Shutdown() {
bs.blockStore.Close()
glog.V(0).Infof("block service: shut down")
}
// SetBlockService wires a BlockService into the VolumeServer so that
// heartbeats include block volume info and the server is marked block-capable.
func (vs *VolumeServer) SetBlockService(bs *BlockService) {
vs.blockService = bs
}
weed/shell/command_block_volume.go
+220
View File
@@ -0,0 +1,220 @@
package shell
import (
"context"
"flag"
"fmt"
"io"
"strings"
"github.com/seaweedfs/seaweedfs/weed/storage/blockvol/blockapi"
)
func init() {
Commands = append(Commands, &commandBlockCreate{})
Commands = append(Commands, &commandBlockList{})
Commands = append(Commands, &commandBlockStatus{})
Commands = append(Commands, &commandBlockDelete{})
Commands = append(Commands, &commandBlockAssign{})
}
// mastersHTTP builds a comma-separated list of "http://host:port" URLs from
// the shell environment's master addresses.
func mastersHTTP(commandEnv *CommandEnv) string {
addrs := commandEnv.MasterClient.GetMasters(context.Background())
parts := make([]string, len(addrs))
for i, a := range addrs {
s := string(a)
if !strings.HasPrefix(s, "http://") && !strings.HasPrefix(s, "https://") {
s = "http://" + s
}
parts[i] = s
}
return strings.Join(parts, ",")
}
// ---- block.create ----
type commandBlockCreate struct{}
func (c *commandBlockCreate) Name() string { return "block.create" }
func (c *commandBlockCreate) Help() string {
return `create a block volume
block.create -name <name> -size <bytes> [-replicaPlacement <xyz>] [-disk <type>]
`
}
func (c *commandBlockCreate) HasTag(CommandTag) bool { return false }
func (c *commandBlockCreate) Do(args []string, commandEnv *CommandEnv, writer io.Writer) error {
f := flag.NewFlagSet(c.Name(), flag.ContinueOnError)
name := f.String("name", "", "volume name")
size := f.Uint64("size", 0, "volume size in bytes")
placement := f.String("replicaPlacement", "000", "placement string: 000, 001, 010, 100")
disk := f.String("disk", "", "disk type (e.g. ssd, hdd)")
if err := f.Parse(args); err != nil {
return nil
}
if *name == "" || *size == 0 {
return fmt.Errorf("both -name and -size are required")
}
client := blockapi.NewClient(mastersHTTP(commandEnv))
info, err := client.CreateVolume(context.Background(), blockapi.CreateVolumeRequest{
Name: *name,
SizeBytes: *size,
ReplicaPlacement: *placement,
DiskType: *disk,
})
if err != nil {
return err
}
fmt.Fprintf(writer, "created block volume %q on %s (iqn=%s, iscsi=%s)\n",
info.Name, info.VolumeServer, info.IQN, info.ISCSIAddr)
if info.ReplicaServer != "" {
fmt.Fprintf(writer, " replica on %s (iqn=%s, iscsi=%s)\n",
info.ReplicaServer, info.ReplicaIQN, info.ReplicaISCSIAddr)
}
return nil
}
// ---- block.list ----
type commandBlockList struct{}
func (c *commandBlockList) Name() string { return "block.list" }
func (c *commandBlockList) Help() string {
return `list all block volumes
block.list
`
}
func (c *commandBlockList) HasTag(CommandTag) bool { return false }
func (c *commandBlockList) Do(args []string, commandEnv *CommandEnv, writer io.Writer) error {
client := blockapi.NewClient(mastersHTTP(commandEnv))
vols, err := client.ListVolumes(context.Background())
if err != nil {
return err
}
if len(vols) == 0 {
fmt.Fprintln(writer, "no block volumes")
return nil
}
fmt.Fprintf(writer, "%-20s %-20s %-12s %-8s %-8s %-20s\n",
"NAME", "SERVER", "SIZE", "EPOCH", "ROLE", "STATUS")
for _, v := range vols {
fmt.Fprintf(writer, "%-20s %-20s %-12d %-8d %-8s %-20s\n",
v.Name, v.VolumeServer, v.SizeBytes, v.Epoch, v.Role, v.Status)
}
return nil
}
// ---- block.status ----
type commandBlockStatus struct{}
func (c *commandBlockStatus) Name() string { return "block.status" }
func (c *commandBlockStatus) Help() string {
return `show status of a block volume
block.status <name>
`
}
func (c *commandBlockStatus) HasTag(CommandTag) bool { return false }
func (c *commandBlockStatus) Do(args []string, commandEnv *CommandEnv, writer io.Writer) error {
if len(args) == 0 {
return fmt.Errorf("usage: block.status <name>")
}
name := args[0]
client := blockapi.NewClient(mastersHTTP(commandEnv))
info, err := client.LookupVolume(context.Background(), name)
if err != nil {
return err
}
fmt.Fprintf(writer, "Name: %s\n", info.Name)
fmt.Fprintf(writer, "VolumeServer: %s\n", info.VolumeServer)
fmt.Fprintf(writer, "SizeBytes: %d\n", info.SizeBytes)
fmt.Fprintf(writer, "Epoch: %d\n", info.Epoch)
fmt.Fprintf(writer, "Role: %s\n", info.Role)
fmt.Fprintf(writer, "Status: %s\n", info.Status)
fmt.Fprintf(writer, "ISCSIAddr: %s\n", info.ISCSIAddr)
fmt.Fprintf(writer, "IQN: %s\n", info.IQN)
if info.ReplicaServer != "" {
fmt.Fprintf(writer, "ReplicaServer: %s\n", info.ReplicaServer)
fmt.Fprintf(writer, "ReplicaISCSI: %s\n", info.ReplicaISCSIAddr)
fmt.Fprintf(writer, "ReplicaIQN: %s\n", info.ReplicaIQN)
fmt.Fprintf(writer, "ReplicaData: %s\n", info.ReplicaDataAddr)
fmt.Fprintf(writer, "ReplicaCtrl: %s\n", info.ReplicaCtrlAddr)
}
return nil
}
// ---- block.delete ----
type commandBlockDelete struct{}
func (c *commandBlockDelete) Name() string { return "block.delete" }
func (c *commandBlockDelete) Help() string {
return `delete a block volume
block.delete <name>
`
}
func (c *commandBlockDelete) HasTag(CommandTag) bool { return false }
func (c *commandBlockDelete) Do(args []string, commandEnv *CommandEnv, writer io.Writer) error {
if len(args) == 0 {
return fmt.Errorf("usage: block.delete <name>")
}
name := args[0]
client := blockapi.NewClient(mastersHTTP(commandEnv))
if err := client.DeleteVolume(context.Background(), name); err != nil {
return err
}
fmt.Fprintf(writer, "deleted block volume %q\n", name)
return nil
}
// ---- block.assign ----
type commandBlockAssign struct{}
func (c *commandBlockAssign) Name() string { return "block.assign" }
func (c *commandBlockAssign) Help() string {
return `assign a role to a block volume
block.assign -name <name> -epoch <n> -role <primary|replica> [-lease <ms>]
`
}
func (c *commandBlockAssign) HasTag(CommandTag) bool { return false }
func (c *commandBlockAssign) Do(args []string, commandEnv *CommandEnv, writer io.Writer) error {
f := flag.NewFlagSet(c.Name(), flag.ContinueOnError)
name := f.String("name", "", "volume name")
epoch := f.Uint64("epoch", 0, "epoch number")
role := f.String("role", "", "role: primary or replica")
leaseTTLMs := f.Uint64("lease", 30000, "lease TTL in milliseconds")
if err := f.Parse(args); err != nil {
return nil
}
if *name == "" || *role == "" {
return fmt.Errorf("both -name and -role are required")
}
client := blockapi.NewClient(mastersHTTP(commandEnv))
if err := client.AssignRole(context.Background(), blockapi.AssignRequest{
Name: *name,
Epoch: *epoch,
Role: *role,
LeaseTTLMs: *leaseTTLMs,
}); err != nil {
return err
}
fmt.Fprintf(writer, "assignment queued: %s epoch=%d role=%s lease=%dms\n",
*name, *epoch, *role, *leaseTTLMs)
return nil
}
weed/storage/blockvol/block_heartbeat.go
+37 -27
View File
@@ -8,17 +8,21 @@ import (
// BlockVolumeInfoMessage is the heartbeat status for one block volume.
// Mirrors the proto message that will be generated from master.proto.
type BlockVolumeInfoMessage struct {
Path string // volume file path (unique ID on this server)
VolumeSize uint64 // logical size in bytes
BlockSize uint32 // block size in bytes
Epoch uint64 // current fencing epoch
Role uint32 // blockvol.Role as uint32 for wire compat
WalHeadLsn uint64 // WAL head LSN
CheckpointLsn uint64 // last flushed LSN
HasLease bool // whether volume holds a valid lease
DiskType string // e.g., "ssd", "hdd"
ReplicaDataAddr string // receiver data listen addr (VS reports in heartbeat)
ReplicaCtrlAddr string // receiver ctrl listen addr
Path string // volume file path (unique ID on this server)
VolumeSize uint64 // logical size in bytes
BlockSize uint32 // block size in bytes
Epoch uint64 // current fencing epoch
Role uint32 // blockvol.Role as uint32 for wire compat
WalHeadLsn uint64 // WAL head LSN
CheckpointLsn uint64 // last flushed LSN
HasLease bool // whether volume holds a valid lease
DiskType string // e.g., "ssd", "hdd"
ReplicaDataAddr string // receiver data listen addr (VS reports in heartbeat)
ReplicaCtrlAddr string // receiver ctrl listen addr
HealthScore float64 // CP8-2: 0.0-1.0
ScrubErrors int64 // CP8-2: lifetime scrub error count
LastScrubTime int64 // CP8-2: unix seconds
ReplicaDegraded bool // CP8-2: true if any replica shipper degraded
}
// BlockVolumeShortInfoMessage is used for delta heartbeats
@@ -33,13 +37,14 @@ type BlockVolumeShortInfoMessage struct {
// BlockVolumeAssignment carries a role/epoch/lease assignment
// from master to volume server for one block volume.
type BlockVolumeAssignment struct {
Path string // which block volume
Epoch uint64 // new epoch
Role uint32 // target role (blockvol.Role as uint32)
LeaseTtlMs uint32 // lease TTL in milliseconds (0 = no lease)
ReplicaDataAddr string // where primary ships WAL data
ReplicaCtrlAddr string // where primary sends barriers
RebuildAddr string // where rebuild server listens
Path string // which block volume
Epoch uint64 // new epoch
Role uint32 // target role (blockvol.Role as uint32)
LeaseTtlMs uint32 // lease TTL in milliseconds (0 = no lease)
ReplicaDataAddr string // where primary ships WAL data (scalar, RF=2 compat)
ReplicaCtrlAddr string // where primary sends barriers (scalar, RF=2 compat)
RebuildAddr string // where rebuild server listens
ReplicaAddrs []ReplicaAddr // CP8-2: multi-replica addrs (precedence over scalar)
}
// ToBlockVolumeInfoMessage converts a BlockVol's current state
@@ -48,16 +53,21 @@ type BlockVolumeAssignment struct {
func ToBlockVolumeInfoMessage(path, diskType string, vol *BlockVol) BlockVolumeInfoMessage {
info := vol.Info()
status := vol.Status()
hs := vol.HealthStats()
return BlockVolumeInfoMessage{
Path: path,
VolumeSize: info.VolumeSize,
BlockSize: info.BlockSize,
Epoch: status.Epoch,
Role: RoleToWire(status.Role),
WalHeadLsn: status.WALHeadLSN,
CheckpointLsn: status.CheckpointLSN,
HasLease: status.HasLease,
DiskType: diskType,
Path: path,
VolumeSize: info.VolumeSize,
BlockSize: info.BlockSize,
Epoch: status.Epoch,
Role: RoleToWire(status.Role),
WalHeadLsn: status.WALHeadLSN,
CheckpointLsn: status.CheckpointLSN,
HasLease: status.HasLease,
DiskType: diskType,
HealthScore: status.HealthScore,
ScrubErrors: hs.ScrubErrors,
LastScrubTime: hs.LastScrubTime,
ReplicaDegraded: status.ReplicaDegraded,
}
}
weed/storage/blockvol/block_heartbeat_proto.go
+37 -9
View File
@@ -18,6 +18,10 @@ func InfoMessageToProto(m BlockVolumeInfoMessage) *master_pb.BlockVolumeInfoMess
DiskType: m.DiskType,
ReplicaDataAddr: m.ReplicaDataAddr,
ReplicaCtrlAddr: m.ReplicaCtrlAddr,
HealthScore: m.HealthScore,
ScrubErrors: m.ScrubErrors,
LastScrubTime: m.LastScrubTime,
ReplicaDegraded: m.ReplicaDegraded,
}
}
@@ -38,6 +42,10 @@ func InfoMessageFromProto(p *master_pb.BlockVolumeInfoMessage) BlockVolumeInfoMe
DiskType: p.DiskType,
ReplicaDataAddr: p.ReplicaDataAddr,
ReplicaCtrlAddr: p.ReplicaCtrlAddr,
HealthScore: p.HealthScore,
ScrubErrors: p.ScrubErrors,
LastScrubTime: p.LastScrubTime,
ReplicaDegraded: p.ReplicaDegraded,
}
}
@@ -84,7 +92,7 @@ func ShortInfoFromProto(p *master_pb.BlockVolumeShortInfoMessage) BlockVolumeSho
// AssignmentToProto converts a Go assignment to proto.
func AssignmentToProto(a BlockVolumeAssignment) *master_pb.BlockVolumeAssignment {
return &master_pb.BlockVolumeAssignment{
pb := &master_pb.BlockVolumeAssignment{
Path: a.Path,
Epoch: a.Epoch,
Role: a.Role,
@@ -93,22 +101,42 @@ func AssignmentToProto(a BlockVolumeAssignment) *master_pb.BlockVolumeAssignment
ReplicaCtrlAddr: a.ReplicaCtrlAddr,
RebuildAddr: a.RebuildAddr,
}
for _, ra := range a.ReplicaAddrs {
pb.ReplicaAddrs = append(pb.ReplicaAddrs, &master_pb.ReplicaAddrMessage{
DataAddr: ra.DataAddr,
CtrlAddr: ra.CtrlAddr,
})
}
return pb
}
// AssignmentFromProto converts a proto assignment to Go wire type.
// Precedence: if ReplicaAddrs is non-empty, use it and ignore scalar fields.
func AssignmentFromProto(p *master_pb.BlockVolumeAssignment) BlockVolumeAssignment {
if p == nil {
return BlockVolumeAssignment{}
}
return BlockVolumeAssignment{
Path: p.Path,
Epoch: p.Epoch,
Role: p.Role,
LeaseTtlMs: p.LeaseTtlMs,
ReplicaDataAddr: p.ReplicaDataAddr,
ReplicaCtrlAddr: p.ReplicaCtrlAddr,
RebuildAddr: p.RebuildAddr,
a := BlockVolumeAssignment{
Path: p.Path,
Epoch: p.Epoch,
Role: p.Role,
LeaseTtlMs: p.LeaseTtlMs,
RebuildAddr: p.RebuildAddr,
}
if len(p.ReplicaAddrs) > 0 {
// Multi-replica: populate ReplicaAddrs, leave scalar fields empty.
for _, ra := range p.ReplicaAddrs {
a.ReplicaAddrs = append(a.ReplicaAddrs, ReplicaAddr{
DataAddr: ra.DataAddr,
CtrlAddr: ra.CtrlAddr,
})
}
} else {
// Backward compat: use scalar fields.
a.ReplicaDataAddr = p.ReplicaDataAddr
a.ReplicaCtrlAddr = p.ReplicaCtrlAddr
}
return a
}
// AssignmentsToProto converts a slice of Go assignments to proto.
weed/storage/blockvol/block_heartbeat_proto_test.go
+196 -3
View File
@@ -1,6 +1,7 @@
package blockvol
import (
"reflect"
"testing"
)
@@ -46,7 +47,7 @@ func TestAssignmentRoundTrip(t *testing.T) {
}
pb := AssignmentToProto(orig)
back := AssignmentFromProto(pb)
if back != orig {
if !reflect.DeepEqual(back, orig) {
t.Fatalf("round-trip mismatch:\n got %+v\n want %+v", back, orig)
}
}
@@ -80,7 +81,7 @@ func TestAssignmentRoundTripWithReplicaAddrs(t *testing.T) {
}
pb := AssignmentToProto(orig)
back := AssignmentFromProto(pb)
if back != orig {
if !reflect.DeepEqual(back, orig) {
t.Fatalf("round-trip mismatch:\n got %+v\n want %+v", back, orig)
}
}
@@ -195,7 +196,199 @@ func TestNilProtoConversions(t *testing.T) {
t.Fatalf("nil short proto should yield zero value, got %+v", short)
}
assign := AssignmentFromProto(nil)
if assign != (BlockVolumeAssignment{}) {
if !reflect.DeepEqual(assign, BlockVolumeAssignment{}) {
t.Fatalf("nil assignment proto should yield zero value, got %+v", assign)
}
}
// --- CP8-2 new tests ---
func TestInfoMessage_HealthScoreRoundTrip(t *testing.T) {
orig := BlockVolumeInfoMessage{
Path: "/data/vol.blk",
VolumeSize: 1 << 30,
BlockSize: 4096,
Epoch: 5,
HealthScore: 0.85,
ScrubErrors: 3,
LastScrubTime: 1709000000,
}
pb := InfoMessageToProto(orig)
back := InfoMessageFromProto(pb)
if back.HealthScore != 0.85 {
t.Fatalf("HealthScore: got %f, want 0.85", back.HealthScore)
}
if back.ScrubErrors != 3 {
t.Fatalf("ScrubErrors: got %d, want 3", back.ScrubErrors)
}
if back.LastScrubTime != 1709000000 {
t.Fatalf("LastScrubTime: got %d, want 1709000000", back.LastScrubTime)
}
}
func TestInfoMessage_ReplicaDegradedRoundTrip(t *testing.T) {
orig := BlockVolumeInfoMessage{
Path: "/data/vol.blk",
Epoch: 1,
ReplicaDegraded: true,
}
pb := InfoMessageToProto(orig)
back := InfoMessageFromProto(pb)
if !back.ReplicaDegraded {
t.Fatal("ReplicaDegraded should be true after round-trip")
}
// Also verify false round-trips.
orig.ReplicaDegraded = false
pb = InfoMessageToProto(orig)
back = InfoMessageFromProto(pb)
if back.ReplicaDegraded {
t.Fatal("ReplicaDegraded should be false after round-trip")
}
}
func TestAssignment_MultiReplicaRoundTrip(t *testing.T) {
orig := BlockVolumeAssignment{
Path: "/data/vol.blk",
Epoch: 10,
Role: RoleToWire(RolePrimary),
LeaseTtlMs: 30000,
ReplicaAddrs: []ReplicaAddr{
{DataAddr: "10.0.0.2:14260", CtrlAddr: "10.0.0.2:14261"},
{DataAddr: "10.0.0.3:14260", CtrlAddr: "10.0.0.3:14261"},
},
}
pb := AssignmentToProto(orig)
back := AssignmentFromProto(pb)
if len(back.ReplicaAddrs) != 2 {
t.Fatalf("ReplicaAddrs len: got %d, want 2", len(back.ReplicaAddrs))
}
if back.ReplicaAddrs[0].DataAddr != "10.0.0.2:14260" {
t.Fatalf("ReplicaAddrs[0].DataAddr: got %q", back.ReplicaAddrs[0].DataAddr)
}
if back.ReplicaAddrs[1].CtrlAddr != "10.0.0.3:14261" {
t.Fatalf("ReplicaAddrs[1].CtrlAddr: got %q", back.ReplicaAddrs[1].CtrlAddr)
}
// Scalar fields should be empty when ReplicaAddrs is populated.
if back.ReplicaDataAddr != "" || back.ReplicaCtrlAddr != "" {
t.Fatalf("scalar addrs should be empty when ReplicaAddrs set, got data=%q ctrl=%q",
back.ReplicaDataAddr, back.ReplicaCtrlAddr)
}
}
func TestAssignment_PrecedenceRule(t *testing.T) {
// When proto has BOTH ReplicaAddrs AND scalar fields,
// ReplicaAddrs takes precedence and scalar fields are ignored.
orig := BlockVolumeAssignment{
Path: "/data/vol.blk",
Epoch: 5,
Role: RoleToWire(RolePrimary),
ReplicaDataAddr: "scalar-data:1234",
ReplicaCtrlAddr: "scalar-ctrl:1235",
ReplicaAddrs: []ReplicaAddr{
{DataAddr: "multi-data:4260", CtrlAddr: "multi-ctrl:4261"},
},
}
pb := AssignmentToProto(orig)
// Proto will have both scalar and repeated fields set.
if pb.ReplicaDataAddr != "scalar-data:1234" {
t.Fatalf("proto should preserve scalar, got %q", pb.ReplicaDataAddr)
}
if len(pb.ReplicaAddrs) != 1 {
t.Fatalf("proto should have 1 ReplicaAddr, got %d", len(pb.ReplicaAddrs))
}
// FromProto: ReplicaAddrs non-empty → use it, ignore scalar.
back := AssignmentFromProto(pb)
if len(back.ReplicaAddrs) != 1 {
t.Fatalf("back.ReplicaAddrs len: got %d, want 1", len(back.ReplicaAddrs))
}
if back.ReplicaDataAddr != "" {
t.Fatalf("scalar ReplicaDataAddr should be empty due to precedence, got %q", back.ReplicaDataAddr)
}
if back.ReplicaCtrlAddr != "" {
t.Fatalf("scalar ReplicaCtrlAddr should be empty due to precedence, got %q", back.ReplicaCtrlAddr)
}
}
func TestAssignment_BackwardCompatScalar(t *testing.T) {
// When proto has only scalar fields (no ReplicaAddrs), backward compat path.
orig := BlockVolumeAssignment{
Path: "/data/vol.blk",
Epoch: 3,
Role: RoleToWire(RolePrimary),
LeaseTtlMs: 5000,
ReplicaDataAddr: "host:4260",
ReplicaCtrlAddr: "host:4261",
}
pb := AssignmentToProto(orig)
back := AssignmentFromProto(pb)
if back.ReplicaDataAddr != "host:4260" {
t.Fatalf("scalar ReplicaDataAddr: got %q, want %q", back.ReplicaDataAddr, "host:4260")
}
if back.ReplicaCtrlAddr != "host:4261" {
t.Fatalf("scalar ReplicaCtrlAddr: got %q, want %q", back.ReplicaCtrlAddr, "host:4261")
}
if len(back.ReplicaAddrs) != 0 {
t.Fatalf("ReplicaAddrs should be empty for backward compat, got %d", len(back.ReplicaAddrs))
}
}
func TestAssignmentsSlice_MultiReplicaRoundTrip(t *testing.T) {
as := []BlockVolumeAssignment{
{
Path: "/a.blk",
Epoch: 1,
ReplicaAddrs: []ReplicaAddr{
{DataAddr: "h1:1", CtrlAddr: "h1:2"},
{DataAddr: "h2:1", CtrlAddr: "h2:2"},
},
},
{
Path: "/b.blk",
Epoch: 2,
ReplicaDataAddr: "scalar:1",
},
}
pbs := AssignmentsToProto(as)
back := AssignmentsFromProto(pbs)
if len(back) != 2 {
t.Fatalf("len: got %d, want 2", len(back))
}
// First: multi-replica path.
if len(back[0].ReplicaAddrs) != 2 {
t.Fatalf("back[0].ReplicaAddrs len: got %d, want 2", len(back[0].ReplicaAddrs))
}
// Second: scalar path.
if back[1].ReplicaDataAddr != "scalar:1" {
t.Fatalf("back[1].ReplicaDataAddr: got %q", back[1].ReplicaDataAddr)
}
if len(back[1].ReplicaAddrs) != 0 {
t.Fatalf("back[1].ReplicaAddrs should be empty, got %d", len(back[1].ReplicaAddrs))
}
}
func TestInfoMessage_HealthFieldsZeroDefault(t *testing.T) {
// Verify zero-valued health fields round-trip correctly (backward compat).
orig := BlockVolumeInfoMessage{
Path: "/data/old.blk",
Epoch: 1,
// All CP8-2 fields at zero default.
}
pb := InfoMessageToProto(orig)
back := InfoMessageFromProto(pb)
if back.HealthScore != 0 {
t.Fatalf("HealthScore zero default: got %f", back.HealthScore)
}
if back.ScrubErrors != 0 {
t.Fatalf("ScrubErrors zero default: got %d", back.ScrubErrors)
}
if back.LastScrubTime != 0 {
t.Fatalf("LastScrubTime zero default: got %d", back.LastScrubTime)
}
if back.ReplicaDegraded {
t.Fatal("ReplicaDegraded zero default should be false")
}
}
weed/storage/blockvol/blockapi/client.go
+189
View File
@@ -0,0 +1,189 @@
package blockapi
import (
"bytes"
"context"
"encoding/json"
"fmt"
"io"
"net/http"
"strings"
"time"
)
// Client is a Go HTTP client for the master's block volume REST API.
// It supports multiple master addresses for failover.
type Client struct {
Masters []string
HTTPClient *http.Client
}
// NewClient creates a Client from a comma-separated list of master URLs.
// Example: "http://m1:9333,http://m2:9333"
func NewClient(masters string) *Client {
var addrs []string
for _, m := range strings.Split(masters, ",") {
m = strings.TrimSpace(m)
if m != "" {
addrs = append(addrs, m)
}
}
return &Client{
Masters: addrs,
HTTPClient: &http.Client{
Timeout: 30 * time.Second,
},
}
}
// CreateVolume creates a new block volume.
func (c *Client) CreateVolume(ctx context.Context, req CreateVolumeRequest) (*VolumeInfo, error) {
body, err := json.Marshal(req)
if err != nil {
return nil, fmt.Errorf("marshal request: %w", err)
}
resp, err := c.doRequest(ctx, http.MethodPost, "/block/volume", bytes.NewReader(body))
if err != nil {
return nil, err
}
defer resp.Body.Close()
if err := checkStatus(resp, http.StatusOK, http.StatusCreated); err != nil {
return nil, err
}
var info VolumeInfo
if err := json.NewDecoder(resp.Body).Decode(&info); err != nil {
return nil, fmt.Errorf("decode response: %w", err)
}
return &info, nil
}
// DeleteVolume deletes a block volume by name.
func (c *Client) DeleteVolume(ctx context.Context, name string) error {
resp, err := c.doRequest(ctx, http.MethodDelete, "/block/volume/"+name, nil)
if err != nil {
return err
}
defer resp.Body.Close()
return checkStatus(resp, http.StatusOK)
}
// LookupVolume looks up a single block volume by name.
func (c *Client) LookupVolume(ctx context.Context, name string) (*VolumeInfo, error) {
resp, err := c.doRequest(ctx, http.MethodGet, "/block/volume/"+name, nil)
if err != nil {
return nil, err
}
defer resp.Body.Close()
if err := checkStatus(resp, http.StatusOK); err != nil {
return nil, err
}
var info VolumeInfo
if err := json.NewDecoder(resp.Body).Decode(&info); err != nil {
return nil, fmt.Errorf("decode response: %w", err)
}
return &info, nil
}
// ListVolumes lists all block volumes.
func (c *Client) ListVolumes(ctx context.Context) ([]VolumeInfo, error) {
resp, err := c.doRequest(ctx, http.MethodGet, "/block/volumes", nil)
if err != nil {
return nil, err
}
defer resp.Body.Close()
if err := checkStatus(resp, http.StatusOK); err != nil {
return nil, err
}
var infos []VolumeInfo
if err := json.NewDecoder(resp.Body).Decode(&infos); err != nil {
return nil, fmt.Errorf("decode response: %w", err)
}
return infos, nil
}
// AssignRole enqueues a role assignment for a block volume.
func (c *Client) AssignRole(ctx context.Context, req AssignRequest) error {
body, err := json.Marshal(req)
if err != nil {
return fmt.Errorf("marshal request: %w", err)
}
resp, err := c.doRequest(ctx, http.MethodPost, "/block/assign", bytes.NewReader(body))
if err != nil {
return err
}
defer resp.Body.Close()
return checkStatus(resp, http.StatusOK)
}
// ListServers lists all block-capable volume servers.
func (c *Client) ListServers(ctx context.Context) ([]ServerInfo, error) {
resp, err := c.doRequest(ctx, http.MethodGet, "/block/servers", nil)
if err != nil {
return nil, err
}
defer resp.Body.Close()
if err := checkStatus(resp, http.StatusOK); err != nil {
return nil, err
}
var infos []ServerInfo
if err := json.NewDecoder(resp.Body).Decode(&infos); err != nil {
return nil, fmt.Errorf("decode response: %w", err)
}
return infos, nil
}
// doRequest tries each master in order until one responds.
func (c *Client) doRequest(ctx context.Context, method, path string, body io.Reader) (*http.Response, error) {
var lastErr error
for _, master := range c.Masters {
url := strings.TrimRight(master, "/") + path
var bodyReader io.Reader
if body != nil {
// Re-read body for retries: buffer it
if lastErr != nil {
// body was already consumed on previous attempt
// We need a seekable body — caller should pass bytes.Reader
if seeker, ok := body.(io.Seeker); ok {
seeker.Seek(0, io.SeekStart)
}
}
bodyReader = body
}
req, err := http.NewRequestWithContext(ctx, method, url, bodyReader)
if err != nil {
lastErr = fmt.Errorf("master %s: %w", master, err)
continue
}
if method == http.MethodPost || method == http.MethodPut {
req.Header.Set("Content-Type", "application/json")
}
resp, err := c.HTTPClient.Do(req)
if err != nil {
lastErr = fmt.Errorf("master %s: %w", master, err)
continue
}
return resp, nil
}
if lastErr != nil {
return nil, lastErr
}
return nil, fmt.Errorf("no master addresses configured")
}
// checkStatus returns an error if the response status is not in the accepted list.
func checkStatus(resp *http.Response, accepted ...int) error {
for _, code := range accepted {
if resp.StatusCode == code {
return nil
}
}
body, _ := io.ReadAll(resp.Body)
// Try to extract error message from JSON.
var errResp struct {
Error string `json:"error"`
}
if json.Unmarshal(body, &errResp) == nil && errResp.Error != "" {
return fmt.Errorf("HTTP %d: %s", resp.StatusCode, errResp.Error)
}
return fmt.Errorf("HTTP %d: %s", resp.StatusCode, string(body))
}
weed/storage/blockvol/blockapi/client_test.go
+103
View File
@@ -0,0 +1,103 @@
package blockapi
import (
"context"
"encoding/json"
"net/http"
"net/http/httptest"
"testing"
)
func TestClientCreateVolume(t *testing.T) {
ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if r.Method != "POST" || r.URL.Path != "/block/volume" {
t.Errorf("unexpected request: %s %s", r.Method, r.URL.Path)
w.WriteHeader(http.StatusNotFound)
return
}
var req CreateVolumeRequest
if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
t.Errorf("decode request: %v", err)
w.WriteHeader(http.StatusBadRequest)
return
}
if req.Name != "test-vol" {
t.Errorf("expected name test-vol, got %s", req.Name)
}
w.Header().Set("Content-Type", "application/json")
json.NewEncoder(w).Encode(VolumeInfo{
Name: req.Name,
VolumeServer: "vs1:9333",
SizeBytes: req.SizeBytes,
Epoch: 1,
Role: "primary",
Status: "active",
})
}))
defer ts.Close()
client := NewClient(ts.URL)
info, err := client.CreateVolume(context.Background(), CreateVolumeRequest{
Name: "test-vol",
SizeBytes: 1 << 30,
})
if err != nil {
t.Fatal(err)
}
if info.Name != "test-vol" {
t.Errorf("expected name test-vol, got %s", info.Name)
}
if info.VolumeServer != "vs1:9333" {
t.Errorf("expected vs1:9333, got %s", info.VolumeServer)
}
}
func TestClientListVolumes(t *testing.T) {
ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if r.Method != "GET" || r.URL.Path != "/block/volumes" {
w.WriteHeader(http.StatusNotFound)
return
}
w.Header().Set("Content-Type", "application/json")
json.NewEncoder(w).Encode([]VolumeInfo{
{Name: "alpha", VolumeServer: "vs1:9333"},
{Name: "beta", VolumeServer: "vs2:9333"},
})
}))
defer ts.Close()
client := NewClient(ts.URL)
vols, err := client.ListVolumes(context.Background())
if err != nil {
t.Fatal(err)
}
if len(vols) != 2 {
t.Fatalf("expected 2 volumes, got %d", len(vols))
}
if vols[0].Name != "alpha" || vols[1].Name != "beta" {
t.Errorf("unexpected volumes: %+v", vols)
}
}
func TestClientMultiMasterFallback(t *testing.T) {
// First server immediately rejects connections (closed listener).
dead := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {}))
deadURL := dead.URL
dead.Close() // close it so connections are refused
// Second server responds normally.
ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "application/json")
json.NewEncoder(w).Encode([]VolumeInfo{})
}))
defer ts.Close()
client := NewClient(deadURL + "," + ts.URL)
vols, err := client.ListVolumes(context.Background())
if err != nil {
t.Fatal(err)
}
if vols == nil {
t.Error("expected non-nil result")
}
}
weed/storage/blockvol/blockapi/types.go
+81
View File
@@ -0,0 +1,81 @@
package blockapi
import (
"strings"
"github.com/seaweedfs/seaweedfs/weed/storage/blockvol"
)
// CreateVolumeRequest is the request body for POST /block/volume.
type CreateVolumeRequest struct {
Name string `json:"name"`
SizeBytes uint64 `json:"size_bytes"`
ReplicaPlacement string `json:"replica_placement"` // SeaweedFS placement string: "000", "001", "010", "100"
DiskType string `json:"disk_type"` // e.g. "ssd", "hdd"
}
// VolumeInfo describes a block volume.
type VolumeInfo struct {
Name string `json:"name"`
VolumeServer string `json:"volume_server"`
SizeBytes uint64 `json:"size_bytes"`
ReplicaPlacement string `json:"replica_placement,omitempty"`
Epoch uint64 `json:"epoch"`
Role string `json:"role"`
Status string `json:"status"`
ISCSIAddr string `json:"iscsi_addr"`
IQN string `json:"iqn"`
ReplicaServer string `json:"replica_server,omitempty"`
ReplicaISCSIAddr string `json:"replica_iscsi_addr,omitempty"`
ReplicaIQN string `json:"replica_iqn,omitempty"`
ReplicaDataAddr string `json:"replica_data_addr,omitempty"`
ReplicaCtrlAddr string `json:"replica_ctrl_addr,omitempty"`
// CP8-2: Multi-replica fields.
ReplicaFactor int `json:"replica_factor"`
Replicas []ReplicaDetail `json:"replicas,omitempty"`
HealthScore float64 `json:"health_score"`
ReplicaDegraded bool `json:"replica_degraded,omitempty"`
}
// ReplicaDetail describes one replica in the API response.
type ReplicaDetail struct {
Server string `json:"server"`
ISCSIAddr string `json:"iscsi_addr,omitempty"`
IQN string `json:"iqn,omitempty"`
HealthScore float64 `json:"health_score"`
WALLag uint64 `json:"wal_lag,omitempty"`
}
// AssignRequest is the request body for POST /block/assign.
type AssignRequest struct {
Name string `json:"name"`
Epoch uint64 `json:"epoch"`
Role string `json:"role"` // "primary" | "replica"
LeaseTTLMs uint64 `json:"lease_ttl_ms"` // lease TTL in milliseconds
}
// ServerInfo describes a block-capable volume server.
type ServerInfo struct {
Address string `json:"address"`
VolumeCount int `json:"volume_count"`
BlockCapable bool `json:"block_capable"`
}
// RoleFromString converts a role string to its uint32 wire value.
// Returns 0 (RoleNone) for unrecognized strings.
func RoleFromString(s string) uint32 {
switch strings.ToLower(s) {
case "primary":
return uint32(blockvol.RolePrimary)
case "replica":
return uint32(blockvol.RoleReplica)
case "stale":
return uint32(blockvol.RoleStale)
case "rebuilding":
return uint32(blockvol.RoleRebuilding)
case "draining":
return uint32(blockvol.RoleDraining)
default:
return uint32(blockvol.RoleNone)
}
}
weed/storage/blockvol/blockvol.go
+151 -46
View File
@@ -44,9 +44,9 @@ type BlockVol struct {
opsOutstanding atomic.Int64 // in-flight Read/Write/Trim/SyncCache ops
opsDrained chan struct{}
// Replication fields (Phase 4A CP2).
shipper *WALShipper
replRecv *ReplicaReceiver
// Replication fields (Phase 4A CP2, generalized to N replicas in CP8-2).
shipperGroup *ShipperGroup
replRecv *ReplicaReceiver
// Fencing fields (Phase 4A).
epoch atomic.Uint64 // current persisted epoch
@@ -60,6 +60,13 @@ type BlockVol struct {
assignMu sync.Mutex // serializes HandleAssignment calls
drainTimeout time.Duration // default 10s, for demote drain
// Health score and scrub (CP8-2).
healthScore *HealthScore
scrubber *Scrubber
// Observability (CP8-4).
Metrics *EngineMetrics
// Snapshot fields (Phase 5 CP5-2).
snapMu sync.RWMutex
snapshots map[uint32]*activeSnapshot
@@ -116,14 +123,16 @@ func CreateBlockVol(path string, opts CreateOptions, cfgs ...BlockVolConfig) (*B
dm := NewDirtyMap(cfg.DirtyMapShards)
wal := NewWALWriter(fd, sb.WALOffset, sb.WALSize, 0, 0)
v := &BlockVol{
fd: fd,
path: path,
super: sb,
config: cfg,
wal: wal,
dirtyMap: dm,
opsDrained: make(chan struct{}, 1),
snapshots: make(map[uint32]*activeSnapshot),
fd: fd,
path: path,
super: sb,
config: cfg,
wal: wal,
dirtyMap: dm,
healthScore: NewHealthScore(),
Metrics: NewEngineMetrics(),
opsDrained: make(chan struct{}, 1),
snapshots: make(map[uint32]*activeSnapshot),
}
v.nextLSN.Store(1)
v.healthy.Store(true)
@@ -134,6 +143,7 @@ func CreateBlockVol(path string, opts CreateOptions, cfgs ...BlockVolConfig) (*B
LowWatermark: cfg.GroupCommitLowWatermark,
OnDegraded: func() { v.healthy.Store(false) },
PostSyncCheck: v.writeGate,
Metrics: v.Metrics,
})
go v.groupCommit.Run()
v.flusher = NewFlusher(FlusherConfig{
@@ -142,6 +152,7 @@ func CreateBlockVol(path string, opts CreateOptions, cfgs ...BlockVolConfig) (*B
WAL: wal,
DirtyMap: dm,
Interval: cfg.FlushInterval,
Metrics: v.Metrics,
})
go v.flusher.Run()
return v, nil
@@ -189,13 +200,15 @@ func OpenBlockVol(path string, cfgs ...BlockVolConfig) (*BlockVol, error) {
wal := NewWALWriter(fd, sb.WALOffset, sb.WALSize, sb.WALHead, sb.WALTail)
v := &BlockVol{
fd: fd,
path: path,
super: sb,
config: cfg,
wal: wal,
dirtyMap: dirtyMap,
opsDrained: make(chan struct{}, 1),
fd: fd,
path: path,
super: sb,
config: cfg,
wal: wal,
dirtyMap: dirtyMap,
healthScore: NewHealthScore(),
Metrics: NewEngineMetrics(),
opsDrained: make(chan struct{}, 1),
}
v.nextLSN.Store(nextLSN)
v.epoch.Store(sb.Epoch)
@@ -207,6 +220,7 @@ func OpenBlockVol(path string, cfgs ...BlockVolConfig) (*BlockVol, error) {
LowWatermark: cfg.GroupCommitLowWatermark,
OnDegraded: func() { v.healthy.Store(false) },
PostSyncCheck: v.writeGate,
Metrics: v.Metrics,
})
go v.groupCommit.Run()
v.flusher = NewFlusher(FlusherConfig{
@@ -215,6 +229,7 @@ func OpenBlockVol(path string, cfgs ...BlockVolConfig) (*BlockVol, error) {
WAL: wal,
DirtyMap: dirtyMap,
Interval: cfg.FlushInterval,
Metrics: v.Metrics,
})
go v.flusher.Run()
@@ -292,9 +307,14 @@ func (v *BlockVol) appendWithRetry(entry *WALEntry) (uint64, error) {
return 0, ErrVolumeClosed
}
// Ship to replica if configured (fire-and-forget).
if v.shipper != nil {
v.shipper.Ship(entry)
// Ship to replicas if configured (fire-and-forget).
if v.shipperGroup != nil {
v.shipperGroup.ShipAll(entry)
}
// Notify scrubber of written LBAs for CRC invalidation.
if v.scrubber != nil && entry.Type == EntryTypeWrite {
v.scrubber.NotifyWrite(entry.LBA, entry.Length/v.super.BlockSize)
}
return walOff, nil
@@ -567,17 +587,33 @@ func (v *BlockVol) SyncCache() error {
return v.groupCommit.Submit()
}
// SetReplicaAddr configures the replica endpoint and creates a WAL shipper
// with distributed group commit. Must be called before any writes.
// ReplicaAddr holds the data and control addresses for one replica.
type ReplicaAddr struct {
DataAddr string
CtrlAddr string
}
// SetReplicaAddr configures a single replica endpoint. Backward-compatible wrapper
// around SetReplicaAddrs for RF=2 callers.
func (v *BlockVol) SetReplicaAddr(dataAddr, ctrlAddr string) {
v.shipper = NewWALShipper(dataAddr, ctrlAddr, func() uint64 {
return v.epoch.Load()
})
v.SetReplicaAddrs([]ReplicaAddr{{DataAddr: dataAddr, CtrlAddr: ctrlAddr}})
}
// SetReplicaAddrs configures N replica endpoints and creates a ShipperGroup
// with distributed group commit. Creates fresh shippers (old group is GC'd).
func (v *BlockVol) SetReplicaAddrs(addrs []ReplicaAddr) {
shippers := make([]*WALShipper, len(addrs))
for i, a := range addrs {
shippers[i] = NewWALShipper(a.DataAddr, a.CtrlAddr, func() uint64 {
return v.epoch.Load()
}, v.Metrics)
}
v.shipperGroup = NewShipperGroup(shippers)
// Replace the group committer's sync function with a distributed version.
v.groupCommit.Stop()
v.groupCommit = NewGroupCommitter(GroupCommitterConfig{
SyncFunc: MakeDistributedSync(v.fd.Sync, v.shipper, v),
SyncFunc: MakeDistributedSync(v.fd.Sync, v.shipperGroup, v),
MaxDelay: v.config.GroupCommitMaxDelay,
MaxBatch: v.config.GroupCommitMaxBatch,
LowWatermark: v.config.GroupCommitLowWatermark,
@@ -622,21 +658,21 @@ func (v *BlockVol) StopRebuildServer() {
}
}
// degradeReplica marks the shipper as degraded and logs a warning.
// degradeReplica logs a warning about a replica barrier failure.
// Individual shippers are already degraded by Barrier itself.
func (v *BlockVol) degradeReplica(err error) {
if v.shipper != nil {
v.shipper.degraded.Store(true)
}
log.Printf("blockvol: replica degraded: %v", err)
}
// BlockVolumeStatus contains block volume state for heartbeat reporting.
type BlockVolumeStatus struct {
Epoch uint64
WALHeadLSN uint64
Role Role
CheckpointLSN uint64
HasLease bool
Epoch uint64
WALHeadLSN uint64
Role Role
CheckpointLSN uint64
HasLease bool
HealthScore float64
ReplicaDegraded bool
}
// Status returns the current block volume status for heartbeat reporting.
@@ -645,13 +681,79 @@ func (v *BlockVol) Status() BlockVolumeStatus {
if v.flusher != nil {
cpLSN = v.flusher.CheckpointLSN()
}
return BlockVolumeStatus{
Epoch: v.epoch.Load(),
WALHeadLSN: v.nextLSN.Load() - 1,
Role: v.Role(),
CheckpointLSN: cpLSN,
HasLease: v.lease.IsValid(),
var hs float64 = 1.0
if v.healthScore != nil {
hs = v.healthScore.Score()
}
var degraded bool
if v.shipperGroup != nil {
degraded = v.shipperGroup.AnyDegraded()
}
return BlockVolumeStatus{
Epoch: v.epoch.Load(),
WALHeadLSN: v.nextLSN.Load() - 1,
Role: v.Role(),
CheckpointLSN: cpLSN,
HasLease: v.lease.IsValid(),
HealthScore: hs,
ReplicaDegraded: degraded,
}
}
// CheckpointLSN returns the last LSN flushed to the extent region.
func (v *BlockVol) CheckpointLSN() uint64 {
if v.flusher != nil {
return v.flusher.CheckpointLSN()
}
return 0
}
// HealthScore returns the current health score (0.0-1.0).
func (v *BlockVol) HealthScore() float64 {
if v.healthScore == nil {
return 1.0
}
return v.healthScore.Score()
}
// HealthStats returns detailed health statistics.
func (v *BlockVol) HealthStats() HealthStats {
if v.healthScore == nil {
return HealthStats{Score: 1.0}
}
return v.healthScore.Stats()
}
// StartScrub starts background scrubbing with the given interval.
func (v *BlockVol) StartScrub(interval time.Duration) {
if v.scrubber != nil {
return
}
v.scrubber = NewScrubber(v, interval)
v.scrubber.Start()
}
// StopScrub stops the background scrubber.
func (v *BlockVol) StopScrub() {
if v.scrubber != nil {
v.scrubber.Stop()
v.scrubber = nil
}
}
// TriggerScrub triggers an immediate scrub pass.
func (v *BlockVol) TriggerScrub() {
if v.scrubber != nil {
v.scrubber.TriggerNow()
}
}
// ScrubStats returns the current scrub statistics.
func (v *BlockVol) ScrubStats() ScrubStats {
if v.scrubber == nil {
return ScrubStats{}
}
return v.scrubber.Stats()
}
// CreateSnapshot creates a point-in-time snapshot. The snapshot captures the
@@ -945,13 +1047,16 @@ func (v *BlockVol) persistSuperblock() error {
}
// Close shuts down the block volume and closes the file.
// Shutdown order: shipper -> replica receiver -> rebuild server -> drain ops -> group committer -> flusher -> final flush -> close fd.
// Shutdown order: shippers -> replica receiver -> rebuild server -> drain ops -> group committer -> flusher -> final flush -> close fd.
func (v *BlockVol) Close() error {
v.closed.Store(true)
// Stop shipper first: no more Ship calls.
if v.shipper != nil {
v.shipper.Stop()
// Stop scrubber first (no more extent reads).
v.StopScrub()
// Stop shippers: no more Ship calls.
if v.shipperGroup != nil {
v.shipperGroup.StopAll()
}
// Stop replica receiver: no more barrier waits.
if v.replRecv != nil {
weed/storage/blockvol/blockvol_test.go
+13 -12
View File
@@ -2982,7 +2982,7 @@ func testDistCommitBothPass(t *testing.T) {
shipper := NewWALShipper(dataAddr, ctrlAddr, func() uint64 { return 0 })
defer shipper.Stop()
distSync := MakeDistributedSync(walSync, shipper, v)
distSync := MakeDistributedSync(walSync, NewShipperGroup([]*WALShipper{shipper}), v)
if err := distSync(); err != nil {
t.Fatalf("distSync: %v", err)
}
@@ -3005,7 +3005,7 @@ func testDistCommitLocalFail(t *testing.T) {
shipper := NewWALShipper(dataAddr, ctrlAddr, func() uint64 { return 0 })
defer shipper.Stop()
distSync := MakeDistributedSync(walSync, shipper, v)
distSync := MakeDistributedSync(walSync, NewShipperGroup([]*WALShipper{shipper}), v)
err := distSync()
if err == nil {
t.Fatal("expected error from local sync failure")
@@ -3025,10 +3025,10 @@ func testDistCommitRemoteFailDegrades(t *testing.T) {
shipper := NewWALShipper(dataAddr, ctrlAddr, func() uint64 { return 0 })
defer shipper.Stop()
// Set shipper on vol so degradeReplica can mark it degraded.
v.shipper = shipper
// Set shipper group on vol so degradeReplica can see it.
v.shipperGroup = NewShipperGroup([]*WALShipper{shipper})
distSync := MakeDistributedSync(walSync, shipper, v)
distSync := MakeDistributedSync(walSync, v.shipperGroup, v)
// Should NOT return error (local succeeded, remote degraded).
if err := distSync(); err != nil {
t.Fatalf("distSync should not fail on remote error: %v", err)
@@ -3054,7 +3054,7 @@ func testDistCommitNoReplica(t *testing.T) {
v := createTestVol(t)
defer v.Close()
// nil shipper --local-only mode.
// nil group --local-only mode.
distSync := MakeDistributedSync(walSync, nil, v)
if err := distSync(); err != nil {
t.Fatalf("distSync: %v", err)
@@ -3133,9 +3133,9 @@ func testBlockvolNoReplicaCompat(t *testing.T) {
t.Error("data mismatch in no-replica mode")
}
// Verify shipper is nil.
if v.shipper != nil {
t.Error("shipper should be nil without SetReplicaAddr")
// Verify shipperGroup is nil.
if v.shipperGroup != nil {
t.Error("shipperGroup should be nil without SetReplicaAddr")
}
}
@@ -3541,15 +3541,16 @@ func testDemoteStopsShipper(t *testing.T) {
v := setupPrimary(t)
defer v.Close()
// Create a shipper (won't connect but that's fine for this test).
v.shipper = NewWALShipper("127.0.0.1:0", "127.0.0.1:0", func() uint64 {
// Create a shipper group (won't connect but that's fine for this test).
shipper := NewWALShipper("127.0.0.1:0", "127.0.0.1:0", func() uint64 {
return v.epoch.Load()
})
v.shipperGroup = NewShipperGroup([]*WALShipper{shipper})
if err := v.HandleAssignment(2, RoleStale, 0); err != nil {
t.Fatalf("demote: %v", err)
}
if !v.shipper.stopped.Load() {
if !shipper.stopped.Load() {
t.Error("shipper should be stopped after demotion")
}
}
weed/storage/blockvol/csi/controller.go
+199 -18
View File
@@ -3,10 +3,14 @@ package csi
import (
"context"
"errors"
"fmt"
"strings"
"time"
"github.com/container-storage-interface/spec/lib/go/csi"
"google.golang.org/grpc/codes"
"google.golang.org/grpc/status"
"google.golang.org/protobuf/types/known/timestamppb"
)
const (
@@ -32,7 +36,7 @@ func (s *controllerServer) CreateVolume(_ context.Context, req *csi.CreateVolume
if req.CapacityRange != nil && req.CapacityRange.RequiredBytes > 0 {
sizeBytes = req.CapacityRange.RequiredBytes
} else if req.CapacityRange != nil && req.CapacityRange.LimitBytes > 0 {
// No RequiredBytes set — use LimitBytes as the target size.
// No RequiredBytes set -- use LimitBytes as the target size.
sizeBytes = req.CapacityRange.LimitBytes
}
if req.CapacityRange != nil && req.CapacityRange.LimitBytes > 0 {
@@ -126,27 +130,201 @@ func (s *controllerServer) ControllerUnpublishVolume(_ context.Context, req *csi
return &csi.ControllerUnpublishVolumeResponse{}, nil
}
func (s *controllerServer) ControllerGetCapabilities(_ context.Context, _ *csi.ControllerGetCapabilitiesRequest) (*csi.ControllerGetCapabilitiesResponse, error) {
return &csi.ControllerGetCapabilitiesResponse{
Capabilities: []*csi.ControllerServiceCapability{
{
Type: &csi.ControllerServiceCapability_Rpc{
Rpc: &csi.ControllerServiceCapability_RPC{
Type: csi.ControllerServiceCapability_RPC_CREATE_DELETE_VOLUME,
},
},
},
{
Type: &csi.ControllerServiceCapability_Rpc{
Rpc: &csi.ControllerServiceCapability_RPC{
Type: csi.ControllerServiceCapability_RPC_PUBLISH_UNPUBLISH_VOLUME,
},
},
},
func (s *controllerServer) CreateSnapshot(_ context.Context, req *csi.CreateSnapshotRequest) (*csi.CreateSnapshotResponse, error) {
if req.SourceVolumeId == "" {
return nil, status.Error(codes.InvalidArgument, "source volume ID is required")
}
if req.Name == "" {
return nil, status.Error(codes.InvalidArgument, "snapshot name is required")
}
snapID := snapshotNameToID(req.Name)
info, err := s.backend.CreateSnapshot(context.Background(), req.SourceVolumeId, snapID)
if err != nil {
errMsg := err.Error()
// CSI idempotency: if the snapshot already exists on the same volume,
// return it as success. Only return AlreadyExists for true collisions
// (different source volume using the same FNV hash).
if strings.Contains(errMsg, "snapshot exists") || strings.Contains(errMsg, "already exists") {
snaps, lerr := s.backend.ListSnapshots(context.Background(), req.SourceVolumeId)
if lerr == nil {
for _, snap := range snaps {
if snap.SnapshotID == snapID {
// Same name retried on same volume -- idempotent success.
csiSnapID := FormatSnapshotID(req.SourceVolumeId, snapID)
return &csi.CreateSnapshotResponse{
Snapshot: &csi.Snapshot{
SnapshotId: csiSnapID,
SourceVolumeId: req.SourceVolumeId,
CreationTime: timestamppb.New(time.Unix(snap.CreatedAt, 0)),
ReadyToUse: true,
SizeBytes: int64(snap.SizeBytes),
},
}, nil
}
}
}
// Snapshot exists but not on this volume -- true FNV collision.
return nil, status.Errorf(codes.AlreadyExists,
"snapshot ID collision: FNV hash %d already used by a different snapshot", snapID)
}
if strings.Contains(errMsg, "not found") {
return nil, status.Errorf(codes.NotFound, "volume %q not found", req.SourceVolumeId)
}
return nil, status.Errorf(codes.Internal, "create snapshot: %v", err)
}
csiSnapID := FormatSnapshotID(req.SourceVolumeId, snapID)
return &csi.CreateSnapshotResponse{
Snapshot: &csi.Snapshot{
SnapshotId: csiSnapID,
SourceVolumeId: req.SourceVolumeId,
CreationTime: timestamppb.New(time.Unix(info.CreatedAt, 0)),
ReadyToUse: true,
SizeBytes: int64(info.SizeBytes),
},
}, nil
}
func (s *controllerServer) DeleteSnapshot(_ context.Context, req *csi.DeleteSnapshotRequest) (*csi.DeleteSnapshotResponse, error) {
if req.SnapshotId == "" {
return nil, status.Error(codes.InvalidArgument, "snapshot ID is required")
}
volumeID, snapID, err := ParseSnapshotID(req.SnapshotId)
if err != nil {
// Can't parse → treat as already deleted (idempotent).
return &csi.DeleteSnapshotResponse{}, nil
}
if berr := s.backend.DeleteSnapshot(context.Background(), volumeID, snapID); berr != nil {
// Volume not found → snapshot doesn't exist → success (idempotent).
if strings.Contains(berr.Error(), "not found") {
return &csi.DeleteSnapshotResponse{}, nil
}
return nil, status.Errorf(codes.Internal, "delete snapshot: %v", berr)
}
return &csi.DeleteSnapshotResponse{}, nil
}
func (s *controllerServer) ListSnapshots(_ context.Context, req *csi.ListSnapshotsRequest) (*csi.ListSnapshotsResponse, error) {
// If filtering by snapshot_id, parse it to get the volume.
if req.SnapshotId != "" {
volumeID, snapID, err := ParseSnapshotID(req.SnapshotId)
if err != nil {
return &csi.ListSnapshotsResponse{}, nil
}
snaps, lerr := s.backend.ListSnapshots(context.Background(), volumeID)
if lerr != nil {
if strings.Contains(lerr.Error(), "not found") {
return &csi.ListSnapshotsResponse{}, nil
}
return nil, status.Errorf(codes.Internal, "list snapshots: %v", lerr)
}
for _, snap := range snaps {
if snap.SnapshotID == snapID {
return &csi.ListSnapshotsResponse{
Entries: []*csi.ListSnapshotsResponse_Entry{{
Snapshot: &csi.Snapshot{
SnapshotId: req.SnapshotId,
SourceVolumeId: volumeID,
SizeBytes: int64(snap.SizeBytes),
ReadyToUse: true,
},
}},
}, nil
}
}
return &csi.ListSnapshotsResponse{}, nil
}
// If filtering by source_volume_id, list snapshots for that volume.
if req.SourceVolumeId != "" {
snaps, err := s.backend.ListSnapshots(context.Background(), req.SourceVolumeId)
if err != nil {
if strings.Contains(err.Error(), "not found") {
return &csi.ListSnapshotsResponse{}, nil
}
return nil, status.Errorf(codes.Internal, "list snapshots: %v", err)
}
resp := &csi.ListSnapshotsResponse{}
for _, snap := range snaps {
resp.Entries = append(resp.Entries, &csi.ListSnapshotsResponse_Entry{
Snapshot: &csi.Snapshot{
SnapshotId: FormatSnapshotID(req.SourceVolumeId, snap.SnapshotID),
SourceVolumeId: req.SourceVolumeId,
SizeBytes: int64(snap.SizeBytes),
ReadyToUse: true,
},
})
}
return resp, nil
}
// No filter: not supported for distributed mode (would need to enumerate all volumes).
return &csi.ListSnapshotsResponse{}, nil
}
func (s *controllerServer) ControllerExpandVolume(_ context.Context, req *csi.ControllerExpandVolumeRequest) (*csi.ControllerExpandVolumeResponse, error) {
if req.VolumeId == "" {
return nil, status.Error(codes.InvalidArgument, "volume ID is required")
}
if req.CapacityRange == nil || req.CapacityRange.RequiredBytes <= 0 {
return nil, status.Error(codes.InvalidArgument, "capacity_range.required_bytes must be > 0")
}
newSize := uint64(req.CapacityRange.RequiredBytes)
// Round up to block size.
if newSize%blockSize != 0 {
newSize = (newSize/blockSize + 1) * blockSize
}
capacity, err := s.backend.ExpandVolume(context.Background(), req.VolumeId, newSize)
if err != nil {
errMsg := err.Error()
if strings.Contains(errMsg, "shrink") {
return nil, status.Errorf(codes.InvalidArgument, "cannot shrink volume")
}
if strings.Contains(errMsg, "snapshots") {
return nil, status.Errorf(codes.FailedPrecondition,
"cannot resize with active snapshots: delete snapshots first")
}
if strings.Contains(errMsg, "not found") {
return nil, status.Errorf(codes.NotFound, "volume %q not found", req.VolumeId)
}
return nil, status.Errorf(codes.Internal, "expand volume: %v", err)
}
return &csi.ControllerExpandVolumeResponse{
CapacityBytes: int64(capacity),
NodeExpansionRequired: true,
}, nil
}
func (s *controllerServer) ControllerGetCapabilities(_ context.Context, _ *csi.ControllerGetCapabilitiesRequest) (*csi.ControllerGetCapabilitiesResponse, error) {
caps := []csi.ControllerServiceCapability_RPC_Type{
csi.ControllerServiceCapability_RPC_CREATE_DELETE_VOLUME,
csi.ControllerServiceCapability_RPC_PUBLISH_UNPUBLISH_VOLUME,
csi.ControllerServiceCapability_RPC_CREATE_DELETE_SNAPSHOT,
csi.ControllerServiceCapability_RPC_LIST_SNAPSHOTS,
csi.ControllerServiceCapability_RPC_EXPAND_VOLUME,
}
var result []*csi.ControllerServiceCapability
for _, c := range caps {
result = append(result, &csi.ControllerServiceCapability{
Type: &csi.ControllerServiceCapability_Rpc{
Rpc: &csi.ControllerServiceCapability_RPC{Type: c},
},
})
}
return &csi.ControllerGetCapabilitiesResponse{Capabilities: result}, nil
}
func (s *controllerServer) ValidateVolumeCapabilities(_ context.Context, req *csi.ValidateVolumeCapabilitiesRequest) (*csi.ValidateVolumeCapabilitiesResponse, error) {
if req.VolumeId == "" {
return nil, status.Error(codes.InvalidArgument, "volume ID is required")
@@ -164,3 +342,6 @@ func (s *controllerServer) ValidateVolumeCapabilities(_ context.Context, req *cs
},
}, nil
}
// errSnapshotsPreventResize is used by the mock backend for testing.
var errSnapshotsPreventResize = fmt.Errorf("blockvol: cannot resize with active snapshots")
weed/storage/blockvol/csi/controller_test.go
+216 -15
View File
@@ -220,7 +220,7 @@ func TestControllerUnpublish_Success(t *testing.T) {
}
}
func TestController_Capabilities_IncludesPublish(t *testing.T) {
func TestController_Capabilities(t *testing.T) {
mgr := newTestManager(t)
backend := NewLocalVolumeBackend(mgr)
cs := &controllerServer{backend: backend}
@@ -230,24 +230,225 @@ func TestController_Capabilities_IncludesPublish(t *testing.T) {
t.Fatalf("ControllerGetCapabilities: %v", err)
}
hasCreate := false
hasPublish := false
want := map[csi.ControllerServiceCapability_RPC_Type]bool{
csi.ControllerServiceCapability_RPC_CREATE_DELETE_VOLUME: true,
csi.ControllerServiceCapability_RPC_PUBLISH_UNPUBLISH_VOLUME: true,
csi.ControllerServiceCapability_RPC_CREATE_DELETE_SNAPSHOT: true,
csi.ControllerServiceCapability_RPC_LIST_SNAPSHOTS: true,
csi.ControllerServiceCapability_RPC_EXPAND_VOLUME: true,
}
got := make(map[csi.ControllerServiceCapability_RPC_Type]bool)
for _, cap := range resp.Capabilities {
rpc := cap.GetRpc()
if rpc == nil {
continue
}
switch rpc.Type {
case csi.ControllerServiceCapability_RPC_CREATE_DELETE_VOLUME:
hasCreate = true
case csi.ControllerServiceCapability_RPC_PUBLISH_UNPUBLISH_VOLUME:
hasPublish = true
if rpc != nil {
got[rpc.Type] = true
}
}
if !hasCreate {
t.Fatal("expected CREATE_DELETE_VOLUME capability")
for capType := range want {
if !got[capType] {
t.Fatalf("missing capability: %v", capType)
}
}
if !hasPublish {
t.Fatal("expected PUBLISH_UNPUBLISH_VOLUME capability")
}
func TestController_CreateSnapshot(t *testing.T) {
mgr := newTestManager(t)
backend := NewLocalVolumeBackend(mgr)
cs := &controllerServer{backend: backend}
// Create volume first.
cs.CreateVolume(context.Background(), &csi.CreateVolumeRequest{
Name: "snap-vol",
VolumeCapabilities: testVolCaps(),
CapacityRange: &csi.CapacityRange{RequiredBytes: 4 * 1024 * 1024},
})
resp, err := cs.CreateSnapshot(context.Background(), &csi.CreateSnapshotRequest{
Name: "my-snapshot",
SourceVolumeId: "snap-vol",
})
if err != nil {
t.Fatalf("CreateSnapshot: %v", err)
}
if resp.Snapshot.SnapshotId == "" {
t.Fatal("snapshot ID should not be empty")
}
if resp.Snapshot.SourceVolumeId != "snap-vol" {
t.Fatalf("SourceVolumeId: got %q, want snap-vol", resp.Snapshot.SourceVolumeId)
}
if !resp.Snapshot.ReadyToUse {
t.Fatal("snapshot should be ready")
}
}
func TestController_CreateSnapshotIdempotent(t *testing.T) {
mgr := newTestManager(t)
backend := NewLocalVolumeBackend(mgr)
cs := &controllerServer{backend: backend}
cs.CreateVolume(context.Background(), &csi.CreateVolumeRequest{
Name: "snap-vol",
VolumeCapabilities: testVolCaps(),
CapacityRange: &csi.CapacityRange{RequiredBytes: 4 * 1024 * 1024},
})
// First create.
resp1, err := cs.CreateSnapshot(context.Background(), &csi.CreateSnapshotRequest{
Name: "my-snapshot",
SourceVolumeId: "snap-vol",
})
if err != nil {
t.Fatalf("first CreateSnapshot: %v", err)
}
// Same name on same volume -- CSI idempotency: should return existing snapshot.
resp2, err := cs.CreateSnapshot(context.Background(), &csi.CreateSnapshotRequest{
Name: "my-snapshot",
SourceVolumeId: "snap-vol",
})
if err != nil {
t.Fatalf("idempotent CreateSnapshot: %v", err)
}
if resp2.Snapshot.SnapshotId != resp1.Snapshot.SnapshotId {
t.Fatalf("snapshot IDs differ: %q vs %q", resp2.Snapshot.SnapshotId, resp1.Snapshot.SnapshotId)
}
if resp2.Snapshot.SourceVolumeId != "snap-vol" {
t.Fatalf("source volume: got %q", resp2.Snapshot.SourceVolumeId)
}
}
func TestController_DeleteSnapshot(t *testing.T) {
mgr := newTestManager(t)
backend := NewLocalVolumeBackend(mgr)
cs := &controllerServer{backend: backend}
cs.CreateVolume(context.Background(), &csi.CreateVolumeRequest{
Name: "snap-vol",
VolumeCapabilities: testVolCaps(),
CapacityRange: &csi.CapacityRange{RequiredBytes: 4 * 1024 * 1024},
})
resp, _ := cs.CreateSnapshot(context.Background(), &csi.CreateSnapshotRequest{
Name: "del-snap",
SourceVolumeId: "snap-vol",
})
_, err := cs.DeleteSnapshot(context.Background(), &csi.DeleteSnapshotRequest{
SnapshotId: resp.Snapshot.SnapshotId,
})
if err != nil {
t.Fatalf("DeleteSnapshot: %v", err)
}
}
func TestController_DeleteSnapshotNotFound(t *testing.T) {
mgr := newTestManager(t)
backend := NewLocalVolumeBackend(mgr)
cs := &controllerServer{backend: backend}
// Delete non-existent snapshot should succeed (idempotent).
_, err := cs.DeleteSnapshot(context.Background(), &csi.DeleteSnapshotRequest{
SnapshotId: "snap-nonexistent-vol-42",
})
if err != nil {
t.Fatalf("DeleteSnapshot non-existent: %v", err)
}
}
func TestController_ListSnapshotsByVolume(t *testing.T) {
mgr := newTestManager(t)
backend := NewLocalVolumeBackend(mgr)
cs := &controllerServer{backend: backend}
cs.CreateVolume(context.Background(), &csi.CreateVolumeRequest{
Name: "list-vol",
VolumeCapabilities: testVolCaps(),
CapacityRange: &csi.CapacityRange{RequiredBytes: 4 * 1024 * 1024},
})
cs.CreateSnapshot(context.Background(), &csi.CreateSnapshotRequest{
Name: "snap-a", SourceVolumeId: "list-vol",
})
cs.CreateSnapshot(context.Background(), &csi.CreateSnapshotRequest{
Name: "snap-b", SourceVolumeId: "list-vol",
})
resp, err := cs.ListSnapshots(context.Background(), &csi.ListSnapshotsRequest{
SourceVolumeId: "list-vol",
})
if err != nil {
t.Fatalf("ListSnapshots: %v", err)
}
if len(resp.Entries) != 2 {
t.Fatalf("expected 2 snapshots, got %d", len(resp.Entries))
}
}
func TestController_ExpandVolume(t *testing.T) {
mgr := newTestManager(t)
backend := NewLocalVolumeBackend(mgr)
cs := &controllerServer{backend: backend}
cs.CreateVolume(context.Background(), &csi.CreateVolumeRequest{
Name: "expand-vol",
VolumeCapabilities: testVolCaps(),
CapacityRange: &csi.CapacityRange{RequiredBytes: 4 * 1024 * 1024},
})
resp, err := cs.ControllerExpandVolume(context.Background(), &csi.ControllerExpandVolumeRequest{
VolumeId: "expand-vol",
CapacityRange: &csi.CapacityRange{RequiredBytes: 8 * 1024 * 1024},
})
if err != nil {
t.Fatalf("ControllerExpandVolume: %v", err)
}
if resp.CapacityBytes != 8*1024*1024 {
t.Fatalf("CapacityBytes: got %d, want %d", resp.CapacityBytes, 8*1024*1024)
}
if !resp.NodeExpansionRequired {
t.Fatal("NodeExpansionRequired should be true")
}
}
func TestController_ExpandNoShrink(t *testing.T) {
mgr := newTestManager(t)
backend := NewLocalVolumeBackend(mgr)
cs := &controllerServer{backend: backend}
cs.CreateVolume(context.Background(), &csi.CreateVolumeRequest{
Name: "noshrink-vol",
VolumeCapabilities: testVolCaps(),
CapacityRange: &csi.CapacityRange{RequiredBytes: 8 * 1024 * 1024},
})
_, err := cs.ControllerExpandVolume(context.Background(), &csi.ControllerExpandVolumeRequest{
VolumeId: "noshrink-vol",
CapacityRange: &csi.CapacityRange{RequiredBytes: 4 * 1024 * 1024},
})
if err == nil {
t.Fatal("expected error for shrink")
}
st, _ := status.FromError(err)
if st.Code() != codes.InvalidArgument {
t.Fatalf("expected InvalidArgument, got %v", st.Code())
}
}
func TestController_ExpandMissingVolumeID(t *testing.T) {
mgr := newTestManager(t)
backend := NewLocalVolumeBackend(mgr)
cs := &controllerServer{backend: backend}
_, err := cs.ControllerExpandVolume(context.Background(), &csi.ControllerExpandVolumeRequest{
CapacityRange: &csi.CapacityRange{RequiredBytes: 8 * 1024 * 1024},
})
if err == nil {
t.Fatal("expected error for missing volume ID")
}
st, _ := status.FromError(err)
if st.Code() != codes.InvalidArgument {
t.Fatalf("expected InvalidArgument, got %v", st.Code())
}
}
weed/storage/blockvol/csi/iscsi_util.go
+43
View File
@@ -16,6 +16,7 @@ type ISCSIUtil interface {
Logout(ctx context.Context, iqn string) error
GetDeviceByIQN(ctx context.Context, iqn string) (string, error)
IsLoggedIn(ctx context.Context, iqn string) (bool, error)
RescanDevice(ctx context.Context, iqn string) error
}
// MountUtil provides filesystem mount operations.
@@ -26,6 +27,7 @@ type MountUtil interface {
Unmount(ctx context.Context, target string) error
IsFormatted(ctx context.Context, device string) (bool, error)
IsMounted(ctx context.Context, target string) (bool, error)
ResizeFS(ctx context.Context, devicePath, mountPath, fsType string) error
}
// realISCSIUtil uses iscsiadm CLI.
@@ -115,6 +117,15 @@ func (r *realISCSIUtil) IsLoggedIn(ctx context.Context, iqn string) (bool, error
return strings.Contains(string(out), iqn), nil
}
func (r *realISCSIUtil) RescanDevice(ctx context.Context, iqn string) error {
cmd := exec.CommandContext(ctx, "iscsiadm", "-m", "node", "-T", iqn, "--rescan")
out, err := cmd.CombinedOutput()
if err != nil {
return fmt.Errorf("iscsiadm rescan: %s: %w", string(out), err)
}
return nil
}
// realMountUtil uses mount/umount/mkfs CLI.
type realMountUtil struct{}
@@ -200,11 +211,32 @@ func (r *realMountUtil) IsMounted(ctx context.Context, target string) (bool, err
return false, nil
}
func (r *realMountUtil) ResizeFS(ctx context.Context, devicePath, mountPath, fsType string) error {
switch fsType {
case "ext4", "ext3", "ext2":
cmd := exec.CommandContext(ctx, "resize2fs", devicePath)
out, err := cmd.CombinedOutput()
if err != nil {
return fmt.Errorf("resize2fs: %s: %w", string(out), err)
}
case "xfs":
cmd := exec.CommandContext(ctx, "xfs_growfs", mountPath)
out, err := cmd.CombinedOutput()
if err != nil {
return fmt.Errorf("xfs_growfs: %s: %w", string(out), err)
}
default:
return fmt.Errorf("unsupported filesystem type for resize: %s", fsType)
}
return nil
}
// mockISCSIUtil is a test double for ISCSIUtil.
type mockISCSIUtil struct {
discoveryErr error
loginErr error
logoutErr error
rescanErr error
getDeviceResult string
getDeviceErr error
loggedIn map[string]bool
@@ -247,11 +279,17 @@ func (m *mockISCSIUtil) IsLoggedIn(_ context.Context, iqn string) (bool, error)
return m.loggedIn[iqn], nil
}
func (m *mockISCSIUtil) RescanDevice(_ context.Context, iqn string) error {
m.calls = append(m.calls, "rescan:"+iqn)
return m.rescanErr
}
// mockMountUtil is a test double for MountUtil.
type mockMountUtil struct {
formatAndMountErr error
mountErr error
unmountErr error
resizeFSErr error
isFormattedResult bool
isMountedTargets map[string]bool
calls []string
@@ -304,3 +342,8 @@ func (m *mockMountUtil) IsFormatted(_ context.Context, device string) (bool, err
func (m *mockMountUtil) IsMounted(_ context.Context, target string) (bool, error) {
return m.isMountedTargets[target], nil
}
func (m *mockMountUtil) ResizeFS(_ context.Context, devicePath, mountPath, fsType string) error {
m.calls = append(m.calls, "resizefs:"+devicePath+":"+mountPath+":"+fsType)
return m.resizeFSErr
}
weed/storage/blockvol/csi/node.go
+84 -18
View File
@@ -13,11 +13,13 @@ import (
"google.golang.org/grpc/status"
)
// stagedVolumeInfo tracks info needed for NodeUnstageVolume.
// stagedVolumeInfo tracks info needed for NodeUnstageVolume and NodeExpandVolume.
type stagedVolumeInfo struct {
iqn string
iscsiAddr string
isLocal bool // true if volume is served by local VolumeManager
iqn string
iscsiAddr string
isLocal bool // true if volume is served by local VolumeManager
fsType string // filesystem type (ext4, xfs, etc.)
stagingPath string // staging mount path
}
type nodeServer struct {
@@ -140,9 +142,11 @@ func (s *nodeServer) NodeStageVolume(ctx context.Context, req *csi.NodeStageVolu
s.staged = make(map[string]*stagedVolumeInfo)
}
s.staged[volumeID] = &stagedVolumeInfo{
iqn: iqn,
iscsiAddr: portal,
isLocal: isLocal,
iqn: iqn,
iscsiAddr: portal,
isLocal: isLocal,
fsType: fsType,
stagingPath: stagingPath,
}
s.stagedMu.Unlock()
@@ -293,20 +297,82 @@ func (s *nodeServer) NodeUnpublishVolume(ctx context.Context, req *csi.NodeUnpub
return &csi.NodeUnpublishVolumeResponse{}, nil
}
func (s *nodeServer) NodeGetCapabilities(_ context.Context, _ *csi.NodeGetCapabilitiesRequest) (*csi.NodeGetCapabilitiesResponse, error) {
return &csi.NodeGetCapabilitiesResponse{
Capabilities: []*csi.NodeServiceCapability{
{
Type: &csi.NodeServiceCapability_Rpc{
Rpc: &csi.NodeServiceCapability_RPC{
Type: csi.NodeServiceCapability_RPC_STAGE_UNSTAGE_VOLUME,
},
},
},
},
func (s *nodeServer) NodeExpandVolume(ctx context.Context, req *csi.NodeExpandVolumeRequest) (*csi.NodeExpandVolumeResponse, error) {
if req.VolumeId == "" {
return nil, status.Error(codes.InvalidArgument, "volume ID is required")
}
// Look up staged volume info.
s.stagedMu.Lock()
info, ok := s.staged[req.VolumeId]
s.stagedMu.Unlock()
if !ok {
return nil, status.Errorf(codes.FailedPrecondition, "volume %q not staged", req.VolumeId)
}
// Check that iSCSI session is active.
loggedIn, err := s.iscsiUtil.IsLoggedIn(ctx, info.iqn)
if err != nil {
return nil, status.Errorf(codes.Internal, "check iSCSI session: %v", err)
}
if !loggedIn {
return nil, status.Errorf(codes.FailedPrecondition, "volume %q not staged: iSCSI session not active", req.VolumeId)
}
// Rescan device to pick up new size.
if err := s.iscsiUtil.RescanDevice(ctx, info.iqn); err != nil {
return nil, status.Errorf(codes.Internal, "rescan iSCSI device: %v", err)
}
// Find the device path.
device, err := s.iscsiUtil.GetDeviceByIQN(ctx, info.iqn)
if err != nil {
return nil, status.Errorf(codes.Internal, "find device: %v", err)
}
// Determine mount path and fsType.
mountPath := info.stagingPath
if req.VolumePath != "" {
mountPath = req.VolumePath
}
fsType := info.fsType
if fsType == "" {
fsType = "ext4"
}
// Resize the filesystem.
if err := s.mountUtil.ResizeFS(ctx, device, mountPath, fsType); err != nil {
return nil, status.Errorf(codes.Internal, "resize filesystem: %v", err)
}
s.logger.Printf("NodeExpandVolume: %s expanded (device=%s, fs=%s)", req.VolumeId, device, fsType)
capacity := int64(0)
if req.CapacityRange != nil {
capacity = req.CapacityRange.RequiredBytes
}
return &csi.NodeExpandVolumeResponse{
CapacityBytes: capacity,
}, nil
}
func (s *nodeServer) NodeGetCapabilities(_ context.Context, _ *csi.NodeGetCapabilitiesRequest) (*csi.NodeGetCapabilitiesResponse, error) {
caps := []csi.NodeServiceCapability_RPC_Type{
csi.NodeServiceCapability_RPC_STAGE_UNSTAGE_VOLUME,
csi.NodeServiceCapability_RPC_EXPAND_VOLUME,
}
var result []*csi.NodeServiceCapability
for _, c := range caps {
result = append(result, &csi.NodeServiceCapability{
Type: &csi.NodeServiceCapability_Rpc{
Rpc: &csi.NodeServiceCapability_RPC{Type: c},
},
})
}
return &csi.NodeGetCapabilitiesResponse{Capabilities: result}, nil
}
func (s *nodeServer) NodeGetInfo(_ context.Context, _ *csi.NodeGetInfoRequest) (*csi.NodeGetInfoResponse, error) {
return &csi.NodeGetInfoResponse{
NodeId: s.nodeID,
weed/storage/blockvol/csi/node_test.go
+234
View File
@@ -5,6 +5,7 @@ import (
"errors"
"log"
"os"
"strings"
"testing"
"github.com/container-storage-interface/spec/lib/go/csi"
@@ -511,6 +512,239 @@ func TestNode_UnstageRetryKeepsStagedEntry(t *testing.T) {
}
}
// TestNode_ExpandVolume verifies happy path for NodeExpandVolume.
func TestNode_ExpandVolume(t *testing.T) {
ns, mi, mm := newTestNodeServer(t)
stagingPath := t.TempDir()
// Stage the volume first.
_, err := ns.NodeStageVolume(context.Background(), &csi.NodeStageVolumeRequest{
VolumeId: "test-vol",
StagingTargetPath: stagingPath,
VolumeCapability: testVolCap(),
})
if err != nil {
t.Fatalf("NodeStageVolume: %v", err)
}
// Reset calls to track expand-specific calls.
mi.calls = nil
mm.calls = nil
_, err = ns.NodeExpandVolume(context.Background(), &csi.NodeExpandVolumeRequest{
VolumeId: "test-vol",
CapacityRange: &csi.CapacityRange{RequiredBytes: 8 * 1024 * 1024},
})
if err != nil {
t.Fatalf("NodeExpandVolume: %v", err)
}
// Verify rescan was called.
foundRescan := false
for _, c := range mi.calls {
if strings.HasPrefix(c, "rescan:") {
foundRescan = true
}
}
if !foundRescan {
t.Fatalf("expected rescan call, got: %v", mi.calls)
}
// Verify resize was called.
foundResize := false
for _, c := range mm.calls {
if strings.HasPrefix(c, "resizefs:") {
foundResize = true
}
}
if !foundResize {
t.Fatalf("expected resizefs call, got: %v", mm.calls)
}
}
// TestNode_ExpandNotStaged verifies error when volume is not staged.
func TestNode_ExpandNotStaged(t *testing.T) {
ns, _, _ := newTestNodeServer(t)
_, err := ns.NodeExpandVolume(context.Background(), &csi.NodeExpandVolumeRequest{
VolumeId: "not-staged",
CapacityRange: &csi.CapacityRange{RequiredBytes: 8 * 1024 * 1024},
})
if err == nil {
t.Fatal("expected error for not-staged volume")
}
st, _ := status.FromError(err)
if st.Code() != codes.FailedPrecondition {
t.Fatalf("expected FailedPrecondition, got %v", st.Code())
}
}
// TestNode_ExpandMissingVolumeID verifies error for missing volume ID.
func TestNode_ExpandMissingVolumeID(t *testing.T) {
ns, _, _ := newTestNodeServer(t)
_, err := ns.NodeExpandVolume(context.Background(), &csi.NodeExpandVolumeRequest{})
if err == nil {
t.Fatal("expected error for missing volume ID")
}
st, _ := status.FromError(err)
if st.Code() != codes.InvalidArgument {
t.Fatalf("expected InvalidArgument, got %v", st.Code())
}
}
// TestNode_ExpandXFSUsesMountPath verifies xfs_growfs uses mount path.
func TestNode_ExpandXFSUsesMountPath(t *testing.T) {
mi := newMockISCSIUtil()
mi.getDeviceResult = "/dev/sdb"
mm := newMockMountUtil()
ns := &nodeServer{
mgr: nil,
nodeID: "test-node-1",
iqnPrefix: "iqn.2024.com.seaweedfs",
iscsiUtil: mi,
mountUtil: mm,
logger: log.New(os.Stderr, "[test-node] ", log.LstdFlags),
staged: map[string]*stagedVolumeInfo{
"xfs-vol": {
iqn: "iqn.2024.com.seaweedfs:xfs-vol",
iscsiAddr: "10.0.0.5:3260",
isLocal: false,
fsType: "xfs",
stagingPath: "/staging/xfs-vol",
},
},
}
// Mark as logged in so the check passes.
mi.loggedIn["iqn.2024.com.seaweedfs:xfs-vol"] = true
_, err := ns.NodeExpandVolume(context.Background(), &csi.NodeExpandVolumeRequest{
VolumeId: "xfs-vol",
CapacityRange: &csi.CapacityRange{RequiredBytes: 16 * 1024 * 1024},
})
if err != nil {
t.Fatalf("NodeExpandVolume: %v", err)
}
// Verify resizefs was called with xfs type.
foundXFS := false
for _, c := range mm.calls {
if strings.Contains(c, "resizefs:") && strings.Contains(c, ":xfs") {
foundXFS = true
// Verify mount path is used (not device path).
if !strings.Contains(c, "/staging/xfs-vol") {
t.Fatalf("xfs_growfs should use mount path, got: %s", c)
}
}
}
if !foundXFS {
t.Fatalf("expected resizefs call with xfs, got: %v", mm.calls)
}
}
// TestNode_ExpandRejectsNoISCSISession verifies error when iSCSI not logged in.
func TestNode_ExpandRejectsNoISCSISession(t *testing.T) {
mi := newMockISCSIUtil()
mm := newMockMountUtil()
ns := &nodeServer{
mgr: nil,
nodeID: "test-node-1",
iqnPrefix: "iqn.2024.com.seaweedfs",
iscsiUtil: mi,
mountUtil: mm,
logger: log.New(os.Stderr, "[test-node] ", log.LstdFlags),
staged: map[string]*stagedVolumeInfo{
"no-session-vol": {
iqn: "iqn.2024.com.seaweedfs:no-session-vol",
iscsiAddr: "10.0.0.5:3260",
isLocal: false,
fsType: "ext4",
stagingPath: "/staging/no-session-vol",
},
},
}
// NOT logged in (loggedIn map is empty).
_, err := ns.NodeExpandVolume(context.Background(), &csi.NodeExpandVolumeRequest{
VolumeId: "no-session-vol",
})
if err == nil {
t.Fatal("expected error when iSCSI session not active")
}
st, _ := status.FromError(err)
if st.Code() != codes.FailedPrecondition {
t.Fatalf("expected FailedPrecondition, got %v", st.Code())
}
}
// TestNode_NodeGetCapabilities verifies EXPAND_VOLUME capability is present.
func TestNode_NodeGetCapabilities(t *testing.T) {
ns, _, _ := newTestNodeServer(t)
resp, err := ns.NodeGetCapabilities(context.Background(), &csi.NodeGetCapabilitiesRequest{})
if err != nil {
t.Fatalf("NodeGetCapabilities: %v", err)
}
hasStage := false
hasExpand := false
for _, cap := range resp.Capabilities {
rpc := cap.GetRpc()
if rpc == nil {
continue
}
switch rpc.Type {
case csi.NodeServiceCapability_RPC_STAGE_UNSTAGE_VOLUME:
hasStage = true
case csi.NodeServiceCapability_RPC_EXPAND_VOLUME:
hasExpand = true
}
}
if !hasStage {
t.Fatal("expected STAGE_UNSTAGE_VOLUME capability")
}
if !hasExpand {
t.Fatal("expected EXPAND_VOLUME capability")
}
}
// TestNode_StageCapturesFSType verifies that fsType is captured in staged info.
func TestNode_StageCapturesFSType(t *testing.T) {
ns, _, _ := newTestNodeServer(t)
stagingPath := t.TempDir()
_, err := ns.NodeStageVolume(context.Background(), &csi.NodeStageVolumeRequest{
VolumeId: "test-vol",
StagingTargetPath: stagingPath,
VolumeCapability: &csi.VolumeCapability{
AccessType: &csi.VolumeCapability_Mount{
Mount: &csi.VolumeCapability_MountVolume{FsType: "xfs"},
},
},
})
if err != nil {
t.Fatalf("NodeStageVolume: %v", err)
}
ns.stagedMu.Lock()
info := ns.staged["test-vol"]
ns.stagedMu.Unlock()
if info == nil {
t.Fatal("expected test-vol in staged map")
}
if info.fsType != "xfs" {
t.Fatalf("fsType: got %q, want xfs", info.fsType)
}
if info.stagingPath != stagingPath {
t.Fatalf("stagingPath: got %q, want %q", info.stagingPath, stagingPath)
}
}
// TestNode_StageFallbackLocal verifies local fallback when no volume_context is provided.
func TestNode_StageFallbackLocal(t *testing.T) {
ns, mi, _ := newTestNodeServer(t)
weed/storage/blockvol/csi/qa_cp83_adversarial_test.go
+690
View File
@@ -0,0 +1,690 @@
package csi
import (
"context"
"fmt"
"sync"
"sync/atomic"
"testing"
"github.com/container-storage-interface/spec/lib/go/csi"
"google.golang.org/grpc/codes"
"google.golang.org/grpc/status"
)
// ============================================================================
// QA Adversarial Tests for CP8-3: CSI Snapshots + Expansion
// Covers concurrency, edge cases, error injection, and lifecycle races.
// ============================================================================
// --- A. Concurrent Snapshot Operations ---
// TestQA_CP83_ConcurrentCreateSameSnapshot: 10 goroutines create the same
// snapshot name on the same volume. Exactly one should create; the rest
// should get idempotent success (same snapID on same volume).
func TestQA_CP83_ConcurrentCreateSameSnapshot(t *testing.T) {
mgr := newTestManager(t)
backend := NewLocalVolumeBackend(mgr)
cs := &controllerServer{backend: backend}
cs.CreateVolume(context.Background(), &csi.CreateVolumeRequest{
Name: "conc-snap-vol",
VolumeCapabilities: testVolCaps(),
CapacityRange: &csi.CapacityRange{RequiredBytes: 4 * 1024 * 1024},
})
const N = 10
var wg sync.WaitGroup
errs := make([]error, N)
resps := make([]*csi.CreateSnapshotResponse, N)
wg.Add(N)
for i := 0; i < N; i++ {
go func(idx int) {
defer wg.Done()
resp, err := cs.CreateSnapshot(context.Background(), &csi.CreateSnapshotRequest{
Name: "same-snap-name",
SourceVolumeId: "conc-snap-vol",
})
errs[idx] = err
resps[idx] = resp
}(i)
}
wg.Wait()
// All should succeed (one creates, rest get idempotent).
var snapID string
for i := 0; i < N; i++ {
if errs[i] != nil {
t.Errorf("goroutine %d: unexpected error: %v", i, errs[i])
continue
}
if snapID == "" {
snapID = resps[i].Snapshot.SnapshotId
} else if resps[i].Snapshot.SnapshotId != snapID {
t.Errorf("goroutine %d: snapshot ID mismatch: %s vs %s", i, resps[i].Snapshot.SnapshotId, snapID)
}
}
}
// TestQA_CP83_ConcurrentCreateDifferentSnapshots: 5 goroutines each create
// a uniquely named snapshot on the same volume. All should succeed.
func TestQA_CP83_ConcurrentCreateDifferentSnapshots(t *testing.T) {
mgr := newTestManager(t)
backend := NewLocalVolumeBackend(mgr)
cs := &controllerServer{backend: backend}
cs.CreateVolume(context.Background(), &csi.CreateVolumeRequest{
Name: "multi-snap-vol",
VolumeCapabilities: testVolCaps(),
CapacityRange: &csi.CapacityRange{RequiredBytes: 4 * 1024 * 1024},
})
const N = 5
var wg sync.WaitGroup
errs := make([]error, N)
wg.Add(N)
for i := 0; i < N; i++ {
go func(idx int) {
defer wg.Done()
_, errs[idx] = cs.CreateSnapshot(context.Background(), &csi.CreateSnapshotRequest{
Name: fmt.Sprintf("snap-%d", idx),
SourceVolumeId: "multi-snap-vol",
})
}(i)
}
wg.Wait()
for i := 0; i < N; i++ {
if errs[i] != nil {
t.Errorf("goroutine %d: unexpected error: %v", i, errs[i])
}
}
// Verify all snapshots exist.
listResp, err := cs.ListSnapshots(context.Background(), &csi.ListSnapshotsRequest{
SourceVolumeId: "multi-snap-vol",
})
if err != nil {
t.Fatal(err)
}
if len(listResp.Entries) != N {
t.Fatalf("expected %d snapshots, got %d", N, len(listResp.Entries))
}
}
// TestQA_CP83_ConcurrentSnapshotAndExpand: one goroutine creates a snapshot,
// another tries to expand. The expand should fail (FailedPrecondition) if
// the snapshot was created first, or the snapshot should succeed.
func TestQA_CP83_ConcurrentSnapshotAndExpand(t *testing.T) {
mgr := newTestManager(t)
backend := NewLocalVolumeBackend(mgr)
cs := &controllerServer{backend: backend}
cs.CreateVolume(context.Background(), &csi.CreateVolumeRequest{
Name: "race-vol",
VolumeCapabilities: testVolCaps(),
CapacityRange: &csi.CapacityRange{RequiredBytes: 4 * 1024 * 1024},
})
var wg sync.WaitGroup
var snapErr, expandErr error
wg.Add(2)
go func() {
defer wg.Done()
_, snapErr = cs.CreateSnapshot(context.Background(), &csi.CreateSnapshotRequest{
Name: "race-snap",
SourceVolumeId: "race-vol",
})
}()
go func() {
defer wg.Done()
_, expandErr = cs.ControllerExpandVolume(context.Background(), &csi.ControllerExpandVolumeRequest{
VolumeId: "race-vol",
CapacityRange: &csi.CapacityRange{RequiredBytes: 8 * 1024 * 1024},
})
}()
wg.Wait()
// One of two outcomes:
// 1. Snapshot wins → expand gets FailedPrecondition
// 2. Expand wins → snapshot still succeeds (taken after expand)
if snapErr != nil {
t.Fatalf("snapshot should always succeed, got: %v", snapErr)
}
if expandErr != nil {
st, _ := status.FromError(expandErr)
if st.Code() != codes.FailedPrecondition {
t.Fatalf("expand error should be FailedPrecondition, got %v: %s", st.Code(), st.Message())
}
}
// No panic = pass.
}
// --- B. Expand Edge Cases ---
// TestQA_CP83_ExpandToSameSize: expand to current size should succeed (no-op).
func TestQA_CP83_ExpandToSameSize(t *testing.T) {
mgr := newTestManager(t)
backend := NewLocalVolumeBackend(mgr)
cs := &controllerServer{backend: backend}
cs.CreateVolume(context.Background(), &csi.CreateVolumeRequest{
Name: "same-size-vol",
VolumeCapabilities: testVolCaps(),
CapacityRange: &csi.CapacityRange{RequiredBytes: 4 * 1024 * 1024},
})
resp, err := cs.ControllerExpandVolume(context.Background(), &csi.ControllerExpandVolumeRequest{
VolumeId: "same-size-vol",
CapacityRange: &csi.CapacityRange{RequiredBytes: 4 * 1024 * 1024},
})
// Engine may return success (no-op) or error (can't shrink to same).
// Either way, no panic.
if err != nil {
st, _ := status.FromError(err)
// InvalidArgument (shrink guard) is acceptable for same-size.
if st.Code() != codes.InvalidArgument {
t.Fatalf("unexpected error: %v", err)
}
} else if resp.CapacityBytes != 4*1024*1024 {
t.Fatalf("expected 4MB, got %d", resp.CapacityBytes)
}
}
// TestQA_CP83_ExpandToSmallerSize: shrink should be rejected.
func TestQA_CP83_ExpandToSmallerSize(t *testing.T) {
mgr := newTestManager(t)
backend := NewLocalVolumeBackend(mgr)
cs := &controllerServer{backend: backend}
cs.CreateVolume(context.Background(), &csi.CreateVolumeRequest{
Name: "shrink-vol",
VolumeCapabilities: testVolCaps(),
CapacityRange: &csi.CapacityRange{RequiredBytes: 8 * 1024 * 1024},
})
_, err := cs.ControllerExpandVolume(context.Background(), &csi.ControllerExpandVolumeRequest{
VolumeId: "shrink-vol",
CapacityRange: &csi.CapacityRange{RequiredBytes: 4 * 1024 * 1024},
})
if err == nil {
t.Fatal("expected error for shrink")
}
st, _ := status.FromError(err)
if st.Code() != codes.InvalidArgument {
t.Fatalf("expected InvalidArgument for shrink, got %v", st.Code())
}
}
// TestQA_CP83_ExpandWithZeroBytes: expand with 0 bytes should be rejected.
func TestQA_CP83_ExpandWithZeroBytes(t *testing.T) {
mgr := newTestManager(t)
backend := NewLocalVolumeBackend(mgr)
cs := &controllerServer{backend: backend}
cs.CreateVolume(context.Background(), &csi.CreateVolumeRequest{
Name: "zero-expand-vol",
VolumeCapabilities: testVolCaps(),
CapacityRange: &csi.CapacityRange{RequiredBytes: 4 * 1024 * 1024},
})
_, err := cs.ControllerExpandVolume(context.Background(), &csi.ControllerExpandVolumeRequest{
VolumeId: "zero-expand-vol",
CapacityRange: &csi.CapacityRange{RequiredBytes: 0},
})
if err == nil {
t.Fatal("expected error for zero expand")
}
st, _ := status.FromError(err)
if st.Code() != codes.InvalidArgument {
t.Fatalf("expected InvalidArgument, got %v", st.Code())
}
}
// TestQA_CP83_ExpandNilCapacity: expand with nil CapacityRange should be rejected.
func TestQA_CP83_ExpandNilCapacity(t *testing.T) {
mgr := newTestManager(t)
backend := NewLocalVolumeBackend(mgr)
cs := &controllerServer{backend: backend}
_, err := cs.ControllerExpandVolume(context.Background(), &csi.ControllerExpandVolumeRequest{
VolumeId: "nil-cap-vol",
CapacityRange: nil,
})
if err == nil {
t.Fatal("expected error for nil capacity range")
}
st, _ := status.FromError(err)
if st.Code() != codes.InvalidArgument {
t.Fatalf("expected InvalidArgument, got %v", st.Code())
}
}
// TestQA_CP83_ExpandNonAligned: expand to a non-block-aligned size should
// round up and succeed.
func TestQA_CP83_ExpandNonAligned(t *testing.T) {
mgr := newTestManager(t)
backend := NewLocalVolumeBackend(mgr)
cs := &controllerServer{backend: backend}
cs.CreateVolume(context.Background(), &csi.CreateVolumeRequest{
Name: "align-vol",
VolumeCapabilities: testVolCaps(),
CapacityRange: &csi.CapacityRange{RequiredBytes: 4 * 1024 * 1024},
})
// Request non-aligned size (5MB + 1 byte).
requested := int64(5*1024*1024 + 1)
resp, err := cs.ControllerExpandVolume(context.Background(), &csi.ControllerExpandVolumeRequest{
VolumeId: "align-vol",
CapacityRange: &csi.CapacityRange{RequiredBytes: requested},
})
if err != nil {
t.Fatalf("non-aligned expand should succeed: %v", err)
}
// Result should be rounded up to next 4096 boundary.
expected := int64((5*1024*1024+1)/4096+1) * 4096
if resp.CapacityBytes != expected {
t.Fatalf("expected rounded %d, got %d", expected, resp.CapacityBytes)
}
}
// TestQA_CP83_ExpandMissingVolume: expand on nonexistent volume.
func TestQA_CP83_ExpandMissingVolume(t *testing.T) {
mgr := newTestManager(t)
backend := NewLocalVolumeBackend(mgr)
cs := &controllerServer{backend: backend}
_, err := cs.ControllerExpandVolume(context.Background(), &csi.ControllerExpandVolumeRequest{
VolumeId: "no-such-vol",
CapacityRange: &csi.CapacityRange{RequiredBytes: 8 * 1024 * 1024},
})
if err == nil {
t.Fatal("expected error for missing volume")
}
st, _ := status.FromError(err)
if st.Code() != codes.NotFound {
t.Fatalf("expected NotFound, got %v: %s", st.Code(), st.Message())
}
}
// --- C. Snapshot Lifecycle ---
// TestQA_CP83_DoubleSnapshot: create two snapshots, delete first, verify
// second still exists, delete second, verify empty.
func TestQA_CP83_DoubleSnapshot(t *testing.T) {
mgr := newTestManager(t)
backend := NewLocalVolumeBackend(mgr)
cs := &controllerServer{backend: backend}
ctx := context.Background()
cs.CreateVolume(ctx, &csi.CreateVolumeRequest{
Name: "double-snap-vol",
VolumeCapabilities: testVolCaps(),
CapacityRange: &csi.CapacityRange{RequiredBytes: 4 * 1024 * 1024},
})
snap1, err := cs.CreateSnapshot(ctx, &csi.CreateSnapshotRequest{
Name: "snap-alpha", SourceVolumeId: "double-snap-vol",
})
if err != nil {
t.Fatal(err)
}
snap2, err := cs.CreateSnapshot(ctx, &csi.CreateSnapshotRequest{
Name: "snap-beta", SourceVolumeId: "double-snap-vol",
})
if err != nil {
t.Fatal(err)
}
// Delete first.
cs.DeleteSnapshot(ctx, &csi.DeleteSnapshotRequest{SnapshotId: snap1.Snapshot.SnapshotId})
// List should show 1.
list, _ := cs.ListSnapshots(ctx, &csi.ListSnapshotsRequest{SourceVolumeId: "double-snap-vol"})
if len(list.Entries) != 1 {
t.Fatalf("expected 1 snapshot, got %d", len(list.Entries))
}
if list.Entries[0].Snapshot.SnapshotId != snap2.Snapshot.SnapshotId {
t.Fatalf("wrong snapshot survived")
}
// Delete second.
cs.DeleteSnapshot(ctx, &csi.DeleteSnapshotRequest{SnapshotId: snap2.Snapshot.SnapshotId})
// List should be empty.
list, _ = cs.ListSnapshots(ctx, &csi.ListSnapshotsRequest{SourceVolumeId: "double-snap-vol"})
if len(list.Entries) != 0 {
t.Fatalf("expected 0 snapshots, got %d", len(list.Entries))
}
}
// TestQA_CP83_SnapshotOnDeletedVolume: delete volume, then try snapshot.
func TestQA_CP83_SnapshotOnDeletedVolume(t *testing.T) {
mgr := newTestManager(t)
backend := NewLocalVolumeBackend(mgr)
cs := &controllerServer{backend: backend}
ctx := context.Background()
cs.CreateVolume(ctx, &csi.CreateVolumeRequest{
Name: "doomed-vol",
VolumeCapabilities: testVolCaps(),
CapacityRange: &csi.CapacityRange{RequiredBytes: 4 * 1024 * 1024},
})
cs.DeleteVolume(ctx, &csi.DeleteVolumeRequest{VolumeId: "doomed-vol"})
_, err := cs.CreateSnapshot(ctx, &csi.CreateSnapshotRequest{
Name: "orphan", SourceVolumeId: "doomed-vol",
})
if err == nil {
t.Fatal("expected error for snapshot on deleted volume")
}
st, _ := status.FromError(err)
if st.Code() != codes.NotFound {
t.Fatalf("expected NotFound, got %v", st.Code())
}
}
// TestQA_CP83_DeleteVolumeWithActiveSnapshots: volume with snapshots can be
// deleted (engine cleans up snap files).
func TestQA_CP83_DeleteVolumeWithActiveSnapshots(t *testing.T) {
mgr := newTestManager(t)
backend := NewLocalVolumeBackend(mgr)
cs := &controllerServer{backend: backend}
ctx := context.Background()
cs.CreateVolume(ctx, &csi.CreateVolumeRequest{
Name: "snap-then-delete-vol",
VolumeCapabilities: testVolCaps(),
CapacityRange: &csi.CapacityRange{RequiredBytes: 4 * 1024 * 1024},
})
cs.CreateSnapshot(ctx, &csi.CreateSnapshotRequest{
Name: "pre-delete-snap", SourceVolumeId: "snap-then-delete-vol",
})
_, err := cs.DeleteVolume(ctx, &csi.DeleteVolumeRequest{VolumeId: "snap-then-delete-vol"})
if err != nil {
t.Fatalf("delete volume with snapshots should succeed: %v", err)
}
// Verify volume is gone.
_, err = cs.CreateSnapshot(ctx, &csi.CreateSnapshotRequest{
Name: "after-delete-snap", SourceVolumeId: "snap-then-delete-vol",
})
if err == nil {
t.Fatal("expected error for snapshot on deleted volume")
}
}
// TestQA_CP83_FullCycleSnapExpandSnap: create → snap → delete snap → expand →
// snap at new size → verify snapshot reports new size.
func TestQA_CP83_FullCycleSnapExpandSnap(t *testing.T) {
mgr := newTestManager(t)
backend := NewLocalVolumeBackend(mgr)
cs := &controllerServer{backend: backend}
ctx := context.Background()
cs.CreateVolume(ctx, &csi.CreateVolumeRequest{
Name: "cycle-vol",
VolumeCapabilities: testVolCaps(),
CapacityRange: &csi.CapacityRange{RequiredBytes: 4 * 1024 * 1024},
})
// Snapshot at 4MB.
snap1, _ := cs.CreateSnapshot(ctx, &csi.CreateSnapshotRequest{
Name: "pre-expand-snap", SourceVolumeId: "cycle-vol",
})
if snap1.Snapshot.SizeBytes != 4*1024*1024 {
t.Fatalf("snap1 size: got %d, want %d", snap1.Snapshot.SizeBytes, 4*1024*1024)
}
// Delete snap so we can expand.
cs.DeleteSnapshot(ctx, &csi.DeleteSnapshotRequest{SnapshotId: snap1.Snapshot.SnapshotId})
// Expand to 8MB.
_, err := cs.ControllerExpandVolume(ctx, &csi.ControllerExpandVolumeRequest{
VolumeId: "cycle-vol",
CapacityRange: &csi.CapacityRange{RequiredBytes: 8 * 1024 * 1024},
})
if err != nil {
t.Fatalf("expand: %v", err)
}
// Snapshot at 8MB.
snap2, _ := cs.CreateSnapshot(ctx, &csi.CreateSnapshotRequest{
Name: "post-expand-snap", SourceVolumeId: "cycle-vol",
})
if snap2.Snapshot.SizeBytes != 8*1024*1024 {
t.Fatalf("snap2 size: got %d, want %d", snap2.Snapshot.SizeBytes, 8*1024*1024)
}
}
// --- D. Error Injection ---
// TestQA_CP83_ExpandBackendFailure: backend returns error on expand.
func TestQA_CP83_ExpandBackendFailure(t *testing.T) {
mock := &expandFailBackend{expandErr: fmt.Errorf("disk full")}
cs := &controllerServer{backend: mock}
_, err := cs.ControllerExpandVolume(context.Background(), &csi.ControllerExpandVolumeRequest{
VolumeId: "fail-vol",
CapacityRange: &csi.CapacityRange{RequiredBytes: 8 * 1024 * 1024},
})
if err == nil {
t.Fatal("expected error from backend")
}
st, _ := status.FromError(err)
if st.Code() != codes.Internal {
t.Fatalf("expected Internal, got %v", st.Code())
}
}
// TestQA_CP83_SnapshotBackendFailure: backend returns error on create snapshot.
func TestQA_CP83_SnapshotBackendFailure(t *testing.T) {
mock := &expandFailBackend{snapErr: fmt.Errorf("I/O error")}
cs := &controllerServer{backend: mock}
_, err := cs.CreateSnapshot(context.Background(), &csi.CreateSnapshotRequest{
Name: "fail-snap",
SourceVolumeId: "fail-vol",
})
if err == nil {
t.Fatal("expected error from backend")
}
st, _ := status.FromError(err)
if st.Code() != codes.Internal {
t.Fatalf("expected Internal, got %v", st.Code())
}
}
// --- E. ListSnapshots Edge Cases ---
// TestQA_CP83_ListSnapshotsBySnapshotID: filter by snapshot ID returns exactly 1.
func TestQA_CP83_ListSnapshotsBySnapshotID(t *testing.T) {
mgr := newTestManager(t)
backend := NewLocalVolumeBackend(mgr)
cs := &controllerServer{backend: backend}
ctx := context.Background()
cs.CreateVolume(ctx, &csi.CreateVolumeRequest{
Name: "list-by-id-vol",
VolumeCapabilities: testVolCaps(),
CapacityRange: &csi.CapacityRange{RequiredBytes: 4 * 1024 * 1024},
})
snapResp, _ := cs.CreateSnapshot(ctx, &csi.CreateSnapshotRequest{
Name: "list-snap", SourceVolumeId: "list-by-id-vol",
})
list, err := cs.ListSnapshots(ctx, &csi.ListSnapshotsRequest{
SnapshotId: snapResp.Snapshot.SnapshotId,
})
if err != nil {
t.Fatal(err)
}
if len(list.Entries) != 1 {
t.Fatalf("expected 1, got %d", len(list.Entries))
}
if list.Entries[0].Snapshot.SnapshotId != snapResp.Snapshot.SnapshotId {
t.Fatalf("wrong snapshot")
}
}
// TestQA_CP83_ListSnapshotsBySnapshotID_NotFound: filter by non-existent
// snapshot ID returns empty.
func TestQA_CP83_ListSnapshotsBySnapshotID_NotFound(t *testing.T) {
mgr := newTestManager(t)
backend := NewLocalVolumeBackend(mgr)
cs := &controllerServer{backend: backend}
ctx := context.Background()
cs.CreateVolume(ctx, &csi.CreateVolumeRequest{
Name: "list-miss-vol",
VolumeCapabilities: testVolCaps(),
CapacityRange: &csi.CapacityRange{RequiredBytes: 4 * 1024 * 1024},
})
list, err := cs.ListSnapshots(ctx, &csi.ListSnapshotsRequest{
SnapshotId: FormatSnapshotID("list-miss-vol", 99999),
})
if err != nil {
t.Fatal(err)
}
if len(list.Entries) != 0 {
t.Fatalf("expected 0, got %d", len(list.Entries))
}
}
// TestQA_CP83_ListSnapshotsNoFilter: no filter returns empty (documented behavior).
func TestQA_CP83_ListSnapshotsNoFilter(t *testing.T) {
mgr := newTestManager(t)
backend := NewLocalVolumeBackend(mgr)
cs := &controllerServer{backend: backend}
ctx := context.Background()
cs.CreateVolume(ctx, &csi.CreateVolumeRequest{
Name: "no-filter-vol",
VolumeCapabilities: testVolCaps(),
CapacityRange: &csi.CapacityRange{RequiredBytes: 4 * 1024 * 1024},
})
cs.CreateSnapshot(ctx, &csi.CreateSnapshotRequest{
Name: "no-filter-snap", SourceVolumeId: "no-filter-vol",
})
list, err := cs.ListSnapshots(ctx, &csi.ListSnapshotsRequest{})
if err != nil {
t.Fatal(err)
}
if len(list.Entries) != 0 {
t.Fatalf("expected 0 (no filter not supported), got %d", len(list.Entries))
}
}
// --- F. Concurrent Expand ---
// TestQA_CP83_ConcurrentExpand: 5 goroutines expand the same volume to
// increasing sizes. No panics, final size should be the largest requested.
func TestQA_CP83_ConcurrentExpand(t *testing.T) {
mgr := newTestManager(t)
backend := NewLocalVolumeBackend(mgr)
cs := &controllerServer{backend: backend}
cs.CreateVolume(context.Background(), &csi.CreateVolumeRequest{
Name: "conc-expand-vol",
VolumeCapabilities: testVolCaps(),
CapacityRange: &csi.CapacityRange{RequiredBytes: 4 * 1024 * 1024},
})
const N = 5
var wg sync.WaitGroup
var maxCapacity atomic.Int64
wg.Add(N)
for i := 0; i < N; i++ {
go func(idx int) {
defer wg.Done()
newSize := int64((idx + 2) * 4 * 1024 * 1024) // 8MB, 12MB, 16MB, 20MB, 24MB
resp, err := cs.ControllerExpandVolume(context.Background(), &csi.ControllerExpandVolumeRequest{
VolumeId: "conc-expand-vol",
CapacityRange: &csi.CapacityRange{RequiredBytes: newSize},
})
if err != nil {
// Some may fail with "shrink" if a larger expand completed first.
return
}
for {
old := maxCapacity.Load()
if resp.CapacityBytes <= old {
break
}
if maxCapacity.CompareAndSwap(old, resp.CapacityBytes) {
break
}
}
}(i)
}
wg.Wait()
// Final capacity should be at least 24MB.
if maxCapacity.Load() < 24*1024*1024 {
t.Fatalf("expected max capacity >= 24MB, got %d", maxCapacity.Load())
}
}
// --- G. Snapshot ID Helpers ---
// TestQA_CP83_FormatParseRoundTrip: verify format/parse round-trip for various names.
func TestQA_CP83_FormatParseRoundTrip(t *testing.T) {
cases := []struct {
vol string
snapID uint32
}{
{"simple", 42},
{"pvc-12345", 99},
{"vol-with-many-dashes", 1},
{"vol", 0}, // ID 0
{"vol", 0xFFFFFFFF}, // max uint32
}
for _, tc := range cases {
formatted := FormatSnapshotID(tc.vol, tc.snapID)
gotVol, gotSnap, err := ParseSnapshotID(formatted)
if err != nil {
t.Errorf("parse(%q): %v", formatted, err)
continue
}
if gotVol != tc.vol || gotSnap != tc.snapID {
t.Errorf("round-trip failed: vol=%q snap=%d → %q → vol=%q snap=%d",
tc.vol, tc.snapID, formatted, gotVol, gotSnap)
}
}
}
// --- Mock backends for error injection ---
type expandFailBackend struct {
expandErr error
snapErr error
}
func (m *expandFailBackend) CreateVolume(_ context.Context, name string, sizeBytes uint64) (*VolumeInfo, error) {
return &VolumeInfo{VolumeID: name, CapacityBytes: sizeBytes}, nil
}
func (m *expandFailBackend) DeleteVolume(_ context.Context, _ string) error { return nil }
func (m *expandFailBackend) LookupVolume(_ context.Context, name string) (*VolumeInfo, error) {
return &VolumeInfo{VolumeID: name}, nil
}
func (m *expandFailBackend) CreateSnapshot(_ context.Context, volumeID string, snapID uint32) (*SnapshotInfo, error) {
if m.snapErr != nil {
return nil, m.snapErr
}
return &SnapshotInfo{SnapshotID: snapID, VolumeID: volumeID}, nil
}
func (m *expandFailBackend) DeleteSnapshot(_ context.Context, _ string, _ uint32) error { return nil }
func (m *expandFailBackend) ListSnapshots(_ context.Context, _ string) ([]*SnapshotInfo, error) {
return nil, nil
}
func (m *expandFailBackend) ExpandVolume(_ context.Context, _ string, _ uint64) (uint64, error) {
return 0, m.expandErr
}
weed/storage/blockvol/csi/qa_cp83_test.go
+446
View File
@@ -0,0 +1,446 @@
package csi
import (
"context"
"errors"
"fmt"
"log"
"os"
"strings"
"testing"
"github.com/container-storage-interface/spec/lib/go/csi"
"google.golang.org/grpc/codes"
"google.golang.org/grpc/status"
)
// TestQA_Snap_CreateOnNonExistentVolume verifies NotFound when creating snapshot
// on a volume that doesn't exist.
func TestQA_Snap_CreateOnNonExistentVolume(t *testing.T) {
mgr := newTestManager(t)
backend := NewLocalVolumeBackend(mgr)
cs := &controllerServer{backend: backend}
_, err := cs.CreateSnapshot(context.Background(), &csi.CreateSnapshotRequest{
Name: "orphan-snap",
SourceVolumeId: "nonexistent-volume",
})
if err == nil {
t.Fatal("expected error for snapshot on nonexistent volume")
}
st, _ := status.FromError(err)
if st.Code() != codes.NotFound {
t.Fatalf("expected NotFound, got %v: %s", st.Code(), st.Message())
}
}
// TestQA_Snap_DeleteNonExistentSnapshot verifies idempotent success when deleting
// a snapshot that doesn't exist (CSI spec: delete should succeed).
func TestQA_Snap_DeleteNonExistentSnapshot(t *testing.T) {
mgr := newTestManager(t)
backend := NewLocalVolumeBackend(mgr)
cs := &controllerServer{backend: backend}
// Create volume but no snapshot.
cs.CreateVolume(context.Background(), &csi.CreateVolumeRequest{
Name: "del-snap-vol",
VolumeCapabilities: testVolCaps(),
CapacityRange: &csi.CapacityRange{RequiredBytes: 4 * 1024 * 1024},
})
// Delete a well-formed snapshot ID for an existing volume, but snapshot doesn't exist.
snapID := FormatSnapshotID("del-snap-vol", 99999)
_, err := cs.DeleteSnapshot(context.Background(), &csi.DeleteSnapshotRequest{
SnapshotId: snapID,
})
if err != nil {
t.Fatalf("delete nonexistent snapshot should succeed (idempotent), got: %v", err)
}
}
// TestQA_Snap_SnapshotIDParsing exercises ParseSnapshotID with weird, empty,
// and unicode strings. Delete should treat unparseable IDs as idempotent success.
func TestQA_Snap_SnapshotIDParsing(t *testing.T) {
mgr := newTestManager(t)
backend := NewLocalVolumeBackend(mgr)
cs := &controllerServer{backend: backend}
cases := []struct {
name string
id string
}{
{"empty", ""},
{"no prefix", "garbage-123"},
{"only prefix", "snap-"},
{"no numeric suffix", "snap-vol-abc"},
{"unicode volume", "snap-日本語ボリューム-42"},
{"missing separator", "snap-noseparator"},
{"double dash", "snap--42"},
{"huge number", "snap-vol-99999999999999999999"},
}
for _, tc := range cases {
t.Run(tc.name, func(t *testing.T) {
if tc.id == "" {
// Empty ID → InvalidArgument from DeleteSnapshot.
_, err := cs.DeleteSnapshot(context.Background(), &csi.DeleteSnapshotRequest{
SnapshotId: tc.id,
})
if err == nil {
t.Fatal("expected error for empty snapshot ID")
}
st, _ := status.FromError(err)
if st.Code() != codes.InvalidArgument {
t.Fatalf("expected InvalidArgument for empty ID, got %v", st.Code())
}
return
}
// Non-empty but unparseable → idempotent success.
_, err := cs.DeleteSnapshot(context.Background(), &csi.DeleteSnapshotRequest{
SnapshotId: tc.id,
})
if err != nil {
t.Fatalf("delete unparseable snapshot ID should succeed (idempotent), got: %v", err)
}
})
}
}
// TestQA_Snap_ExpandWithActiveSnapshots verifies FailedPrecondition when expanding
// a volume that has active snapshots.
func TestQA_Snap_ExpandWithActiveSnapshots(t *testing.T) {
mgr := newTestManager(t)
backend := NewLocalVolumeBackend(mgr)
cs := &controllerServer{backend: backend}
// Create volume.
cs.CreateVolume(context.Background(), &csi.CreateVolumeRequest{
Name: "snap-expand-vol",
VolumeCapabilities: testVolCaps(),
CapacityRange: &csi.CapacityRange{RequiredBytes: 4 * 1024 * 1024},
})
// Create snapshot.
_, err := cs.CreateSnapshot(context.Background(), &csi.CreateSnapshotRequest{
Name: "blocking-snap",
SourceVolumeId: "snap-expand-vol",
})
if err != nil {
t.Fatalf("CreateSnapshot: %v", err)
}
// Now try to expand --engine should reject due to active snapshots.
_, err = cs.ControllerExpandVolume(context.Background(), &csi.ControllerExpandVolumeRequest{
VolumeId: "snap-expand-vol",
CapacityRange: &csi.CapacityRange{RequiredBytes: 8 * 1024 * 1024},
})
if err == nil {
t.Fatal("expected error for expand with active snapshots")
}
st, _ := status.FromError(err)
if st.Code() != codes.FailedPrecondition {
t.Fatalf("expected FailedPrecondition, got %v: %s", st.Code(), st.Message())
}
if !strings.Contains(st.Message(), "snapshots") {
t.Fatalf("error message should mention snapshots: %s", st.Message())
}
}
// TestQA_Snap_SnapshotCollisionHandled tests that two different names hashing to
// the same FNV-32a ID are handled with AlreadyExists.
func TestQA_Snap_SnapshotCollisionHandled(t *testing.T) {
// Use a mock backend that always returns "already exists" after the first create,
// simulating what happens when two names produce the same FNV-32a hash.
mock := &collisionMockBackend{
createCount: 0,
}
cs := &controllerServer{backend: mock}
// First snapshot succeeds.
_, err := cs.CreateSnapshot(context.Background(), &csi.CreateSnapshotRequest{
Name: "first-name",
SourceVolumeId: "collision-vol",
})
if err != nil {
t.Fatalf("first CreateSnapshot: %v", err)
}
// Second snapshot: mock simulates FNV collision by returning "already exists".
_, err = cs.CreateSnapshot(context.Background(), &csi.CreateSnapshotRequest{
Name: "second-name",
SourceVolumeId: "collision-vol",
})
if err == nil {
t.Fatal("expected error for collision")
}
st, _ := status.FromError(err)
if st.Code() != codes.AlreadyExists {
t.Fatalf("expected AlreadyExists, got %v: %s", st.Code(), st.Message())
}
if !strings.Contains(st.Message(), "collision") {
t.Fatalf("error message should mention collision: %s", st.Message())
}
}
// TestQA_Expand_RescanFailure verifies that NodeExpandVolume handles rescan failure
// by returning Internal error.
func TestQA_Expand_RescanFailure(t *testing.T) {
mi := newMockISCSIUtil()
mi.getDeviceResult = "/dev/sdb"
mi.rescanErr = errors.New("iscsiadm rescan: device not found")
mm := newMockMountUtil()
ns := &nodeServer{
mgr: nil,
nodeID: "test-node-1",
iqnPrefix: "iqn.2024.com.seaweedfs",
iscsiUtil: mi,
mountUtil: mm,
logger: log.New(os.Stderr, "[test-node] ", log.LstdFlags),
staged: map[string]*stagedVolumeInfo{
"rescan-fail-vol": {
iqn: "iqn.2024.com.seaweedfs:rescan-fail-vol",
iscsiAddr: "10.0.0.5:3260",
isLocal: false,
fsType: "ext4",
stagingPath: "/staging/rescan-fail-vol",
},
},
}
// Mark as logged in so session check passes.
mi.loggedIn["iqn.2024.com.seaweedfs:rescan-fail-vol"] = true
_, err := ns.NodeExpandVolume(context.Background(), &csi.NodeExpandVolumeRequest{
VolumeId: "rescan-fail-vol",
CapacityRange: &csi.CapacityRange{RequiredBytes: 16 * 1024 * 1024},
})
if err == nil {
t.Fatal("expected error from rescan failure")
}
st, _ := status.FromError(err)
if st.Code() != codes.Internal {
t.Fatalf("expected Internal, got %v: %s", st.Code(), st.Message())
}
}
// TestQA_FullLifecycleWithSnapshotExpand tests the full lifecycle:
// create volume → snapshot → delete snapshot → expand → verify.
func TestQA_FullLifecycleWithSnapshotExpand(t *testing.T) {
mgr := newTestManager(t)
backend := NewLocalVolumeBackend(mgr)
cs := &controllerServer{backend: backend}
ctx := context.Background()
// 1. Create volume.
volResp, err := cs.CreateVolume(ctx, &csi.CreateVolumeRequest{
Name: "lifecycle-vol",
VolumeCapabilities: testVolCaps(),
CapacityRange: &csi.CapacityRange{RequiredBytes: 4 * 1024 * 1024},
})
if err != nil {
t.Fatalf("CreateVolume: %v", err)
}
if volResp.Volume.CapacityBytes != 4*1024*1024 {
t.Fatalf("initial capacity: got %d", volResp.Volume.CapacityBytes)
}
// 2. Create snapshot.
snapResp, err := cs.CreateSnapshot(ctx, &csi.CreateSnapshotRequest{
Name: "lifecycle-snap",
SourceVolumeId: "lifecycle-vol",
})
if err != nil {
t.Fatalf("CreateSnapshot: %v", err)
}
if !snapResp.Snapshot.ReadyToUse {
t.Fatal("snapshot should be ready")
}
snapID := snapResp.Snapshot.SnapshotId
// 3. List snapshots --should find it.
listResp, err := cs.ListSnapshots(ctx, &csi.ListSnapshotsRequest{
SourceVolumeId: "lifecycle-vol",
})
if err != nil {
t.Fatalf("ListSnapshots: %v", err)
}
if len(listResp.Entries) != 1 {
t.Fatalf("expected 1 snapshot, got %d", len(listResp.Entries))
}
// 4. Expand should fail with active snapshot.
_, err = cs.ControllerExpandVolume(ctx, &csi.ControllerExpandVolumeRequest{
VolumeId: "lifecycle-vol",
CapacityRange: &csi.CapacityRange{RequiredBytes: 8 * 1024 * 1024},
})
if err == nil {
t.Fatal("expected expand to fail with active snapshot")
}
st, _ := status.FromError(err)
if st.Code() != codes.FailedPrecondition {
t.Fatalf("expected FailedPrecondition, got %v", st.Code())
}
// 5. Delete snapshot.
_, err = cs.DeleteSnapshot(ctx, &csi.DeleteSnapshotRequest{
SnapshotId: snapID,
})
if err != nil {
t.Fatalf("DeleteSnapshot: %v", err)
}
// 6. Now expand should succeed.
expandResp, err := cs.ControllerExpandVolume(ctx, &csi.ControllerExpandVolumeRequest{
VolumeId: "lifecycle-vol",
CapacityRange: &csi.CapacityRange{RequiredBytes: 8 * 1024 * 1024},
})
if err != nil {
t.Fatalf("ControllerExpandVolume: %v", err)
}
if expandResp.CapacityBytes != 8*1024*1024 {
t.Fatalf("expanded capacity: got %d, want %d", expandResp.CapacityBytes, 8*1024*1024)
}
if !expandResp.NodeExpansionRequired {
t.Fatal("NodeExpansionRequired should be true")
}
// 7. List snapshots --should be empty now.
listResp, err = cs.ListSnapshots(ctx, &csi.ListSnapshotsRequest{
SourceVolumeId: "lifecycle-vol",
})
if err != nil {
t.Fatalf("ListSnapshots after delete: %v", err)
}
if len(listResp.Entries) != 0 {
t.Fatalf("expected 0 snapshots after delete, got %d", len(listResp.Entries))
}
}
// TestQA_Snap_ListSnapshotsBackendError verifies that ListSnapshots propagates
// backend errors as Internal instead of silently returning empty list.
func TestQA_Snap_ListSnapshotsBackendError(t *testing.T) {
mock := &errorBackend{
listErr: fmt.Errorf("connection refused"),
}
cs := &controllerServer{backend: mock}
// By snapshot_id.
_, err := cs.ListSnapshots(context.Background(), &csi.ListSnapshotsRequest{
SnapshotId: FormatSnapshotID("vol1", 42),
})
if err == nil {
t.Fatal("expected error from backend failure")
}
st, _ := status.FromError(err)
if st.Code() != codes.Internal {
t.Fatalf("expected Internal, got %v: %s", st.Code(), st.Message())
}
// By source_volume_id.
_, err = cs.ListSnapshots(context.Background(), &csi.ListSnapshotsRequest{
SourceVolumeId: "vol1",
})
if err == nil {
t.Fatal("expected error from backend failure")
}
st, _ = status.FromError(err)
if st.Code() != codes.Internal {
t.Fatalf("expected Internal, got %v: %s", st.Code(), st.Message())
}
}
// TestQA_Snap_ListSnapshotsNotFoundVolume verifies that ListSnapshots returns
// empty list (not error) when volume is not found.
func TestQA_Snap_ListSnapshotsNotFoundVolume(t *testing.T) {
mgr := newTestManager(t)
backend := NewLocalVolumeBackend(mgr)
cs := &controllerServer{backend: backend}
resp, err := cs.ListSnapshots(context.Background(), &csi.ListSnapshotsRequest{
SourceVolumeId: "nonexistent-vol",
})
if err != nil {
t.Fatalf("ListSnapshots on missing volume should succeed, got: %v", err)
}
if len(resp.Entries) != 0 {
t.Fatalf("expected 0 entries, got %d", len(resp.Entries))
}
}
// --- collisionMockBackend ---
// collisionMockBackend is a VolumeBackend that simulates FNV-32a hash collision
// on CreateSnapshot. The first CreateSnapshot succeeds; all subsequent ones
// return "snapshot already exists" to simulate what happens when two different
// names produce the same FNV-32a hash.
type collisionMockBackend struct {
createCount int
}
func (m *collisionMockBackend) CreateVolume(_ context.Context, name string, sizeBytes uint64) (*VolumeInfo, error) {
return &VolumeInfo{VolumeID: name, CapacityBytes: sizeBytes}, nil
}
func (m *collisionMockBackend) DeleteVolume(_ context.Context, name string) error {
return nil
}
func (m *collisionMockBackend) LookupVolume(_ context.Context, name string) (*VolumeInfo, error) {
return &VolumeInfo{VolumeID: name}, nil
}
func (m *collisionMockBackend) CreateSnapshot(_ context.Context, volumeID string, snapID uint32) (*SnapshotInfo, error) {
m.createCount++
if m.createCount > 1 {
return nil, fmt.Errorf("snapshot already exists")
}
return &SnapshotInfo{SnapshotID: snapID, VolumeID: volumeID, SizeBytes: 4 * 1024 * 1024}, nil
}
func (m *collisionMockBackend) DeleteSnapshot(_ context.Context, volumeID string, snapID uint32) error {
return nil
}
func (m *collisionMockBackend) ListSnapshots(_ context.Context, volumeID string) ([]*SnapshotInfo, error) {
return nil, nil
}
func (m *collisionMockBackend) ExpandVolume(_ context.Context, volumeID string, newSizeBytes uint64) (uint64, error) {
return newSizeBytes, nil
}
// --- errorBackend ---
// errorBackend is a VolumeBackend that returns configurable errors for testing
// error propagation paths.
type errorBackend struct {
listErr error
}
func (m *errorBackend) CreateVolume(_ context.Context, name string, sizeBytes uint64) (*VolumeInfo, error) {
return &VolumeInfo{VolumeID: name, CapacityBytes: sizeBytes}, nil
}
func (m *errorBackend) DeleteVolume(_ context.Context, name string) error { return nil }
func (m *errorBackend) LookupVolume(_ context.Context, name string) (*VolumeInfo, error) {
return &VolumeInfo{VolumeID: name}, nil
}
func (m *errorBackend) CreateSnapshot(_ context.Context, volumeID string, snapID uint32) (*SnapshotInfo, error) {
return &SnapshotInfo{SnapshotID: snapID, VolumeID: volumeID}, nil
}
func (m *errorBackend) DeleteSnapshot(_ context.Context, volumeID string, snapID uint32) error {
return nil
}
func (m *errorBackend) ListSnapshots(_ context.Context, volumeID string) ([]*SnapshotInfo, error) {
return nil, m.listErr
}
func (m *errorBackend) ExpandVolume(_ context.Context, volumeID string, newSizeBytes uint64) (uint64, error) {
return newSizeBytes, nil
}
weed/storage/blockvol/csi/snapshot_ids.go
+44
View File
@@ -0,0 +1,44 @@
package csi
import (
"fmt"
"hash/fnv"
"strconv"
"strings"
)
// FormatSnapshotID creates a CSI-standard snapshot ID: "snap-{volumeID}-{snapID}".
func FormatSnapshotID(volumeID string, snapID uint32) string {
return fmt.Sprintf("snap-%s-%d", volumeID, snapID)
}
// ParseSnapshotID reverses FormatSnapshotID, returning volumeID and snapID.
// The volumeID may contain dashes, so we split on the last dash.
func ParseSnapshotID(csiSnapID string) (string, uint32, error) {
if !strings.HasPrefix(csiSnapID, "snap-") {
return "", 0, fmt.Errorf("invalid snapshot ID format: missing snap- prefix")
}
rest := csiSnapID[len("snap-"):]
lastDash := strings.LastIndex(rest, "-")
if lastDash < 0 || lastDash == 0 {
return "", 0, fmt.Errorf("invalid snapshot ID format: no volume/snap separator")
}
volumeID := rest[:lastDash]
numStr := rest[lastDash+1:]
id, err := strconv.ParseUint(numStr, 10, 32)
if err != nil {
return "", 0, fmt.Errorf("invalid snapshot ID format: %w", err)
}
return volumeID, uint32(id), nil
}
// snapshotNameToID converts a snapshot name to a deterministic uint32 via FNV-32a hash.
func snapshotNameToID(name string) uint32 {
h := fnv.New32a()
h.Write([]byte(name))
return h.Sum32()
}
weed/storage/blockvol/csi/snapshot_ids_test.go
+84
View File
@@ -0,0 +1,84 @@
package csi
import (
"testing"
)
func TestSnapshotID_FormatParseRoundTrip(t *testing.T) {
tests := []struct {
volumeID string
snapID uint32
}{
{"pvc-abc123", 42},
{"vol-with-dashes-in-name", 999},
{"simple", 0},
{"a", 4294967295}, // max uint32
}
for _, tt := range tests {
csiID := FormatSnapshotID(tt.volumeID, tt.snapID)
gotVol, gotSnap, err := ParseSnapshotID(csiID)
if err != nil {
t.Errorf("ParseSnapshotID(%q): %v", csiID, err)
continue
}
if gotVol != tt.volumeID {
t.Errorf("volumeID: got %q, want %q", gotVol, tt.volumeID)
}
if gotSnap != tt.snapID {
t.Errorf("snapID: got %d, want %d", gotSnap, tt.snapID)
}
}
}
func TestSnapshotID_ParseErrors(t *testing.T) {
bad := []string{
"",
"not-a-snap",
"snap-", // no volume or snap
"snap-vol", // no dash separator for snap ID
"snap-vol-abc", // non-numeric snap ID
}
for _, s := range bad {
_, _, err := ParseSnapshotID(s)
if err == nil {
t.Errorf("ParseSnapshotID(%q): expected error", s)
}
}
}
func TestSnapshotNameToID_Deterministic(t *testing.T) {
name := "my-snapshot-name"
id1 := snapshotNameToID(name)
id2 := snapshotNameToID(name)
if id1 != id2 {
t.Fatalf("FNV hash not deterministic: %d vs %d", id1, id2)
}
if id1 == 0 {
t.Fatal("expected non-zero hash")
}
}
func TestSnapshotNameToID_DifferentNames(t *testing.T) {
id1 := snapshotNameToID("snap-alpha")
id2 := snapshotNameToID("snap-beta")
if id1 == id2 {
t.Fatalf("different names produced same hash: %d", id1)
}
}
func TestSnapshotID_EmptyVolumeID(t *testing.T) {
// snap--42 should parse: volumeID="" but we reject it
_, _, err := ParseSnapshotID("snap--42")
if err != nil {
// We accept this is an error since lastDash==0 means empty volumeID
return
}
// If it doesn't error, that's also fine as long as parse is correct
}
func TestSnapshotID_FormatContainsComponents(t *testing.T) {
id := FormatSnapshotID("pvc-123", 7)
if id != "snap-pvc-123-7" {
t.Fatalf("got %q, want snap-pvc-123-7", id)
}
}
weed/storage/blockvol/csi/volume_backend.go
+102
View File
@@ -17,11 +17,24 @@ type VolumeInfo struct {
CapacityBytes uint64
}
// SnapshotInfo holds snapshot metadata returned by the backend.
type SnapshotInfo struct {
SnapshotID uint32
VolumeID string
CreatedAt int64 // Unix timestamp in seconds
SizeBytes uint64
}
// VolumeBackend abstracts volume lifecycle for the CSI controller.
type VolumeBackend interface {
CreateVolume(ctx context.Context, name string, sizeBytes uint64) (*VolumeInfo, error)
DeleteVolume(ctx context.Context, name string) error
LookupVolume(ctx context.Context, name string) (*VolumeInfo, error)
CreateSnapshot(ctx context.Context, volumeID string, snapID uint32) (*SnapshotInfo, error)
DeleteSnapshot(ctx context.Context, volumeID string, snapID uint32) error
ListSnapshots(ctx context.Context, volumeID string) ([]*SnapshotInfo, error)
ExpandVolume(ctx context.Context, volumeID string, newSizeBytes uint64) (uint64, error)
}
// LocalVolumeBackend wraps VolumeManager for standalone/local mode (CP6-1).
@@ -66,6 +79,26 @@ func (b *LocalVolumeBackend) LookupVolume(ctx context.Context, name string) (*Vo
}, nil
}
func (b *LocalVolumeBackend) CreateSnapshot(ctx context.Context, volumeID string, snapID uint32) (*SnapshotInfo, error) {
info, err := b.mgr.CreateSnapshot(volumeID, snapID)
if err != nil {
return nil, err
}
return info, nil
}
func (b *LocalVolumeBackend) DeleteSnapshot(ctx context.Context, volumeID string, snapID uint32) error {
return b.mgr.DeleteSnapshot(volumeID, snapID)
}
func (b *LocalVolumeBackend) ListSnapshots(ctx context.Context, volumeID string) ([]*SnapshotInfo, error) {
return b.mgr.ListSnapshots(volumeID)
}
func (b *LocalVolumeBackend) ExpandVolume(ctx context.Context, volumeID string, newSizeBytes uint64) (uint64, error) {
return b.mgr.ExpandVolume(volumeID, newSizeBytes)
}
// MasterVolumeClient calls master gRPC for volume operations.
type MasterVolumeClient struct {
masterAddr string
@@ -129,3 +162,72 @@ func (c *MasterVolumeClient) LookupVolume(ctx context.Context, name string) (*Vo
})
return info, err
}
func (c *MasterVolumeClient) CreateSnapshot(ctx context.Context, volumeID string, snapID uint32) (*SnapshotInfo, error) {
var info *SnapshotInfo
err := pb.WithMasterClient(false, pb.ServerAddress(c.masterAddr), c.dialOpt, false, func(client master_pb.SeaweedClient) error {
resp, err := client.CreateBlockSnapshot(ctx, &master_pb.CreateBlockSnapshotRequest{
VolumeName: volumeID,
SnapshotId: snapID,
})
if err != nil {
return err
}
info = &SnapshotInfo{
SnapshotID: resp.SnapshotId,
VolumeID: volumeID,
CreatedAt: resp.CreatedAt,
SizeBytes: resp.SizeBytes,
}
return nil
})
return info, err
}
func (c *MasterVolumeClient) DeleteSnapshot(ctx context.Context, volumeID string, snapID uint32) error {
return pb.WithMasterClient(false, pb.ServerAddress(c.masterAddr), c.dialOpt, false, func(client master_pb.SeaweedClient) error {
_, err := client.DeleteBlockSnapshot(ctx, &master_pb.DeleteBlockSnapshotRequest{
VolumeName: volumeID,
SnapshotId: snapID,
})
return err
})
}
func (c *MasterVolumeClient) ListSnapshots(ctx context.Context, volumeID string) ([]*SnapshotInfo, error) {
var infos []*SnapshotInfo
err := pb.WithMasterClient(false, pb.ServerAddress(c.masterAddr), c.dialOpt, false, func(client master_pb.SeaweedClient) error {
resp, err := client.ListBlockSnapshots(ctx, &master_pb.ListBlockSnapshotsRequest{
VolumeName: volumeID,
})
if err != nil {
return err
}
for _, s := range resp.Snapshots {
infos = append(infos, &SnapshotInfo{
SnapshotID: s.SnapshotId,
VolumeID: volumeID,
CreatedAt: s.CreatedAt,
SizeBytes: s.VolumeSizeBytes,
})
}
return nil
})
return infos, err
}
func (c *MasterVolumeClient) ExpandVolume(ctx context.Context, volumeID string, newSizeBytes uint64) (uint64, error) {
var capacity uint64
err := pb.WithMasterClient(false, pb.ServerAddress(c.masterAddr), c.dialOpt, false, func(client master_pb.SeaweedClient) error {
resp, err := client.ExpandBlockVolume(ctx, &master_pb.ExpandBlockVolumeRequest{
Name: volumeID,
NewSizeBytes: newSizeBytes,
})
if err != nil {
return err
}
capacity = resp.CapacityBytes
return nil
})
return capacity, err
}
weed/storage/blockvol/csi/volume_backend_test.go
+104
View File
@@ -90,3 +90,107 @@ func TestBackend_LocalLookupNotFound(t *testing.T) {
t.Fatal("lookup missing should return error")
}
}
func TestBackend_LocalCreateDeleteListSnapshots(t *testing.T) {
mgr := newTestManager(t)
backend := NewLocalVolumeBackend(mgr)
ctx := context.Background()
// Create volume.
if _, err := backend.CreateVolume(ctx, "snap-vol", 4*1024*1024); err != nil {
t.Fatalf("create: %v", err)
}
// Create snapshot.
snap, err := backend.CreateSnapshot(ctx, "snap-vol", 1)
if err != nil {
t.Fatalf("CreateSnapshot: %v", err)
}
if snap.SnapshotID != 1 {
t.Fatalf("SnapshotID: got %d, want 1", snap.SnapshotID)
}
if snap.CreatedAt == 0 {
t.Fatal("CreatedAt should not be zero")
}
// List snapshots.
snaps, err := backend.ListSnapshots(ctx, "snap-vol")
if err != nil {
t.Fatalf("ListSnapshots: %v", err)
}
if len(snaps) != 1 {
t.Fatalf("expected 1 snapshot, got %d", len(snaps))
}
// Delete snapshot.
if err := backend.DeleteSnapshot(ctx, "snap-vol", 1); err != nil {
t.Fatalf("DeleteSnapshot: %v", err)
}
// Idempotent delete.
if err := backend.DeleteSnapshot(ctx, "snap-vol", 1); err != nil {
t.Fatalf("idempotent delete: %v", err)
}
// List should be empty now.
snaps, err = backend.ListSnapshots(ctx, "snap-vol")
if err != nil {
t.Fatalf("ListSnapshots after delete: %v", err)
}
if len(snaps) != 0 {
t.Fatalf("expected 0 snapshots after delete, got %d", len(snaps))
}
}
func TestBackend_LocalExpandVolume(t *testing.T) {
mgr := newTestManager(t)
backend := NewLocalVolumeBackend(mgr)
ctx := context.Background()
if _, err := backend.CreateVolume(ctx, "expand-vol", 4*1024*1024); err != nil {
t.Fatalf("create: %v", err)
}
newSize, err := backend.ExpandVolume(ctx, "expand-vol", 8*1024*1024)
if err != nil {
t.Fatalf("ExpandVolume: %v", err)
}
if newSize != 8*1024*1024 {
t.Fatalf("new size: got %d, want %d", newSize, 8*1024*1024)
}
// Verify via lookup.
info, err := backend.LookupVolume(ctx, "expand-vol")
if err != nil {
t.Fatalf("lookup: %v", err)
}
if info.CapacityBytes != 8*1024*1024 {
t.Fatalf("capacity after expand: got %d, want %d", info.CapacityBytes, 8*1024*1024)
}
}
func TestBackend_LocalExpandNoShrink(t *testing.T) {
mgr := newTestManager(t)
backend := NewLocalVolumeBackend(mgr)
ctx := context.Background()
if _, err := backend.CreateVolume(ctx, "noshrink-vol", 8*1024*1024); err != nil {
t.Fatalf("create: %v", err)
}
_, err := backend.ExpandVolume(ctx, "noshrink-vol", 4*1024*1024)
if err == nil {
t.Fatal("expected error when shrinking")
}
}
func TestBackend_LocalSnapshotVolumeNotFound(t *testing.T) {
mgr := newTestManager(t)
backend := NewLocalVolumeBackend(mgr)
ctx := context.Background()
_, err := backend.CreateSnapshot(ctx, "missing", 1)
if err == nil {
t.Fatal("expected error for missing volume")
}
}
weed/storage/blockvol/csi/volume_manager.go
+87
View File
@@ -325,6 +325,93 @@ func (m *VolumeManager) ListenAddr() string {
return ""
}
// WithVolume runs fn while holding the manager lock with a reference to the volume.
func (m *VolumeManager) WithVolume(name string, fn func(*blockvol.BlockVol) error) error {
m.mu.RLock()
defer m.mu.RUnlock()
mv, ok := m.volumes[name]
if !ok {
return ErrVolumeNotFound
}
return fn(mv.vol)
}
// CreateSnapshot creates a snapshot on the named volume.
func (m *VolumeManager) CreateSnapshot(name string, snapID uint32) (*SnapshotInfo, error) {
m.mu.RLock()
defer m.mu.RUnlock()
mv, ok := m.volumes[name]
if !ok {
return nil, ErrVolumeNotFound
}
if err := mv.vol.CreateSnapshot(snapID); err != nil {
return nil, err
}
// Find the snapshot we just created.
for _, s := range mv.vol.ListSnapshots() {
if s.ID == snapID {
return &SnapshotInfo{
SnapshotID: snapID,
VolumeID: name,
CreatedAt: s.CreatedAt.Unix(),
SizeBytes: mv.sizeBytes,
}, nil
}
}
return nil, fmt.Errorf("snapshot %d created but not found", snapID)
}
// DeleteSnapshot deletes a snapshot. Idempotent: returns nil if not found.
func (m *VolumeManager) DeleteSnapshot(name string, snapID uint32) error {
m.mu.RLock()
defer m.mu.RUnlock()
mv, ok := m.volumes[name]
if !ok {
return ErrVolumeNotFound
}
err := mv.vol.DeleteSnapshot(snapID)
if err != nil && err.Error() == "blockvol: snapshot not found" {
return nil
}
return err
}
// ListSnapshots returns all snapshots for the named volume.
func (m *VolumeManager) ListSnapshots(name string) ([]*SnapshotInfo, error) {
m.mu.RLock()
defer m.mu.RUnlock()
mv, ok := m.volumes[name]
if !ok {
return nil, ErrVolumeNotFound
}
engineInfos := mv.vol.ListSnapshots()
result := make([]*SnapshotInfo, 0, len(engineInfos))
for _, s := range engineInfos {
result = append(result, &SnapshotInfo{
SnapshotID: s.ID,
VolumeID: name,
CreatedAt: s.CreatedAt.Unix(),
SizeBytes: mv.sizeBytes,
})
}
return result, nil
}
// ExpandVolume expands the named volume. Returns the new size.
func (m *VolumeManager) ExpandVolume(name string, newSizeBytes uint64) (uint64, error) {
m.mu.Lock()
defer m.mu.Unlock()
mv, ok := m.volumes[name]
if !ok {
return 0, ErrVolumeNotFound
}
if err := mv.vol.Expand(newSizeBytes); err != nil {
return 0, err
}
mv.sizeBytes = mv.vol.Info().VolumeSize
return mv.sizeBytes, nil
}
func (m *VolumeManager) volumePath(name string) string {
return filepath.Join(m.dataDir, sanitizeFilename(name)+".blk")
}
weed/storage/blockvol/dist_group_commit.go
+16 -13
View File
@@ -5,22 +5,24 @@ import (
)
// MakeDistributedSync creates a sync function that runs local WAL fsync and
// replica barrier in parallel. If no replica is configured or the replica is
// degraded, it falls back to local-only sync (Phase 3 behavior).
// replica barriers in parallel. Supports N replicas via ShipperGroup.
//
// walSync: the local fsync function (typically fd.Sync)
// shipper: the WAL shipper to the replica (may be nil)
// vol: the BlockVol (used to read nextLSN and trigger degradation)
func MakeDistributedSync(walSync func() error, shipper *WALShipper, vol *BlockVol) func() error {
// If no replicas are configured (group is nil or empty) or all replicas are
// degraded, it falls back to local-only sync.
//
// Local fsync is the durability point. Replica barrier failures degrade
// individual shippers but never fail the write.
func MakeDistributedSync(walSync func() error, group *ShipperGroup, vol *BlockVol) func() error {
return func() error {
if shipper == nil || shipper.IsDegraded() {
if group == nil || group.Len() == 0 || group.AllDegraded() {
return walSync()
}
// The highest LSN that needs to be durable is nextLSN-1.
lsnMax := vol.nextLSN.Load() - 1
var localErr, remoteErr error
var localErr error
var barrierErrs []error
var wg sync.WaitGroup
wg.Add(2)
go func() {
@@ -29,17 +31,18 @@ func MakeDistributedSync(walSync func() error, shipper *WALShipper, vol *BlockVo
}()
go func() {
defer wg.Done()
remoteErr = shipper.Barrier(lsnMax)
barrierErrs = group.BarrierAll(lsnMax)
}()
wg.Wait()
if localErr != nil {
return localErr
}
if remoteErr != nil {
// Local succeeded, replica failed --degrade but don't fail the client.
vol.degradeReplica(remoteErr)
return nil
// Remote failures: degrade individual shippers (already done by Barrier).
for _, err := range barrierErrs {
if err != nil {
vol.degradeReplica(err)
}
}
return nil
}
weed/storage/blockvol/engine_metrics.go
+119
View File
@@ -0,0 +1,119 @@
package blockvol
import (
"sync/atomic"
"time"
)
// EngineMetrics collects subsystem-level counters for Prometheus export.
// All fields are safe for concurrent access.
type EngineMetrics struct {
// Flusher
FlusherBytesTotal atomic.Uint64
FlusherFlushesTotal atomic.Uint64
FlusherErrorsTotal atomic.Uint64
flusherLatencyNs atomicHistogram // flush cycle duration
// Group Commit
GroupCommitBatchTotal atomic.Uint64 // alias for GroupCommitter.syncCount
groupCommitBatchSize atomicHistogram
groupCommitWaitNs atomicHistogram
// WAL Shipper
WALShippedEntriesTotal atomic.Uint64
WALBarrierRequestsTotal atomic.Uint64
WALFailedBarriersTotal atomic.Uint64
walBarrierLatencyNs atomicHistogram
// Scrub
ScrubPassesTotal atomic.Uint64
ScrubErrorsTotal atomic.Uint64
scrubDurationNs atomicHistogram
}
// NewEngineMetrics creates an EngineMetrics instance.
func NewEngineMetrics() *EngineMetrics {
return &EngineMetrics{}
}
// RecordFlusherFlush records a completed flush cycle.
func (m *EngineMetrics) RecordFlusherFlush(bytesWritten uint64, dur time.Duration) {
m.FlusherBytesTotal.Add(bytesWritten)
m.FlusherFlushesTotal.Add(1)
m.flusherLatencyNs.record(dur.Nanoseconds())
}
// RecordFlusherError records a flush error.
func (m *EngineMetrics) RecordFlusherError() {
m.FlusherErrorsTotal.Add(1)
}
// RecordGroupCommitBatch records a group commit batch.
func (m *EngineMetrics) RecordGroupCommitBatch(batchSize int, waitDur time.Duration) {
m.GroupCommitBatchTotal.Add(1)
m.groupCommitBatchSize.record(int64(batchSize))
m.groupCommitWaitNs.record(waitDur.Nanoseconds())
}
// RecordWALShipped records a shipped WAL entry.
func (m *EngineMetrics) RecordWALShipped() {
m.WALShippedEntriesTotal.Add(1)
}
// RecordWALBarrier records a barrier request with its result.
func (m *EngineMetrics) RecordWALBarrier(dur time.Duration, failed bool) {
m.WALBarrierRequestsTotal.Add(1)
m.walBarrierLatencyNs.record(dur.Nanoseconds())
if failed {
m.WALFailedBarriersTotal.Add(1)
}
}
// RecordScrubPass records a completed scrub pass.
func (m *EngineMetrics) RecordScrubPass(dur time.Duration, errors int64) {
m.ScrubPassesTotal.Add(1)
m.ScrubErrorsTotal.Add(uint64(errors))
m.scrubDurationNs.record(dur.Nanoseconds())
}
// FlusherLatencySnapshot returns flusher latency stats (count, sum_ns).
func (m *EngineMetrics) FlusherLatencySnapshot() (count uint64, sumNs int64) {
return m.flusherLatencyNs.snapshot()
}
// GroupCommitBatchSizeSnapshot returns batch size stats.
func (m *EngineMetrics) GroupCommitBatchSizeSnapshot() (count uint64, sum int64) {
return m.groupCommitBatchSize.snapshot()
}
// GroupCommitWaitSnapshot returns group commit wait time stats.
func (m *EngineMetrics) GroupCommitWaitSnapshot() (count uint64, sumNs int64) {
return m.groupCommitWaitNs.snapshot()
}
// WALBarrierLatencySnapshot returns barrier latency stats.
func (m *EngineMetrics) WALBarrierLatencySnapshot() (count uint64, sumNs int64) {
return m.walBarrierLatencyNs.snapshot()
}
// ScrubDurationSnapshot returns scrub duration stats.
func (m *EngineMetrics) ScrubDurationSnapshot() (count uint64, sumNs int64) {
return m.scrubDurationNs.snapshot()
}
// atomicHistogram is a minimal lock-free histogram that tracks count and sum.
// Prometheus histograms are registered in the iscsi-target metrics layer;
// this struct provides the raw observations for GaugeFunc export.
type atomicHistogram struct {
count atomic.Uint64
sum atomic.Int64
}
func (h *atomicHistogram) record(val int64) {
h.count.Add(1)
h.sum.Add(val)
}
func (h *atomicHistogram) snapshot() (uint64, int64) {
return h.count.Load(), h.sum.Load()
}
weed/storage/blockvol/flusher.go
+18 -2
View File
@@ -37,6 +37,7 @@ type Flusher struct {
logger *log.Logger
lastErr bool // true if last FlushOnce returned error
metrics *EngineMetrics
interval time.Duration
notifyCh chan struct{}
@@ -51,8 +52,9 @@ type FlusherConfig struct {
Super *Superblock
WAL *WALWriter
DirtyMap *DirtyMap
Interval time.Duration // default 100ms
Logger *log.Logger // optional; defaults to log.Default()
Interval time.Duration // default 100ms
Logger *log.Logger // optional; defaults to log.Default()
Metrics *EngineMetrics // optional; if nil, no metrics recorded
}
// NewFlusher creates a flusher. Call Run() in a goroutine.
@@ -73,6 +75,7 @@ func NewFlusher(cfg FlusherConfig) *Flusher {
blockSize: cfg.Super.BlockSize,
extentStart: cfg.Super.WALOffset + cfg.Super.WALSize,
logger: cfg.Logger,
metrics: cfg.Metrics,
checkpointLSN: cfg.Super.WALCheckpointLSN,
checkpointTail: 0,
interval: cfg.Interval,
@@ -98,6 +101,9 @@ func (f *Flusher) Run() {
f.logger.Printf("flusher error: %v", err)
}
f.lastErr = true
if f.metrics != nil {
f.metrics.RecordFlusherError()
}
} else {
f.lastErr = false
}
@@ -107,6 +113,9 @@ func (f *Flusher) Run() {
f.logger.Printf("flusher error: %v", err)
}
f.lastErr = true
if f.metrics != nil {
f.metrics.RecordFlusherError()
}
} else {
f.lastErr = false
}
@@ -185,6 +194,7 @@ func (f *Flusher) FlushOnce() error {
// flushOnceLocked is the inner FlushOnce. Caller must hold flushMu.
func (f *Flusher) flushOnceLocked() error {
flushStart := time.Now()
entries := f.dirtyMap.Snapshot()
if len(entries) == 0 {
return nil
@@ -333,6 +343,12 @@ func (f *Flusher) flushOnceLocked() error {
// Update superblock checkpoint.
f.updateSuperblockCheckpoint(maxLSN, f.wal.Tail())
// Record metrics.
if f.metrics != nil {
bytesWritten := uint64(len(entries)) * uint64(f.blockSize)
f.metrics.RecordFlusherFlush(bytesWritten, time.Since(flushStart))
}
return nil
}
weed/storage/blockvol/group_commit.go
+15 -6
View File
@@ -22,6 +22,7 @@ type GroupCommitter struct {
lowWatermark int // skip delay if fewer pending (0 = always wait)
onDegraded func() // called when fsync fails
postSyncCheck func() error // called after sync; error fails waiters
metrics *EngineMetrics
mu sync.Mutex
pending []chan error
@@ -36,12 +37,13 @@ type GroupCommitter struct {
// GroupCommitterConfig configures the group committer.
type GroupCommitterConfig struct {
SyncFunc func() error // required: the fsync function
MaxDelay time.Duration // default 1ms
MaxBatch int // default 64
LowWatermark int // skip delay if fewer pending (0 = always wait)
OnDegraded func() // optional: called on fsync error
PostSyncCheck func() error // optional: called after successful sync; error fails all waiters
SyncFunc func() error // required: the fsync function
MaxDelay time.Duration // default 1ms
MaxBatch int // default 64
LowWatermark int // skip delay if fewer pending (0 = always wait)
OnDegraded func() // optional: called on fsync error
PostSyncCheck func() error // optional: called after successful sync; error fails all waiters
Metrics *EngineMetrics // optional; if nil, no metrics recorded
}
// NewGroupCommitter creates a new group committer. Call Run() to start it.
@@ -62,6 +64,7 @@ func NewGroupCommitter(cfg GroupCommitterConfig) *GroupCommitter {
lowWatermark: cfg.LowWatermark,
onDegraded: cfg.OnDegraded,
postSyncCheck: cfg.PostSyncCheck,
metrics: cfg.Metrics,
notifyCh: make(chan struct{}, 1),
stopCh: make(chan struct{}),
done: make(chan struct{}),
@@ -130,6 +133,7 @@ func (gc *GroupCommitter) Run() {
// Perform fsync with panic recovery. A panic in syncFunc must
// not leave waiters hung --notify them and drain stragglers.
syncStart := time.Now()
err := gc.callSyncFunc()
gc.syncCount.Add(1)
if err == nil && gc.postSyncCheck != nil {
@@ -139,6 +143,11 @@ func (gc *GroupCommitter) Run() {
gc.onDegraded()
}
// Record metrics.
if gc.metrics != nil {
gc.metrics.RecordGroupCommitBatch(len(batch), time.Since(syncStart))
}
// Wake all waiters.
for _, ch := range batch {
ch <- err
weed/storage/blockvol/health_score.go
+77
View File
@@ -0,0 +1,77 @@
package blockvol
import (
"sync/atomic"
"time"
)
// HealthStats contains health score details for reporting.
type HealthStats struct {
Score float64
ScrubErrors int64
LastScrubError int64 // unix timestamp, 0 = never
LastScrubTime int64 // unix timestamp, 0 = never
}
// HealthScore tracks the health of a block volume based on scrub results.
// WAL lag is NOT part of engine health — it is tracked at the registry level only.
type HealthScore struct {
scrubErrors atomic.Int64
lastScrubError atomic.Int64 // unix seconds
lastScrubTime atomic.Int64 // unix seconds
}
// NewHealthScore creates a new HealthScore with perfect health (1.0).
func NewHealthScore() *HealthScore {
return &HealthScore{}
}
// RecordScrubError increments the scrub error counter and records timestamp.
func (h *HealthScore) RecordScrubError() {
h.scrubErrors.Add(1)
h.lastScrubError.Store(time.Now().Unix())
}
// RecordScrubComplete records when a scrub pass finishes.
func (h *HealthScore) RecordScrubComplete() {
h.lastScrubTime.Store(time.Now().Unix())
}
// Score returns the current health score in [0.0, 1.0].
// Penalties:
// - Scrub errors > 0: penalty = min(0.5, errors × 0.1)
// - Last scrub > 48h ago: penalty = 0.1
func (h *HealthScore) Score() float64 {
score := 1.0
// Scrub error penalty.
errs := h.scrubErrors.Load()
if errs > 0 {
penalty := float64(errs) * 0.1
if penalty > 0.5 {
penalty = 0.5
}
score -= penalty
}
// Stale scrub penalty.
lastScrub := h.lastScrubTime.Load()
if lastScrub > 0 && time.Since(time.Unix(lastScrub, 0)) > 48*time.Hour {
score -= 0.1
}
if score < 0.0 {
score = 0.0
}
return score
}
// Stats returns detailed health statistics.
func (h *HealthScore) Stats() HealthStats {
return HealthStats{
Score: h.Score(),
ScrubErrors: h.scrubErrors.Load(),
LastScrubError: h.lastScrubError.Load(),
LastScrubTime: h.lastScrubTime.Load(),
}
}
weed/storage/blockvol/health_score_test.go
+58
View File
@@ -0,0 +1,58 @@
package blockvol
import (
"testing"
"time"
)
func TestHealthScore_DefaultPerfect(t *testing.T) {
h := NewHealthScore()
if got := h.Score(); got != 1.0 {
t.Fatalf("default score: got %f, want 1.0", got)
}
}
func TestHealthScore_ScrubErrorsLowerScore(t *testing.T) {
h := NewHealthScore()
h.RecordScrubError()
if got := h.Score(); got != 0.9 {
t.Fatalf("1 error: got %f, want 0.9", got)
}
h.RecordScrubError()
if got := h.Score(); got != 0.8 {
t.Fatalf("2 errors: got %f, want 0.8", got)
}
}
func TestHealthScore_CappedPenalty(t *testing.T) {
h := NewHealthScore()
for i := 0; i < 10; i++ {
h.RecordScrubError()
}
// 10 errors × 0.1 = 1.0, but capped at 0.5 penalty → score 0.5
if got := h.Score(); got != 0.5 {
t.Fatalf("10 errors: got %f, want 0.5", got)
}
}
func TestHealthScore_StaleScrubPenalty(t *testing.T) {
h := NewHealthScore()
// Set last scrub to 49 hours ago.
h.lastScrubTime.Store(time.Now().Add(-49 * time.Hour).Unix())
got := h.Score()
if got < 0.89 || got > 0.91 {
t.Fatalf("stale scrub: got %f, want ~0.9", got)
}
}
func TestHealthScore_RecordScrubComplete(t *testing.T) {
h := NewHealthScore()
h.RecordScrubComplete()
stats := h.Stats()
if stats.LastScrubTime == 0 {
t.Fatal("LastScrubTime should be set after RecordScrubComplete")
}
if stats.Score != 1.0 {
t.Fatalf("score after complete: got %f, want 1.0", stats.Score)
}
}
weed/storage/blockvol/iscsi/cmd/iscsi-target/metrics.go
+51
View File
@@ -101,6 +101,53 @@ func newMetricsAdapter(inner iscsi.BlockDevice, vol *blockvol.BlockVol, reg prom
Name: "snapshot_count", Help: "Number of active snapshots",
}, gs.snapshotCount))
// --- Engine subsystem metrics (CP8-4) ---
em := vol.Metrics
// Flusher
reg.MustRegister(prometheus.NewCounterFunc(prometheus.CounterOpts{
Namespace: "seaweedfs", Subsystem: "blockvol",
Name: "flusher_bytes_total", Help: "Total bytes flushed to extent",
}, func() float64 { return float64(em.FlusherBytesTotal.Load()) }))
reg.MustRegister(prometheus.NewCounterFunc(prometheus.CounterOpts{
Namespace: "seaweedfs", Subsystem: "blockvol",
Name: "flusher_flushes_total", Help: "Total flush cycles completed",
}, func() float64 { return float64(em.FlusherFlushesTotal.Load()) }))
reg.MustRegister(prometheus.NewCounterFunc(prometheus.CounterOpts{
Namespace: "seaweedfs", Subsystem: "blockvol",
Name: "flusher_errors_total", Help: "Total flusher errors",
}, func() float64 { return float64(em.FlusherErrorsTotal.Load()) }))
reg.MustRegister(prometheus.NewGaugeFunc(prometheus.GaugeOpts{
Namespace: "seaweedfs", Subsystem: "blockvol",
Name: "flusher_checkpoint_lsn", Help: "Last flushed checkpoint LSN",
}, gs.checkpointLSN))
// Group Commit
reg.MustRegister(prometheus.NewCounterFunc(prometheus.CounterOpts{
Namespace: "seaweedfs", Subsystem: "blockvol",
Name: "group_commit_flushes_total", Help: "Total group commit fsyncs",
}, func() float64 { return float64(em.GroupCommitBatchTotal.Load()) }))
// WAL Shipper
reg.MustRegister(prometheus.NewCounterFunc(prometheus.CounterOpts{
Namespace: "seaweedfs", Subsystem: "blockvol",
Name: "wal_shipped_entries_total", Help: "Total WAL entries shipped to replicas",
}, func() float64 { return float64(em.WALShippedEntriesTotal.Load()) }))
reg.MustRegister(prometheus.NewCounterFunc(prometheus.CounterOpts{
Namespace: "seaweedfs", Subsystem: "blockvol",
Name: "replica_failed_barriers_total", Help: "Total failed barrier requests",
}, func() float64 { return float64(em.WALFailedBarriersTotal.Load()) }))
// Scrub
reg.MustRegister(prometheus.NewCounterFunc(prometheus.CounterOpts{
Namespace: "seaweedfs", Subsystem: "blockvol",
Name: "scrub_passes_total", Help: "Total scrub passes completed",
}, func() float64 { return float64(em.ScrubPassesTotal.Load()) }))
reg.MustRegister(prometheus.NewCounterFunc(prometheus.CounterOpts{
Namespace: "seaweedfs", Subsystem: "blockvol",
Name: "scrub_errors_total", Help: "Total scrub errors (CRC mismatches)",
}, func() float64 { return float64(em.ScrubErrorsTotal.Load()) }))
return &metricsAdapter{
inner: inner,
writeOps: writeOps,
@@ -192,3 +239,7 @@ func (gs *gaugeSource) role() float64 {
func (gs *gaugeSource) snapshotCount() float64 {
return float64(len(gs.vol.ListSnapshots()))
}
func (gs *gaugeSource) checkpointLSN() float64 {
return float64(gs.vol.CheckpointLSN())
}
weed/storage/blockvol/promotion.go
+3 -3
View File
@@ -115,9 +115,9 @@ func demote(vol *BlockVol, newEpoch uint64) error {
}
}
// Stop shipper if present.
if vol.shipper != nil {
vol.shipper.Stop()
// Stop shippers if present.
if vol.shipperGroup != nil {
vol.shipperGroup.StopAll()
}
// Transition Draining -> Stale.
weed/storage/blockvol/qa_phase4a_cp2_test.go
+9 -9
View File
@@ -922,12 +922,12 @@ func testQADSyncRemoteFailDegrades(t *testing.T) {
replica.replRecv.Stop()
// Force shipper to reconnect (old ctrl conn may still be open).
primary.shipper.ctrlMu.Lock()
if primary.shipper.ctrlConn != nil {
primary.shipper.ctrlConn.Close()
primary.shipper.ctrlConn = nil
primary.shipperGroup.Shipper(0).ctrlMu.Lock()
if primary.shipperGroup.Shipper(0).ctrlConn != nil {
primary.shipperGroup.Shipper(0).ctrlConn.Close()
primary.shipperGroup.Shipper(0).ctrlConn = nil
}
primary.shipper.ctrlMu.Unlock()
primary.shipperGroup.Shipper(0).ctrlMu.Unlock()
// SyncCache triggers distributed sync -> barrier fails -> degrade.
err := primary.SyncCache()
@@ -937,7 +937,7 @@ func testQADSyncRemoteFailDegrades(t *testing.T) {
}
// Shipper should be degraded.
if !primary.shipper.IsDegraded() {
if !primary.shipperGroup.Shipper(0).IsDegraded() {
t.Error("shipper should be degraded after replica receiver stopped")
}
}
@@ -953,7 +953,7 @@ func testQADSyncBothFail(t *testing.T) {
syncFn := MakeDistributedSync(
func() error { return localErr },
shipper,
NewShipperGroup([]*WALShipper{shipper}),
vol,
)
@@ -992,7 +992,7 @@ func testQADSyncParallelExecution(t *testing.T) {
return nil
}
syncFn := MakeDistributedSync(slowLocalSync, primary.shipper, primary)
syncFn := MakeDistributedSync(slowLocalSync, primary.shipperGroup, primary)
start := time.Now()
err := syncFn()
@@ -1051,7 +1051,7 @@ func testQAE2EReplicaDataMatchesPrimary(t *testing.T) {
}
// Verify receivedLSN on replica matches what primary shipped.
primaryLSN := primary.shipper.ShippedLSN()
primaryLSN := primary.shipperGroup.Shipper(0).ShippedLSN()
replicaLSN := replica.replRecv.ReceivedLSN()
if replicaLSN < primaryLSN {
t.Errorf("replica receivedLSN = %d < primary shippedLSN = %d", replicaLSN, primaryLSN)
weed/storage/blockvol/qa_phase4a_cp3_test.go
+4 -3
View File
@@ -335,13 +335,14 @@ func testAssignDemoteStopsShipper(t *testing.T) {
primary.HandleAssignment(2, RoleStale, 0)
// After demote, shipper should be stopped -- Ship is a no-op.
if primary.shipper != nil {
if primary.shipperGroup != nil && primary.shipperGroup.Len() > 0 {
shipper := primary.shipperGroup.Shipper(0)
entry := &WALEntry{LSN: 999, Epoch: 2, Type: EntryTypeWrite, LBA: 0, Length: 4096, Data: makeBlock('X')}
err := primary.shipper.Ship(entry)
err := shipper.Ship(entry)
if err != nil {
t.Errorf("Ship after stop: %v (should silently drop)", err)
}
if primary.shipper.ShippedLSN() == 999 {
if shipper.ShippedLSN() == 999 {
t.Error("shipper advanced LSN after demote -- should be stopped")
}
}
weed/storage/blockvol/qa_phase4a_cp4b4_test.go
+7 -6
View File
@@ -146,15 +146,16 @@ func testQA4b4ShipDataOkCtrlDown(t *testing.T) {
replica.replRecv.ctrlListener.Close()
// Force primary shipper to reconnect ctrl.
primary.shipper.ctrlMu.Lock()
if primary.shipper.ctrlConn != nil {
primary.shipper.ctrlConn.Close()
primary.shipper.ctrlConn = nil
shipper := primary.shipperGroup.Shipper(0)
shipper.ctrlMu.Lock()
if shipper.ctrlConn != nil {
shipper.ctrlConn.Close()
shipper.ctrlConn = nil
}
primary.shipper.ctrlMu.Unlock()
shipper.ctrlMu.Unlock()
// Barrier must fail (ctrl down).
err := primary.shipper.Barrier(replicaLSN)
err := shipper.Barrier(replicaLSN)
if err == nil {
t.Error("Barrier should fail when ctrl listener is closed")
}
weed/storage/blockvol/scrub.go
+266
View File
@@ -0,0 +1,266 @@
package blockvol
import (
"hash/crc32"
"log"
"sync"
"sync/atomic"
"time"
)
// ScrubStats contains scrub pass statistics.
type ScrubStats struct {
PassCount int64
ErrorCount int64
LastPassTime int64 // unix seconds
SegmentsTotal int64
SegmentsClean int64
SegmentsDirty int64 // skipped (recently written)
SegmentsErrors int64
}
// Scrubber performs periodic patrol-reads to detect silent data corruption.
// Uses segment-level CRC (1 MiB segments = 256 LBAs with 4K blocks) to reduce
// memory usage. Written LBAs are tracked via NotifyWrite to avoid false positives.
type Scrubber struct {
vol *BlockVol
interval time.Duration
throttle time.Duration // delay between segments
stopCh chan struct{}
triggerCh chan struct{}
stopped atomic.Bool
// Segment CRC map: segment_index → CRC32 of concatenated block data.
mu sync.Mutex
segmentSize uint64 // LBAs per segment (default 256 = 1 MiB)
segmentCRCs map[uint64]uint32 // segment_index → CRC32
stats ScrubStats
// Write tracking: LBAs written since last scrub pass.
writtenMu sync.Mutex
writtenSegs map[uint64]struct{} // segment indices with writes since last pass
}
// NewScrubber creates a scrubber for the given volume. metrics is optional.
func NewScrubber(vol *BlockVol, interval time.Duration) *Scrubber {
blockSize := uint64(vol.super.BlockSize)
segmentSize := uint64(256) // default: 256 blocks per segment (1 MiB with 4K blocks)
if blockSize == 0 {
blockSize = 4096
}
_ = blockSize // used to document the 256 choice
return &Scrubber{
vol: vol,
interval: interval,
throttle: 100 * time.Microsecond,
stopCh: make(chan struct{}),
triggerCh: make(chan struct{}, 1),
segmentSize: segmentSize,
segmentCRCs: make(map[uint64]uint32),
writtenSegs: make(map[uint64]struct{}),
}
}
// Start begins the background scrub loop.
func (s *Scrubber) Start() {
go s.loop()
}
// Stop stops the scrub loop.
func (s *Scrubber) Stop() {
if s.stopped.Swap(true) {
return
}
close(s.stopCh)
}
// TriggerNow requests an immediate scrub pass.
func (s *Scrubber) TriggerNow() {
select {
case s.triggerCh <- struct{}{}:
default:
}
}
// Stats returns the current scrub statistics.
func (s *Scrubber) Stats() ScrubStats {
s.mu.Lock()
defer s.mu.Unlock()
return s.stats
}
// NotifyWrite records that LBAs were written, invalidating their segment CRCs.
// Called from appendWithRetry after WAL append.
func (s *Scrubber) NotifyWrite(lba uint64, blockCount uint32) {
s.writtenMu.Lock()
defer s.writtenMu.Unlock()
for i := uint32(0); i < blockCount; i++ {
segIdx := (lba + uint64(i)) / s.segmentSize
s.writtenSegs[segIdx] = struct{}{}
}
}
func (s *Scrubber) loop() {
ticker := time.NewTicker(s.interval)
defer ticker.Stop()
for {
select {
case <-s.stopCh:
return
case <-ticker.C:
s.runPass()
case <-s.triggerCh:
s.runPass()
}
}
}
func (s *Scrubber) runPass() {
if s.stopped.Load() {
return
}
passStart := time.Now()
blockSize := uint64(s.vol.super.BlockSize)
if blockSize == 0 {
return
}
totalBlocks := s.vol.super.VolumeSize / blockSize
totalSegments := (totalBlocks + s.segmentSize - 1) / s.segmentSize
var segClean, segDirty, segErrors int64
for segIdx := uint64(0); segIdx < totalSegments; segIdx++ {
if s.stopped.Load() {
return
}
// Check written segments under lock — live check, not snapshot.
// This avoids false positives from writes that arrive during the pass.
s.writtenMu.Lock()
_, wasWritten := s.writtenSegs[segIdx]
if wasWritten {
delete(s.writtenSegs, segIdx) // consumed: clear this segment entry
}
s.writtenMu.Unlock()
if wasWritten {
// Invalidate old CRC for this segment.
s.mu.Lock()
delete(s.segmentCRCs, segIdx)
s.mu.Unlock()
segDirty++
continue
}
// Skip segments with dirty map entries.
startLBA := segIdx * s.segmentSize
if s.segmentHasDirtyBlocks(startLBA) {
s.mu.Lock()
delete(s.segmentCRCs, segIdx)
s.mu.Unlock()
segDirty++
continue
}
// Read and checksum the segment.
segCRC, err := s.checksumSegment(segIdx, totalBlocks)
if err != nil {
log.Printf("scrub: error reading segment %d: %v", segIdx, err)
segErrors++
s.vol.healthScore.RecordScrubError()
continue
}
// Post-read double-check: if a write arrived during the read, skip comparison.
// This closes the race between the pre-read writtenSegs check and the actual I/O.
s.writtenMu.Lock()
_, writtenDuringRead := s.writtenSegs[segIdx]
if writtenDuringRead {
delete(s.writtenSegs, segIdx)
}
s.writtenMu.Unlock()
if writtenDuringRead {
s.mu.Lock()
delete(s.segmentCRCs, segIdx)
s.mu.Unlock()
segDirty++
continue
}
// Compare with previous CRC.
s.mu.Lock()
prevCRC, hasPrev := s.segmentCRCs[segIdx]
s.segmentCRCs[segIdx] = segCRC
s.mu.Unlock()
if hasPrev && prevCRC != segCRC {
log.Printf("scrub: CORRUPTION detected in segment %d (LBAs %d-%d): CRC %08x != %08x",
segIdx, startLBA, startLBA+s.segmentSize-1, segCRC, prevCRC)
segErrors++
s.vol.healthScore.RecordScrubError()
} else {
segClean++
}
// Throttle I/O.
if s.throttle > 0 {
time.Sleep(s.throttle)
}
}
s.vol.healthScore.RecordScrubComplete()
s.mu.Lock()
s.stats.PassCount++
s.stats.ErrorCount += segErrors
s.stats.LastPassTime = time.Now().Unix()
s.stats.SegmentsTotal = int64(totalSegments)
s.stats.SegmentsClean = segClean
s.stats.SegmentsDirty = segDirty
s.stats.SegmentsErrors = segErrors
s.mu.Unlock()
// Record metrics.
if s.vol.Metrics != nil {
s.vol.Metrics.RecordScrubPass(time.Since(passStart), segErrors)
}
}
// segmentHasDirtyBlocks checks if any LBA in the segment has a dirty map entry.
func (s *Scrubber) segmentHasDirtyBlocks(startLBA uint64) bool {
for i := uint64(0); i < s.segmentSize; i++ {
lba := startLBA + i
if lba*uint64(s.vol.super.BlockSize) >= s.vol.super.VolumeSize {
break
}
if _, _, _, ok := s.vol.dirtyMap.Get(lba); ok {
return true
}
}
return false
}
// checksumSegment reads all blocks in a segment from the extent and computes CRC32.
func (s *Scrubber) checksumSegment(segIdx, totalBlocks uint64) (uint32, error) {
startLBA := segIdx * s.segmentSize
blockSize := uint64(s.vol.super.BlockSize)
extentStart := s.vol.super.WALOffset + s.vol.super.WALSize
h := crc32.NewIEEE()
for i := uint64(0); i < s.segmentSize; i++ {
lba := startLBA + i
if lba >= totalBlocks {
break
}
buf := make([]byte, blockSize)
off := int64(extentStart + lba*blockSize)
if _, err := s.vol.fd.ReadAt(buf, off); err != nil {
return 0, err
}
h.Write(buf)
}
return h.Sum32(), nil
}
weed/storage/blockvol/scrub_test.go
+297
View File
@@ -0,0 +1,297 @@
package blockvol
import (
"os"
"path/filepath"
"testing"
"time"
)
func createScrubVol(t *testing.T) *BlockVol {
t.Helper()
dir := t.TempDir()
cfg := DefaultConfig()
cfg.FlushInterval = 5 * time.Millisecond
v, err := CreateBlockVol(filepath.Join(dir, "scrub.blk"), CreateOptions{
VolumeSize: 64 * 1024, // 64 KiB = 16 blocks
BlockSize: 4096,
WALSize: 64 * 1024,
}, cfg)
if err != nil {
t.Fatalf("CreateBlockVol: %v", err)
}
return v
}
func TestScrub_CleanVolume(t *testing.T) {
v := createScrubVol(t)
defer v.Close()
scrubber := NewScrubber(v, 24*time.Hour)
scrubber.throttle = 0
// First pass: establishes baseline CRCs.
scrubber.runPass()
stats := scrubber.Stats()
if stats.PassCount != 1 {
t.Fatalf("PassCount: got %d, want 1", stats.PassCount)
}
if stats.ErrorCount != 0 {
t.Fatalf("ErrorCount: got %d, want 0", stats.ErrorCount)
}
// Second pass: same data, no errors.
scrubber.runPass()
stats = scrubber.Stats()
if stats.PassCount != 2 {
t.Fatalf("PassCount: got %d, want 2", stats.PassCount)
}
if stats.ErrorCount != 0 {
t.Fatalf("ErrorCount: got %d, want 0", stats.ErrorCount)
}
}
func TestScrub_DetectCorruption(t *testing.T) {
v := createScrubVol(t)
defer v.Close()
// Write some data and flush it to extent.
data := make([]byte, 4096)
for i := range data {
data[i] = 0xAA
}
if err := v.WriteLBA(0, data); err != nil {
t.Fatalf("WriteLBA: %v", err)
}
if err := v.SyncCache(); err != nil {
t.Fatalf("SyncCache: %v", err)
}
// Wait for flusher to flush.
time.Sleep(50 * time.Millisecond)
scrubber := NewScrubber(v, 24*time.Hour)
scrubber.throttle = 0
// First pass: establish baseline.
scrubber.runPass()
if scrubber.Stats().ErrorCount != 0 {
t.Fatalf("first pass errors: %d", scrubber.Stats().ErrorCount)
}
// Corrupt the extent data directly.
extentStart := v.super.WALOffset + v.super.WALSize
corruptBuf := []byte{0xFF, 0xFF, 0xFF, 0xFF}
v.fd.WriteAt(corruptBuf, int64(extentStart))
v.fd.Sync()
// Second pass: should detect corruption.
scrubber.runPass()
stats := scrubber.Stats()
if stats.ErrorCount == 0 {
t.Fatal("expected corruption error but got none")
}
}
func TestScrub_SkipDirtyBlocks(t *testing.T) {
v := createScrubVol(t)
defer v.Close()
scrubber := NewScrubber(v, 24*time.Hour)
scrubber.throttle = 0
// First pass: baseline.
scrubber.runPass()
// Write data (stays in dirty map since flusher interval is slow relative to us).
data := make([]byte, 4096)
for i := range data {
data[i] = 0xBB
}
if err := v.WriteLBA(0, data); err != nil {
t.Fatalf("WriteLBA: %v", err)
}
// Second pass: segment 0 has dirty blocks, should be skipped.
scrubber.runPass()
stats := scrubber.Stats()
if stats.ErrorCount != 0 {
t.Fatalf("dirty block should be skipped, not flagged as error: errors=%d", stats.ErrorCount)
}
}
func TestScrub_SkipRecentlyWritten(t *testing.T) {
v := createScrubVol(t)
defer v.Close()
scrubber := NewScrubber(v, 24*time.Hour)
scrubber.throttle = 0
// First pass: baseline.
scrubber.runPass()
// Write, sync, flush (data in extent, dirty map cleared).
data := make([]byte, 4096)
for i := range data {
data[i] = 0xCC
}
if err := v.WriteLBA(0, data); err != nil {
t.Fatalf("WriteLBA: %v", err)
}
v.SyncCache()
time.Sleep(50 * time.Millisecond) // let flusher clear dirty map
// Manually notify scrubber of write (simulates hook in appendWithRetry).
scrubber.NotifyWrite(0, 1)
// Second pass: segment 0 was recently written, skip CRC comparison.
scrubber.runPass()
stats := scrubber.Stats()
if stats.ErrorCount != 0 {
t.Fatalf("recently written segment should be skipped: errors=%d", stats.ErrorCount)
}
}
func TestScrub_StatsUpdated(t *testing.T) {
v := createScrubVol(t)
defer v.Close()
scrubber := NewScrubber(v, 24*time.Hour)
scrubber.throttle = 0
scrubber.runPass()
stats := scrubber.Stats()
if stats.LastPassTime == 0 {
t.Fatal("LastPassTime should be set")
}
if stats.SegmentsTotal == 0 {
t.Fatal("SegmentsTotal should be > 0")
}
}
func TestScrub_TriggerNow(t *testing.T) {
v := createScrubVol(t)
defer v.Close()
scrubber := NewScrubber(v, 1*time.Hour) // long interval
scrubber.throttle = 0
scrubber.Start()
defer scrubber.Stop()
scrubber.TriggerNow()
// Wait for the triggered pass to complete.
time.Sleep(100 * time.Millisecond)
stats := scrubber.Stats()
if stats.PassCount < 1 {
t.Fatalf("TriggerNow did not cause a pass: PassCount=%d", stats.PassCount)
}
}
func TestScrub_StopIdempotent(t *testing.T) {
v := createScrubVol(t)
defer v.Close()
scrubber := NewScrubber(v, 24*time.Hour)
scrubber.Start()
scrubber.Stop()
scrubber.Stop() // second stop should be no-op
}
func TestScrub_HealthScoreImpact(t *testing.T) {
dir := t.TempDir()
cfg := DefaultConfig()
cfg.FlushInterval = 5 * time.Millisecond
path := filepath.Join(dir, "health.blk")
v, err := CreateBlockVol(path, CreateOptions{
VolumeSize: 64 * 1024,
BlockSize: 4096,
WALSize: 64 * 1024,
}, cfg)
if err != nil {
t.Fatalf("CreateBlockVol: %v", err)
}
defer v.Close()
if v.HealthScore() != 1.0 {
t.Fatalf("initial health: got %f, want 1.0", v.HealthScore())
}
// Write + flush to extent.
data := make([]byte, 4096)
for i := range data {
data[i] = 0xDD
}
v.WriteLBA(0, data)
v.SyncCache()
time.Sleep(50 * time.Millisecond)
scrubber := NewScrubber(v, 24*time.Hour)
scrubber.throttle = 0
scrubber.runPass()
// Corrupt extent.
extentStart := v.super.WALOffset + v.super.WALSize
corruptBuf := []byte{0xFF}
v.fd.WriteAt(corruptBuf, int64(extentStart))
v.fd.Sync()
scrubber.runPass()
if v.HealthScore() >= 1.0 {
t.Fatalf("health should drop after corruption: got %f", v.HealthScore())
}
// Cleanup temp files on Windows: remove delta files if any.
_ = os.RemoveAll(dir)
}
// Fix #5: Write arriving during a scrub pass must not cause a false corruption detection.
func TestScrub_InPassWrite_NoFalsePositive(t *testing.T) {
v := createScrubVol(t)
defer v.Close()
// Write initial data and flush to extent.
data := make([]byte, 4096)
for i := range data {
data[i] = 0xAA
}
if err := v.WriteLBA(0, data); err != nil {
t.Fatalf("WriteLBA: %v", err)
}
v.SyncCache()
time.Sleep(50 * time.Millisecond) // let flusher clear dirty map
scrubber := NewScrubber(v, 24*time.Hour)
scrubber.throttle = 0
// First pass: establish baseline CRC for segment 0.
scrubber.runPass()
if scrubber.Stats().ErrorCount != 0 {
t.Fatalf("baseline pass should have no errors: %d", scrubber.Stats().ErrorCount)
}
// Now write different data and flush — this changes the extent.
for i := range data {
data[i] = 0xBB
}
if err := v.WriteLBA(0, data); err != nil {
t.Fatalf("WriteLBA: %v", err)
}
v.SyncCache()
time.Sleep(50 * time.Millisecond) // flushed to extent, dirty map cleared
// Simulate in-pass write: NotifyWrite arrives AFTER pass would have
// snapshotted writtenSegs (old bug), but BEFORE segment 0 is read.
// With the fix, writtenSegs is checked live during the pass.
scrubber.NotifyWrite(0, 1)
// Second pass: segment 0 extent data changed, but NotifyWrite should
// prevent false corruption — segment should be skipped.
scrubber.runPass()
stats := scrubber.Stats()
if stats.ErrorCount != 0 {
t.Fatalf("in-pass write caused false corruption: errors=%d", stats.ErrorCount)
}
if stats.SegmentsDirty == 0 {
t.Fatal("segment 0 should be counted as dirty (recently written)")
}
}
weed/storage/blockvol/shipper_group.go
+114
View File
@@ -0,0 +1,114 @@
package blockvol
import (
"sync"
)
// ShipperGroup wraps multiple WALShippers for fan-out replication to N replicas.
// Len()==0 means standalone (no replicas), Len()==1 is RF=2, Len()==2 is RF=3.
type ShipperGroup struct {
mu sync.RWMutex
shippers []*WALShipper
}
// NewShipperGroup creates a ShipperGroup from the given shippers.
func NewShipperGroup(shippers []*WALShipper) *ShipperGroup {
return &ShipperGroup{shippers: shippers}
}
// ShipAll sends a WAL entry to every non-degraded shipper (fire-and-forget).
func (sg *ShipperGroup) ShipAll(entry *WALEntry) {
sg.mu.RLock()
defer sg.mu.RUnlock()
for _, s := range sg.shippers {
s.Ship(entry)
}
}
// BarrierAll sends barriers to all shippers in parallel.
// Returns a per-shipper error slice (nil entry = success).
func (sg *ShipperGroup) BarrierAll(lsnMax uint64) []error {
sg.mu.RLock()
n := len(sg.shippers)
shippers := sg.shippers
sg.mu.RUnlock()
errs := make([]error, n)
var wg sync.WaitGroup
wg.Add(n)
for i := 0; i < n; i++ {
go func(idx int) {
defer wg.Done()
errs[idx] = shippers[idx].Barrier(lsnMax)
}(i)
}
wg.Wait()
return errs
}
// AllDegraded returns true only if every shipper is degraded.
// Returns false when there are no shippers (standalone).
func (sg *ShipperGroup) AllDegraded() bool {
sg.mu.RLock()
defer sg.mu.RUnlock()
if len(sg.shippers) == 0 {
return false
}
for _, s := range sg.shippers {
if !s.IsDegraded() {
return false
}
}
return true
}
// AnyDegraded returns true if at least one shipper is degraded.
func (sg *ShipperGroup) AnyDegraded() bool {
sg.mu.RLock()
defer sg.mu.RUnlock()
for _, s := range sg.shippers {
if s.IsDegraded() {
return true
}
}
return false
}
// DegradedCount returns the number of degraded shippers.
func (sg *ShipperGroup) DegradedCount() int {
sg.mu.RLock()
defer sg.mu.RUnlock()
count := 0
for _, s := range sg.shippers {
if s.IsDegraded() {
count++
}
}
return count
}
// StopAll stops all shippers and closes their connections.
func (sg *ShipperGroup) StopAll() {
sg.mu.Lock()
defer sg.mu.Unlock()
for _, s := range sg.shippers {
s.Stop()
}
}
// Len returns the number of shippers. 0=standalone, 1=RF2, 2=RF3.
func (sg *ShipperGroup) Len() int {
sg.mu.RLock()
defer sg.mu.RUnlock()
return len(sg.shippers)
}
// Shipper returns the shipper at index i. For internal/test use.
func (sg *ShipperGroup) Shipper(i int) *WALShipper {
sg.mu.RLock()
defer sg.mu.RUnlock()
if i < 0 || i >= len(sg.shippers) {
return nil
}
return sg.shippers[i]
}
weed/storage/blockvol/shipper_group_test.go
+116
View File
@@ -0,0 +1,116 @@
package blockvol
import (
"sync/atomic"
"testing"
)
func TestShipperGroup_ShipAll_Single(t *testing.T) {
s := newTestShipper()
sg := NewShipperGroup([]*WALShipper{s})
entry := &WALEntry{LSN: 1, Epoch: 1, Type: EntryTypeWrite}
sg.ShipAll(entry)
// Ship on a non-connected shipper degrades it but doesn't panic.
if sg.Len() != 1 {
t.Fatalf("Len: got %d, want 1", sg.Len())
}
}
func TestShipperGroup_ShipAll_Two(t *testing.T) {
s1 := newTestShipper()
s2 := newTestShipper()
sg := NewShipperGroup([]*WALShipper{s1, s2})
entry := &WALEntry{LSN: 1, Epoch: 1, Type: EntryTypeWrite}
sg.ShipAll(entry)
if sg.Len() != 2 {
t.Fatalf("Len: got %d, want 2", sg.Len())
}
}
func TestShipperGroup_BarrierAll_AllSucceed(t *testing.T) {
// Create shippers that are stopped (Barrier returns ErrShipperStopped).
// We test the fan-out logic: all return errors, which is fine.
s1 := newTestShipper()
s2 := newTestShipper()
sg := NewShipperGroup([]*WALShipper{s1, s2})
errs := sg.BarrierAll(10)
if len(errs) != 2 {
t.Fatalf("BarrierAll: got %d errors, want 2", len(errs))
}
}
func TestShipperGroup_BarrierAll_OneFail(t *testing.T) {
s1 := newTestShipper()
s1.degraded.Store(true)
s2 := newTestShipper()
sg := NewShipperGroup([]*WALShipper{s1, s2})
errs := sg.BarrierAll(10)
if errs[0] == nil {
t.Fatalf("expected error for degraded shipper[0]")
}
}
func TestShipperGroup_BarrierAll_AllFail(t *testing.T) {
s1 := newTestShipper()
s1.degraded.Store(true)
s2 := newTestShipper()
s2.degraded.Store(true)
sg := NewShipperGroup([]*WALShipper{s1, s2})
errs := sg.BarrierAll(10)
for i, e := range errs {
if e == nil {
t.Fatalf("expected error for shipper[%d]", i)
}
}
}
func TestShipperGroup_AllDegraded_Empty(t *testing.T) {
sg := NewShipperGroup(nil)
if sg.AllDegraded() {
t.Fatal("empty group should not be AllDegraded")
}
}
func TestShipperGroup_AllDegraded_Mixed(t *testing.T) {
s1 := newTestShipper()
s1.degraded.Store(true)
s2 := newTestShipper()
sg := NewShipperGroup([]*WALShipper{s1, s2})
if sg.AllDegraded() {
t.Fatal("mixed group should not be AllDegraded")
}
if !sg.AnyDegraded() {
t.Fatal("mixed group should be AnyDegraded")
}
}
func TestShipperGroup_StopAll(t *testing.T) {
s1 := newTestShipper()
s2 := newTestShipper()
sg := NewShipperGroup([]*WALShipper{s1, s2})
sg.StopAll()
if !s1.stopped.Load() || !s2.stopped.Load() {
t.Fatal("StopAll should stop all shippers")
}
}
func TestShipperGroup_DegradedCount(t *testing.T) {
s1 := newTestShipper()
s2 := newTestShipper()
s1.degraded.Store(true)
sg := NewShipperGroup([]*WALShipper{s1, s2})
if got := sg.DegradedCount(); got != 1 {
t.Fatalf("DegradedCount: got %d, want 1", got)
}
}
// newTestShipper creates a WALShipper with a fixed epoch, not connected to anything.
func newTestShipper() *WALShipper {
var epoch atomic.Uint64
epoch.Store(1)
return &WALShipper{
dataAddr: "127.0.0.1:0",
controlAddr: "127.0.0.1:0",
epochFn: func() uint64 { return epoch.Load() },
}
}
weed/storage/blockvol/wal_shipper.go
+32 -1
View File
@@ -23,6 +23,7 @@ type WALShipper struct {
dataAddr string
controlAddr string
epochFn func() uint64
metrics *EngineMetrics
mu sync.Mutex // protects dataConn
dataConn net.Conn
@@ -37,11 +38,17 @@ type WALShipper struct {
// NewWALShipper creates a WAL shipper. Connections are established lazily on
// first Ship/Barrier call. epochFn returns the current epoch for validation.
func NewWALShipper(dataAddr, controlAddr string, epochFn func() uint64) *WALShipper {
// metrics is optional; if nil, no metrics are recorded.
func NewWALShipper(dataAddr, controlAddr string, epochFn func() uint64, metrics ...*EngineMetrics) *WALShipper {
var m *EngineMetrics
if len(metrics) > 0 {
m = metrics[0]
}
return &WALShipper{
dataAddr: dataAddr,
controlAddr: controlAddr,
epochFn: epochFn,
metrics: m,
}
}
@@ -83,6 +90,9 @@ func (s *WALShipper) Ship(entry *WALEntry) error {
s.dataConn.SetWriteDeadline(time.Time{}) // clear deadline
s.shippedLSN.Store(entry.LSN)
if s.metrics != nil {
s.metrics.RecordWALShipped()
}
return nil
}
@@ -97,6 +107,8 @@ func (s *WALShipper) Barrier(lsnMax uint64) error {
return ErrReplicaDegraded
}
barrierStart := time.Now()
req := EncodeBarrierRequest(BarrierRequest{
LSN: lsnMax,
Epoch: s.epochFn(),
@@ -107,6 +119,7 @@ func (s *WALShipper) Barrier(lsnMax uint64) error {
if err := s.ensureCtrlConn(); err != nil {
s.markDegraded()
s.recordBarrierMetric(barrierStart, true)
return ErrReplicaDegraded
}
@@ -114,34 +127,52 @@ func (s *WALShipper) Barrier(lsnMax uint64) error {
if err := WriteFrame(s.ctrlConn, MsgBarrierReq, req); err != nil {
s.markDegraded()
s.recordBarrierMetric(barrierStart, true)
return ErrReplicaDegraded
}
msgType, payload, err := ReadFrame(s.ctrlConn)
if err != nil {
s.markDegraded()
s.recordBarrierMetric(barrierStart, true)
return ErrReplicaDegraded
}
if msgType != MsgBarrierResp || len(payload) < 1 {
s.markDegraded()
s.recordBarrierMetric(barrierStart, true)
return ErrReplicaDegraded
}
switch payload[0] {
case BarrierOK:
s.recordBarrierMetric(barrierStart, false)
return nil
case BarrierEpochMismatch:
s.markDegraded()
s.recordBarrierMetric(barrierStart, true)
return fmt.Errorf("wal_shipper: barrier epoch mismatch")
case BarrierTimeout:
s.markDegraded()
s.recordBarrierMetric(barrierStart, true)
return fmt.Errorf("wal_shipper: barrier timeout on replica")
case BarrierFsyncFailed:
s.markDegraded()
s.recordBarrierMetric(barrierStart, true)
return fmt.Errorf("wal_shipper: barrier fsync failed on replica")
default:
s.markDegraded()
s.recordBarrierMetric(barrierStart, true)
return fmt.Errorf("wal_shipper: unknown barrier status %d", payload[0])
}
}
func (s *WALShipper) recordBarrierMetric(start time.Time, failed bool) {
if s.metrics != nil {
s.metrics.RecordWALBarrier(time.Since(start), failed)
}
}
// ShippedLSN returns the highest LSN successfully sent to the replica.
func (s *WALShipper) ShippedLSN() uint64 {
return s.shippedLSN.Load()