diff --git a/weed/command/master.go b/weed/command/master.go index 565c9cc581..6be079c991 100644 --- a/weed/command/master.go +++ b/weed/command/master.go @@ -71,8 +71,9 @@ type MasterOptions struct { raftBootstrap *bool telemetryUrl *string telemetryEnabled *bool - debug *bool - debugPort *int + debug *bool + debugPort *int + blockPromotionLSNTolerance *int } func init() { @@ -104,6 +105,7 @@ func init() { m.telemetryEnabled = cmdMaster.Flag.Bool("telemetry", false, "enable telemetry reporting") m.debug = cmdMaster.Flag.Bool("debug", false, "serves runtime profiling data via pprof on the port specified by -debug.port") m.debugPort = cmdMaster.Flag.Int("debug.port", 6060, "http port for debugging") + m.blockPromotionLSNTolerance = cmdMaster.Flag.Int("block.promotion.lsnTolerance", 100, "max WAL LSN lag for block volume replica promotion eligibility") } var cmdMaster = &Command{ @@ -410,7 +412,8 @@ func (m *MasterOptions) toMasterOption(whiteList []string) *weed_server.MasterOp DisableHttp: *m.disableHttp, MetricsAddress: *m.metricsAddress, MetricsIntervalSec: *m.metricsIntervalSec, - TelemetryUrl: *m.telemetryUrl, - TelemetryEnabled: *m.telemetryEnabled, + TelemetryUrl: *m.telemetryUrl, + TelemetryEnabled: *m.telemetryEnabled, + BlockPromotionLSNTolerance: *m.blockPromotionLSNTolerance, } } diff --git a/weed/command/volume.go b/weed/command/volume.go index 3ec6fce728..5d607d269b 100644 --- a/weed/command/volume.go +++ b/weed/command/volume.go @@ -310,6 +310,9 @@ func (v VolumeServerOptions) startVolumeServer(volumeFolders, maxVolumeCounts, v // Start block volume iSCSI service (disabled if block.dir is empty). blockService := weed_server.StartBlockService(*v.blockListen, *v.blockDir, *v.blockIQNPrefix) + if blockService != nil { + volumeServer.SetBlockService(blockService) + } grace.OnReload(volumeServer.LoadNewVolumes) grace.OnReload(volumeServer.Reload) diff --git a/weed/pb/master.proto b/weed/pb/master.proto index 7673b0a9a5..1332e5119f 100644 --- a/weed/pb/master.proto +++ b/weed/pb/master.proto @@ -63,6 +63,14 @@ service Seaweed { } rpc LookupBlockVolume (LookupBlockVolumeRequest) returns (LookupBlockVolumeResponse) { } + rpc CreateBlockSnapshot (CreateBlockSnapshotRequest) returns (CreateBlockSnapshotResponse) { + } + rpc DeleteBlockSnapshot (DeleteBlockSnapshotRequest) returns (DeleteBlockSnapshotResponse) { + } + rpc ListBlockSnapshots (ListBlockSnapshotsRequest) returns (ListBlockSnapshotsResponse) { + } + rpc ExpandBlockVolume (ExpandBlockVolumeRequest) returns (ExpandBlockVolumeResponse) { + } } ////////////////////////////////////////////////// @@ -493,6 +501,10 @@ message BlockVolumeInfoMessage { string disk_type = 9; string replica_data_addr = 10; string replica_ctrl_addr = 11; + double health_score = 12; + int64 scrub_errors = 13; + int64 last_scrub_time = 14; + bool replica_degraded = 15; } message BlockVolumeShortInfoMessage { @@ -510,12 +522,19 @@ message BlockVolumeAssignment { string replica_data_addr = 5; string replica_ctrl_addr = 6; string rebuild_addr = 7; + repeated ReplicaAddrMessage replica_addrs = 8; +} + +message ReplicaAddrMessage { + string data_addr = 1; + string ctrl_addr = 2; } message CreateBlockVolumeRequest { string name = 1; uint64 size_bytes = 2; string disk_type = 3; + uint32 replica_factor = 4; } message CreateBlockVolumeResponse { string volume_id = 1; @@ -524,6 +543,7 @@ message CreateBlockVolumeResponse { string iqn = 4; uint64 capacity_bytes = 5; string replica_server = 6; + repeated string replica_servers = 7; } message DeleteBlockVolumeRequest { @@ -541,4 +561,43 @@ message LookupBlockVolumeResponse { string iqn = 3; uint64 capacity_bytes = 4; string replica_server = 5; + uint32 replica_factor = 6; + repeated string replica_servers = 7; +} + +message CreateBlockSnapshotRequest { + string volume_name = 1; + uint32 snapshot_id = 2; +} +message CreateBlockSnapshotResponse { + uint32 snapshot_id = 1; + int64 created_at = 2; // Unix timestamp in seconds + uint64 size_bytes = 3; +} + +message DeleteBlockSnapshotRequest { + string volume_name = 1; + uint32 snapshot_id = 2; +} +message DeleteBlockSnapshotResponse { +} + +message ListBlockSnapshotsRequest { + string volume_name = 1; +} +message ListBlockSnapshotsResponse { + repeated BlockSnapshotInfo snapshots = 1; +} +message BlockSnapshotInfo { + uint32 snapshot_id = 1; + int64 created_at = 2; // Unix timestamp in seconds + uint64 volume_size_bytes = 3; +} + +message ExpandBlockVolumeRequest { + string name = 1; + uint64 new_size_bytes = 2; +} +message ExpandBlockVolumeResponse { + uint64 capacity_bytes = 1; } \ No newline at end of file diff --git a/weed/pb/master_pb/master.pb.go b/weed/pb/master_pb/master.pb.go index 99b37ff813..0d16c4646e 100644 --- a/weed/pb/master_pb/master.pb.go +++ b/weed/pb/master_pb/master.pb.go @@ -3895,6 +3895,10 @@ type BlockVolumeInfoMessage struct { DiskType string `protobuf:"bytes,9,opt,name=disk_type,json=diskType,proto3" json:"disk_type,omitempty"` ReplicaDataAddr string `protobuf:"bytes,10,opt,name=replica_data_addr,json=replicaDataAddr,proto3" json:"replica_data_addr,omitempty"` ReplicaCtrlAddr string `protobuf:"bytes,11,opt,name=replica_ctrl_addr,json=replicaCtrlAddr,proto3" json:"replica_ctrl_addr,omitempty"` + HealthScore float64 `protobuf:"fixed64,12,opt,name=health_score,json=healthScore,proto3" json:"health_score,omitempty"` + ScrubErrors int64 `protobuf:"varint,13,opt,name=scrub_errors,json=scrubErrors,proto3" json:"scrub_errors,omitempty"` + LastScrubTime int64 `protobuf:"varint,14,opt,name=last_scrub_time,json=lastScrubTime,proto3" json:"last_scrub_time,omitempty"` + ReplicaDegraded bool `protobuf:"varint,15,opt,name=replica_degraded,json=replicaDegraded,proto3" json:"replica_degraded,omitempty"` unknownFields protoimpl.UnknownFields sizeCache protoimpl.SizeCache } @@ -4006,6 +4010,34 @@ func (x *BlockVolumeInfoMessage) GetReplicaCtrlAddr() string { return "" } +func (x *BlockVolumeInfoMessage) GetHealthScore() float64 { + if x != nil { + return x.HealthScore + } + return 0 +} + +func (x *BlockVolumeInfoMessage) GetScrubErrors() int64 { + if x != nil { + return x.ScrubErrors + } + return 0 +} + +func (x *BlockVolumeInfoMessage) GetLastScrubTime() int64 { + if x != nil { + return x.LastScrubTime + } + return 0 +} + +func (x *BlockVolumeInfoMessage) GetReplicaDegraded() bool { + if x != nil { + return x.ReplicaDegraded + } + return false +} + type BlockVolumeShortInfoMessage struct { state protoimpl.MessageState `protogen:"open.v1"` Path string `protobuf:"bytes,1,opt,name=path,proto3" json:"path,omitempty"` @@ -4083,6 +4115,7 @@ type BlockVolumeAssignment struct { ReplicaDataAddr string `protobuf:"bytes,5,opt,name=replica_data_addr,json=replicaDataAddr,proto3" json:"replica_data_addr,omitempty"` ReplicaCtrlAddr string `protobuf:"bytes,6,opt,name=replica_ctrl_addr,json=replicaCtrlAddr,proto3" json:"replica_ctrl_addr,omitempty"` RebuildAddr string `protobuf:"bytes,7,opt,name=rebuild_addr,json=rebuildAddr,proto3" json:"rebuild_addr,omitempty"` + ReplicaAddrs []*ReplicaAddrMessage `protobuf:"bytes,8,rep,name=replica_addrs,json=replicaAddrs,proto3" json:"replica_addrs,omitempty"` unknownFields protoimpl.UnknownFields sizeCache protoimpl.SizeCache } @@ -4166,11 +4199,46 @@ func (x *BlockVolumeAssignment) GetRebuildAddr() string { return "" } +func (x *BlockVolumeAssignment) GetReplicaAddrs() []*ReplicaAddrMessage { + if x != nil { + return x.ReplicaAddrs + } + return nil +} + +// ReplicaAddrMessage holds data and control addresses for one replica. +type ReplicaAddrMessage struct { + state protoimpl.MessageState `protogen:"open.v1"` + DataAddr string `protobuf:"bytes,1,opt,name=data_addr,json=dataAddr,proto3" json:"data_addr,omitempty"` + CtrlAddr string `protobuf:"bytes,2,opt,name=ctrl_addr,json=ctrlAddr,proto3" json:"ctrl_addr,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *ReplicaAddrMessage) Reset() { *x = ReplicaAddrMessage{} } +func (x *ReplicaAddrMessage) String() string { return "ReplicaAddr{" + x.DataAddr + "," + x.CtrlAddr + "}" } +func (*ReplicaAddrMessage) ProtoMessage() {} + +func (x *ReplicaAddrMessage) GetDataAddr() string { + if x != nil { + return x.DataAddr + } + return "" +} + +func (x *ReplicaAddrMessage) GetCtrlAddr() string { + if x != nil { + return x.CtrlAddr + } + return "" +} + type CreateBlockVolumeRequest struct { state protoimpl.MessageState `protogen:"open.v1"` Name string `protobuf:"bytes,1,opt,name=name,proto3" json:"name,omitempty"` SizeBytes uint64 `protobuf:"varint,2,opt,name=size_bytes,json=sizeBytes,proto3" json:"size_bytes,omitempty"` DiskType string `protobuf:"bytes,3,opt,name=disk_type,json=diskType,proto3" json:"disk_type,omitempty"` + ReplicaFactor uint32 `protobuf:"varint,4,opt,name=replica_factor,json=replicaFactor,proto3" json:"replica_factor,omitempty"` unknownFields protoimpl.UnknownFields sizeCache protoimpl.SizeCache } @@ -4226,6 +4294,13 @@ func (x *CreateBlockVolumeRequest) GetDiskType() string { return "" } +func (x *CreateBlockVolumeRequest) GetReplicaFactor() uint32 { + if x != nil { + return x.ReplicaFactor + } + return 0 +} + type CreateBlockVolumeResponse struct { state protoimpl.MessageState `protogen:"open.v1"` VolumeId string `protobuf:"bytes,1,opt,name=volume_id,json=volumeId,proto3" json:"volume_id,omitempty"` @@ -4233,9 +4308,10 @@ type CreateBlockVolumeResponse struct { IscsiAddr string `protobuf:"bytes,3,opt,name=iscsi_addr,json=iscsiAddr,proto3" json:"iscsi_addr,omitempty"` Iqn string `protobuf:"bytes,4,opt,name=iqn,proto3" json:"iqn,omitempty"` CapacityBytes uint64 `protobuf:"varint,5,opt,name=capacity_bytes,json=capacityBytes,proto3" json:"capacity_bytes,omitempty"` - ReplicaServer string `protobuf:"bytes,6,opt,name=replica_server,json=replicaServer,proto3" json:"replica_server,omitempty"` - unknownFields protoimpl.UnknownFields - sizeCache protoimpl.SizeCache + ReplicaServer string `protobuf:"bytes,6,opt,name=replica_server,json=replicaServer,proto3" json:"replica_server,omitempty"` + ReplicaServers []string `protobuf:"bytes,7,rep,name=replica_servers,json=replicaServers,proto3" json:"replica_servers,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache } func (x *CreateBlockVolumeResponse) Reset() { @@ -4310,6 +4386,13 @@ func (x *CreateBlockVolumeResponse) GetReplicaServer() string { return "" } +func (x *CreateBlockVolumeResponse) GetReplicaServers() []string { + if x != nil { + return x.ReplicaServers + } + return nil +} + type DeleteBlockVolumeRequest struct { state protoimpl.MessageState `protogen:"open.v1"` Name string `protobuf:"bytes,1,opt,name=name,proto3" json:"name,omitempty"` @@ -4435,14 +4518,16 @@ func (x *LookupBlockVolumeRequest) GetName() string { } type LookupBlockVolumeResponse struct { - state protoimpl.MessageState `protogen:"open.v1"` - VolumeServer string `protobuf:"bytes,1,opt,name=volume_server,json=volumeServer,proto3" json:"volume_server,omitempty"` - IscsiAddr string `protobuf:"bytes,2,opt,name=iscsi_addr,json=iscsiAddr,proto3" json:"iscsi_addr,omitempty"` - Iqn string `protobuf:"bytes,3,opt,name=iqn,proto3" json:"iqn,omitempty"` - CapacityBytes uint64 `protobuf:"varint,4,opt,name=capacity_bytes,json=capacityBytes,proto3" json:"capacity_bytes,omitempty"` - ReplicaServer string `protobuf:"bytes,5,opt,name=replica_server,json=replicaServer,proto3" json:"replica_server,omitempty"` - unknownFields protoimpl.UnknownFields - sizeCache protoimpl.SizeCache + state protoimpl.MessageState `protogen:"open.v1"` + VolumeServer string `protobuf:"bytes,1,opt,name=volume_server,json=volumeServer,proto3" json:"volume_server,omitempty"` + IscsiAddr string `protobuf:"bytes,2,opt,name=iscsi_addr,json=iscsiAddr,proto3" json:"iscsi_addr,omitempty"` + Iqn string `protobuf:"bytes,3,opt,name=iqn,proto3" json:"iqn,omitempty"` + CapacityBytes uint64 `protobuf:"varint,4,opt,name=capacity_bytes,json=capacityBytes,proto3" json:"capacity_bytes,omitempty"` + ReplicaServer string `protobuf:"bytes,5,opt,name=replica_server,json=replicaServer,proto3" json:"replica_server,omitempty"` + ReplicaFactor uint32 `protobuf:"varint,6,opt,name=replica_factor,json=replicaFactor,proto3" json:"replica_factor,omitempty"` + ReplicaServers []string `protobuf:"bytes,7,rep,name=replica_servers,json=replicaServers,proto3" json:"replica_servers,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache } func (x *LookupBlockVolumeResponse) Reset() { @@ -4510,6 +4595,464 @@ func (x *LookupBlockVolumeResponse) GetReplicaServer() string { return "" } +func (x *LookupBlockVolumeResponse) GetReplicaFactor() uint32 { + if x != nil { + return x.ReplicaFactor + } + return 0 +} + +func (x *LookupBlockVolumeResponse) GetReplicaServers() []string { + if x != nil { + return x.ReplicaServers + } + return nil +} + +type CreateBlockSnapshotRequest struct { + state protoimpl.MessageState `protogen:"open.v1"` + VolumeName string `protobuf:"bytes,1,opt,name=volume_name,json=volumeName,proto3" json:"volume_name,omitempty"` + SnapshotId uint32 `protobuf:"varint,2,opt,name=snapshot_id,json=snapshotId,proto3" json:"snapshot_id,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *CreateBlockSnapshotRequest) Reset() { + *x = CreateBlockSnapshotRequest{} + mi := &file_master_proto_msgTypes[70] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *CreateBlockSnapshotRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*CreateBlockSnapshotRequest) ProtoMessage() {} + +func (x *CreateBlockSnapshotRequest) ProtoReflect() protoreflect.Message { + mi := &file_master_proto_msgTypes[70] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use CreateBlockSnapshotRequest.ProtoReflect.Descriptor instead. +func (*CreateBlockSnapshotRequest) Descriptor() ([]byte, []int) { + return file_master_proto_rawDescGZIP(), []int{70} +} + +func (x *CreateBlockSnapshotRequest) GetVolumeName() string { + if x != nil { + return x.VolumeName + } + return "" +} + +func (x *CreateBlockSnapshotRequest) GetSnapshotId() uint32 { + if x != nil { + return x.SnapshotId + } + return 0 +} + +type CreateBlockSnapshotResponse struct { + state protoimpl.MessageState `protogen:"open.v1"` + SnapshotId uint32 `protobuf:"varint,1,opt,name=snapshot_id,json=snapshotId,proto3" json:"snapshot_id,omitempty"` + CreatedAt int64 `protobuf:"varint,2,opt,name=created_at,json=createdAt,proto3" json:"created_at,omitempty"` // Unix timestamp in seconds + SizeBytes uint64 `protobuf:"varint,3,opt,name=size_bytes,json=sizeBytes,proto3" json:"size_bytes,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *CreateBlockSnapshotResponse) Reset() { + *x = CreateBlockSnapshotResponse{} + mi := &file_master_proto_msgTypes[71] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *CreateBlockSnapshotResponse) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*CreateBlockSnapshotResponse) ProtoMessage() {} + +func (x *CreateBlockSnapshotResponse) ProtoReflect() protoreflect.Message { + mi := &file_master_proto_msgTypes[71] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use CreateBlockSnapshotResponse.ProtoReflect.Descriptor instead. +func (*CreateBlockSnapshotResponse) Descriptor() ([]byte, []int) { + return file_master_proto_rawDescGZIP(), []int{71} +} + +func (x *CreateBlockSnapshotResponse) GetSnapshotId() uint32 { + if x != nil { + return x.SnapshotId + } + return 0 +} + +func (x *CreateBlockSnapshotResponse) GetCreatedAt() int64 { + if x != nil { + return x.CreatedAt + } + return 0 +} + +func (x *CreateBlockSnapshotResponse) GetSizeBytes() uint64 { + if x != nil { + return x.SizeBytes + } + return 0 +} + +type DeleteBlockSnapshotRequest struct { + state protoimpl.MessageState `protogen:"open.v1"` + VolumeName string `protobuf:"bytes,1,opt,name=volume_name,json=volumeName,proto3" json:"volume_name,omitempty"` + SnapshotId uint32 `protobuf:"varint,2,opt,name=snapshot_id,json=snapshotId,proto3" json:"snapshot_id,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *DeleteBlockSnapshotRequest) Reset() { + *x = DeleteBlockSnapshotRequest{} + mi := &file_master_proto_msgTypes[72] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *DeleteBlockSnapshotRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*DeleteBlockSnapshotRequest) ProtoMessage() {} + +func (x *DeleteBlockSnapshotRequest) ProtoReflect() protoreflect.Message { + mi := &file_master_proto_msgTypes[72] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use DeleteBlockSnapshotRequest.ProtoReflect.Descriptor instead. +func (*DeleteBlockSnapshotRequest) Descriptor() ([]byte, []int) { + return file_master_proto_rawDescGZIP(), []int{72} +} + +func (x *DeleteBlockSnapshotRequest) GetVolumeName() string { + if x != nil { + return x.VolumeName + } + return "" +} + +func (x *DeleteBlockSnapshotRequest) GetSnapshotId() uint32 { + if x != nil { + return x.SnapshotId + } + return 0 +} + +type DeleteBlockSnapshotResponse struct { + state protoimpl.MessageState `protogen:"open.v1"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *DeleteBlockSnapshotResponse) Reset() { + *x = DeleteBlockSnapshotResponse{} + mi := &file_master_proto_msgTypes[73] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *DeleteBlockSnapshotResponse) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*DeleteBlockSnapshotResponse) ProtoMessage() {} + +func (x *DeleteBlockSnapshotResponse) ProtoReflect() protoreflect.Message { + mi := &file_master_proto_msgTypes[73] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use DeleteBlockSnapshotResponse.ProtoReflect.Descriptor instead. +func (*DeleteBlockSnapshotResponse) Descriptor() ([]byte, []int) { + return file_master_proto_rawDescGZIP(), []int{73} +} + +type ListBlockSnapshotsRequest struct { + state protoimpl.MessageState `protogen:"open.v1"` + VolumeName string `protobuf:"bytes,1,opt,name=volume_name,json=volumeName,proto3" json:"volume_name,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *ListBlockSnapshotsRequest) Reset() { + *x = ListBlockSnapshotsRequest{} + mi := &file_master_proto_msgTypes[74] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *ListBlockSnapshotsRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*ListBlockSnapshotsRequest) ProtoMessage() {} + +func (x *ListBlockSnapshotsRequest) ProtoReflect() protoreflect.Message { + mi := &file_master_proto_msgTypes[74] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use ListBlockSnapshotsRequest.ProtoReflect.Descriptor instead. +func (*ListBlockSnapshotsRequest) Descriptor() ([]byte, []int) { + return file_master_proto_rawDescGZIP(), []int{74} +} + +func (x *ListBlockSnapshotsRequest) GetVolumeName() string { + if x != nil { + return x.VolumeName + } + return "" +} + +type ListBlockSnapshotsResponse struct { + state protoimpl.MessageState `protogen:"open.v1"` + Snapshots []*BlockSnapshotInfo `protobuf:"bytes,1,rep,name=snapshots,proto3" json:"snapshots,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *ListBlockSnapshotsResponse) Reset() { + *x = ListBlockSnapshotsResponse{} + mi := &file_master_proto_msgTypes[75] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *ListBlockSnapshotsResponse) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*ListBlockSnapshotsResponse) ProtoMessage() {} + +func (x *ListBlockSnapshotsResponse) ProtoReflect() protoreflect.Message { + mi := &file_master_proto_msgTypes[75] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use ListBlockSnapshotsResponse.ProtoReflect.Descriptor instead. +func (*ListBlockSnapshotsResponse) Descriptor() ([]byte, []int) { + return file_master_proto_rawDescGZIP(), []int{75} +} + +func (x *ListBlockSnapshotsResponse) GetSnapshots() []*BlockSnapshotInfo { + if x != nil { + return x.Snapshots + } + return nil +} + +type BlockSnapshotInfo struct { + state protoimpl.MessageState `protogen:"open.v1"` + SnapshotId uint32 `protobuf:"varint,1,opt,name=snapshot_id,json=snapshotId,proto3" json:"snapshot_id,omitempty"` + CreatedAt int64 `protobuf:"varint,2,opt,name=created_at,json=createdAt,proto3" json:"created_at,omitempty"` // Unix timestamp in seconds + VolumeSizeBytes uint64 `protobuf:"varint,3,opt,name=volume_size_bytes,json=volumeSizeBytes,proto3" json:"volume_size_bytes,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *BlockSnapshotInfo) Reset() { + *x = BlockSnapshotInfo{} + mi := &file_master_proto_msgTypes[76] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *BlockSnapshotInfo) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*BlockSnapshotInfo) ProtoMessage() {} + +func (x *BlockSnapshotInfo) ProtoReflect() protoreflect.Message { + mi := &file_master_proto_msgTypes[76] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use BlockSnapshotInfo.ProtoReflect.Descriptor instead. +func (*BlockSnapshotInfo) Descriptor() ([]byte, []int) { + return file_master_proto_rawDescGZIP(), []int{76} +} + +func (x *BlockSnapshotInfo) GetSnapshotId() uint32 { + if x != nil { + return x.SnapshotId + } + return 0 +} + +func (x *BlockSnapshotInfo) GetCreatedAt() int64 { + if x != nil { + return x.CreatedAt + } + return 0 +} + +func (x *BlockSnapshotInfo) GetVolumeSizeBytes() uint64 { + if x != nil { + return x.VolumeSizeBytes + } + return 0 +} + +type ExpandBlockVolumeRequest struct { + state protoimpl.MessageState `protogen:"open.v1"` + Name string `protobuf:"bytes,1,opt,name=name,proto3" json:"name,omitempty"` + NewSizeBytes uint64 `protobuf:"varint,2,opt,name=new_size_bytes,json=newSizeBytes,proto3" json:"new_size_bytes,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *ExpandBlockVolumeRequest) Reset() { + *x = ExpandBlockVolumeRequest{} + mi := &file_master_proto_msgTypes[77] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *ExpandBlockVolumeRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*ExpandBlockVolumeRequest) ProtoMessage() {} + +func (x *ExpandBlockVolumeRequest) ProtoReflect() protoreflect.Message { + mi := &file_master_proto_msgTypes[77] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use ExpandBlockVolumeRequest.ProtoReflect.Descriptor instead. +func (*ExpandBlockVolumeRequest) Descriptor() ([]byte, []int) { + return file_master_proto_rawDescGZIP(), []int{77} +} + +func (x *ExpandBlockVolumeRequest) GetName() string { + if x != nil { + return x.Name + } + return "" +} + +func (x *ExpandBlockVolumeRequest) GetNewSizeBytes() uint64 { + if x != nil { + return x.NewSizeBytes + } + return 0 +} + +type ExpandBlockVolumeResponse struct { + state protoimpl.MessageState `protogen:"open.v1"` + CapacityBytes uint64 `protobuf:"varint,1,opt,name=capacity_bytes,json=capacityBytes,proto3" json:"capacity_bytes,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *ExpandBlockVolumeResponse) Reset() { + *x = ExpandBlockVolumeResponse{} + mi := &file_master_proto_msgTypes[78] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *ExpandBlockVolumeResponse) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*ExpandBlockVolumeResponse) ProtoMessage() {} + +func (x *ExpandBlockVolumeResponse) ProtoReflect() protoreflect.Message { + mi := &file_master_proto_msgTypes[78] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use ExpandBlockVolumeResponse.ProtoReflect.Descriptor instead. +func (*ExpandBlockVolumeResponse) Descriptor() ([]byte, []int) { + return file_master_proto_rawDescGZIP(), []int{78} +} + +func (x *ExpandBlockVolumeResponse) GetCapacityBytes() uint64 { + if x != nil { + return x.CapacityBytes + } + return 0 +} + type SuperBlockExtra_ErasureCoding struct { state protoimpl.MessageState `protogen:"open.v1"` Data uint32 `protobuf:"varint,1,opt,name=data,proto3" json:"data,omitempty"` @@ -4521,7 +5064,7 @@ type SuperBlockExtra_ErasureCoding struct { func (x *SuperBlockExtra_ErasureCoding) Reset() { *x = SuperBlockExtra_ErasureCoding{} - mi := &file_master_proto_msgTypes[72] + mi := &file_master_proto_msgTypes[81] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -4533,7 +5076,7 @@ func (x *SuperBlockExtra_ErasureCoding) String() string { func (*SuperBlockExtra_ErasureCoding) ProtoMessage() {} func (x *SuperBlockExtra_ErasureCoding) ProtoReflect() protoreflect.Message { - mi := &file_master_proto_msgTypes[72] + mi := &file_master_proto_msgTypes[81] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -4582,7 +5125,7 @@ type LookupVolumeResponse_VolumeIdLocation struct { func (x *LookupVolumeResponse_VolumeIdLocation) Reset() { *x = LookupVolumeResponse_VolumeIdLocation{} - mi := &file_master_proto_msgTypes[73] + mi := &file_master_proto_msgTypes[82] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -4594,7 +5137,7 @@ func (x *LookupVolumeResponse_VolumeIdLocation) String() string { func (*LookupVolumeResponse_VolumeIdLocation) ProtoMessage() {} func (x *LookupVolumeResponse_VolumeIdLocation) ProtoReflect() protoreflect.Message { - mi := &file_master_proto_msgTypes[73] + mi := &file_master_proto_msgTypes[82] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -4648,7 +5191,7 @@ type LookupEcVolumeResponse_EcShardIdLocation struct { func (x *LookupEcVolumeResponse_EcShardIdLocation) Reset() { *x = LookupEcVolumeResponse_EcShardIdLocation{} - mi := &file_master_proto_msgTypes[78] + mi := &file_master_proto_msgTypes[87] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -4660,7 +5203,7 @@ func (x *LookupEcVolumeResponse_EcShardIdLocation) String() string { func (*LookupEcVolumeResponse_EcShardIdLocation) ProtoMessage() {} func (x *LookupEcVolumeResponse_EcShardIdLocation) ProtoReflect() protoreflect.Message { - mi := &file_master_proto_msgTypes[78] + mi := &file_master_proto_msgTypes[87] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -4703,7 +5246,7 @@ type ListClusterNodesResponse_ClusterNode struct { func (x *ListClusterNodesResponse_ClusterNode) Reset() { *x = ListClusterNodesResponse_ClusterNode{} - mi := &file_master_proto_msgTypes[79] + mi := &file_master_proto_msgTypes[88] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -4715,7 +5258,7 @@ func (x *ListClusterNodesResponse_ClusterNode) String() string { func (*ListClusterNodesResponse_ClusterNode) ProtoMessage() {} func (x *ListClusterNodesResponse_ClusterNode) ProtoReflect() protoreflect.Message { - mi := &file_master_proto_msgTypes[79] + mi := &file_master_proto_msgTypes[88] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -4778,7 +5321,7 @@ type RaftListClusterServersResponse_ClusterServers struct { func (x *RaftListClusterServersResponse_ClusterServers) Reset() { *x = RaftListClusterServersResponse_ClusterServers{} - mi := &file_master_proto_msgTypes[80] + mi := &file_master_proto_msgTypes[89] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -4790,7 +5333,7 @@ func (x *RaftListClusterServersResponse_ClusterServers) String() string { func (*RaftListClusterServersResponse_ClusterServers) ProtoMessage() {} func (x *RaftListClusterServersResponse_ClusterServers) ProtoReflect() protoreflect.Message { - mi := &file_master_proto_msgTypes[80] + mi := &file_master_proto_msgTypes[89] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -5205,7 +5748,7 @@ const file_master_proto_rawDesc = "" + "\x0fprevious_leader\x18\x01 \x01(\tR\x0epreviousLeader\x12\x1d\n" + "\n" + "new_leader\x18\x02 \x01(\tR\tnewLeader\"\x14\n" + - "\x12VolumeGrowResponse\"\x99\x02\n" + + "\x12VolumeGrowResponse\"\xf1\x02\n" + "\x16BlockVolumeInfoMessage\x12\x12\n" + "\x04path\x18\x01 \x01(\tR\x04path\x12\x1f\n" + "\vvolume_size\x18\x02 \x01(\x04R\n" + @@ -5218,43 +5761,85 @@ const file_master_proto_rawDesc = "" + "walHeadLsn\x12%\n" + "\x0echeckpoint_lsn\x18\a \x01(\x04R\rcheckpointLsn\x12\x1b\n" + "\thas_lease\x18\b \x01(\bR\bhasLease\x12\x1b\n" + - "\tdisk_type\x18\t \x01(\tR\bdiskType\"\x8e\x01\n" + + "\tdisk_type\x18\t \x01(\tR\bdiskType\x12*\n" + + "\x11replica_data_addr\x18\n" + + " \x01(\tR\x0freplicaDataAddr\x12*\n" + + "\x11replica_ctrl_addr\x18\v \x01(\tR\x0freplicaCtrlAddr\"\x8e\x01\n" + "\x1bBlockVolumeShortInfoMessage\x12\x12\n" + "\x04path\x18\x01 \x01(\tR\x04path\x12\x1f\n" + "\vvolume_size\x18\x02 \x01(\x04R\n" + "volumeSize\x12\x1d\n" + "\n" + "block_size\x18\x03 \x01(\rR\tblockSize\x12\x1b\n" + - "\tdisk_type\x18\x04 \x01(\tR\bdiskType\"w\n" + + "\tdisk_type\x18\x04 \x01(\tR\bdiskType\"\xf2\x01\n" + "\x15BlockVolumeAssignment\x12\x12\n" + "\x04path\x18\x01 \x01(\tR\x04path\x12\x14\n" + "\x05epoch\x18\x02 \x01(\x04R\x05epoch\x12\x12\n" + "\x04role\x18\x03 \x01(\rR\x04role\x12 \n" + "\flease_ttl_ms\x18\x04 \x01(\rR\n" + - "leaseTtlMs\"j\n" + + "leaseTtlMs\x12*\n" + + "\x11replica_data_addr\x18\x05 \x01(\tR\x0freplicaDataAddr\x12*\n" + + "\x11replica_ctrl_addr\x18\x06 \x01(\tR\x0freplicaCtrlAddr\x12!\n" + + "\frebuild_addr\x18\a \x01(\tR\vrebuildAddr\"j\n" + "\x18CreateBlockVolumeRequest\x12\x12\n" + "\x04name\x18\x01 \x01(\tR\x04name\x12\x1d\n" + "\n" + "size_bytes\x18\x02 \x01(\x04R\tsizeBytes\x12\x1b\n" + - "\tdisk_type\x18\x03 \x01(\tR\bdiskType\"\xb5\x01\n" + + "\tdisk_type\x18\x03 \x01(\tR\bdiskType\"\xdc\x01\n" + "\x19CreateBlockVolumeResponse\x12\x1b\n" + "\tvolume_id\x18\x01 \x01(\tR\bvolumeId\x12#\n" + "\rvolume_server\x18\x02 \x01(\tR\fvolumeServer\x12\x1d\n" + "\n" + "iscsi_addr\x18\x03 \x01(\tR\tiscsiAddr\x12\x10\n" + "\x03iqn\x18\x04 \x01(\tR\x03iqn\x12%\n" + - "\x0ecapacity_bytes\x18\x05 \x01(\x04R\rcapacityBytes\".\n" + + "\x0ecapacity_bytes\x18\x05 \x01(\x04R\rcapacityBytes\x12%\n" + + "\x0ereplica_server\x18\x06 \x01(\tR\rreplicaServer\".\n" + "\x18DeleteBlockVolumeRequest\x12\x12\n" + "\x04name\x18\x01 \x01(\tR\x04name\"\x1b\n" + "\x19DeleteBlockVolumeResponse\".\n" + "\x18LookupBlockVolumeRequest\x12\x12\n" + - "\x04name\x18\x01 \x01(\tR\x04name\"\x98\x01\n" + + "\x04name\x18\x01 \x01(\tR\x04name\"\xbf\x01\n" + "\x19LookupBlockVolumeResponse\x12#\n" + "\rvolume_server\x18\x01 \x01(\tR\fvolumeServer\x12\x1d\n" + "\n" + "iscsi_addr\x18\x02 \x01(\tR\tiscsiAddr\x12\x10\n" + "\x03iqn\x18\x03 \x01(\tR\x03iqn\x12%\n" + - "\x0ecapacity_bytes\x18\x04 \x01(\x04R\rcapacityBytes2\xec\x12\n" + + "\x0ecapacity_bytes\x18\x04 \x01(\x04R\rcapacityBytes\x12%\n" + + "\x0ereplica_server\x18\x05 \x01(\tR\rreplicaServer\"^\n" + + "\x1aCreateBlockSnapshotRequest\x12\x1f\n" + + "\vvolume_name\x18\x01 \x01(\tR\n" + + "volumeName\x12\x1f\n" + + "\vsnapshot_id\x18\x02 \x01(\rR\n" + + "snapshotId\"|\n" + + "\x1bCreateBlockSnapshotResponse\x12\x1f\n" + + "\vsnapshot_id\x18\x01 \x01(\rR\n" + + "snapshotId\x12\x1d\n" + + "\n" + + "created_at\x18\x02 \x01(\x03R\tcreatedAt\x12\x1d\n" + + "\n" + + "size_bytes\x18\x03 \x01(\x04R\tsizeBytes\"^\n" + + "\x1aDeleteBlockSnapshotRequest\x12\x1f\n" + + "\vvolume_name\x18\x01 \x01(\tR\n" + + "volumeName\x12\x1f\n" + + "\vsnapshot_id\x18\x02 \x01(\rR\n" + + "snapshotId\"\x1d\n" + + "\x1bDeleteBlockSnapshotResponse\"<\n" + + "\x19ListBlockSnapshotsRequest\x12\x1f\n" + + "\vvolume_name\x18\x01 \x01(\tR\n" + + "volumeName\"X\n" + + "\x1aListBlockSnapshotsResponse\x12:\n" + + "\tsnapshots\x18\x01 \x03(\v2\x1c.master_pb.BlockSnapshotInfoR\tsnapshots\"\x7f\n" + + "\x11BlockSnapshotInfo\x12\x1f\n" + + "\vsnapshot_id\x18\x01 \x01(\rR\n" + + "snapshotId\x12\x1d\n" + + "\n" + + "created_at\x18\x02 \x01(\x03R\tcreatedAt\x12*\n" + + "\x11volume_size_bytes\x18\x03 \x01(\x04R\x0fvolumeSizeBytes\"T\n" + + "\x18ExpandBlockVolumeRequest\x12\x12\n" + + "\x04name\x18\x01 \x01(\tR\x04name\x12$\n" + + "\x0enew_size_bytes\x18\x02 \x01(\x04R\fnewSizeBytes\"B\n" + + "\x19ExpandBlockVolumeResponse\x12%\n" + + "\x0ecapacity_bytes\x18\x01 \x01(\x04R\rcapacityBytes2\x83\x16\n" + "\aSeaweed\x12I\n" + "\rSendHeartbeat\x12\x14.master_pb.Heartbeat\x1a\x1c.master_pb.HeartbeatResponse\"\x00(\x010\x01\x12X\n" + "\rKeepConnected\x12\x1f.master_pb.KeepConnectedRequest\x1a .master_pb.KeepConnectedResponse\"\x00(\x010\x01\x12Q\n" + @@ -5285,7 +5870,11 @@ const file_master_proto_rawDesc = "" + "VolumeGrow\x12\x1c.master_pb.VolumeGrowRequest\x1a\x1d.master_pb.VolumeGrowResponse\"\x00\x12`\n" + "\x11CreateBlockVolume\x12#.master_pb.CreateBlockVolumeRequest\x1a$.master_pb.CreateBlockVolumeResponse\"\x00\x12`\n" + "\x11DeleteBlockVolume\x12#.master_pb.DeleteBlockVolumeRequest\x1a$.master_pb.DeleteBlockVolumeResponse\"\x00\x12`\n" + - "\x11LookupBlockVolume\x12#.master_pb.LookupBlockVolumeRequest\x1a$.master_pb.LookupBlockVolumeResponse\"\x00B2Z0github.com/seaweedfs/seaweedfs/weed/pb/master_pbb\x06proto3" + "\x11LookupBlockVolume\x12#.master_pb.LookupBlockVolumeRequest\x1a$.master_pb.LookupBlockVolumeResponse\"\x00\x12f\n" + + "\x13CreateBlockSnapshot\x12%.master_pb.CreateBlockSnapshotRequest\x1a&.master_pb.CreateBlockSnapshotResponse\"\x00\x12f\n" + + "\x13DeleteBlockSnapshot\x12%.master_pb.DeleteBlockSnapshotRequest\x1a&.master_pb.DeleteBlockSnapshotResponse\"\x00\x12c\n" + + "\x12ListBlockSnapshots\x12$.master_pb.ListBlockSnapshotsRequest\x1a%.master_pb.ListBlockSnapshotsResponse\"\x00\x12`\n" + + "\x11ExpandBlockVolume\x12#.master_pb.ExpandBlockVolumeRequest\x1a$.master_pb.ExpandBlockVolumeResponse\"\x00B2Z0github.com/seaweedfs/seaweedfs/weed/pb/master_pbb\x06proto3" var ( file_master_proto_rawDescOnce sync.Once @@ -5299,7 +5888,7 @@ func file_master_proto_rawDescGZIP() []byte { return file_master_proto_rawDescData } -var file_master_proto_msgTypes = make([]protoimpl.MessageInfo, 81) +var file_master_proto_msgTypes = make([]protoimpl.MessageInfo, 90) var file_master_proto_goTypes = []any{ (*Heartbeat)(nil), // 0: master_pb.Heartbeat (*HeartbeatResponse)(nil), // 1: master_pb.HeartbeatResponse @@ -5371,18 +5960,27 @@ var file_master_proto_goTypes = []any{ (*DeleteBlockVolumeResponse)(nil), // 67: master_pb.DeleteBlockVolumeResponse (*LookupBlockVolumeRequest)(nil), // 68: master_pb.LookupBlockVolumeRequest (*LookupBlockVolumeResponse)(nil), // 69: master_pb.LookupBlockVolumeResponse - nil, // 70: master_pb.Heartbeat.MaxVolumeCountsEntry - nil, // 71: master_pb.StorageBackend.PropertiesEntry - (*SuperBlockExtra_ErasureCoding)(nil), // 72: master_pb.SuperBlockExtra.ErasureCoding - (*LookupVolumeResponse_VolumeIdLocation)(nil), // 73: master_pb.LookupVolumeResponse.VolumeIdLocation - nil, // 74: master_pb.DataNodeInfo.DiskInfosEntry - nil, // 75: master_pb.RackInfo.DiskInfosEntry - nil, // 76: master_pb.DataCenterInfo.DiskInfosEntry - nil, // 77: master_pb.TopologyInfo.DiskInfosEntry - (*LookupEcVolumeResponse_EcShardIdLocation)(nil), // 78: master_pb.LookupEcVolumeResponse.EcShardIdLocation - (*ListClusterNodesResponse_ClusterNode)(nil), // 79: master_pb.ListClusterNodesResponse.ClusterNode - (*RaftListClusterServersResponse_ClusterServers)(nil), // 80: master_pb.RaftListClusterServersResponse.ClusterServers - (*volume_server_pb.VolumeServerState)(nil), // 81: volume_server_pb.VolumeServerState + (*CreateBlockSnapshotRequest)(nil), // 70: master_pb.CreateBlockSnapshotRequest + (*CreateBlockSnapshotResponse)(nil), // 71: master_pb.CreateBlockSnapshotResponse + (*DeleteBlockSnapshotRequest)(nil), // 72: master_pb.DeleteBlockSnapshotRequest + (*DeleteBlockSnapshotResponse)(nil), // 73: master_pb.DeleteBlockSnapshotResponse + (*ListBlockSnapshotsRequest)(nil), // 74: master_pb.ListBlockSnapshotsRequest + (*ListBlockSnapshotsResponse)(nil), // 75: master_pb.ListBlockSnapshotsResponse + (*BlockSnapshotInfo)(nil), // 76: master_pb.BlockSnapshotInfo + (*ExpandBlockVolumeRequest)(nil), // 77: master_pb.ExpandBlockVolumeRequest + (*ExpandBlockVolumeResponse)(nil), // 78: master_pb.ExpandBlockVolumeResponse + nil, // 79: master_pb.Heartbeat.MaxVolumeCountsEntry + nil, // 80: master_pb.StorageBackend.PropertiesEntry + (*SuperBlockExtra_ErasureCoding)(nil), // 81: master_pb.SuperBlockExtra.ErasureCoding + (*LookupVolumeResponse_VolumeIdLocation)(nil), // 82: master_pb.LookupVolumeResponse.VolumeIdLocation + nil, // 83: master_pb.DataNodeInfo.DiskInfosEntry + nil, // 84: master_pb.RackInfo.DiskInfosEntry + nil, // 85: master_pb.DataCenterInfo.DiskInfosEntry + nil, // 86: master_pb.TopologyInfo.DiskInfosEntry + (*LookupEcVolumeResponse_EcShardIdLocation)(nil), // 87: master_pb.LookupEcVolumeResponse.EcShardIdLocation + (*ListClusterNodesResponse_ClusterNode)(nil), // 88: master_pb.ListClusterNodesResponse.ClusterNode + (*RaftListClusterServersResponse_ClusterServers)(nil), // 89: master_pb.RaftListClusterServersResponse.ClusterServers + (*volume_server_pb.VolumeServerState)(nil), // 90: volume_server_pb.VolumeServerState } var file_master_proto_depIdxs = []int32{ 2, // 0: master_pb.Heartbeat.volumes:type_name -> master_pb.VolumeInformationMessage @@ -5391,100 +5989,109 @@ var file_master_proto_depIdxs = []int32{ 4, // 3: master_pb.Heartbeat.ec_shards:type_name -> master_pb.VolumeEcShardInformationMessage 4, // 4: master_pb.Heartbeat.new_ec_shards:type_name -> master_pb.VolumeEcShardInformationMessage 4, // 5: master_pb.Heartbeat.deleted_ec_shards:type_name -> master_pb.VolumeEcShardInformationMessage - 70, // 6: master_pb.Heartbeat.max_volume_counts:type_name -> master_pb.Heartbeat.MaxVolumeCountsEntry - 81, // 7: master_pb.Heartbeat.state:type_name -> volume_server_pb.VolumeServerState + 79, // 6: master_pb.Heartbeat.max_volume_counts:type_name -> master_pb.Heartbeat.MaxVolumeCountsEntry + 90, // 7: master_pb.Heartbeat.state:type_name -> volume_server_pb.VolumeServerState 61, // 8: master_pb.Heartbeat.block_volume_infos:type_name -> master_pb.BlockVolumeInfoMessage 62, // 9: master_pb.Heartbeat.new_block_volumes:type_name -> master_pb.BlockVolumeShortInfoMessage 62, // 10: master_pb.Heartbeat.deleted_block_volumes:type_name -> master_pb.BlockVolumeShortInfoMessage 5, // 11: master_pb.HeartbeatResponse.storage_backends:type_name -> master_pb.StorageBackend 63, // 12: master_pb.HeartbeatResponse.block_volume_assignments:type_name -> master_pb.BlockVolumeAssignment - 71, // 13: master_pb.StorageBackend.properties:type_name -> master_pb.StorageBackend.PropertiesEntry - 72, // 14: master_pb.SuperBlockExtra.erasure_coding:type_name -> master_pb.SuperBlockExtra.ErasureCoding + 80, // 13: master_pb.StorageBackend.properties:type_name -> master_pb.StorageBackend.PropertiesEntry + 81, // 14: master_pb.SuperBlockExtra.erasure_coding:type_name -> master_pb.SuperBlockExtra.ErasureCoding 9, // 15: master_pb.KeepConnectedResponse.volume_location:type_name -> master_pb.VolumeLocation 10, // 16: master_pb.KeepConnectedResponse.cluster_node_update:type_name -> master_pb.ClusterNodeUpdate - 73, // 17: master_pb.LookupVolumeResponse.volume_id_locations:type_name -> master_pb.LookupVolumeResponse.VolumeIdLocation + 82, // 17: master_pb.LookupVolumeResponse.volume_id_locations:type_name -> master_pb.LookupVolumeResponse.VolumeIdLocation 14, // 18: master_pb.AssignResponse.replicas:type_name -> master_pb.Location 14, // 19: master_pb.AssignResponse.location:type_name -> master_pb.Location 20, // 20: master_pb.CollectionListResponse.collections:type_name -> master_pb.Collection 2, // 21: master_pb.DiskInfo.volume_infos:type_name -> master_pb.VolumeInformationMessage 4, // 22: master_pb.DiskInfo.ec_shard_infos:type_name -> master_pb.VolumeEcShardInformationMessage - 74, // 23: master_pb.DataNodeInfo.diskInfos:type_name -> master_pb.DataNodeInfo.DiskInfosEntry + 83, // 23: master_pb.DataNodeInfo.diskInfos:type_name -> master_pb.DataNodeInfo.DiskInfosEntry 26, // 24: master_pb.RackInfo.data_node_infos:type_name -> master_pb.DataNodeInfo - 75, // 25: master_pb.RackInfo.diskInfos:type_name -> master_pb.RackInfo.DiskInfosEntry + 84, // 25: master_pb.RackInfo.diskInfos:type_name -> master_pb.RackInfo.DiskInfosEntry 27, // 26: master_pb.DataCenterInfo.rack_infos:type_name -> master_pb.RackInfo - 76, // 27: master_pb.DataCenterInfo.diskInfos:type_name -> master_pb.DataCenterInfo.DiskInfosEntry + 85, // 27: master_pb.DataCenterInfo.diskInfos:type_name -> master_pb.DataCenterInfo.DiskInfosEntry 28, // 28: master_pb.TopologyInfo.data_center_infos:type_name -> master_pb.DataCenterInfo - 77, // 29: master_pb.TopologyInfo.diskInfos:type_name -> master_pb.TopologyInfo.DiskInfosEntry + 86, // 29: master_pb.TopologyInfo.diskInfos:type_name -> master_pb.TopologyInfo.DiskInfosEntry 29, // 30: master_pb.VolumeListResponse.topology_info:type_name -> master_pb.TopologyInfo - 78, // 31: master_pb.LookupEcVolumeResponse.shard_id_locations:type_name -> master_pb.LookupEcVolumeResponse.EcShardIdLocation + 87, // 31: master_pb.LookupEcVolumeResponse.shard_id_locations:type_name -> master_pb.LookupEcVolumeResponse.EcShardIdLocation 5, // 32: master_pb.GetMasterConfigurationResponse.storage_backends:type_name -> master_pb.StorageBackend - 79, // 33: master_pb.ListClusterNodesResponse.cluster_nodes:type_name -> master_pb.ListClusterNodesResponse.ClusterNode - 80, // 34: master_pb.RaftListClusterServersResponse.cluster_servers:type_name -> master_pb.RaftListClusterServersResponse.ClusterServers - 14, // 35: master_pb.LookupVolumeResponse.VolumeIdLocation.locations:type_name -> master_pb.Location - 25, // 36: master_pb.DataNodeInfo.DiskInfosEntry.value:type_name -> master_pb.DiskInfo - 25, // 37: master_pb.RackInfo.DiskInfosEntry.value:type_name -> master_pb.DiskInfo - 25, // 38: master_pb.DataCenterInfo.DiskInfosEntry.value:type_name -> master_pb.DiskInfo - 25, // 39: master_pb.TopologyInfo.DiskInfosEntry.value:type_name -> master_pb.DiskInfo - 14, // 40: master_pb.LookupEcVolumeResponse.EcShardIdLocation.locations:type_name -> master_pb.Location - 0, // 41: master_pb.Seaweed.SendHeartbeat:input_type -> master_pb.Heartbeat - 8, // 42: master_pb.Seaweed.KeepConnected:input_type -> master_pb.KeepConnectedRequest - 12, // 43: master_pb.Seaweed.LookupVolume:input_type -> master_pb.LookupVolumeRequest - 15, // 44: master_pb.Seaweed.Assign:input_type -> master_pb.AssignRequest - 15, // 45: master_pb.Seaweed.StreamAssign:input_type -> master_pb.AssignRequest - 18, // 46: master_pb.Seaweed.Statistics:input_type -> master_pb.StatisticsRequest - 21, // 47: master_pb.Seaweed.CollectionList:input_type -> master_pb.CollectionListRequest - 23, // 48: master_pb.Seaweed.CollectionDelete:input_type -> master_pb.CollectionDeleteRequest - 30, // 49: master_pb.Seaweed.VolumeList:input_type -> master_pb.VolumeListRequest - 32, // 50: master_pb.Seaweed.LookupEcVolume:input_type -> master_pb.LookupEcVolumeRequest - 34, // 51: master_pb.Seaweed.VacuumVolume:input_type -> master_pb.VacuumVolumeRequest - 36, // 52: master_pb.Seaweed.DisableVacuum:input_type -> master_pb.DisableVacuumRequest - 38, // 53: master_pb.Seaweed.EnableVacuum:input_type -> master_pb.EnableVacuumRequest - 40, // 54: master_pb.Seaweed.VolumeMarkReadonly:input_type -> master_pb.VolumeMarkReadonlyRequest - 42, // 55: master_pb.Seaweed.GetMasterConfiguration:input_type -> master_pb.GetMasterConfigurationRequest - 44, // 56: master_pb.Seaweed.ListClusterNodes:input_type -> master_pb.ListClusterNodesRequest - 46, // 57: master_pb.Seaweed.LeaseAdminToken:input_type -> master_pb.LeaseAdminTokenRequest - 48, // 58: master_pb.Seaweed.ReleaseAdminToken:input_type -> master_pb.ReleaseAdminTokenRequest - 50, // 59: master_pb.Seaweed.Ping:input_type -> master_pb.PingRequest - 56, // 60: master_pb.Seaweed.RaftListClusterServers:input_type -> master_pb.RaftListClusterServersRequest - 52, // 61: master_pb.Seaweed.RaftAddServer:input_type -> master_pb.RaftAddServerRequest - 54, // 62: master_pb.Seaweed.RaftRemoveServer:input_type -> master_pb.RaftRemoveServerRequest - 58, // 63: master_pb.Seaweed.RaftLeadershipTransfer:input_type -> master_pb.RaftLeadershipTransferRequest - 16, // 64: master_pb.Seaweed.VolumeGrow:input_type -> master_pb.VolumeGrowRequest - 64, // 65: master_pb.Seaweed.CreateBlockVolume:input_type -> master_pb.CreateBlockVolumeRequest - 66, // 66: master_pb.Seaweed.DeleteBlockVolume:input_type -> master_pb.DeleteBlockVolumeRequest - 68, // 67: master_pb.Seaweed.LookupBlockVolume:input_type -> master_pb.LookupBlockVolumeRequest - 1, // 68: master_pb.Seaweed.SendHeartbeat:output_type -> master_pb.HeartbeatResponse - 11, // 69: master_pb.Seaweed.KeepConnected:output_type -> master_pb.KeepConnectedResponse - 13, // 70: master_pb.Seaweed.LookupVolume:output_type -> master_pb.LookupVolumeResponse - 17, // 71: master_pb.Seaweed.Assign:output_type -> master_pb.AssignResponse - 17, // 72: master_pb.Seaweed.StreamAssign:output_type -> master_pb.AssignResponse - 19, // 73: master_pb.Seaweed.Statistics:output_type -> master_pb.StatisticsResponse - 22, // 74: master_pb.Seaweed.CollectionList:output_type -> master_pb.CollectionListResponse - 24, // 75: master_pb.Seaweed.CollectionDelete:output_type -> master_pb.CollectionDeleteResponse - 31, // 76: master_pb.Seaweed.VolumeList:output_type -> master_pb.VolumeListResponse - 33, // 77: master_pb.Seaweed.LookupEcVolume:output_type -> master_pb.LookupEcVolumeResponse - 35, // 78: master_pb.Seaweed.VacuumVolume:output_type -> master_pb.VacuumVolumeResponse - 37, // 79: master_pb.Seaweed.DisableVacuum:output_type -> master_pb.DisableVacuumResponse - 39, // 80: master_pb.Seaweed.EnableVacuum:output_type -> master_pb.EnableVacuumResponse - 41, // 81: master_pb.Seaweed.VolumeMarkReadonly:output_type -> master_pb.VolumeMarkReadonlyResponse - 43, // 82: master_pb.Seaweed.GetMasterConfiguration:output_type -> master_pb.GetMasterConfigurationResponse - 45, // 83: master_pb.Seaweed.ListClusterNodes:output_type -> master_pb.ListClusterNodesResponse - 47, // 84: master_pb.Seaweed.LeaseAdminToken:output_type -> master_pb.LeaseAdminTokenResponse - 49, // 85: master_pb.Seaweed.ReleaseAdminToken:output_type -> master_pb.ReleaseAdminTokenResponse - 51, // 86: master_pb.Seaweed.Ping:output_type -> master_pb.PingResponse - 57, // 87: master_pb.Seaweed.RaftListClusterServers:output_type -> master_pb.RaftListClusterServersResponse - 53, // 88: master_pb.Seaweed.RaftAddServer:output_type -> master_pb.RaftAddServerResponse - 55, // 89: master_pb.Seaweed.RaftRemoveServer:output_type -> master_pb.RaftRemoveServerResponse - 59, // 90: master_pb.Seaweed.RaftLeadershipTransfer:output_type -> master_pb.RaftLeadershipTransferResponse - 60, // 91: master_pb.Seaweed.VolumeGrow:output_type -> master_pb.VolumeGrowResponse - 65, // 92: master_pb.Seaweed.CreateBlockVolume:output_type -> master_pb.CreateBlockVolumeResponse - 67, // 93: master_pb.Seaweed.DeleteBlockVolume:output_type -> master_pb.DeleteBlockVolumeResponse - 69, // 94: master_pb.Seaweed.LookupBlockVolume:output_type -> master_pb.LookupBlockVolumeResponse - 68, // [68:95] is the sub-list for method output_type - 41, // [41:68] is the sub-list for method input_type - 41, // [41:41] is the sub-list for extension type_name - 41, // [41:41] is the sub-list for extension extendee - 0, // [0:41] is the sub-list for field type_name + 88, // 33: master_pb.ListClusterNodesResponse.cluster_nodes:type_name -> master_pb.ListClusterNodesResponse.ClusterNode + 89, // 34: master_pb.RaftListClusterServersResponse.cluster_servers:type_name -> master_pb.RaftListClusterServersResponse.ClusterServers + 76, // 35: master_pb.ListBlockSnapshotsResponse.snapshots:type_name -> master_pb.BlockSnapshotInfo + 14, // 36: master_pb.LookupVolumeResponse.VolumeIdLocation.locations:type_name -> master_pb.Location + 25, // 37: master_pb.DataNodeInfo.DiskInfosEntry.value:type_name -> master_pb.DiskInfo + 25, // 38: master_pb.RackInfo.DiskInfosEntry.value:type_name -> master_pb.DiskInfo + 25, // 39: master_pb.DataCenterInfo.DiskInfosEntry.value:type_name -> master_pb.DiskInfo + 25, // 40: master_pb.TopologyInfo.DiskInfosEntry.value:type_name -> master_pb.DiskInfo + 14, // 41: master_pb.LookupEcVolumeResponse.EcShardIdLocation.locations:type_name -> master_pb.Location + 0, // 42: master_pb.Seaweed.SendHeartbeat:input_type -> master_pb.Heartbeat + 8, // 43: master_pb.Seaweed.KeepConnected:input_type -> master_pb.KeepConnectedRequest + 12, // 44: master_pb.Seaweed.LookupVolume:input_type -> master_pb.LookupVolumeRequest + 15, // 45: master_pb.Seaweed.Assign:input_type -> master_pb.AssignRequest + 15, // 46: master_pb.Seaweed.StreamAssign:input_type -> master_pb.AssignRequest + 18, // 47: master_pb.Seaweed.Statistics:input_type -> master_pb.StatisticsRequest + 21, // 48: master_pb.Seaweed.CollectionList:input_type -> master_pb.CollectionListRequest + 23, // 49: master_pb.Seaweed.CollectionDelete:input_type -> master_pb.CollectionDeleteRequest + 30, // 50: master_pb.Seaweed.VolumeList:input_type -> master_pb.VolumeListRequest + 32, // 51: master_pb.Seaweed.LookupEcVolume:input_type -> master_pb.LookupEcVolumeRequest + 34, // 52: master_pb.Seaweed.VacuumVolume:input_type -> master_pb.VacuumVolumeRequest + 36, // 53: master_pb.Seaweed.DisableVacuum:input_type -> master_pb.DisableVacuumRequest + 38, // 54: master_pb.Seaweed.EnableVacuum:input_type -> master_pb.EnableVacuumRequest + 40, // 55: master_pb.Seaweed.VolumeMarkReadonly:input_type -> master_pb.VolumeMarkReadonlyRequest + 42, // 56: master_pb.Seaweed.GetMasterConfiguration:input_type -> master_pb.GetMasterConfigurationRequest + 44, // 57: master_pb.Seaweed.ListClusterNodes:input_type -> master_pb.ListClusterNodesRequest + 46, // 58: master_pb.Seaweed.LeaseAdminToken:input_type -> master_pb.LeaseAdminTokenRequest + 48, // 59: master_pb.Seaweed.ReleaseAdminToken:input_type -> master_pb.ReleaseAdminTokenRequest + 50, // 60: master_pb.Seaweed.Ping:input_type -> master_pb.PingRequest + 56, // 61: master_pb.Seaweed.RaftListClusterServers:input_type -> master_pb.RaftListClusterServersRequest + 52, // 62: master_pb.Seaweed.RaftAddServer:input_type -> master_pb.RaftAddServerRequest + 54, // 63: master_pb.Seaweed.RaftRemoveServer:input_type -> master_pb.RaftRemoveServerRequest + 58, // 64: master_pb.Seaweed.RaftLeadershipTransfer:input_type -> master_pb.RaftLeadershipTransferRequest + 16, // 65: master_pb.Seaweed.VolumeGrow:input_type -> master_pb.VolumeGrowRequest + 64, // 66: master_pb.Seaweed.CreateBlockVolume:input_type -> master_pb.CreateBlockVolumeRequest + 66, // 67: master_pb.Seaweed.DeleteBlockVolume:input_type -> master_pb.DeleteBlockVolumeRequest + 68, // 68: master_pb.Seaweed.LookupBlockVolume:input_type -> master_pb.LookupBlockVolumeRequest + 70, // 69: master_pb.Seaweed.CreateBlockSnapshot:input_type -> master_pb.CreateBlockSnapshotRequest + 72, // 70: master_pb.Seaweed.DeleteBlockSnapshot:input_type -> master_pb.DeleteBlockSnapshotRequest + 74, // 71: master_pb.Seaweed.ListBlockSnapshots:input_type -> master_pb.ListBlockSnapshotsRequest + 77, // 72: master_pb.Seaweed.ExpandBlockVolume:input_type -> master_pb.ExpandBlockVolumeRequest + 1, // 73: master_pb.Seaweed.SendHeartbeat:output_type -> master_pb.HeartbeatResponse + 11, // 74: master_pb.Seaweed.KeepConnected:output_type -> master_pb.KeepConnectedResponse + 13, // 75: master_pb.Seaweed.LookupVolume:output_type -> master_pb.LookupVolumeResponse + 17, // 76: master_pb.Seaweed.Assign:output_type -> master_pb.AssignResponse + 17, // 77: master_pb.Seaweed.StreamAssign:output_type -> master_pb.AssignResponse + 19, // 78: master_pb.Seaweed.Statistics:output_type -> master_pb.StatisticsResponse + 22, // 79: master_pb.Seaweed.CollectionList:output_type -> master_pb.CollectionListResponse + 24, // 80: master_pb.Seaweed.CollectionDelete:output_type -> master_pb.CollectionDeleteResponse + 31, // 81: master_pb.Seaweed.VolumeList:output_type -> master_pb.VolumeListResponse + 33, // 82: master_pb.Seaweed.LookupEcVolume:output_type -> master_pb.LookupEcVolumeResponse + 35, // 83: master_pb.Seaweed.VacuumVolume:output_type -> master_pb.VacuumVolumeResponse + 37, // 84: master_pb.Seaweed.DisableVacuum:output_type -> master_pb.DisableVacuumResponse + 39, // 85: master_pb.Seaweed.EnableVacuum:output_type -> master_pb.EnableVacuumResponse + 41, // 86: master_pb.Seaweed.VolumeMarkReadonly:output_type -> master_pb.VolumeMarkReadonlyResponse + 43, // 87: master_pb.Seaweed.GetMasterConfiguration:output_type -> master_pb.GetMasterConfigurationResponse + 45, // 88: master_pb.Seaweed.ListClusterNodes:output_type -> master_pb.ListClusterNodesResponse + 47, // 89: master_pb.Seaweed.LeaseAdminToken:output_type -> master_pb.LeaseAdminTokenResponse + 49, // 90: master_pb.Seaweed.ReleaseAdminToken:output_type -> master_pb.ReleaseAdminTokenResponse + 51, // 91: master_pb.Seaweed.Ping:output_type -> master_pb.PingResponse + 57, // 92: master_pb.Seaweed.RaftListClusterServers:output_type -> master_pb.RaftListClusterServersResponse + 53, // 93: master_pb.Seaweed.RaftAddServer:output_type -> master_pb.RaftAddServerResponse + 55, // 94: master_pb.Seaweed.RaftRemoveServer:output_type -> master_pb.RaftRemoveServerResponse + 59, // 95: master_pb.Seaweed.RaftLeadershipTransfer:output_type -> master_pb.RaftLeadershipTransferResponse + 60, // 96: master_pb.Seaweed.VolumeGrow:output_type -> master_pb.VolumeGrowResponse + 65, // 97: master_pb.Seaweed.CreateBlockVolume:output_type -> master_pb.CreateBlockVolumeResponse + 67, // 98: master_pb.Seaweed.DeleteBlockVolume:output_type -> master_pb.DeleteBlockVolumeResponse + 69, // 99: master_pb.Seaweed.LookupBlockVolume:output_type -> master_pb.LookupBlockVolumeResponse + 71, // 100: master_pb.Seaweed.CreateBlockSnapshot:output_type -> master_pb.CreateBlockSnapshotResponse + 73, // 101: master_pb.Seaweed.DeleteBlockSnapshot:output_type -> master_pb.DeleteBlockSnapshotResponse + 75, // 102: master_pb.Seaweed.ListBlockSnapshots:output_type -> master_pb.ListBlockSnapshotsResponse + 78, // 103: master_pb.Seaweed.ExpandBlockVolume:output_type -> master_pb.ExpandBlockVolumeResponse + 73, // [73:104] is the sub-list for method output_type + 42, // [42:73] is the sub-list for method input_type + 42, // [42:42] is the sub-list for extension type_name + 42, // [42:42] is the sub-list for extension extendee + 0, // [0:42] is the sub-list for field type_name } func init() { file_master_proto_init() } @@ -5498,7 +6105,7 @@ func file_master_proto_init() { GoPackagePath: reflect.TypeOf(x{}).PkgPath(), RawDescriptor: unsafe.Slice(unsafe.StringData(file_master_proto_rawDesc), len(file_master_proto_rawDesc)), NumEnums: 0, - NumMessages: 81, + NumMessages: 90, NumExtensions: 0, NumServices: 1, }, diff --git a/weed/pb/master_pb/master_grpc.pb.go b/weed/pb/master_pb/master_grpc.pb.go index ad2359292e..cd8a3ad458 100644 --- a/weed/pb/master_pb/master_grpc.pb.go +++ b/weed/pb/master_pb/master_grpc.pb.go @@ -46,6 +46,10 @@ const ( Seaweed_CreateBlockVolume_FullMethodName = "/master_pb.Seaweed/CreateBlockVolume" Seaweed_DeleteBlockVolume_FullMethodName = "/master_pb.Seaweed/DeleteBlockVolume" Seaweed_LookupBlockVolume_FullMethodName = "/master_pb.Seaweed/LookupBlockVolume" + Seaweed_CreateBlockSnapshot_FullMethodName = "/master_pb.Seaweed/CreateBlockSnapshot" + Seaweed_DeleteBlockSnapshot_FullMethodName = "/master_pb.Seaweed/DeleteBlockSnapshot" + Seaweed_ListBlockSnapshots_FullMethodName = "/master_pb.Seaweed/ListBlockSnapshots" + Seaweed_ExpandBlockVolume_FullMethodName = "/master_pb.Seaweed/ExpandBlockVolume" ) // SeaweedClient is the client API for Seaweed service. @@ -79,6 +83,10 @@ type SeaweedClient interface { CreateBlockVolume(ctx context.Context, in *CreateBlockVolumeRequest, opts ...grpc.CallOption) (*CreateBlockVolumeResponse, error) DeleteBlockVolume(ctx context.Context, in *DeleteBlockVolumeRequest, opts ...grpc.CallOption) (*DeleteBlockVolumeResponse, error) LookupBlockVolume(ctx context.Context, in *LookupBlockVolumeRequest, opts ...grpc.CallOption) (*LookupBlockVolumeResponse, error) + CreateBlockSnapshot(ctx context.Context, in *CreateBlockSnapshotRequest, opts ...grpc.CallOption) (*CreateBlockSnapshotResponse, error) + DeleteBlockSnapshot(ctx context.Context, in *DeleteBlockSnapshotRequest, opts ...grpc.CallOption) (*DeleteBlockSnapshotResponse, error) + ListBlockSnapshots(ctx context.Context, in *ListBlockSnapshotsRequest, opts ...grpc.CallOption) (*ListBlockSnapshotsResponse, error) + ExpandBlockVolume(ctx context.Context, in *ExpandBlockVolumeRequest, opts ...grpc.CallOption) (*ExpandBlockVolumeResponse, error) } type seaweedClient struct { @@ -368,6 +376,46 @@ func (c *seaweedClient) LookupBlockVolume(ctx context.Context, in *LookupBlockVo return out, nil } +func (c *seaweedClient) CreateBlockSnapshot(ctx context.Context, in *CreateBlockSnapshotRequest, opts ...grpc.CallOption) (*CreateBlockSnapshotResponse, error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) + out := new(CreateBlockSnapshotResponse) + err := c.cc.Invoke(ctx, Seaweed_CreateBlockSnapshot_FullMethodName, in, out, cOpts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *seaweedClient) DeleteBlockSnapshot(ctx context.Context, in *DeleteBlockSnapshotRequest, opts ...grpc.CallOption) (*DeleteBlockSnapshotResponse, error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) + out := new(DeleteBlockSnapshotResponse) + err := c.cc.Invoke(ctx, Seaweed_DeleteBlockSnapshot_FullMethodName, in, out, cOpts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *seaweedClient) ListBlockSnapshots(ctx context.Context, in *ListBlockSnapshotsRequest, opts ...grpc.CallOption) (*ListBlockSnapshotsResponse, error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) + out := new(ListBlockSnapshotsResponse) + err := c.cc.Invoke(ctx, Seaweed_ListBlockSnapshots_FullMethodName, in, out, cOpts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *seaweedClient) ExpandBlockVolume(ctx context.Context, in *ExpandBlockVolumeRequest, opts ...grpc.CallOption) (*ExpandBlockVolumeResponse, error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) + out := new(ExpandBlockVolumeResponse) + err := c.cc.Invoke(ctx, Seaweed_ExpandBlockVolume_FullMethodName, in, out, cOpts...) + if err != nil { + return nil, err + } + return out, nil +} + // SeaweedServer is the server API for Seaweed service. // All implementations must embed UnimplementedSeaweedServer // for forward compatibility. @@ -399,6 +447,10 @@ type SeaweedServer interface { CreateBlockVolume(context.Context, *CreateBlockVolumeRequest) (*CreateBlockVolumeResponse, error) DeleteBlockVolume(context.Context, *DeleteBlockVolumeRequest) (*DeleteBlockVolumeResponse, error) LookupBlockVolume(context.Context, *LookupBlockVolumeRequest) (*LookupBlockVolumeResponse, error) + CreateBlockSnapshot(context.Context, *CreateBlockSnapshotRequest) (*CreateBlockSnapshotResponse, error) + DeleteBlockSnapshot(context.Context, *DeleteBlockSnapshotRequest) (*DeleteBlockSnapshotResponse, error) + ListBlockSnapshots(context.Context, *ListBlockSnapshotsRequest) (*ListBlockSnapshotsResponse, error) + ExpandBlockVolume(context.Context, *ExpandBlockVolumeRequest) (*ExpandBlockVolumeResponse, error) mustEmbedUnimplementedSeaweedServer() } @@ -490,6 +542,18 @@ func (UnimplementedSeaweedServer) DeleteBlockVolume(context.Context, *DeleteBloc func (UnimplementedSeaweedServer) LookupBlockVolume(context.Context, *LookupBlockVolumeRequest) (*LookupBlockVolumeResponse, error) { return nil, status.Error(codes.Unimplemented, "method LookupBlockVolume not implemented") } +func (UnimplementedSeaweedServer) CreateBlockSnapshot(context.Context, *CreateBlockSnapshotRequest) (*CreateBlockSnapshotResponse, error) { + return nil, status.Error(codes.Unimplemented, "method CreateBlockSnapshot not implemented") +} +func (UnimplementedSeaweedServer) DeleteBlockSnapshot(context.Context, *DeleteBlockSnapshotRequest) (*DeleteBlockSnapshotResponse, error) { + return nil, status.Error(codes.Unimplemented, "method DeleteBlockSnapshot not implemented") +} +func (UnimplementedSeaweedServer) ListBlockSnapshots(context.Context, *ListBlockSnapshotsRequest) (*ListBlockSnapshotsResponse, error) { + return nil, status.Error(codes.Unimplemented, "method ListBlockSnapshots not implemented") +} +func (UnimplementedSeaweedServer) ExpandBlockVolume(context.Context, *ExpandBlockVolumeRequest) (*ExpandBlockVolumeResponse, error) { + return nil, status.Error(codes.Unimplemented, "method ExpandBlockVolume not implemented") +} func (UnimplementedSeaweedServer) mustEmbedUnimplementedSeaweedServer() {} func (UnimplementedSeaweedServer) testEmbeddedByValue() {} @@ -964,6 +1028,78 @@ func _Seaweed_LookupBlockVolume_Handler(srv interface{}, ctx context.Context, de return interceptor(ctx, in, info, handler) } +func _Seaweed_CreateBlockSnapshot_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(CreateBlockSnapshotRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(SeaweedServer).CreateBlockSnapshot(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: Seaweed_CreateBlockSnapshot_FullMethodName, + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(SeaweedServer).CreateBlockSnapshot(ctx, req.(*CreateBlockSnapshotRequest)) + } + return interceptor(ctx, in, info, handler) +} + +func _Seaweed_DeleteBlockSnapshot_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(DeleteBlockSnapshotRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(SeaweedServer).DeleteBlockSnapshot(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: Seaweed_DeleteBlockSnapshot_FullMethodName, + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(SeaweedServer).DeleteBlockSnapshot(ctx, req.(*DeleteBlockSnapshotRequest)) + } + return interceptor(ctx, in, info, handler) +} + +func _Seaweed_ListBlockSnapshots_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(ListBlockSnapshotsRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(SeaweedServer).ListBlockSnapshots(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: Seaweed_ListBlockSnapshots_FullMethodName, + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(SeaweedServer).ListBlockSnapshots(ctx, req.(*ListBlockSnapshotsRequest)) + } + return interceptor(ctx, in, info, handler) +} + +func _Seaweed_ExpandBlockVolume_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(ExpandBlockVolumeRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(SeaweedServer).ExpandBlockVolume(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: Seaweed_ExpandBlockVolume_FullMethodName, + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(SeaweedServer).ExpandBlockVolume(ctx, req.(*ExpandBlockVolumeRequest)) + } + return interceptor(ctx, in, info, handler) +} + // Seaweed_ServiceDesc is the grpc.ServiceDesc for Seaweed service. // It's only intended for direct use with grpc.RegisterService, // and not to be introspected or modified (even as a copy) @@ -1067,6 +1203,22 @@ var Seaweed_ServiceDesc = grpc.ServiceDesc{ MethodName: "LookupBlockVolume", Handler: _Seaweed_LookupBlockVolume_Handler, }, + { + MethodName: "CreateBlockSnapshot", + Handler: _Seaweed_CreateBlockSnapshot_Handler, + }, + { + MethodName: "DeleteBlockSnapshot", + Handler: _Seaweed_DeleteBlockSnapshot_Handler, + }, + { + MethodName: "ListBlockSnapshots", + Handler: _Seaweed_ListBlockSnapshots_Handler, + }, + { + MethodName: "ExpandBlockVolume", + Handler: _Seaweed_ExpandBlockVolume_Handler, + }, }, Streams: []grpc.StreamDesc{ { diff --git a/weed/pb/volume_server.proto b/weed/pb/volume_server.proto index ea55374966..28e1b67408 100644 --- a/weed/pb/volume_server.proto +++ b/weed/pb/volume_server.proto @@ -143,6 +143,14 @@ service VolumeServer { } rpc VolumeServerDeleteBlockVolume (VolumeServerDeleteBlockVolumeRequest) returns (VolumeServerDeleteBlockVolumeResponse) { } + rpc SnapshotBlockVolume (SnapshotBlockVolumeRequest) returns (SnapshotBlockVolumeResponse) { + } + rpc DeleteBlockSnapshot (DeleteBlockSnapshotRequest) returns (DeleteBlockSnapshotResponse) { + } + rpc ListBlockSnapshots (ListBlockSnapshotsRequest) returns (ListBlockSnapshotsResponse) { + } + rpc ExpandBlockVolume (ExpandBlockVolumeRequest) returns (ExpandBlockVolumeResponse) { + } } @@ -786,3 +794,40 @@ message VolumeServerDeleteBlockVolumeRequest { } message VolumeServerDeleteBlockVolumeResponse { } + +message SnapshotBlockVolumeRequest { + string name = 1; + uint32 snapshot_id = 2; +} +message SnapshotBlockVolumeResponse { + uint32 snapshot_id = 1; + int64 created_at = 2; // Unix timestamp in seconds + uint64 size_bytes = 3; +} + +message DeleteBlockSnapshotRequest { + string name = 1; + uint32 snapshot_id = 2; +} +message DeleteBlockSnapshotResponse { +} + +message ListBlockSnapshotsRequest { + string name = 1; +} +message ListBlockSnapshotsResponse { + repeated BlockSnapshotInfo snapshots = 1; +} +message BlockSnapshotInfo { + uint32 snapshot_id = 1; + int64 created_at = 2; // Unix timestamp in seconds + uint64 volume_size_bytes = 3; +} + +message ExpandBlockVolumeRequest { + string name = 1; + uint64 new_size_bytes = 2; +} +message ExpandBlockVolumeResponse { + uint64 capacity_bytes = 1; +} diff --git a/weed/pb/volume_server_pb/volume_server.pb.go b/weed/pb/volume_server_pb/volume_server.pb.go index cba59a36bf..88f77f2e33 100644 --- a/weed/pb/volume_server_pb/volume_server.pb.go +++ b/weed/pb/volume_server_pb/volume_server.pb.go @@ -6246,15 +6246,15 @@ func (x *AllocateBlockVolumeRequest) GetDiskType() string { } type AllocateBlockVolumeResponse struct { - state protoimpl.MessageState `protogen:"open.v1"` - Path string `protobuf:"bytes,1,opt,name=path,proto3" json:"path,omitempty"` - Iqn string `protobuf:"bytes,2,opt,name=iqn,proto3" json:"iqn,omitempty"` - IscsiAddr string `protobuf:"bytes,3,opt,name=iscsi_addr,json=iscsiAddr,proto3" json:"iscsi_addr,omitempty"` - ReplicaDataAddr string `protobuf:"bytes,4,opt,name=replica_data_addr,json=replicaDataAddr,proto3" json:"replica_data_addr,omitempty"` - ReplicaCtrlAddr string `protobuf:"bytes,5,opt,name=replica_ctrl_addr,json=replicaCtrlAddr,proto3" json:"replica_ctrl_addr,omitempty"` - RebuildListenAddr string `protobuf:"bytes,6,opt,name=rebuild_listen_addr,json=rebuildListenAddr,proto3" json:"rebuild_listen_addr,omitempty"` - unknownFields protoimpl.UnknownFields - sizeCache protoimpl.SizeCache + state protoimpl.MessageState `protogen:"open.v1"` + Path string `protobuf:"bytes,1,opt,name=path,proto3" json:"path,omitempty"` + Iqn string `protobuf:"bytes,2,opt,name=iqn,proto3" json:"iqn,omitempty"` + IscsiAddr string `protobuf:"bytes,3,opt,name=iscsi_addr,json=iscsiAddr,proto3" json:"iscsi_addr,omitempty"` + ReplicaDataAddr string `protobuf:"bytes,4,opt,name=replica_data_addr,json=replicaDataAddr,proto3" json:"replica_data_addr,omitempty"` + ReplicaCtrlAddr string `protobuf:"bytes,5,opt,name=replica_ctrl_addr,json=replicaCtrlAddr,proto3" json:"replica_ctrl_addr,omitempty"` + RebuildListenAddr string `protobuf:"bytes,6,opt,name=rebuild_listen_addr,json=rebuildListenAddr,proto3" json:"rebuild_listen_addr,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache } func (x *AllocateBlockVolumeResponse) Reset() { @@ -6409,6 +6409,450 @@ func (*VolumeServerDeleteBlockVolumeResponse) Descriptor() ([]byte, []int) { return file_volume_server_proto_rawDescGZIP(), []int{110} } +type SnapshotBlockVolumeRequest struct { + state protoimpl.MessageState `protogen:"open.v1"` + Name string `protobuf:"bytes,1,opt,name=name,proto3" json:"name,omitempty"` + SnapshotId uint32 `protobuf:"varint,2,opt,name=snapshot_id,json=snapshotId,proto3" json:"snapshot_id,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *SnapshotBlockVolumeRequest) Reset() { + *x = SnapshotBlockVolumeRequest{} + mi := &file_volume_server_proto_msgTypes[111] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *SnapshotBlockVolumeRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*SnapshotBlockVolumeRequest) ProtoMessage() {} + +func (x *SnapshotBlockVolumeRequest) ProtoReflect() protoreflect.Message { + mi := &file_volume_server_proto_msgTypes[111] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use SnapshotBlockVolumeRequest.ProtoReflect.Descriptor instead. +func (*SnapshotBlockVolumeRequest) Descriptor() ([]byte, []int) { + return file_volume_server_proto_rawDescGZIP(), []int{111} +} + +func (x *SnapshotBlockVolumeRequest) GetName() string { + if x != nil { + return x.Name + } + return "" +} + +func (x *SnapshotBlockVolumeRequest) GetSnapshotId() uint32 { + if x != nil { + return x.SnapshotId + } + return 0 +} + +type SnapshotBlockVolumeResponse struct { + state protoimpl.MessageState `protogen:"open.v1"` + SnapshotId uint32 `protobuf:"varint,1,opt,name=snapshot_id,json=snapshotId,proto3" json:"snapshot_id,omitempty"` + CreatedAt int64 `protobuf:"varint,2,opt,name=created_at,json=createdAt,proto3" json:"created_at,omitempty"` // Unix timestamp in seconds + SizeBytes uint64 `protobuf:"varint,3,opt,name=size_bytes,json=sizeBytes,proto3" json:"size_bytes,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *SnapshotBlockVolumeResponse) Reset() { + *x = SnapshotBlockVolumeResponse{} + mi := &file_volume_server_proto_msgTypes[112] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *SnapshotBlockVolumeResponse) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*SnapshotBlockVolumeResponse) ProtoMessage() {} + +func (x *SnapshotBlockVolumeResponse) ProtoReflect() protoreflect.Message { + mi := &file_volume_server_proto_msgTypes[112] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use SnapshotBlockVolumeResponse.ProtoReflect.Descriptor instead. +func (*SnapshotBlockVolumeResponse) Descriptor() ([]byte, []int) { + return file_volume_server_proto_rawDescGZIP(), []int{112} +} + +func (x *SnapshotBlockVolumeResponse) GetSnapshotId() uint32 { + if x != nil { + return x.SnapshotId + } + return 0 +} + +func (x *SnapshotBlockVolumeResponse) GetCreatedAt() int64 { + if x != nil { + return x.CreatedAt + } + return 0 +} + +func (x *SnapshotBlockVolumeResponse) GetSizeBytes() uint64 { + if x != nil { + return x.SizeBytes + } + return 0 +} + +type DeleteBlockSnapshotRequest struct { + state protoimpl.MessageState `protogen:"open.v1"` + Name string `protobuf:"bytes,1,opt,name=name,proto3" json:"name,omitempty"` + SnapshotId uint32 `protobuf:"varint,2,opt,name=snapshot_id,json=snapshotId,proto3" json:"snapshot_id,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *DeleteBlockSnapshotRequest) Reset() { + *x = DeleteBlockSnapshotRequest{} + mi := &file_volume_server_proto_msgTypes[113] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *DeleteBlockSnapshotRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*DeleteBlockSnapshotRequest) ProtoMessage() {} + +func (x *DeleteBlockSnapshotRequest) ProtoReflect() protoreflect.Message { + mi := &file_volume_server_proto_msgTypes[113] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use DeleteBlockSnapshotRequest.ProtoReflect.Descriptor instead. +func (*DeleteBlockSnapshotRequest) Descriptor() ([]byte, []int) { + return file_volume_server_proto_rawDescGZIP(), []int{113} +} + +func (x *DeleteBlockSnapshotRequest) GetName() string { + if x != nil { + return x.Name + } + return "" +} + +func (x *DeleteBlockSnapshotRequest) GetSnapshotId() uint32 { + if x != nil { + return x.SnapshotId + } + return 0 +} + +type DeleteBlockSnapshotResponse struct { + state protoimpl.MessageState `protogen:"open.v1"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *DeleteBlockSnapshotResponse) Reset() { + *x = DeleteBlockSnapshotResponse{} + mi := &file_volume_server_proto_msgTypes[114] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *DeleteBlockSnapshotResponse) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*DeleteBlockSnapshotResponse) ProtoMessage() {} + +func (x *DeleteBlockSnapshotResponse) ProtoReflect() protoreflect.Message { + mi := &file_volume_server_proto_msgTypes[114] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use DeleteBlockSnapshotResponse.ProtoReflect.Descriptor instead. +func (*DeleteBlockSnapshotResponse) Descriptor() ([]byte, []int) { + return file_volume_server_proto_rawDescGZIP(), []int{114} +} + +type ListBlockSnapshotsRequest struct { + state protoimpl.MessageState `protogen:"open.v1"` + Name string `protobuf:"bytes,1,opt,name=name,proto3" json:"name,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *ListBlockSnapshotsRequest) Reset() { + *x = ListBlockSnapshotsRequest{} + mi := &file_volume_server_proto_msgTypes[115] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *ListBlockSnapshotsRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*ListBlockSnapshotsRequest) ProtoMessage() {} + +func (x *ListBlockSnapshotsRequest) ProtoReflect() protoreflect.Message { + mi := &file_volume_server_proto_msgTypes[115] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use ListBlockSnapshotsRequest.ProtoReflect.Descriptor instead. +func (*ListBlockSnapshotsRequest) Descriptor() ([]byte, []int) { + return file_volume_server_proto_rawDescGZIP(), []int{115} +} + +func (x *ListBlockSnapshotsRequest) GetName() string { + if x != nil { + return x.Name + } + return "" +} + +type ListBlockSnapshotsResponse struct { + state protoimpl.MessageState `protogen:"open.v1"` + Snapshots []*BlockSnapshotInfo `protobuf:"bytes,1,rep,name=snapshots,proto3" json:"snapshots,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *ListBlockSnapshotsResponse) Reset() { + *x = ListBlockSnapshotsResponse{} + mi := &file_volume_server_proto_msgTypes[116] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *ListBlockSnapshotsResponse) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*ListBlockSnapshotsResponse) ProtoMessage() {} + +func (x *ListBlockSnapshotsResponse) ProtoReflect() protoreflect.Message { + mi := &file_volume_server_proto_msgTypes[116] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use ListBlockSnapshotsResponse.ProtoReflect.Descriptor instead. +func (*ListBlockSnapshotsResponse) Descriptor() ([]byte, []int) { + return file_volume_server_proto_rawDescGZIP(), []int{116} +} + +func (x *ListBlockSnapshotsResponse) GetSnapshots() []*BlockSnapshotInfo { + if x != nil { + return x.Snapshots + } + return nil +} + +type BlockSnapshotInfo struct { + state protoimpl.MessageState `protogen:"open.v1"` + SnapshotId uint32 `protobuf:"varint,1,opt,name=snapshot_id,json=snapshotId,proto3" json:"snapshot_id,omitempty"` + CreatedAt int64 `protobuf:"varint,2,opt,name=created_at,json=createdAt,proto3" json:"created_at,omitempty"` // Unix timestamp in seconds + VolumeSizeBytes uint64 `protobuf:"varint,3,opt,name=volume_size_bytes,json=volumeSizeBytes,proto3" json:"volume_size_bytes,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *BlockSnapshotInfo) Reset() { + *x = BlockSnapshotInfo{} + mi := &file_volume_server_proto_msgTypes[117] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *BlockSnapshotInfo) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*BlockSnapshotInfo) ProtoMessage() {} + +func (x *BlockSnapshotInfo) ProtoReflect() protoreflect.Message { + mi := &file_volume_server_proto_msgTypes[117] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use BlockSnapshotInfo.ProtoReflect.Descriptor instead. +func (*BlockSnapshotInfo) Descriptor() ([]byte, []int) { + return file_volume_server_proto_rawDescGZIP(), []int{117} +} + +func (x *BlockSnapshotInfo) GetSnapshotId() uint32 { + if x != nil { + return x.SnapshotId + } + return 0 +} + +func (x *BlockSnapshotInfo) GetCreatedAt() int64 { + if x != nil { + return x.CreatedAt + } + return 0 +} + +func (x *BlockSnapshotInfo) GetVolumeSizeBytes() uint64 { + if x != nil { + return x.VolumeSizeBytes + } + return 0 +} + +type ExpandBlockVolumeRequest struct { + state protoimpl.MessageState `protogen:"open.v1"` + Name string `protobuf:"bytes,1,opt,name=name,proto3" json:"name,omitempty"` + NewSizeBytes uint64 `protobuf:"varint,2,opt,name=new_size_bytes,json=newSizeBytes,proto3" json:"new_size_bytes,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *ExpandBlockVolumeRequest) Reset() { + *x = ExpandBlockVolumeRequest{} + mi := &file_volume_server_proto_msgTypes[118] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *ExpandBlockVolumeRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*ExpandBlockVolumeRequest) ProtoMessage() {} + +func (x *ExpandBlockVolumeRequest) ProtoReflect() protoreflect.Message { + mi := &file_volume_server_proto_msgTypes[118] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use ExpandBlockVolumeRequest.ProtoReflect.Descriptor instead. +func (*ExpandBlockVolumeRequest) Descriptor() ([]byte, []int) { + return file_volume_server_proto_rawDescGZIP(), []int{118} +} + +func (x *ExpandBlockVolumeRequest) GetName() string { + if x != nil { + return x.Name + } + return "" +} + +func (x *ExpandBlockVolumeRequest) GetNewSizeBytes() uint64 { + if x != nil { + return x.NewSizeBytes + } + return 0 +} + +type ExpandBlockVolumeResponse struct { + state protoimpl.MessageState `protogen:"open.v1"` + CapacityBytes uint64 `protobuf:"varint,1,opt,name=capacity_bytes,json=capacityBytes,proto3" json:"capacity_bytes,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *ExpandBlockVolumeResponse) Reset() { + *x = ExpandBlockVolumeResponse{} + mi := &file_volume_server_proto_msgTypes[119] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *ExpandBlockVolumeResponse) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*ExpandBlockVolumeResponse) ProtoMessage() {} + +func (x *ExpandBlockVolumeResponse) ProtoReflect() protoreflect.Message { + mi := &file_volume_server_proto_msgTypes[119] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use ExpandBlockVolumeResponse.ProtoReflect.Descriptor instead. +func (*ExpandBlockVolumeResponse) Descriptor() ([]byte, []int) { + return file_volume_server_proto_rawDescGZIP(), []int{119} +} + +func (x *ExpandBlockVolumeResponse) GetCapacityBytes() uint64 { + if x != nil { + return x.CapacityBytes + } + return 0 +} + type FetchAndWriteNeedleRequest_Replica struct { state protoimpl.MessageState `protogen:"open.v1"` Url string `protobuf:"bytes,1,opt,name=url,proto3" json:"url,omitempty"` @@ -6420,7 +6864,7 @@ type FetchAndWriteNeedleRequest_Replica struct { func (x *FetchAndWriteNeedleRequest_Replica) Reset() { *x = FetchAndWriteNeedleRequest_Replica{} - mi := &file_volume_server_proto_msgTypes[111] + mi := &file_volume_server_proto_msgTypes[120] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -6432,7 +6876,7 @@ func (x *FetchAndWriteNeedleRequest_Replica) String() string { func (*FetchAndWriteNeedleRequest_Replica) ProtoMessage() {} func (x *FetchAndWriteNeedleRequest_Replica) ProtoReflect() protoreflect.Message { - mi := &file_volume_server_proto_msgTypes[111] + mi := &file_volume_server_proto_msgTypes[120] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -6480,7 +6924,7 @@ type QueryRequest_Filter struct { func (x *QueryRequest_Filter) Reset() { *x = QueryRequest_Filter{} - mi := &file_volume_server_proto_msgTypes[112] + mi := &file_volume_server_proto_msgTypes[121] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -6492,7 +6936,7 @@ func (x *QueryRequest_Filter) String() string { func (*QueryRequest_Filter) ProtoMessage() {} func (x *QueryRequest_Filter) ProtoReflect() protoreflect.Message { - mi := &file_volume_server_proto_msgTypes[112] + mi := &file_volume_server_proto_msgTypes[121] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -6542,7 +6986,7 @@ type QueryRequest_InputSerialization struct { func (x *QueryRequest_InputSerialization) Reset() { *x = QueryRequest_InputSerialization{} - mi := &file_volume_server_proto_msgTypes[113] + mi := &file_volume_server_proto_msgTypes[122] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -6554,7 +6998,7 @@ func (x *QueryRequest_InputSerialization) String() string { func (*QueryRequest_InputSerialization) ProtoMessage() {} func (x *QueryRequest_InputSerialization) ProtoReflect() protoreflect.Message { - mi := &file_volume_server_proto_msgTypes[113] + mi := &file_volume_server_proto_msgTypes[122] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -6608,7 +7052,7 @@ type QueryRequest_OutputSerialization struct { func (x *QueryRequest_OutputSerialization) Reset() { *x = QueryRequest_OutputSerialization{} - mi := &file_volume_server_proto_msgTypes[114] + mi := &file_volume_server_proto_msgTypes[123] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -6620,7 +7064,7 @@ func (x *QueryRequest_OutputSerialization) String() string { func (*QueryRequest_OutputSerialization) ProtoMessage() {} func (x *QueryRequest_OutputSerialization) ProtoReflect() protoreflect.Message { - mi := &file_volume_server_proto_msgTypes[114] + mi := &file_volume_server_proto_msgTypes[123] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -6666,7 +7110,7 @@ type QueryRequest_InputSerialization_CSVInput struct { func (x *QueryRequest_InputSerialization_CSVInput) Reset() { *x = QueryRequest_InputSerialization_CSVInput{} - mi := &file_volume_server_proto_msgTypes[115] + mi := &file_volume_server_proto_msgTypes[124] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -6678,7 +7122,7 @@ func (x *QueryRequest_InputSerialization_CSVInput) String() string { func (*QueryRequest_InputSerialization_CSVInput) ProtoMessage() {} func (x *QueryRequest_InputSerialization_CSVInput) ProtoReflect() protoreflect.Message { - mi := &file_volume_server_proto_msgTypes[115] + mi := &file_volume_server_proto_msgTypes[124] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -6752,7 +7196,7 @@ type QueryRequest_InputSerialization_JSONInput struct { func (x *QueryRequest_InputSerialization_JSONInput) Reset() { *x = QueryRequest_InputSerialization_JSONInput{} - mi := &file_volume_server_proto_msgTypes[116] + mi := &file_volume_server_proto_msgTypes[125] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -6764,7 +7208,7 @@ func (x *QueryRequest_InputSerialization_JSONInput) String() string { func (*QueryRequest_InputSerialization_JSONInput) ProtoMessage() {} func (x *QueryRequest_InputSerialization_JSONInput) ProtoReflect() protoreflect.Message { - mi := &file_volume_server_proto_msgTypes[116] + mi := &file_volume_server_proto_msgTypes[125] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -6795,7 +7239,7 @@ type QueryRequest_InputSerialization_ParquetInput struct { func (x *QueryRequest_InputSerialization_ParquetInput) Reset() { *x = QueryRequest_InputSerialization_ParquetInput{} - mi := &file_volume_server_proto_msgTypes[117] + mi := &file_volume_server_proto_msgTypes[126] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -6807,7 +7251,7 @@ func (x *QueryRequest_InputSerialization_ParquetInput) String() string { func (*QueryRequest_InputSerialization_ParquetInput) ProtoMessage() {} func (x *QueryRequest_InputSerialization_ParquetInput) ProtoReflect() protoreflect.Message { - mi := &file_volume_server_proto_msgTypes[117] + mi := &file_volume_server_proto_msgTypes[126] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -6836,7 +7280,7 @@ type QueryRequest_OutputSerialization_CSVOutput struct { func (x *QueryRequest_OutputSerialization_CSVOutput) Reset() { *x = QueryRequest_OutputSerialization_CSVOutput{} - mi := &file_volume_server_proto_msgTypes[118] + mi := &file_volume_server_proto_msgTypes[127] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -6848,7 +7292,7 @@ func (x *QueryRequest_OutputSerialization_CSVOutput) String() string { func (*QueryRequest_OutputSerialization_CSVOutput) ProtoMessage() {} func (x *QueryRequest_OutputSerialization_CSVOutput) ProtoReflect() protoreflect.Message { - mi := &file_volume_server_proto_msgTypes[118] + mi := &file_volume_server_proto_msgTypes[127] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -6908,7 +7352,7 @@ type QueryRequest_OutputSerialization_JSONOutput struct { func (x *QueryRequest_OutputSerialization_JSONOutput) Reset() { *x = QueryRequest_OutputSerialization_JSONOutput{} - mi := &file_volume_server_proto_msgTypes[119] + mi := &file_volume_server_proto_msgTypes[128] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -6920,7 +7364,7 @@ func (x *QueryRequest_OutputSerialization_JSONOutput) String() string { func (*QueryRequest_OutputSerialization_JSONOutput) ProtoMessage() {} func (x *QueryRequest_OutputSerialization_JSONOutput) ProtoReflect() protoreflect.Message { - mi := &file_volume_server_proto_msgTypes[119] + mi := &file_volume_server_proto_msgTypes[128] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -7451,20 +7895,54 @@ const file_volume_server_proto_rawDesc = "" + "\x04name\x18\x01 \x01(\tR\x04name\x12\x1d\n" + "\n" + "size_bytes\x18\x02 \x01(\x04R\tsizeBytes\x12\x1b\n" + - "\tdisk_type\x18\x03 \x01(\tR\bdiskType\"b\n" + + "\tdisk_type\x18\x03 \x01(\tR\bdiskType\"\xea\x01\n" + "\x1bAllocateBlockVolumeResponse\x12\x12\n" + "\x04path\x18\x01 \x01(\tR\x04path\x12\x10\n" + "\x03iqn\x18\x02 \x01(\tR\x03iqn\x12\x1d\n" + "\n" + - "iscsi_addr\x18\x03 \x01(\tR\tiscsiAddr\":\n" + + "iscsi_addr\x18\x03 \x01(\tR\tiscsiAddr\x12*\n" + + "\x11replica_data_addr\x18\x04 \x01(\tR\x0freplicaDataAddr\x12*\n" + + "\x11replica_ctrl_addr\x18\x05 \x01(\tR\x0freplicaCtrlAddr\x12.\n" + + "\x13rebuild_listen_addr\x18\x06 \x01(\tR\x11rebuildListenAddr\":\n" + "$VolumeServerDeleteBlockVolumeRequest\x12\x12\n" + "\x04name\x18\x01 \x01(\tR\x04name\"'\n" + - "%VolumeServerDeleteBlockVolumeResponse*>\n" + + "%VolumeServerDeleteBlockVolumeResponse\"Q\n" + + "\x1aSnapshotBlockVolumeRequest\x12\x12\n" + + "\x04name\x18\x01 \x01(\tR\x04name\x12\x1f\n" + + "\vsnapshot_id\x18\x02 \x01(\rR\n" + + "snapshotId\"|\n" + + "\x1bSnapshotBlockVolumeResponse\x12\x1f\n" + + "\vsnapshot_id\x18\x01 \x01(\rR\n" + + "snapshotId\x12\x1d\n" + + "\n" + + "created_at\x18\x02 \x01(\x03R\tcreatedAt\x12\x1d\n" + + "\n" + + "size_bytes\x18\x03 \x01(\x04R\tsizeBytes\"Q\n" + + "\x1aDeleteBlockSnapshotRequest\x12\x12\n" + + "\x04name\x18\x01 \x01(\tR\x04name\x12\x1f\n" + + "\vsnapshot_id\x18\x02 \x01(\rR\n" + + "snapshotId\"\x1d\n" + + "\x1bDeleteBlockSnapshotResponse\"/\n" + + "\x19ListBlockSnapshotsRequest\x12\x12\n" + + "\x04name\x18\x01 \x01(\tR\x04name\"_\n" + + "\x1aListBlockSnapshotsResponse\x12A\n" + + "\tsnapshots\x18\x01 \x03(\v2#.volume_server_pb.BlockSnapshotInfoR\tsnapshots\"\x7f\n" + + "\x11BlockSnapshotInfo\x12\x1f\n" + + "\vsnapshot_id\x18\x01 \x01(\rR\n" + + "snapshotId\x12\x1d\n" + + "\n" + + "created_at\x18\x02 \x01(\x03R\tcreatedAt\x12*\n" + + "\x11volume_size_bytes\x18\x03 \x01(\x04R\x0fvolumeSizeBytes\"T\n" + + "\x18ExpandBlockVolumeRequest\x12\x12\n" + + "\x04name\x18\x01 \x01(\tR\x04name\x12$\n" + + "\x0enew_size_bytes\x18\x02 \x01(\x04R\fnewSizeBytes\"B\n" + + "\x19ExpandBlockVolumeResponse\x12%\n" + + "\x0ecapacity_bytes\x18\x01 \x01(\x04R\rcapacityBytes*>\n" + "\x0fVolumeScrubMode\x12\v\n" + "\aUNKNOWN\x10\x00\x12\t\n" + "\x05INDEX\x10\x01\x12\b\n" + "\x04FULL\x10\x02\x12\t\n" + - "\x05LOCAL\x10\x032\x86+\n" + + "\x05LOCAL\x10\x032\xd5.\n" + "\fVolumeServer\x12\\\n" + "\vBatchDelete\x12$.volume_server_pb.BatchDeleteRequest\x1a%.volume_server_pb.BatchDeleteResponse\"\x00\x12n\n" + "\x11VacuumVolumeCheck\x12*.volume_server_pb.VacuumVolumeCheckRequest\x1a+.volume_server_pb.VacuumVolumeCheckResponse\"\x00\x12v\n" + @@ -7516,7 +7994,11 @@ const file_volume_server_proto_rawDesc = "" + "\x12VolumeNeedleStatus\x12+.volume_server_pb.VolumeNeedleStatusRequest\x1a,.volume_server_pb.VolumeNeedleStatusResponse\"\x00\x12G\n" + "\x04Ping\x12\x1d.volume_server_pb.PingRequest\x1a\x1e.volume_server_pb.PingResponse\"\x00\x12t\n" + "\x13AllocateBlockVolume\x12,.volume_server_pb.AllocateBlockVolumeRequest\x1a-.volume_server_pb.AllocateBlockVolumeResponse\"\x00\x12\x92\x01\n" + - "\x1dVolumeServerDeleteBlockVolume\x126.volume_server_pb.VolumeServerDeleteBlockVolumeRequest\x1a7.volume_server_pb.VolumeServerDeleteBlockVolumeResponse\"\x00B9Z7github.com/seaweedfs/seaweedfs/weed/pb/volume_server_pbb\x06proto3" + "\x1dVolumeServerDeleteBlockVolume\x126.volume_server_pb.VolumeServerDeleteBlockVolumeRequest\x1a7.volume_server_pb.VolumeServerDeleteBlockVolumeResponse\"\x00\x12t\n" + + "\x13SnapshotBlockVolume\x12,.volume_server_pb.SnapshotBlockVolumeRequest\x1a-.volume_server_pb.SnapshotBlockVolumeResponse\"\x00\x12t\n" + + "\x13DeleteBlockSnapshot\x12,.volume_server_pb.DeleteBlockSnapshotRequest\x1a-.volume_server_pb.DeleteBlockSnapshotResponse\"\x00\x12q\n" + + "\x12ListBlockSnapshots\x12+.volume_server_pb.ListBlockSnapshotsRequest\x1a,.volume_server_pb.ListBlockSnapshotsResponse\"\x00\x12n\n" + + "\x11ExpandBlockVolume\x12*.volume_server_pb.ExpandBlockVolumeRequest\x1a+.volume_server_pb.ExpandBlockVolumeResponse\"\x00B9Z7github.com/seaweedfs/seaweedfs/weed/pb/volume_server_pbb\x06proto3" var ( file_volume_server_proto_rawDescOnce sync.Once @@ -7531,7 +8013,7 @@ func file_volume_server_proto_rawDescGZIP() []byte { } var file_volume_server_proto_enumTypes = make([]protoimpl.EnumInfo, 1) -var file_volume_server_proto_msgTypes = make([]protoimpl.MessageInfo, 120) +var file_volume_server_proto_msgTypes = make([]protoimpl.MessageInfo, 129) var file_volume_server_proto_goTypes = []any{ (VolumeScrubMode)(0), // 0: volume_server_pb.VolumeScrubMode (*VolumeServerState)(nil), // 1: volume_server_pb.VolumeServerState @@ -7645,17 +8127,26 @@ var file_volume_server_proto_goTypes = []any{ (*AllocateBlockVolumeResponse)(nil), // 109: volume_server_pb.AllocateBlockVolumeResponse (*VolumeServerDeleteBlockVolumeRequest)(nil), // 110: volume_server_pb.VolumeServerDeleteBlockVolumeRequest (*VolumeServerDeleteBlockVolumeResponse)(nil), // 111: volume_server_pb.VolumeServerDeleteBlockVolumeResponse - (*FetchAndWriteNeedleRequest_Replica)(nil), // 112: volume_server_pb.FetchAndWriteNeedleRequest.Replica - (*QueryRequest_Filter)(nil), // 113: volume_server_pb.QueryRequest.Filter - (*QueryRequest_InputSerialization)(nil), // 114: volume_server_pb.QueryRequest.InputSerialization - (*QueryRequest_OutputSerialization)(nil), // 115: volume_server_pb.QueryRequest.OutputSerialization - (*QueryRequest_InputSerialization_CSVInput)(nil), // 116: volume_server_pb.QueryRequest.InputSerialization.CSVInput - (*QueryRequest_InputSerialization_JSONInput)(nil), // 117: volume_server_pb.QueryRequest.InputSerialization.JSONInput - (*QueryRequest_InputSerialization_ParquetInput)(nil), // 118: volume_server_pb.QueryRequest.InputSerialization.ParquetInput - (*QueryRequest_OutputSerialization_CSVOutput)(nil), // 119: volume_server_pb.QueryRequest.OutputSerialization.CSVOutput - (*QueryRequest_OutputSerialization_JSONOutput)(nil), // 120: volume_server_pb.QueryRequest.OutputSerialization.JSONOutput - (*remote_pb.RemoteConf)(nil), // 121: remote_pb.RemoteConf - (*remote_pb.RemoteStorageLocation)(nil), // 122: remote_pb.RemoteStorageLocation + (*SnapshotBlockVolumeRequest)(nil), // 112: volume_server_pb.SnapshotBlockVolumeRequest + (*SnapshotBlockVolumeResponse)(nil), // 113: volume_server_pb.SnapshotBlockVolumeResponse + (*DeleteBlockSnapshotRequest)(nil), // 114: volume_server_pb.DeleteBlockSnapshotRequest + (*DeleteBlockSnapshotResponse)(nil), // 115: volume_server_pb.DeleteBlockSnapshotResponse + (*ListBlockSnapshotsRequest)(nil), // 116: volume_server_pb.ListBlockSnapshotsRequest + (*ListBlockSnapshotsResponse)(nil), // 117: volume_server_pb.ListBlockSnapshotsResponse + (*BlockSnapshotInfo)(nil), // 118: volume_server_pb.BlockSnapshotInfo + (*ExpandBlockVolumeRequest)(nil), // 119: volume_server_pb.ExpandBlockVolumeRequest + (*ExpandBlockVolumeResponse)(nil), // 120: volume_server_pb.ExpandBlockVolumeResponse + (*FetchAndWriteNeedleRequest_Replica)(nil), // 121: volume_server_pb.FetchAndWriteNeedleRequest.Replica + (*QueryRequest_Filter)(nil), // 122: volume_server_pb.QueryRequest.Filter + (*QueryRequest_InputSerialization)(nil), // 123: volume_server_pb.QueryRequest.InputSerialization + (*QueryRequest_OutputSerialization)(nil), // 124: volume_server_pb.QueryRequest.OutputSerialization + (*QueryRequest_InputSerialization_CSVInput)(nil), // 125: volume_server_pb.QueryRequest.InputSerialization.CSVInput + (*QueryRequest_InputSerialization_JSONInput)(nil), // 126: volume_server_pb.QueryRequest.InputSerialization.JSONInput + (*QueryRequest_InputSerialization_ParquetInput)(nil), // 127: volume_server_pb.QueryRequest.InputSerialization.ParquetInput + (*QueryRequest_OutputSerialization_CSVOutput)(nil), // 128: volume_server_pb.QueryRequest.OutputSerialization.CSVOutput + (*QueryRequest_OutputSerialization_JSONOutput)(nil), // 129: volume_server_pb.QueryRequest.OutputSerialization.JSONOutput + (*remote_pb.RemoteConf)(nil), // 130: remote_pb.RemoteConf + (*remote_pb.RemoteStorageLocation)(nil), // 131: remote_pb.RemoteStorageLocation } var file_volume_server_proto_depIdxs = []int32{ 4, // 0: volume_server_pb.BatchDeleteResponse.results:type_name -> volume_server_pb.DeleteResult @@ -7671,125 +8162,134 @@ var file_volume_server_proto_depIdxs = []int32{ 82, // 10: volume_server_pb.VolumeServerStatusResponse.disk_statuses:type_name -> volume_server_pb.DiskStatus 83, // 11: volume_server_pb.VolumeServerStatusResponse.memory_status:type_name -> volume_server_pb.MemStatus 1, // 12: volume_server_pb.VolumeServerStatusResponse.state:type_name -> volume_server_pb.VolumeServerState - 112, // 13: volume_server_pb.FetchAndWriteNeedleRequest.replicas:type_name -> volume_server_pb.FetchAndWriteNeedleRequest.Replica - 121, // 14: volume_server_pb.FetchAndWriteNeedleRequest.remote_conf:type_name -> remote_pb.RemoteConf - 122, // 15: volume_server_pb.FetchAndWriteNeedleRequest.remote_location:type_name -> remote_pb.RemoteStorageLocation + 121, // 13: volume_server_pb.FetchAndWriteNeedleRequest.replicas:type_name -> volume_server_pb.FetchAndWriteNeedleRequest.Replica + 130, // 14: volume_server_pb.FetchAndWriteNeedleRequest.remote_conf:type_name -> remote_pb.RemoteConf + 131, // 15: volume_server_pb.FetchAndWriteNeedleRequest.remote_location:type_name -> remote_pb.RemoteStorageLocation 0, // 16: volume_server_pb.ScrubVolumeRequest.mode:type_name -> volume_server_pb.VolumeScrubMode 0, // 17: volume_server_pb.ScrubEcVolumeRequest.mode:type_name -> volume_server_pb.VolumeScrubMode 79, // 18: volume_server_pb.ScrubEcVolumeResponse.broken_shard_infos:type_name -> volume_server_pb.EcShardInfo - 113, // 19: volume_server_pb.QueryRequest.filter:type_name -> volume_server_pb.QueryRequest.Filter - 114, // 20: volume_server_pb.QueryRequest.input_serialization:type_name -> volume_server_pb.QueryRequest.InputSerialization - 115, // 21: volume_server_pb.QueryRequest.output_serialization:type_name -> volume_server_pb.QueryRequest.OutputSerialization - 116, // 22: volume_server_pb.QueryRequest.InputSerialization.csv_input:type_name -> volume_server_pb.QueryRequest.InputSerialization.CSVInput - 117, // 23: volume_server_pb.QueryRequest.InputSerialization.json_input:type_name -> volume_server_pb.QueryRequest.InputSerialization.JSONInput - 118, // 24: volume_server_pb.QueryRequest.InputSerialization.parquet_input:type_name -> volume_server_pb.QueryRequest.InputSerialization.ParquetInput - 119, // 25: volume_server_pb.QueryRequest.OutputSerialization.csv_output:type_name -> volume_server_pb.QueryRequest.OutputSerialization.CSVOutput - 120, // 26: volume_server_pb.QueryRequest.OutputSerialization.json_output:type_name -> volume_server_pb.QueryRequest.OutputSerialization.JSONOutput - 2, // 27: volume_server_pb.VolumeServer.BatchDelete:input_type -> volume_server_pb.BatchDeleteRequest - 6, // 28: volume_server_pb.VolumeServer.VacuumVolumeCheck:input_type -> volume_server_pb.VacuumVolumeCheckRequest - 8, // 29: volume_server_pb.VolumeServer.VacuumVolumeCompact:input_type -> volume_server_pb.VacuumVolumeCompactRequest - 10, // 30: volume_server_pb.VolumeServer.VacuumVolumeCommit:input_type -> volume_server_pb.VacuumVolumeCommitRequest - 12, // 31: volume_server_pb.VolumeServer.VacuumVolumeCleanup:input_type -> volume_server_pb.VacuumVolumeCleanupRequest - 14, // 32: volume_server_pb.VolumeServer.DeleteCollection:input_type -> volume_server_pb.DeleteCollectionRequest - 16, // 33: volume_server_pb.VolumeServer.AllocateVolume:input_type -> volume_server_pb.AllocateVolumeRequest - 18, // 34: volume_server_pb.VolumeServer.VolumeSyncStatus:input_type -> volume_server_pb.VolumeSyncStatusRequest - 20, // 35: volume_server_pb.VolumeServer.VolumeIncrementalCopy:input_type -> volume_server_pb.VolumeIncrementalCopyRequest - 22, // 36: volume_server_pb.VolumeServer.VolumeMount:input_type -> volume_server_pb.VolumeMountRequest - 24, // 37: volume_server_pb.VolumeServer.VolumeUnmount:input_type -> volume_server_pb.VolumeUnmountRequest - 26, // 38: volume_server_pb.VolumeServer.VolumeDelete:input_type -> volume_server_pb.VolumeDeleteRequest - 28, // 39: volume_server_pb.VolumeServer.VolumeMarkReadonly:input_type -> volume_server_pb.VolumeMarkReadonlyRequest - 30, // 40: volume_server_pb.VolumeServer.VolumeMarkWritable:input_type -> volume_server_pb.VolumeMarkWritableRequest - 32, // 41: volume_server_pb.VolumeServer.VolumeConfigure:input_type -> volume_server_pb.VolumeConfigureRequest - 34, // 42: volume_server_pb.VolumeServer.VolumeStatus:input_type -> volume_server_pb.VolumeStatusRequest - 36, // 43: volume_server_pb.VolumeServer.GetState:input_type -> volume_server_pb.GetStateRequest - 38, // 44: volume_server_pb.VolumeServer.SetState:input_type -> volume_server_pb.SetStateRequest - 40, // 45: volume_server_pb.VolumeServer.VolumeCopy:input_type -> volume_server_pb.VolumeCopyRequest - 80, // 46: volume_server_pb.VolumeServer.ReadVolumeFileStatus:input_type -> volume_server_pb.ReadVolumeFileStatusRequest - 42, // 47: volume_server_pb.VolumeServer.CopyFile:input_type -> volume_server_pb.CopyFileRequest - 44, // 48: volume_server_pb.VolumeServer.ReceiveFile:input_type -> volume_server_pb.ReceiveFileRequest - 47, // 49: volume_server_pb.VolumeServer.ReadNeedleBlob:input_type -> volume_server_pb.ReadNeedleBlobRequest - 49, // 50: volume_server_pb.VolumeServer.ReadNeedleMeta:input_type -> volume_server_pb.ReadNeedleMetaRequest - 51, // 51: volume_server_pb.VolumeServer.WriteNeedleBlob:input_type -> volume_server_pb.WriteNeedleBlobRequest - 53, // 52: volume_server_pb.VolumeServer.ReadAllNeedles:input_type -> volume_server_pb.ReadAllNeedlesRequest - 55, // 53: volume_server_pb.VolumeServer.VolumeTailSender:input_type -> volume_server_pb.VolumeTailSenderRequest - 57, // 54: volume_server_pb.VolumeServer.VolumeTailReceiver:input_type -> volume_server_pb.VolumeTailReceiverRequest - 59, // 55: volume_server_pb.VolumeServer.VolumeEcShardsGenerate:input_type -> volume_server_pb.VolumeEcShardsGenerateRequest - 61, // 56: volume_server_pb.VolumeServer.VolumeEcShardsRebuild:input_type -> volume_server_pb.VolumeEcShardsRebuildRequest - 63, // 57: volume_server_pb.VolumeServer.VolumeEcShardsCopy:input_type -> volume_server_pb.VolumeEcShardsCopyRequest - 65, // 58: volume_server_pb.VolumeServer.VolumeEcShardsDelete:input_type -> volume_server_pb.VolumeEcShardsDeleteRequest - 67, // 59: volume_server_pb.VolumeServer.VolumeEcShardsMount:input_type -> volume_server_pb.VolumeEcShardsMountRequest - 69, // 60: volume_server_pb.VolumeServer.VolumeEcShardsUnmount:input_type -> volume_server_pb.VolumeEcShardsUnmountRequest - 71, // 61: volume_server_pb.VolumeServer.VolumeEcShardRead:input_type -> volume_server_pb.VolumeEcShardReadRequest - 73, // 62: volume_server_pb.VolumeServer.VolumeEcBlobDelete:input_type -> volume_server_pb.VolumeEcBlobDeleteRequest - 75, // 63: volume_server_pb.VolumeServer.VolumeEcShardsToVolume:input_type -> volume_server_pb.VolumeEcShardsToVolumeRequest - 77, // 64: volume_server_pb.VolumeServer.VolumeEcShardsInfo:input_type -> volume_server_pb.VolumeEcShardsInfoRequest - 88, // 65: volume_server_pb.VolumeServer.VolumeTierMoveDatToRemote:input_type -> volume_server_pb.VolumeTierMoveDatToRemoteRequest - 90, // 66: volume_server_pb.VolumeServer.VolumeTierMoveDatFromRemote:input_type -> volume_server_pb.VolumeTierMoveDatFromRemoteRequest - 92, // 67: volume_server_pb.VolumeServer.VolumeServerStatus:input_type -> volume_server_pb.VolumeServerStatusRequest - 94, // 68: volume_server_pb.VolumeServer.VolumeServerLeave:input_type -> volume_server_pb.VolumeServerLeaveRequest - 96, // 69: volume_server_pb.VolumeServer.FetchAndWriteNeedle:input_type -> volume_server_pb.FetchAndWriteNeedleRequest - 98, // 70: volume_server_pb.VolumeServer.ScrubVolume:input_type -> volume_server_pb.ScrubVolumeRequest - 100, // 71: volume_server_pb.VolumeServer.ScrubEcVolume:input_type -> volume_server_pb.ScrubEcVolumeRequest - 102, // 72: volume_server_pb.VolumeServer.Query:input_type -> volume_server_pb.QueryRequest - 104, // 73: volume_server_pb.VolumeServer.VolumeNeedleStatus:input_type -> volume_server_pb.VolumeNeedleStatusRequest - 106, // 74: volume_server_pb.VolumeServer.Ping:input_type -> volume_server_pb.PingRequest - 108, // 75: volume_server_pb.VolumeServer.AllocateBlockVolume:input_type -> volume_server_pb.AllocateBlockVolumeRequest - 110, // 76: volume_server_pb.VolumeServer.VolumeServerDeleteBlockVolume:input_type -> volume_server_pb.VolumeServerDeleteBlockVolumeRequest - 3, // 77: volume_server_pb.VolumeServer.BatchDelete:output_type -> volume_server_pb.BatchDeleteResponse - 7, // 78: volume_server_pb.VolumeServer.VacuumVolumeCheck:output_type -> volume_server_pb.VacuumVolumeCheckResponse - 9, // 79: volume_server_pb.VolumeServer.VacuumVolumeCompact:output_type -> volume_server_pb.VacuumVolumeCompactResponse - 11, // 80: volume_server_pb.VolumeServer.VacuumVolumeCommit:output_type -> volume_server_pb.VacuumVolumeCommitResponse - 13, // 81: volume_server_pb.VolumeServer.VacuumVolumeCleanup:output_type -> volume_server_pb.VacuumVolumeCleanupResponse - 15, // 82: volume_server_pb.VolumeServer.DeleteCollection:output_type -> volume_server_pb.DeleteCollectionResponse - 17, // 83: volume_server_pb.VolumeServer.AllocateVolume:output_type -> volume_server_pb.AllocateVolumeResponse - 19, // 84: volume_server_pb.VolumeServer.VolumeSyncStatus:output_type -> volume_server_pb.VolumeSyncStatusResponse - 21, // 85: volume_server_pb.VolumeServer.VolumeIncrementalCopy:output_type -> volume_server_pb.VolumeIncrementalCopyResponse - 23, // 86: volume_server_pb.VolumeServer.VolumeMount:output_type -> volume_server_pb.VolumeMountResponse - 25, // 87: volume_server_pb.VolumeServer.VolumeUnmount:output_type -> volume_server_pb.VolumeUnmountResponse - 27, // 88: volume_server_pb.VolumeServer.VolumeDelete:output_type -> volume_server_pb.VolumeDeleteResponse - 29, // 89: volume_server_pb.VolumeServer.VolumeMarkReadonly:output_type -> volume_server_pb.VolumeMarkReadonlyResponse - 31, // 90: volume_server_pb.VolumeServer.VolumeMarkWritable:output_type -> volume_server_pb.VolumeMarkWritableResponse - 33, // 91: volume_server_pb.VolumeServer.VolumeConfigure:output_type -> volume_server_pb.VolumeConfigureResponse - 35, // 92: volume_server_pb.VolumeServer.VolumeStatus:output_type -> volume_server_pb.VolumeStatusResponse - 37, // 93: volume_server_pb.VolumeServer.GetState:output_type -> volume_server_pb.GetStateResponse - 39, // 94: volume_server_pb.VolumeServer.SetState:output_type -> volume_server_pb.SetStateResponse - 41, // 95: volume_server_pb.VolumeServer.VolumeCopy:output_type -> volume_server_pb.VolumeCopyResponse - 81, // 96: volume_server_pb.VolumeServer.ReadVolumeFileStatus:output_type -> volume_server_pb.ReadVolumeFileStatusResponse - 43, // 97: volume_server_pb.VolumeServer.CopyFile:output_type -> volume_server_pb.CopyFileResponse - 46, // 98: volume_server_pb.VolumeServer.ReceiveFile:output_type -> volume_server_pb.ReceiveFileResponse - 48, // 99: volume_server_pb.VolumeServer.ReadNeedleBlob:output_type -> volume_server_pb.ReadNeedleBlobResponse - 50, // 100: volume_server_pb.VolumeServer.ReadNeedleMeta:output_type -> volume_server_pb.ReadNeedleMetaResponse - 52, // 101: volume_server_pb.VolumeServer.WriteNeedleBlob:output_type -> volume_server_pb.WriteNeedleBlobResponse - 54, // 102: volume_server_pb.VolumeServer.ReadAllNeedles:output_type -> volume_server_pb.ReadAllNeedlesResponse - 56, // 103: volume_server_pb.VolumeServer.VolumeTailSender:output_type -> volume_server_pb.VolumeTailSenderResponse - 58, // 104: volume_server_pb.VolumeServer.VolumeTailReceiver:output_type -> volume_server_pb.VolumeTailReceiverResponse - 60, // 105: volume_server_pb.VolumeServer.VolumeEcShardsGenerate:output_type -> volume_server_pb.VolumeEcShardsGenerateResponse - 62, // 106: volume_server_pb.VolumeServer.VolumeEcShardsRebuild:output_type -> volume_server_pb.VolumeEcShardsRebuildResponse - 64, // 107: volume_server_pb.VolumeServer.VolumeEcShardsCopy:output_type -> volume_server_pb.VolumeEcShardsCopyResponse - 66, // 108: volume_server_pb.VolumeServer.VolumeEcShardsDelete:output_type -> volume_server_pb.VolumeEcShardsDeleteResponse - 68, // 109: volume_server_pb.VolumeServer.VolumeEcShardsMount:output_type -> volume_server_pb.VolumeEcShardsMountResponse - 70, // 110: volume_server_pb.VolumeServer.VolumeEcShardsUnmount:output_type -> volume_server_pb.VolumeEcShardsUnmountResponse - 72, // 111: volume_server_pb.VolumeServer.VolumeEcShardRead:output_type -> volume_server_pb.VolumeEcShardReadResponse - 74, // 112: volume_server_pb.VolumeServer.VolumeEcBlobDelete:output_type -> volume_server_pb.VolumeEcBlobDeleteResponse - 76, // 113: volume_server_pb.VolumeServer.VolumeEcShardsToVolume:output_type -> volume_server_pb.VolumeEcShardsToVolumeResponse - 78, // 114: volume_server_pb.VolumeServer.VolumeEcShardsInfo:output_type -> volume_server_pb.VolumeEcShardsInfoResponse - 89, // 115: volume_server_pb.VolumeServer.VolumeTierMoveDatToRemote:output_type -> volume_server_pb.VolumeTierMoveDatToRemoteResponse - 91, // 116: volume_server_pb.VolumeServer.VolumeTierMoveDatFromRemote:output_type -> volume_server_pb.VolumeTierMoveDatFromRemoteResponse - 93, // 117: volume_server_pb.VolumeServer.VolumeServerStatus:output_type -> volume_server_pb.VolumeServerStatusResponse - 95, // 118: volume_server_pb.VolumeServer.VolumeServerLeave:output_type -> volume_server_pb.VolumeServerLeaveResponse - 97, // 119: volume_server_pb.VolumeServer.FetchAndWriteNeedle:output_type -> volume_server_pb.FetchAndWriteNeedleResponse - 99, // 120: volume_server_pb.VolumeServer.ScrubVolume:output_type -> volume_server_pb.ScrubVolumeResponse - 101, // 121: volume_server_pb.VolumeServer.ScrubEcVolume:output_type -> volume_server_pb.ScrubEcVolumeResponse - 103, // 122: volume_server_pb.VolumeServer.Query:output_type -> volume_server_pb.QueriedStripe - 105, // 123: volume_server_pb.VolumeServer.VolumeNeedleStatus:output_type -> volume_server_pb.VolumeNeedleStatusResponse - 107, // 124: volume_server_pb.VolumeServer.Ping:output_type -> volume_server_pb.PingResponse - 109, // 125: volume_server_pb.VolumeServer.AllocateBlockVolume:output_type -> volume_server_pb.AllocateBlockVolumeResponse - 111, // 126: volume_server_pb.VolumeServer.VolumeServerDeleteBlockVolume:output_type -> volume_server_pb.VolumeServerDeleteBlockVolumeResponse - 77, // [77:127] is the sub-list for method output_type - 27, // [27:77] is the sub-list for method input_type - 27, // [27:27] is the sub-list for extension type_name - 27, // [27:27] is the sub-list for extension extendee - 0, // [0:27] is the sub-list for field type_name + 122, // 19: volume_server_pb.QueryRequest.filter:type_name -> volume_server_pb.QueryRequest.Filter + 123, // 20: volume_server_pb.QueryRequest.input_serialization:type_name -> volume_server_pb.QueryRequest.InputSerialization + 124, // 21: volume_server_pb.QueryRequest.output_serialization:type_name -> volume_server_pb.QueryRequest.OutputSerialization + 118, // 22: volume_server_pb.ListBlockSnapshotsResponse.snapshots:type_name -> volume_server_pb.BlockSnapshotInfo + 125, // 23: volume_server_pb.QueryRequest.InputSerialization.csv_input:type_name -> volume_server_pb.QueryRequest.InputSerialization.CSVInput + 126, // 24: volume_server_pb.QueryRequest.InputSerialization.json_input:type_name -> volume_server_pb.QueryRequest.InputSerialization.JSONInput + 127, // 25: volume_server_pb.QueryRequest.InputSerialization.parquet_input:type_name -> volume_server_pb.QueryRequest.InputSerialization.ParquetInput + 128, // 26: volume_server_pb.QueryRequest.OutputSerialization.csv_output:type_name -> volume_server_pb.QueryRequest.OutputSerialization.CSVOutput + 129, // 27: volume_server_pb.QueryRequest.OutputSerialization.json_output:type_name -> volume_server_pb.QueryRequest.OutputSerialization.JSONOutput + 2, // 28: volume_server_pb.VolumeServer.BatchDelete:input_type -> volume_server_pb.BatchDeleteRequest + 6, // 29: volume_server_pb.VolumeServer.VacuumVolumeCheck:input_type -> volume_server_pb.VacuumVolumeCheckRequest + 8, // 30: volume_server_pb.VolumeServer.VacuumVolumeCompact:input_type -> volume_server_pb.VacuumVolumeCompactRequest + 10, // 31: volume_server_pb.VolumeServer.VacuumVolumeCommit:input_type -> volume_server_pb.VacuumVolumeCommitRequest + 12, // 32: volume_server_pb.VolumeServer.VacuumVolumeCleanup:input_type -> volume_server_pb.VacuumVolumeCleanupRequest + 14, // 33: volume_server_pb.VolumeServer.DeleteCollection:input_type -> volume_server_pb.DeleteCollectionRequest + 16, // 34: volume_server_pb.VolumeServer.AllocateVolume:input_type -> volume_server_pb.AllocateVolumeRequest + 18, // 35: volume_server_pb.VolumeServer.VolumeSyncStatus:input_type -> volume_server_pb.VolumeSyncStatusRequest + 20, // 36: volume_server_pb.VolumeServer.VolumeIncrementalCopy:input_type -> volume_server_pb.VolumeIncrementalCopyRequest + 22, // 37: volume_server_pb.VolumeServer.VolumeMount:input_type -> volume_server_pb.VolumeMountRequest + 24, // 38: volume_server_pb.VolumeServer.VolumeUnmount:input_type -> volume_server_pb.VolumeUnmountRequest + 26, // 39: volume_server_pb.VolumeServer.VolumeDelete:input_type -> volume_server_pb.VolumeDeleteRequest + 28, // 40: volume_server_pb.VolumeServer.VolumeMarkReadonly:input_type -> volume_server_pb.VolumeMarkReadonlyRequest + 30, // 41: volume_server_pb.VolumeServer.VolumeMarkWritable:input_type -> volume_server_pb.VolumeMarkWritableRequest + 32, // 42: volume_server_pb.VolumeServer.VolumeConfigure:input_type -> volume_server_pb.VolumeConfigureRequest + 34, // 43: volume_server_pb.VolumeServer.VolumeStatus:input_type -> volume_server_pb.VolumeStatusRequest + 36, // 44: volume_server_pb.VolumeServer.GetState:input_type -> volume_server_pb.GetStateRequest + 38, // 45: volume_server_pb.VolumeServer.SetState:input_type -> volume_server_pb.SetStateRequest + 40, // 46: volume_server_pb.VolumeServer.VolumeCopy:input_type -> volume_server_pb.VolumeCopyRequest + 80, // 47: volume_server_pb.VolumeServer.ReadVolumeFileStatus:input_type -> volume_server_pb.ReadVolumeFileStatusRequest + 42, // 48: volume_server_pb.VolumeServer.CopyFile:input_type -> volume_server_pb.CopyFileRequest + 44, // 49: volume_server_pb.VolumeServer.ReceiveFile:input_type -> volume_server_pb.ReceiveFileRequest + 47, // 50: volume_server_pb.VolumeServer.ReadNeedleBlob:input_type -> volume_server_pb.ReadNeedleBlobRequest + 49, // 51: volume_server_pb.VolumeServer.ReadNeedleMeta:input_type -> volume_server_pb.ReadNeedleMetaRequest + 51, // 52: volume_server_pb.VolumeServer.WriteNeedleBlob:input_type -> volume_server_pb.WriteNeedleBlobRequest + 53, // 53: volume_server_pb.VolumeServer.ReadAllNeedles:input_type -> volume_server_pb.ReadAllNeedlesRequest + 55, // 54: volume_server_pb.VolumeServer.VolumeTailSender:input_type -> volume_server_pb.VolumeTailSenderRequest + 57, // 55: volume_server_pb.VolumeServer.VolumeTailReceiver:input_type -> volume_server_pb.VolumeTailReceiverRequest + 59, // 56: volume_server_pb.VolumeServer.VolumeEcShardsGenerate:input_type -> volume_server_pb.VolumeEcShardsGenerateRequest + 61, // 57: volume_server_pb.VolumeServer.VolumeEcShardsRebuild:input_type -> volume_server_pb.VolumeEcShardsRebuildRequest + 63, // 58: volume_server_pb.VolumeServer.VolumeEcShardsCopy:input_type -> volume_server_pb.VolumeEcShardsCopyRequest + 65, // 59: volume_server_pb.VolumeServer.VolumeEcShardsDelete:input_type -> volume_server_pb.VolumeEcShardsDeleteRequest + 67, // 60: volume_server_pb.VolumeServer.VolumeEcShardsMount:input_type -> volume_server_pb.VolumeEcShardsMountRequest + 69, // 61: volume_server_pb.VolumeServer.VolumeEcShardsUnmount:input_type -> volume_server_pb.VolumeEcShardsUnmountRequest + 71, // 62: volume_server_pb.VolumeServer.VolumeEcShardRead:input_type -> volume_server_pb.VolumeEcShardReadRequest + 73, // 63: volume_server_pb.VolumeServer.VolumeEcBlobDelete:input_type -> volume_server_pb.VolumeEcBlobDeleteRequest + 75, // 64: volume_server_pb.VolumeServer.VolumeEcShardsToVolume:input_type -> volume_server_pb.VolumeEcShardsToVolumeRequest + 77, // 65: volume_server_pb.VolumeServer.VolumeEcShardsInfo:input_type -> volume_server_pb.VolumeEcShardsInfoRequest + 88, // 66: volume_server_pb.VolumeServer.VolumeTierMoveDatToRemote:input_type -> volume_server_pb.VolumeTierMoveDatToRemoteRequest + 90, // 67: volume_server_pb.VolumeServer.VolumeTierMoveDatFromRemote:input_type -> volume_server_pb.VolumeTierMoveDatFromRemoteRequest + 92, // 68: volume_server_pb.VolumeServer.VolumeServerStatus:input_type -> volume_server_pb.VolumeServerStatusRequest + 94, // 69: volume_server_pb.VolumeServer.VolumeServerLeave:input_type -> volume_server_pb.VolumeServerLeaveRequest + 96, // 70: volume_server_pb.VolumeServer.FetchAndWriteNeedle:input_type -> volume_server_pb.FetchAndWriteNeedleRequest + 98, // 71: volume_server_pb.VolumeServer.ScrubVolume:input_type -> volume_server_pb.ScrubVolumeRequest + 100, // 72: volume_server_pb.VolumeServer.ScrubEcVolume:input_type -> volume_server_pb.ScrubEcVolumeRequest + 102, // 73: volume_server_pb.VolumeServer.Query:input_type -> volume_server_pb.QueryRequest + 104, // 74: volume_server_pb.VolumeServer.VolumeNeedleStatus:input_type -> volume_server_pb.VolumeNeedleStatusRequest + 106, // 75: volume_server_pb.VolumeServer.Ping:input_type -> volume_server_pb.PingRequest + 108, // 76: volume_server_pb.VolumeServer.AllocateBlockVolume:input_type -> volume_server_pb.AllocateBlockVolumeRequest + 110, // 77: volume_server_pb.VolumeServer.VolumeServerDeleteBlockVolume:input_type -> volume_server_pb.VolumeServerDeleteBlockVolumeRequest + 112, // 78: volume_server_pb.VolumeServer.SnapshotBlockVolume:input_type -> volume_server_pb.SnapshotBlockVolumeRequest + 114, // 79: volume_server_pb.VolumeServer.DeleteBlockSnapshot:input_type -> volume_server_pb.DeleteBlockSnapshotRequest + 116, // 80: volume_server_pb.VolumeServer.ListBlockSnapshots:input_type -> volume_server_pb.ListBlockSnapshotsRequest + 119, // 81: volume_server_pb.VolumeServer.ExpandBlockVolume:input_type -> volume_server_pb.ExpandBlockVolumeRequest + 3, // 82: volume_server_pb.VolumeServer.BatchDelete:output_type -> volume_server_pb.BatchDeleteResponse + 7, // 83: volume_server_pb.VolumeServer.VacuumVolumeCheck:output_type -> volume_server_pb.VacuumVolumeCheckResponse + 9, // 84: volume_server_pb.VolumeServer.VacuumVolumeCompact:output_type -> volume_server_pb.VacuumVolumeCompactResponse + 11, // 85: volume_server_pb.VolumeServer.VacuumVolumeCommit:output_type -> volume_server_pb.VacuumVolumeCommitResponse + 13, // 86: volume_server_pb.VolumeServer.VacuumVolumeCleanup:output_type -> volume_server_pb.VacuumVolumeCleanupResponse + 15, // 87: volume_server_pb.VolumeServer.DeleteCollection:output_type -> volume_server_pb.DeleteCollectionResponse + 17, // 88: volume_server_pb.VolumeServer.AllocateVolume:output_type -> volume_server_pb.AllocateVolumeResponse + 19, // 89: volume_server_pb.VolumeServer.VolumeSyncStatus:output_type -> volume_server_pb.VolumeSyncStatusResponse + 21, // 90: volume_server_pb.VolumeServer.VolumeIncrementalCopy:output_type -> volume_server_pb.VolumeIncrementalCopyResponse + 23, // 91: volume_server_pb.VolumeServer.VolumeMount:output_type -> volume_server_pb.VolumeMountResponse + 25, // 92: volume_server_pb.VolumeServer.VolumeUnmount:output_type -> volume_server_pb.VolumeUnmountResponse + 27, // 93: volume_server_pb.VolumeServer.VolumeDelete:output_type -> volume_server_pb.VolumeDeleteResponse + 29, // 94: volume_server_pb.VolumeServer.VolumeMarkReadonly:output_type -> volume_server_pb.VolumeMarkReadonlyResponse + 31, // 95: volume_server_pb.VolumeServer.VolumeMarkWritable:output_type -> volume_server_pb.VolumeMarkWritableResponse + 33, // 96: volume_server_pb.VolumeServer.VolumeConfigure:output_type -> volume_server_pb.VolumeConfigureResponse + 35, // 97: volume_server_pb.VolumeServer.VolumeStatus:output_type -> volume_server_pb.VolumeStatusResponse + 37, // 98: volume_server_pb.VolumeServer.GetState:output_type -> volume_server_pb.GetStateResponse + 39, // 99: volume_server_pb.VolumeServer.SetState:output_type -> volume_server_pb.SetStateResponse + 41, // 100: volume_server_pb.VolumeServer.VolumeCopy:output_type -> volume_server_pb.VolumeCopyResponse + 81, // 101: volume_server_pb.VolumeServer.ReadVolumeFileStatus:output_type -> volume_server_pb.ReadVolumeFileStatusResponse + 43, // 102: volume_server_pb.VolumeServer.CopyFile:output_type -> volume_server_pb.CopyFileResponse + 46, // 103: volume_server_pb.VolumeServer.ReceiveFile:output_type -> volume_server_pb.ReceiveFileResponse + 48, // 104: volume_server_pb.VolumeServer.ReadNeedleBlob:output_type -> volume_server_pb.ReadNeedleBlobResponse + 50, // 105: volume_server_pb.VolumeServer.ReadNeedleMeta:output_type -> volume_server_pb.ReadNeedleMetaResponse + 52, // 106: volume_server_pb.VolumeServer.WriteNeedleBlob:output_type -> volume_server_pb.WriteNeedleBlobResponse + 54, // 107: volume_server_pb.VolumeServer.ReadAllNeedles:output_type -> volume_server_pb.ReadAllNeedlesResponse + 56, // 108: volume_server_pb.VolumeServer.VolumeTailSender:output_type -> volume_server_pb.VolumeTailSenderResponse + 58, // 109: volume_server_pb.VolumeServer.VolumeTailReceiver:output_type -> volume_server_pb.VolumeTailReceiverResponse + 60, // 110: volume_server_pb.VolumeServer.VolumeEcShardsGenerate:output_type -> volume_server_pb.VolumeEcShardsGenerateResponse + 62, // 111: volume_server_pb.VolumeServer.VolumeEcShardsRebuild:output_type -> volume_server_pb.VolumeEcShardsRebuildResponse + 64, // 112: volume_server_pb.VolumeServer.VolumeEcShardsCopy:output_type -> volume_server_pb.VolumeEcShardsCopyResponse + 66, // 113: volume_server_pb.VolumeServer.VolumeEcShardsDelete:output_type -> volume_server_pb.VolumeEcShardsDeleteResponse + 68, // 114: volume_server_pb.VolumeServer.VolumeEcShardsMount:output_type -> volume_server_pb.VolumeEcShardsMountResponse + 70, // 115: volume_server_pb.VolumeServer.VolumeEcShardsUnmount:output_type -> volume_server_pb.VolumeEcShardsUnmountResponse + 72, // 116: volume_server_pb.VolumeServer.VolumeEcShardRead:output_type -> volume_server_pb.VolumeEcShardReadResponse + 74, // 117: volume_server_pb.VolumeServer.VolumeEcBlobDelete:output_type -> volume_server_pb.VolumeEcBlobDeleteResponse + 76, // 118: volume_server_pb.VolumeServer.VolumeEcShardsToVolume:output_type -> volume_server_pb.VolumeEcShardsToVolumeResponse + 78, // 119: volume_server_pb.VolumeServer.VolumeEcShardsInfo:output_type -> volume_server_pb.VolumeEcShardsInfoResponse + 89, // 120: volume_server_pb.VolumeServer.VolumeTierMoveDatToRemote:output_type -> volume_server_pb.VolumeTierMoveDatToRemoteResponse + 91, // 121: volume_server_pb.VolumeServer.VolumeTierMoveDatFromRemote:output_type -> volume_server_pb.VolumeTierMoveDatFromRemoteResponse + 93, // 122: volume_server_pb.VolumeServer.VolumeServerStatus:output_type -> volume_server_pb.VolumeServerStatusResponse + 95, // 123: volume_server_pb.VolumeServer.VolumeServerLeave:output_type -> volume_server_pb.VolumeServerLeaveResponse + 97, // 124: volume_server_pb.VolumeServer.FetchAndWriteNeedle:output_type -> volume_server_pb.FetchAndWriteNeedleResponse + 99, // 125: volume_server_pb.VolumeServer.ScrubVolume:output_type -> volume_server_pb.ScrubVolumeResponse + 101, // 126: volume_server_pb.VolumeServer.ScrubEcVolume:output_type -> volume_server_pb.ScrubEcVolumeResponse + 103, // 127: volume_server_pb.VolumeServer.Query:output_type -> volume_server_pb.QueriedStripe + 105, // 128: volume_server_pb.VolumeServer.VolumeNeedleStatus:output_type -> volume_server_pb.VolumeNeedleStatusResponse + 107, // 129: volume_server_pb.VolumeServer.Ping:output_type -> volume_server_pb.PingResponse + 109, // 130: volume_server_pb.VolumeServer.AllocateBlockVolume:output_type -> volume_server_pb.AllocateBlockVolumeResponse + 111, // 131: volume_server_pb.VolumeServer.VolumeServerDeleteBlockVolume:output_type -> volume_server_pb.VolumeServerDeleteBlockVolumeResponse + 113, // 132: volume_server_pb.VolumeServer.SnapshotBlockVolume:output_type -> volume_server_pb.SnapshotBlockVolumeResponse + 115, // 133: volume_server_pb.VolumeServer.DeleteBlockSnapshot:output_type -> volume_server_pb.DeleteBlockSnapshotResponse + 117, // 134: volume_server_pb.VolumeServer.ListBlockSnapshots:output_type -> volume_server_pb.ListBlockSnapshotsResponse + 120, // 135: volume_server_pb.VolumeServer.ExpandBlockVolume:output_type -> volume_server_pb.ExpandBlockVolumeResponse + 82, // [82:136] is the sub-list for method output_type + 28, // [28:82] is the sub-list for method input_type + 28, // [28:28] is the sub-list for extension type_name + 28, // [28:28] is the sub-list for extension extendee + 0, // [0:28] is the sub-list for field type_name } func init() { file_volume_server_proto_init() } @@ -7807,7 +8307,7 @@ func file_volume_server_proto_init() { GoPackagePath: reflect.TypeOf(x{}).PkgPath(), RawDescriptor: unsafe.Slice(unsafe.StringData(file_volume_server_proto_rawDesc), len(file_volume_server_proto_rawDesc)), NumEnums: 1, - NumMessages: 120, + NumMessages: 129, NumExtensions: 0, NumServices: 1, }, diff --git a/weed/pb/volume_server_pb/volume_server_grpc.pb.go b/weed/pb/volume_server_pb/volume_server_grpc.pb.go index d4bb488021..1f76c2b285 100644 --- a/weed/pb/volume_server_pb/volume_server_grpc.pb.go +++ b/weed/pb/volume_server_pb/volume_server_grpc.pb.go @@ -69,6 +69,10 @@ const ( VolumeServer_Ping_FullMethodName = "/volume_server_pb.VolumeServer/Ping" VolumeServer_AllocateBlockVolume_FullMethodName = "/volume_server_pb.VolumeServer/AllocateBlockVolume" VolumeServer_VolumeServerDeleteBlockVolume_FullMethodName = "/volume_server_pb.VolumeServer/VolumeServerDeleteBlockVolume" + VolumeServer_SnapshotBlockVolume_FullMethodName = "/volume_server_pb.VolumeServer/SnapshotBlockVolume" + VolumeServer_DeleteBlockSnapshot_FullMethodName = "/volume_server_pb.VolumeServer/DeleteBlockSnapshot" + VolumeServer_ListBlockSnapshots_FullMethodName = "/volume_server_pb.VolumeServer/ListBlockSnapshots" + VolumeServer_ExpandBlockVolume_FullMethodName = "/volume_server_pb.VolumeServer/ExpandBlockVolume" ) // VolumeServerClient is the client API for VolumeServer service. @@ -133,6 +137,10 @@ type VolumeServerClient interface { // block volumes AllocateBlockVolume(ctx context.Context, in *AllocateBlockVolumeRequest, opts ...grpc.CallOption) (*AllocateBlockVolumeResponse, error) VolumeServerDeleteBlockVolume(ctx context.Context, in *VolumeServerDeleteBlockVolumeRequest, opts ...grpc.CallOption) (*VolumeServerDeleteBlockVolumeResponse, error) + SnapshotBlockVolume(ctx context.Context, in *SnapshotBlockVolumeRequest, opts ...grpc.CallOption) (*SnapshotBlockVolumeResponse, error) + DeleteBlockSnapshot(ctx context.Context, in *DeleteBlockSnapshotRequest, opts ...grpc.CallOption) (*DeleteBlockSnapshotResponse, error) + ListBlockSnapshots(ctx context.Context, in *ListBlockSnapshotsRequest, opts ...grpc.CallOption) (*ListBlockSnapshotsResponse, error) + ExpandBlockVolume(ctx context.Context, in *ExpandBlockVolumeRequest, opts ...grpc.CallOption) (*ExpandBlockVolumeResponse, error) } type volumeServerClient struct { @@ -736,6 +744,46 @@ func (c *volumeServerClient) VolumeServerDeleteBlockVolume(ctx context.Context, return out, nil } +func (c *volumeServerClient) SnapshotBlockVolume(ctx context.Context, in *SnapshotBlockVolumeRequest, opts ...grpc.CallOption) (*SnapshotBlockVolumeResponse, error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) + out := new(SnapshotBlockVolumeResponse) + err := c.cc.Invoke(ctx, VolumeServer_SnapshotBlockVolume_FullMethodName, in, out, cOpts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *volumeServerClient) DeleteBlockSnapshot(ctx context.Context, in *DeleteBlockSnapshotRequest, opts ...grpc.CallOption) (*DeleteBlockSnapshotResponse, error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) + out := new(DeleteBlockSnapshotResponse) + err := c.cc.Invoke(ctx, VolumeServer_DeleteBlockSnapshot_FullMethodName, in, out, cOpts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *volumeServerClient) ListBlockSnapshots(ctx context.Context, in *ListBlockSnapshotsRequest, opts ...grpc.CallOption) (*ListBlockSnapshotsResponse, error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) + out := new(ListBlockSnapshotsResponse) + err := c.cc.Invoke(ctx, VolumeServer_ListBlockSnapshots_FullMethodName, in, out, cOpts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *volumeServerClient) ExpandBlockVolume(ctx context.Context, in *ExpandBlockVolumeRequest, opts ...grpc.CallOption) (*ExpandBlockVolumeResponse, error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) + out := new(ExpandBlockVolumeResponse) + err := c.cc.Invoke(ctx, VolumeServer_ExpandBlockVolume_FullMethodName, in, out, cOpts...) + if err != nil { + return nil, err + } + return out, nil +} + // VolumeServerServer is the server API for VolumeServer service. // All implementations must embed UnimplementedVolumeServerServer // for forward compatibility. @@ -798,6 +846,10 @@ type VolumeServerServer interface { // block volumes AllocateBlockVolume(context.Context, *AllocateBlockVolumeRequest) (*AllocateBlockVolumeResponse, error) VolumeServerDeleteBlockVolume(context.Context, *VolumeServerDeleteBlockVolumeRequest) (*VolumeServerDeleteBlockVolumeResponse, error) + SnapshotBlockVolume(context.Context, *SnapshotBlockVolumeRequest) (*SnapshotBlockVolumeResponse, error) + DeleteBlockSnapshot(context.Context, *DeleteBlockSnapshotRequest) (*DeleteBlockSnapshotResponse, error) + ListBlockSnapshots(context.Context, *ListBlockSnapshotsRequest) (*ListBlockSnapshotsResponse, error) + ExpandBlockVolume(context.Context, *ExpandBlockVolumeRequest) (*ExpandBlockVolumeResponse, error) mustEmbedUnimplementedVolumeServerServer() } @@ -958,6 +1010,18 @@ func (UnimplementedVolumeServerServer) AllocateBlockVolume(context.Context, *All func (UnimplementedVolumeServerServer) VolumeServerDeleteBlockVolume(context.Context, *VolumeServerDeleteBlockVolumeRequest) (*VolumeServerDeleteBlockVolumeResponse, error) { return nil, status.Error(codes.Unimplemented, "method VolumeServerDeleteBlockVolume not implemented") } +func (UnimplementedVolumeServerServer) SnapshotBlockVolume(context.Context, *SnapshotBlockVolumeRequest) (*SnapshotBlockVolumeResponse, error) { + return nil, status.Error(codes.Unimplemented, "method SnapshotBlockVolume not implemented") +} +func (UnimplementedVolumeServerServer) DeleteBlockSnapshot(context.Context, *DeleteBlockSnapshotRequest) (*DeleteBlockSnapshotResponse, error) { + return nil, status.Error(codes.Unimplemented, "method DeleteBlockSnapshot not implemented") +} +func (UnimplementedVolumeServerServer) ListBlockSnapshots(context.Context, *ListBlockSnapshotsRequest) (*ListBlockSnapshotsResponse, error) { + return nil, status.Error(codes.Unimplemented, "method ListBlockSnapshots not implemented") +} +func (UnimplementedVolumeServerServer) ExpandBlockVolume(context.Context, *ExpandBlockVolumeRequest) (*ExpandBlockVolumeResponse, error) { + return nil, status.Error(codes.Unimplemented, "method ExpandBlockVolume not implemented") +} func (UnimplementedVolumeServerServer) mustEmbedUnimplementedVolumeServerServer() {} func (UnimplementedVolumeServerServer) testEmbeddedByValue() {} @@ -1798,6 +1862,78 @@ func _VolumeServer_VolumeServerDeleteBlockVolume_Handler(srv interface{}, ctx co return interceptor(ctx, in, info, handler) } +func _VolumeServer_SnapshotBlockVolume_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(SnapshotBlockVolumeRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(VolumeServerServer).SnapshotBlockVolume(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: VolumeServer_SnapshotBlockVolume_FullMethodName, + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(VolumeServerServer).SnapshotBlockVolume(ctx, req.(*SnapshotBlockVolumeRequest)) + } + return interceptor(ctx, in, info, handler) +} + +func _VolumeServer_DeleteBlockSnapshot_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(DeleteBlockSnapshotRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(VolumeServerServer).DeleteBlockSnapshot(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: VolumeServer_DeleteBlockSnapshot_FullMethodName, + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(VolumeServerServer).DeleteBlockSnapshot(ctx, req.(*DeleteBlockSnapshotRequest)) + } + return interceptor(ctx, in, info, handler) +} + +func _VolumeServer_ListBlockSnapshots_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(ListBlockSnapshotsRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(VolumeServerServer).ListBlockSnapshots(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: VolumeServer_ListBlockSnapshots_FullMethodName, + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(VolumeServerServer).ListBlockSnapshots(ctx, req.(*ListBlockSnapshotsRequest)) + } + return interceptor(ctx, in, info, handler) +} + +func _VolumeServer_ExpandBlockVolume_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(ExpandBlockVolumeRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(VolumeServerServer).ExpandBlockVolume(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: VolumeServer_ExpandBlockVolume_FullMethodName, + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(VolumeServerServer).ExpandBlockVolume(ctx, req.(*ExpandBlockVolumeRequest)) + } + return interceptor(ctx, in, info, handler) +} + // VolumeServer_ServiceDesc is the grpc.ServiceDesc for VolumeServer service. // It's only intended for direct use with grpc.RegisterService, // and not to be introspected or modified (even as a copy) @@ -1961,6 +2097,22 @@ var VolumeServer_ServiceDesc = grpc.ServiceDesc{ MethodName: "VolumeServerDeleteBlockVolume", Handler: _VolumeServer_VolumeServerDeleteBlockVolume_Handler, }, + { + MethodName: "SnapshotBlockVolume", + Handler: _VolumeServer_SnapshotBlockVolume_Handler, + }, + { + MethodName: "DeleteBlockSnapshot", + Handler: _VolumeServer_DeleteBlockSnapshot_Handler, + }, + { + MethodName: "ListBlockSnapshots", + Handler: _VolumeServer_ListBlockSnapshots_Handler, + }, + { + MethodName: "ExpandBlockVolume", + Handler: _VolumeServer_ExpandBlockVolume_Handler, + }, }, Streams: []grpc.StreamDesc{ { diff --git a/weed/server/integration_block_test.go b/weed/server/integration_block_test.go index 5b1a970a51..d3a590e85d 100644 --- a/weed/server/integration_block_test.go +++ b/weed/server/integration_block_test.go @@ -641,7 +641,22 @@ func TestIntegration_DoubleFailover(t *testing.T) { t.Fatalf("first failover epoch: got %d, want 2", e1.Epoch) } - // Second failover: vs2 dies → vs1 promoted (it's now the replica). + // CP8-2: PromoteBestReplica does NOT add old primary back as replica. + // Reconnect vs1 first so it becomes a replica (via recoverBlockVolumes). + ms.recoverBlockVolumes(vs1) + + // Simulate heartbeat from vs1 that restores iSCSI addr and health score + // (in production this happens when the VS re-registers after reconnect). + e1, _ = ms.blockRegistry.Lookup("pvc-double-1") + for i := range e1.Replicas { + if e1.Replicas[i].Server == vs1 { + e1.Replicas[i].ISCSIAddr = vs1 + ":3260" + e1.Replicas[i].HealthScore = 1.0 + } + } + + // Now vs1 is back as replica. Second failover: vs2 dies → vs1 promoted. + e1, _ = ms.blockRegistry.Lookup("pvc-double-1") e1.LastLeaseGrant = time.Now().Add(-1 * time.Minute) ms.failoverBlockVolumes(vs2) @@ -724,9 +739,10 @@ func TestIntegration_MultiVolumeFailoverRebuild(t *testing.T) { } } _ = otherServer - // rebuildCount should equal the number of volumes that were primary on deadServer. - if rebuildCount != primaryCounts[deadServer] { - t.Fatalf("expected %d rebuild assignments for %s, got %d", - primaryCounts[deadServer], deadServer, rebuildCount) + // CP8-2: With 2 servers and 3 volumes, deadServer hosts all 3 volumes + // (as primary for some, replica for others). All need rebuild on reconnect. + if rebuildCount != 3 { + t.Fatalf("expected 3 rebuild assignments for %s (all volumes), got %d", + deadServer, rebuildCount) } } diff --git a/weed/server/master_block_assignment_queue.go b/weed/server/master_block_assignment_queue.go index 4b7e8fb958..d55e83cdfa 100644 --- a/weed/server/master_block_assignment_queue.go +++ b/weed/server/master_block_assignment_queue.go @@ -123,3 +123,14 @@ func (q *BlockAssignmentQueue) Pending(server string) int { defer q.mu.Unlock() return len(q.queues[server]) } + +// TotalPending returns the total number of pending assignments across all servers. +func (q *BlockAssignmentQueue) TotalPending() int { + q.mu.Lock() + defer q.mu.Unlock() + total := 0 + for _, queue := range q.queues { + total += len(queue) + } + return total +} diff --git a/weed/server/master_block_failover.go b/weed/server/master_block_failover.go index 5e939ec9ce..09649af62d 100644 --- a/weed/server/master_block_failover.go +++ b/weed/server/master_block_failover.go @@ -32,82 +32,116 @@ func newBlockFailoverState() *blockFailoverState { } // failoverBlockVolumes is called when a volume server disconnects. -// It checks each block volume on that server and promotes the replica -// if the lease has expired (F2). +// It checks each block volume on that server and: +// - If dead server is primary: promote best replica (if lease expired). +// - If dead server hosts a replica: remove from replica list, record pending rebuild. func (ms *MasterServer) failoverBlockVolumes(deadServer string) { if ms.blockRegistry == nil { return } + ms.blockRegistry.FailoversTotal.Add(1) entries := ms.blockRegistry.ListByServer(deadServer) now := time.Now() for _, entry := range entries { - if blockvol.RoleFromWire(entry.Role) != blockvol.RolePrimary { + // Case 1: Dead server is the primary. + if entry.VolumeServer == deadServer && + blockvol.RoleFromWire(entry.Role) == blockvol.RolePrimary { + + if !entry.HasReplica() { + glog.Warningf("failover: %q has no replica, cannot promote", entry.Name) + continue + } + // F2: Wait for lease expiry before promoting. + leaseExpiry := entry.LastLeaseGrant.Add(entry.LeaseTTL) + if now.Before(leaseExpiry) { + delay := leaseExpiry.Sub(now) + glog.V(0).Infof("failover: %q lease expires in %v, deferring promotion", entry.Name, delay) + volumeName := entry.Name + timer := time.AfterFunc(delay, func() { + ms.promoteReplica(volumeName) + }) + ms.blockFailover.mu.Lock() + ms.blockFailover.deferredTimers[deadServer] = append( + ms.blockFailover.deferredTimers[deadServer], timer) + ms.blockFailover.mu.Unlock() + continue + } + // Lease already expired — promote immediately. + ms.promoteReplica(entry.Name) continue } - // Only failover volumes whose primary is the dead server. + + // Case 2: Dead server hosts a replica (not the primary). if entry.VolumeServer != deadServer { - continue + ri := entry.ReplicaByServer(deadServer) + if ri != nil { + replicaPath := ri.Path + // Remove dead replica from registry. + if err := ms.blockRegistry.RemoveReplica(entry.Name, deadServer); err != nil { + glog.Warningf("failover: RemoveReplica %q on %s: %v", entry.Name, deadServer, err) + continue + } + // Record pending rebuild for when dead server reconnects. + ms.recordPendingRebuild(deadServer, pendingRebuild{ + VolumeName: entry.Name, + OldPath: replicaPath, + NewPrimary: entry.VolumeServer, // current primary (unchanged) + Epoch: entry.Epoch, + }) + glog.V(0).Infof("failover: removed dead replica %s for %q, pending rebuild", + deadServer, entry.Name) + } } - if entry.ReplicaServer == "" { - glog.Warningf("failover: %q has no replica, cannot promote", entry.Name) - continue - } - // F2: Wait for lease expiry before promoting. - leaseExpiry := entry.LastLeaseGrant.Add(entry.LeaseTTL) - if now.Before(leaseExpiry) { - delay := leaseExpiry.Sub(now) - glog.V(0).Infof("failover: %q lease expires in %v, deferring promotion", entry.Name, delay) - volumeName := entry.Name - timer := time.AfterFunc(delay, func() { - ms.promoteReplica(volumeName) - }) - // R2-F2: Store timer so it can be cancelled if the server reconnects. - ms.blockFailover.mu.Lock() - ms.blockFailover.deferredTimers[deadServer] = append( - ms.blockFailover.deferredTimers[deadServer], timer) - ms.blockFailover.mu.Unlock() - continue - } - // Lease already expired — promote immediately. - ms.promoteReplica(entry.Name) } } -// promoteReplica swaps primary and replica for the named volume, +// promoteReplica promotes the best replica to primary for the named volume, // enqueues an assignment for the new primary, and records a pending rebuild. func (ms *MasterServer) promoteReplica(volumeName string) { entry, ok := ms.blockRegistry.Lookup(volumeName) if !ok { return } - if entry.ReplicaServer == "" { + if !entry.HasReplica() { return } oldPrimary := entry.VolumeServer oldPath := entry.Path - // R2-F5: Epoch computed atomically inside SwapPrimaryReplica (under lock). - newEpoch, err := ms.blockRegistry.SwapPrimaryReplica(volumeName) + // CP8-2: Use PromoteBestReplica (picks by health score, tie-break by WALHeadLSN). + newEpoch, err := ms.blockRegistry.PromoteBestReplica(volumeName) if err != nil { - glog.Warningf("failover: SwapPrimaryReplica %q: %v", volumeName, err) + glog.Warningf("failover: PromoteBestReplica %q: %v", volumeName, err) return } - // Re-read entry after swap. + // Re-read entry after promotion. entry, ok = ms.blockRegistry.Lookup(volumeName) if !ok { return } - // Enqueue assignment for new primary. + // Build assignment for new primary. Include ReplicaAddrs for remaining replicas. leaseTTLMs := blockvol.LeaseTTLToWire(30 * time.Second) - ms.blockAssignmentQueue.Enqueue(entry.VolumeServer, blockvol.BlockVolumeAssignment{ + assignment := blockvol.BlockVolumeAssignment{ Path: entry.Path, Epoch: newEpoch, Role: blockvol.RoleToWire(blockvol.RolePrimary), LeaseTtlMs: leaseTTLMs, - }) + } + for _, ri := range entry.Replicas { + assignment.ReplicaAddrs = append(assignment.ReplicaAddrs, blockvol.ReplicaAddr{ + DataAddr: ri.DataAddr, + CtrlAddr: ri.CtrlAddr, + }) + } + // Backward compat: also set scalar fields if exactly 1 replica. + if len(entry.Replicas) == 1 { + assignment.ReplicaDataAddr = entry.Replicas[0].DataAddr + assignment.ReplicaCtrlAddr = entry.Replicas[0].CtrlAddr + } + ms.blockAssignmentQueue.Enqueue(entry.VolumeServer, assignment) // Record pending rebuild for when dead server reconnects. ms.recordPendingRebuild(oldPrimary, pendingRebuild{ @@ -117,6 +151,7 @@ func (ms *MasterServer) promoteReplica(volumeName string) { Epoch: newEpoch, }) + ms.blockRegistry.PromotionsTotal.Add(1) glog.V(0).Infof("failover: promoted replica for %q: new primary=%s epoch=%d (old primary=%s)", volumeName, entry.VolumeServer, newEpoch, oldPrimary) } @@ -180,8 +215,11 @@ func (ms *MasterServer) recoverBlockVolumes(reconnectedServer string) { continue } - // Update registry: reconnected server becomes the new replica. - ms.blockRegistry.SetReplica(rb.VolumeName, reconnectedServer, rb.OldPath, "", "") + // Update registry: reconnected server becomes a replica (via AddReplica for RF≥2 support). + ms.blockRegistry.AddReplica(rb.VolumeName, ReplicaInfo{ + Server: reconnectedServer, + Path: rb.OldPath, + }) // Enqueue rebuild assignment for the reconnected server. ms.blockAssignmentQueue.Enqueue(reconnectedServer, blockvol.BlockVolumeAssignment{ @@ -191,6 +229,7 @@ func (ms *MasterServer) recoverBlockVolumes(reconnectedServer string) { RebuildAddr: entry.RebuildListenAddr, }) + ms.blockRegistry.RebuildsTotal.Add(1) glog.V(0).Infof("rebuild: enqueued rebuild for %q on %s (epoch=%d, rebuildAddr=%s)", rb.VolumeName, reconnectedServer, entry.Epoch, entry.RebuildListenAddr) } diff --git a/weed/server/master_block_failover_test.go b/weed/server/master_block_failover_test.go index d36ef2e818..1b60a2f926 100644 --- a/weed/server/master_block_failover_test.go +++ b/weed/server/master_block_failover_test.go @@ -50,6 +50,16 @@ func registerVolumeWithReplica(t *testing.T, ms *MasterServer, name, primary, re ReplicaISCSIAddr: replica + ":3260", LeaseTTL: leaseTTL, LastLeaseGrant: time.Now().Add(-2 * leaseTTL), // expired + // CP8-2: also populate Replicas[] for PromoteBestReplica. + Replicas: []ReplicaInfo{ + { + Server: replica, + Path: fmt.Sprintf("/data/%s.blk", name), + IQN: fmt.Sprintf("iqn.2024.test:%s-replica", name), + ISCSIAddr: replica + ":3260", + HealthScore: 1.0, + }, + }, } if err := ms.blockRegistry.Register(entry); err != nil { t.Fatalf("register %s: %v", name, err) @@ -128,12 +138,14 @@ func TestFailover_RegistryUpdated(t *testing.T) { ms.failoverBlockVolumes("vs1") entry, _ := ms.blockRegistry.Lookup("vol1") - // After swap: new primary = vs2, old primary (vs1) becomes replica. + // After PromoteBestReplica: new primary = vs2. + // Old primary (vs1) is NOT added back to Replicas (needs rebuild). if entry.VolumeServer != "vs2" { t.Fatalf("VolumeServer: got %q, want vs2", entry.VolumeServer) } - if entry.ReplicaServer != "vs1" { - t.Fatalf("ReplicaServer: got %q, want vs1 (old primary)", entry.ReplicaServer) + // Replicas should be empty (only had 1 replica which was promoted). + if len(entry.Replicas) != 0 { + t.Fatalf("Replicas should be empty after promotion, got %d", len(entry.Replicas)) } } @@ -196,6 +208,9 @@ func TestFailover_LeaseNotExpired_DeferredPromotion(t *testing.T) { ReplicaISCSIAddr: "vs2:3260", LeaseTTL: 200 * time.Millisecond, LastLeaseGrant: time.Now(), // just granted, NOT expired yet + Replicas: []ReplicaInfo{ + {Server: "vs2", Path: "/data/vol1.blk", IQN: "iqn:vol1-r", ISCSIAddr: "vs2:3260", HealthScore: 1.0}, + }, } ms.blockRegistry.Register(entry) @@ -369,13 +384,14 @@ func TestRebuild_RegistryUpdatedWithNewReplica(t *testing.T) { ms.failoverBlockVolumes("vs1") ms.recoverBlockVolumes("vs1") - // After recovery, vs1 should be the new replica for vol1. + // After recovery, vs1 should be a replica for vol1 (added back via AddReplica). entry, _ := ms.blockRegistry.Lookup("vol1") if entry.VolumeServer != "vs2" { t.Fatalf("primary should be vs2, got %q", entry.VolumeServer) } - if entry.ReplicaServer != "vs1" { - t.Fatalf("replica should be vs1 (reconnected), got %q", entry.ReplicaServer) + // Replicas[] should contain vs1 (added by recoverBlockVolumes). + if len(entry.Replicas) != 1 || entry.Replicas[0].Server != "vs1" { + t.Fatalf("replica should be vs1 (reconnected), got %+v", entry.Replicas) } } @@ -396,6 +412,9 @@ func TestRebuild_AssignmentContainsRebuildAddr(t *testing.T) { RebuildListenAddr: "vs1:15000", LeaseTTL: 5 * time.Second, LastLeaseGrant: time.Now().Add(-10 * time.Second), + Replicas: []ReplicaInfo{ + {Server: "vs2", Path: "/data/vol1.blk", IQN: "iqn:vol1-r", ISCSIAddr: "vs2:3260", HealthScore: 1.0}, + }, } ms.blockRegistry.Register(entry) @@ -437,6 +456,9 @@ func TestFailover_TransientDisconnect_NoPromotion(t *testing.T) { ReplicaISCSIAddr: "vs2:3260", LeaseTTL: 30 * time.Second, LastLeaseGrant: time.Now(), // just granted + Replicas: []ReplicaInfo{ + {Server: "vs2", Path: "/data/vol1.blk", IQN: "iqn:vol1-r", ISCSIAddr: "vs2:3260", HealthScore: 1.0}, + }, } ms.blockRegistry.Register(entry) @@ -526,3 +548,343 @@ func TestLifecycle_CreateFailoverRebuild(t *testing.T) { t.Fatal("expected rebuild assignment for reconnected server") } } + +// ============================================================ +// CP8-2 T8: RF=3 (N>2) Failover Tests +// ============================================================ + +// registerVolumeRF3 creates a volume entry with primary + 2 replicas for RF=3 tests. +func registerVolumeRF3(t *testing.T, ms *MasterServer, name, primary, replica1, replica2 string, epoch uint64, leaseTTL time.Duration) { + t.Helper() + entry := &BlockVolumeEntry{ + Name: name, + VolumeServer: primary, + Path: fmt.Sprintf("/data/%s.blk", name), + IQN: fmt.Sprintf("iqn.2024.test:%s", name), + ISCSIAddr: primary + ":3260", + SizeBytes: 1 << 30, + Epoch: epoch, + Role: blockvol.RoleToWire(blockvol.RolePrimary), + Status: StatusActive, + ReplicaFactor: 3, + LeaseTTL: leaseTTL, + LastLeaseGrant: time.Now().Add(-2 * leaseTTL), // expired + // Deprecated scalar fields (backward compat): first replica only. + ReplicaServer: replica1, + ReplicaPath: fmt.Sprintf("/data/%s.blk", name), + ReplicaIQN: fmt.Sprintf("iqn.2024.test:%s-r1", name), + ReplicaISCSIAddr: replica1 + ":3260", + Replicas: []ReplicaInfo{ + { + Server: replica1, + Path: fmt.Sprintf("/data/%s.blk", name), + IQN: fmt.Sprintf("iqn.2024.test:%s-r1", name), + ISCSIAddr: replica1 + ":3260", + HealthScore: 1.0, + WALHeadLSN: 100, + }, + { + Server: replica2, + Path: fmt.Sprintf("/data/%s.blk", name), + IQN: fmt.Sprintf("iqn.2024.test:%s-r2", name), + ISCSIAddr: replica2 + ":3260", + HealthScore: 1.0, + WALHeadLSN: 100, + }, + }, + } + if err := ms.blockRegistry.Register(entry); err != nil { + t.Fatalf("register %s: %v", name, err) + } +} + +// RF3: Primary dies → best replica promoted, other replica survives. +func TestRF3_PrimaryDies_BestReplicaPromoted(t *testing.T) { + ms := testMasterServerForFailover(t) + registerVolumeRF3(t, ms, "vol1", "vs1", "vs2", "vs3", 1, 5*time.Second) + + // Give vs3 a higher health score so it should be promoted. + entry, _ := ms.blockRegistry.Lookup("vol1") + entry.Replicas[0].HealthScore = 0.8 // vs2 + entry.Replicas[1].HealthScore = 1.0 // vs3 + + ms.failoverBlockVolumes("vs1") + + entry, _ = ms.blockRegistry.Lookup("vol1") + if entry.VolumeServer != "vs3" { + t.Fatalf("primary should be vs3 (highest health), got %q", entry.VolumeServer) + } + if entry.Epoch != 2 { + t.Fatalf("epoch: got %d, want 2", entry.Epoch) + } + // vs2 should remain as the only replica. + if len(entry.Replicas) != 1 { + t.Fatalf("expected 1 remaining replica, got %d", len(entry.Replicas)) + } + if entry.Replicas[0].Server != "vs2" { + t.Fatalf("remaining replica should be vs2, got %q", entry.Replicas[0].Server) + } +} + +// RF3: One replica dies → primary stays, dead replica removed, pending rebuild. +func TestRF3_OneReplicaDies_PrimaryUnchanged(t *testing.T) { + ms := testMasterServerForFailover(t) + registerVolumeRF3(t, ms, "vol1", "vs1", "vs2", "vs3", 1, 5*time.Second) + + // vs3 dies (a replica, not the primary). + ms.failoverBlockVolumes("vs3") + + entry, _ := ms.blockRegistry.Lookup("vol1") + // Primary should remain vs1. + if entry.VolumeServer != "vs1" { + t.Fatalf("primary should remain vs1, got %q", entry.VolumeServer) + } + // Epoch should NOT change (no promotion). + if entry.Epoch != 1 { + t.Fatalf("epoch should remain 1, got %d", entry.Epoch) + } + // vs3 should be removed from replicas, leaving only vs2. + if len(entry.Replicas) != 1 { + t.Fatalf("expected 1 replica after vs3 death, got %d", len(entry.Replicas)) + } + if entry.Replicas[0].Server != "vs2" { + t.Fatalf("remaining replica should be vs2, got %q", entry.Replicas[0].Server) + } + + // Pending rebuild should be recorded for vs3. + ms.blockFailover.mu.Lock() + rebuilds := ms.blockFailover.pendingRebuilds["vs3"] + ms.blockFailover.mu.Unlock() + if len(rebuilds) != 1 || rebuilds[0].VolumeName != "vol1" { + t.Fatalf("expected 1 pending rebuild for vs3/vol1, got %+v", rebuilds) + } +} + +// RF3: Dead replica reconnects → rebuilds, rejoins replica list. +func TestRF3_RecoverRebuildsDeadReplica(t *testing.T) { + ms := testMasterServerForFailover(t) + registerVolumeRF3(t, ms, "vol1", "vs1", "vs2", "vs3", 1, 5*time.Second) + + // vs3 dies. + ms.failoverBlockVolumes("vs3") + + // vs3 reconnects. + ms.recoverBlockVolumes("vs3") + + entry, _ := ms.blockRegistry.Lookup("vol1") + // Primary still vs1. + if entry.VolumeServer != "vs1" { + t.Fatalf("primary should remain vs1, got %q", entry.VolumeServer) + } + // vs3 should be back in replicas (added by recoverBlockVolumes via AddReplica). + if len(entry.Replicas) != 2 { + t.Fatalf("expected 2 replicas after recovery, got %d", len(entry.Replicas)) + } + // Verify rebuild assignment was queued for vs3. + assignments := ms.blockAssignmentQueue.Peek("vs3") + foundRebuild := false + for _, a := range assignments { + if blockvol.RoleFromWire(a.Role) == blockvol.RoleRebuilding { + foundRebuild = true + } + } + if !foundRebuild { + t.Fatal("expected rebuild assignment for vs3") + } +} + +// RF3: Other replicas survive promotion. +func TestRF3_OtherReplicasSurvivePromotion(t *testing.T) { + ms := testMasterServerForFailover(t) + registerVolumeRF3(t, ms, "vol1", "vs1", "vs2", "vs3", 1, 5*time.Second) + + // Primary (vs1) dies → one replica promoted. + ms.failoverBlockVolumes("vs1") + + entry, _ := ms.blockRegistry.Lookup("vol1") + newPrimary := entry.VolumeServer + // New primary should be vs2 or vs3 (both have equal health/LSN, so first wins). + if newPrimary != "vs2" && newPrimary != "vs3" { + t.Fatalf("new primary should be vs2 or vs3, got %q", newPrimary) + } + // The other replica should still be in the list. + if len(entry.Replicas) != 1 { + t.Fatalf("expected 1 surviving replica, got %d", len(entry.Replicas)) + } + // The surviving replica is whichever wasn't promoted. + expectedSurvivor := "vs3" + if newPrimary == "vs3" { + expectedSurvivor = "vs2" + } + if entry.Replicas[0].Server != expectedSurvivor { + t.Fatalf("surviving replica should be %q, got %q", expectedSurvivor, entry.Replicas[0].Server) + } + + // Assignment for new primary should include ReplicaAddrs for the surviving replica. + assignments := ms.blockAssignmentQueue.Peek(newPrimary) + foundPrimary := false + for _, a := range assignments { + if blockvol.RoleFromWire(a.Role) == blockvol.RolePrimary { + foundPrimary = true + if len(a.ReplicaAddrs) != 1 { + t.Fatalf("expected 1 ReplicaAddr in assignment, got %d", len(a.ReplicaAddrs)) + } + } + } + if !foundPrimary { + t.Fatal("expected primary assignment for new primary") + } +} + +// RF2 unchanged: verify RF=2 still works identically. +func TestRF2_Unchanged_AfterCP82(t *testing.T) { + ms := testMasterServerForFailover(t) + registerVolumeWithReplica(t, ms, "vol1", "vs1", "vs2", 1, 5*time.Second) + + ms.failoverBlockVolumes("vs1") + + entry, _ := ms.blockRegistry.Lookup("vol1") + if entry.VolumeServer != "vs2" { + t.Fatalf("RF2: primary should be vs2, got %q", entry.VolumeServer) + } + if entry.Epoch != 2 { + t.Fatalf("RF2: epoch should be 2, got %d", entry.Epoch) + } + // No replicas left (RF=2, one promoted). + if len(entry.Replicas) != 0 { + t.Fatalf("RF2: expected 0 replicas after promotion, got %d", len(entry.Replicas)) + } + + // Rebuild works the same. + ms.recoverBlockVolumes("vs1") + entry, _ = ms.blockRegistry.Lookup("vol1") + if len(entry.Replicas) != 1 || entry.Replicas[0].Server != "vs1" { + t.Fatalf("RF2: vs1 should be back as replica after recovery, got %+v", entry.Replicas) + } +} + +// RF3: All replicas dead → no promotion possible. +func TestRF3_AllReplicasDead_NoPromotion(t *testing.T) { + ms := testMasterServerForFailover(t) + registerVolumeRF3(t, ms, "vol1", "vs1", "vs2", "vs3", 1, 5*time.Second) + + // Both replicas die first (not primary). + ms.failoverBlockVolumes("vs2") + ms.failoverBlockVolumes("vs3") + + entry, _ := ms.blockRegistry.Lookup("vol1") + // Primary still vs1, replicas removed. + if entry.VolumeServer != "vs1" { + t.Fatalf("primary should remain vs1, got %q", entry.VolumeServer) + } + if len(entry.Replicas) != 0 { + t.Fatalf("expected 0 replicas, got %d", len(entry.Replicas)) + } + + // Now primary dies → no promotion possible (no replicas left). + ms.failoverBlockVolumes("vs1") + + entry, _ = ms.blockRegistry.Lookup("vol1") + if entry.VolumeServer != "vs1" { + t.Fatalf("should remain vs1 (no replicas to promote), got %q", entry.VolumeServer) + } +} + +// RF3: Lease deferred promotion with RF=3. +func TestRF3_LeaseDeferred_Promotion(t *testing.T) { + ms := testMasterServerForFailover(t) + entry := &BlockVolumeEntry{ + Name: "vol1", + VolumeServer: "vs1", + Path: "/data/vol1.blk", + IQN: "iqn:vol1", + ISCSIAddr: "vs1:3260", + SizeBytes: 1 << 30, + Epoch: 1, + Role: blockvol.RoleToWire(blockvol.RolePrimary), + Status: StatusActive, + ReplicaFactor: 3, + LeaseTTL: 200 * time.Millisecond, + LastLeaseGrant: time.Now(), // just granted → NOT expired + Replicas: []ReplicaInfo{ + {Server: "vs2", Path: "/data/vol1.blk", ISCSIAddr: "vs2:3260", HealthScore: 1.0, WALHeadLSN: 50}, + {Server: "vs3", Path: "/data/vol1.blk", ISCSIAddr: "vs3:3260", HealthScore: 0.9, WALHeadLSN: 50}, + }, + // Deprecated scalar fields. + ReplicaServer: "vs2", ReplicaPath: "/data/vol1.blk", ReplicaISCSIAddr: "vs2:3260", + } + ms.blockRegistry.Register(entry) + + ms.failoverBlockVolumes("vs1") + + // Immediately: no promotion (lease not expired). + e, _ := ms.blockRegistry.Lookup("vol1") + if e.VolumeServer != "vs1" { + t.Fatalf("should NOT promote yet (lease active), got %q", e.VolumeServer) + } + + // Wait for lease to expire + deferred timer. + time.Sleep(350 * time.Millisecond) + + e, _ = ms.blockRegistry.Lookup("vol1") + if e.VolumeServer != "vs2" { + t.Fatalf("should promote vs2 (highest health) after lease expires, got %q", e.VolumeServer) + } + // vs3 should survive as the remaining replica. + if len(e.Replicas) != 1 || e.Replicas[0].Server != "vs3" { + t.Fatalf("vs3 should remain as replica, got %+v", e.Replicas) + } +} + +// RF3: Cancel deferred timers on reconnect. +func TestRF3_CancelDeferredOnReconnect(t *testing.T) { + ms := testMasterServerForFailover(t) + entry := &BlockVolumeEntry{ + Name: "vol1", + VolumeServer: "vs1", + Path: "/data/vol1.blk", + IQN: "iqn:vol1", + ISCSIAddr: "vs1:3260", + SizeBytes: 1 << 30, + Epoch: 1, + Role: blockvol.RoleToWire(blockvol.RolePrimary), + Status: StatusActive, + ReplicaFactor: 3, + LeaseTTL: 5 * time.Second, + LastLeaseGrant: time.Now(), // just granted → long lease + Replicas: []ReplicaInfo{ + {Server: "vs2", Path: "/data/vol1.blk", ISCSIAddr: "vs2:3260", HealthScore: 1.0}, + {Server: "vs3", Path: "/data/vol1.blk", ISCSIAddr: "vs3:3260", HealthScore: 1.0}, + }, + ReplicaServer: "vs2", ReplicaPath: "/data/vol1.blk", ReplicaISCSIAddr: "vs2:3260", + } + ms.blockRegistry.Register(entry) + + // vs1 disconnects → deferred timer created (lease not expired). + ms.failoverBlockVolumes("vs1") + + ms.blockFailover.mu.Lock() + timerCount := len(ms.blockFailover.deferredTimers["vs1"]) + ms.blockFailover.mu.Unlock() + if timerCount == 0 { + t.Fatal("expected deferred timer for vs1") + } + + // vs1 reconnects before lease expires → timers cancelled. + ms.recoverBlockVolumes("vs1") + + ms.blockFailover.mu.Lock() + timerCount = len(ms.blockFailover.deferredTimers["vs1"]) + ms.blockFailover.mu.Unlock() + if timerCount != 0 { + t.Fatalf("expected 0 deferred timers after reconnect, got %d", timerCount) + } + + // Wait past original lease time — no promotion should happen. + time.Sleep(200 * time.Millisecond) + + e, _ := ms.blockRegistry.Lookup("vol1") + if e.VolumeServer != "vs1" { + t.Fatalf("vs1 should remain primary (timer cancelled), got %q", e.VolumeServer) + } +} diff --git a/weed/server/master_block_registry.go b/weed/server/master_block_registry.go index 8b05d8a18c..0187ecf24f 100644 --- a/weed/server/master_block_registry.go +++ b/weed/server/master_block_registry.go @@ -2,9 +2,14 @@ package weed_server import ( "fmt" + "path/filepath" + "sort" + "strings" "sync" + "sync/atomic" "time" + "github.com/seaweedfs/seaweedfs/weed/glog" "github.com/seaweedfs/seaweedfs/weed/pb/master_pb" "github.com/seaweedfs/seaweedfs/weed/storage/blockvol" ) @@ -17,32 +22,102 @@ const ( StatusActive // Confirmed by heartbeat from volume server ) +// ReplicaInfo tracks one replica of a block volume (CP8-2). +type ReplicaInfo struct { + Server string // replica VS address + Path string // file path on replica VS + ISCSIAddr string // iSCSI target address + IQN string // iSCSI qualified name + DataAddr string // WAL receiver data listen addr + CtrlAddr string // WAL receiver ctrl listen addr + HealthScore float64 // from heartbeat (0.0-1.0) + WALHeadLSN uint64 // from heartbeat + WALLag uint64 // computed: primary.WALHeadLSN - replica.WALHeadLSN + LastHeartbeat time.Time // last heartbeat received from this replica + Role uint32 // replica role (RoleReplica, RoleRebuilding, etc.) +} + +const ( + // DefaultPromotionLSNTolerance is the max WAL LSN lag allowed for promotion eligibility. + // Configurable per-registry via SetPromotionLSNTolerance. + DefaultPromotionLSNTolerance uint64 = 100 +) + // BlockVolumeEntry tracks one block volume across the cluster. type BlockVolumeEntry struct { - Name string - VolumeServer string // volume server address (ip:port or grpc addr) - Path string // file path on volume server - IQN string - ISCSIAddr string - SizeBytes uint64 - Epoch uint64 - Role uint32 - Status VolumeStatus + Name string + VolumeServer string // volume server address (ip:port or grpc addr) + Path string // file path on volume server + IQN string + ISCSIAddr string + SizeBytes uint64 + ReplicaPlacement string // SeaweedFS placement string: "000", "001", "010", "100" + Epoch uint64 + Role uint32 + Status VolumeStatus - // Replica tracking (CP6-3). - ReplicaServer string // replica VS address - ReplicaPath string // file path on replica VS + // Deprecated scalar replica fields (CP6-3). Use Replicas[] for new code. + ReplicaServer string + ReplicaPath string ReplicaISCSIAddr string ReplicaIQN string - ReplicaDataAddr string // replica receiver data listen addr - ReplicaCtrlAddr string // replica receiver ctrl listen addr + ReplicaDataAddr string + ReplicaCtrlAddr string RebuildListenAddr string // rebuild server listen addr on primary + // CP8-2: Multi-replica support. + ReplicaFactor int // 2 or 3 (default 2) + Replicas []ReplicaInfo // one per replica (RF-1 entries) + HealthScore float64 // primary health score from heartbeat + ReplicaDegraded bool // primary reports degraded replicas + WALHeadLSN uint64 // primary WAL head LSN from heartbeat + // Lease tracking for failover (CP6-3 F2). LastLeaseGrant time.Time LeaseTTL time.Duration } +// HasReplica returns true if this volume has any replica (checks both new and deprecated fields). +func (e *BlockVolumeEntry) HasReplica() bool { + return len(e.Replicas) > 0 || e.ReplicaServer != "" +} + +// FirstReplica returns the first replica info, or nil if none. +func (e *BlockVolumeEntry) FirstReplica() *ReplicaInfo { + if len(e.Replicas) > 0 { + return &e.Replicas[0] + } + return nil +} + +// ReplicaByServer returns the replica hosted on the given server, or nil. +func (e *BlockVolumeEntry) ReplicaByServer(server string) *ReplicaInfo { + for i := range e.Replicas { + if e.Replicas[i].Server == server { + return &e.Replicas[i] + } + } + return nil +} + +// BestReplicaForPromotion returns the best replica for promotion, or nil if none eligible. +// Criteria: highest HealthScore, tie-break by highest WALHeadLSN, then first in list. +func (e *BlockVolumeEntry) BestReplicaForPromotion() *ReplicaInfo { + if len(e.Replicas) == 0 { + return nil + } + best := 0 + for i := 1; i < len(e.Replicas); i++ { + if e.Replicas[i].HealthScore > e.Replicas[best].HealthScore { + best = i + } else if e.Replicas[i].HealthScore == e.Replicas[best].HealthScore && + e.Replicas[i].WALHeadLSN > e.Replicas[best].WALHeadLSN { + best = i + } + } + return &e.Replicas[best] +} + // BlockVolumeRegistry is the in-memory registry of block volumes. // Rebuilt from heartbeats on master restart (no persistence). type BlockVolumeRegistry struct { @@ -51,8 +126,16 @@ type BlockVolumeRegistry struct { byServer map[string]map[string]bool // server -> set of volume names blockServers map[string]bool // servers known to support block volumes + // Promotion eligibility: max WAL LSN lag for replica to be promotable. + promotionLSNTolerance uint64 + // inflight guards concurrent CreateBlockVolume for the same name. inflight sync.Map // name -> *inflightEntry + + // Metrics (CP8-4). + PromotionsTotal atomic.Uint64 + FailoversTotal atomic.Uint64 + RebuildsTotal atomic.Uint64 } type inflightEntry struct{} @@ -60,9 +143,10 @@ type inflightEntry struct{} // NewBlockVolumeRegistry creates an empty registry. func NewBlockVolumeRegistry() *BlockVolumeRegistry { return &BlockVolumeRegistry{ - volumes: make(map[string]*BlockVolumeEntry), - byServer: make(map[string]map[string]bool), - blockServers: make(map[string]bool), + volumes: make(map[string]*BlockVolumeEntry), + byServer: make(map[string]map[string]bool), + blockServers: make(map[string]bool), + promotionLSNTolerance: DefaultPromotionLSNTolerance, } } @@ -76,6 +160,10 @@ func (r *BlockVolumeRegistry) Register(entry *BlockVolumeEntry) error { } r.volumes[entry.Name] = entry r.addToServer(entry.VolumeServer, entry.Name) + // Also index replica servers so ListByServer finds them. + for _, ri := range entry.Replicas { + r.addToServer(ri.Server, entry.Name) + } return nil } @@ -89,9 +177,25 @@ func (r *BlockVolumeRegistry) Unregister(name string) *BlockVolumeEntry { } delete(r.volumes, name) r.removeFromServer(entry.VolumeServer, name) + for _, ri := range entry.Replicas { + r.removeFromServer(ri.Server, name) + } return entry } +// UpdateSize updates the size of a registered volume. +// Called only after a successful VS expand to keep registry in sync. +func (r *BlockVolumeRegistry) UpdateSize(name string, newSizeBytes uint64) error { + r.mu.Lock() + defer r.mu.Unlock() + entry, ok := r.volumes[name] + if !ok { + return fmt.Errorf("block volume %q not found in registry", name) + } + entry.SizeBytes = newSizeBytes + return nil +} + // Lookup returns the entry for the given name. func (r *BlockVolumeRegistry) Lookup(name string) (*BlockVolumeEntry, bool) { r.mu.RLock() @@ -134,51 +238,172 @@ func (r *BlockVolumeRegistry) UpdateFullHeartbeat(server string, infos []*master reported[info.Path] = info } - // Find entries for this server that are NOT reported -> remove them. + // Find entries for this server that are NOT reported -> reconcile. if names, ok := r.byServer[server]; ok { for name := range names { entry := r.volumes[name] if entry == nil { continue } - if _, found := reported[entry.Path]; !found { - delete(r.volumes, name) - delete(names, name) + if entry.VolumeServer == server { + // Server is the primary: check if primary path is reported. + if _, found := reported[entry.Path]; !found { + delete(r.volumes, name) + delete(names, name) + // Also clean up replica entries from byServer. + for _, ri := range entry.Replicas { + r.removeFromServer(ri.Server, name) + } + } + } else { + // Server is a replica: check if replica path is reported. + ri := entry.ReplicaByServer(server) + if ri == nil { + // No replica record — stale byServer index, just clean up. + delete(names, name) + continue + } + if _, found := reported[ri.Path]; !found { + // Replica path not reported — remove this replica, NOT the whole volume. + r.removeReplicaLocked(entry, server, name) + delete(names, name) + glog.V(0).Infof("block registry: removed stale replica %s for %q (path %s not in heartbeat)", + server, name, ri.Path) + } } } } // Update or add entries for reported volumes. for _, info := range infos { - // Find existing entry by path on this server. + // Find existing entry: search byServer index for matching path (primary or replica). var existing *BlockVolumeEntry + var existingName string if names, ok := r.byServer[server]; ok { - for name := range names { - if e := r.volumes[name]; e != nil && e.Path == info.Path { - existing = e - break + for vname := range names { + if e := r.volumes[vname]; e != nil { + if e.VolumeServer == server && e.Path == info.Path { + existing = e + existingName = vname + break + } + if ri := e.ReplicaByServer(server); ri != nil && ri.Path == info.Path { + existing = e + existingName = vname + break + } } } } + // Also try lookup by name derived from path (handles post-restart). + if existing == nil { + name := nameFromPath(info.Path) + if name != "" { + if e, ok := r.volumes[name]; ok { + existing = e + existingName = name + } + } + } + if existing != nil { - // Update fields from heartbeat. - existing.SizeBytes = info.VolumeSize - existing.Epoch = info.Epoch - existing.Role = info.Role - existing.Status = StatusActive - // R1-5: Refresh lease on heartbeat — VS is alive and running this volume. - existing.LastLeaseGrant = time.Now() - // F5: update replica addresses from heartbeat info. - if info.ReplicaDataAddr != "" { - existing.ReplicaDataAddr = info.ReplicaDataAddr + isPrimary := existing.VolumeServer == server + isReplica := existing.ReplicaByServer(server) != nil + + if isPrimary { + // Primary heartbeat: update primary fields. + existing.SizeBytes = info.VolumeSize + existing.Epoch = info.Epoch + existing.Role = info.Role + existing.Status = StatusActive + existing.LastLeaseGrant = time.Now() + existing.HealthScore = info.HealthScore + existing.ReplicaDegraded = info.ReplicaDegraded + existing.WALHeadLSN = info.WalHeadLsn + // F5: update replica addresses from heartbeat info. + if info.ReplicaDataAddr != "" { + existing.ReplicaDataAddr = info.ReplicaDataAddr + } + if info.ReplicaCtrlAddr != "" { + existing.ReplicaCtrlAddr = info.ReplicaCtrlAddr + } + // Sync first replica's data addrs to Replicas[]. + if info.ReplicaDataAddr != "" && len(existing.Replicas) > 0 { + existing.Replicas[0].DataAddr = info.ReplicaDataAddr + existing.Replicas[0].CtrlAddr = info.ReplicaCtrlAddr + } + } else if isReplica { + // Replica heartbeat: update ReplicaInfo fields. + for i := range existing.Replicas { + if existing.Replicas[i].Server == server { + existing.Replicas[i].Path = info.Path + existing.Replicas[i].WALHeadLSN = info.WalHeadLsn + existing.Replicas[i].HealthScore = info.HealthScore + existing.Replicas[i].LastHeartbeat = time.Now() + existing.Replicas[i].Role = info.Role + if existing.WALHeadLSN > info.WalHeadLsn { + existing.Replicas[i].WALLag = existing.WALHeadLSN - info.WalHeadLsn + } else { + existing.Replicas[i].WALLag = 0 + } + break + } + } + } else { + // Fix #3: Server reports a volume that exists but has no record of this server. + // This happens after master restart: primary heartbeat re-created the entry, + // but replica heartbeat arrives and isn't linked. Add as replica. + ri := ReplicaInfo{ + Server: server, + Path: info.Path, + HealthScore: info.HealthScore, + WALHeadLSN: info.WalHeadLsn, + LastHeartbeat: time.Now(), + Role: info.Role, + } + existing.Replicas = append(existing.Replicas, ri) + r.addToServer(server, existingName) + // Sync deprecated scalar fields. + if len(existing.Replicas) == 1 { + existing.ReplicaServer = ri.Server + existing.ReplicaPath = ri.Path + } + glog.V(0).Infof("block registry: attached replica %s for %q from heartbeat (path=%s)", + server, existingName, info.Path) + } + } else { + // Auto-register volumes reported by heartbeat but not in registry. + // This recovers state after master restart. + name := nameFromPath(info.Path) + if name == "" { + continue } - if info.ReplicaCtrlAddr != "" { - existing.ReplicaCtrlAddr = info.ReplicaCtrlAddr + if _, dup := r.volumes[name]; !dup { + entry := &BlockVolumeEntry{ + Name: name, + VolumeServer: server, + Path: info.Path, + SizeBytes: info.VolumeSize, + Epoch: info.Epoch, + Role: info.Role, + Status: StatusActive, + LastLeaseGrant: time.Now(), + LeaseTTL: 30 * time.Second, + HealthScore: info.HealthScore, + WALHeadLSN: info.WalHeadLsn, + } + if info.ReplicaDataAddr != "" { + entry.ReplicaDataAddr = info.ReplicaDataAddr + } + if info.ReplicaCtrlAddr != "" { + entry.ReplicaCtrlAddr = info.ReplicaCtrlAddr + } + r.volumes[name] = entry + r.addToServer(server, name) + glog.V(0).Infof("block registry: auto-registered %q from heartbeat (server=%s, path=%s, size=%d)", + name, server, info.Path, info.VolumeSize) } } - // If no existing entry found by path, it was created outside master - // (e.g., manually). We don't auto-register unknown volumes — they - // must be created via CreateBlockVolume RPC. } } @@ -274,7 +499,38 @@ func (r *BlockVolumeRegistry) removeFromServer(server, name string) { } } +// removeReplicaLocked removes a replica from an entry by server address. +// Caller must hold r.mu. Also syncs deprecated scalar fields. +func (r *BlockVolumeRegistry) removeReplicaLocked(entry *BlockVolumeEntry, server, name string) { + newReplicas := make([]ReplicaInfo, 0, len(entry.Replicas)) + for _, ri := range entry.Replicas { + if ri.Server == server { + continue + } + newReplicas = append(newReplicas, ri) + } + entry.Replicas = newReplicas + // Sync deprecated scalar fields. + if len(entry.Replicas) > 0 { + r0 := &entry.Replicas[0] + entry.ReplicaServer = r0.Server + entry.ReplicaPath = r0.Path + entry.ReplicaISCSIAddr = r0.ISCSIAddr + entry.ReplicaIQN = r0.IQN + entry.ReplicaDataAddr = r0.DataAddr + entry.ReplicaCtrlAddr = r0.CtrlAddr + } else { + entry.ReplicaServer = "" + entry.ReplicaPath = "" + entry.ReplicaISCSIAddr = "" + entry.ReplicaIQN = "" + entry.ReplicaDataAddr = "" + entry.ReplicaCtrlAddr = "" + } +} + // SetReplica sets replica info for a registered volume. +// Deprecated: use AddReplica for new code. This method syncs both scalar and Replicas[]. func (r *BlockVolumeRegistry) SetReplica(name, server, path, iscsiAddr, iqn string) error { r.mu.Lock() defer r.mu.Unlock() @@ -290,12 +546,31 @@ func (r *BlockVolumeRegistry) SetReplica(name, server, path, iscsiAddr, iqn stri entry.ReplicaPath = path entry.ReplicaISCSIAddr = iscsiAddr entry.ReplicaIQN = iqn - // Also add to byServer index for the replica server. r.addToServer(server, name) + + // CP8-2: also sync to Replicas[]. + info := ReplicaInfo{Server: server, Path: path, ISCSIAddr: iscsiAddr, IQN: iqn} + replaced := false + for i := range entry.Replicas { + if entry.Replicas[i].Server == server { + // Preserve existing health/LSN data. + info.HealthScore = entry.Replicas[i].HealthScore + info.WALHeadLSN = entry.Replicas[i].WALHeadLSN + info.DataAddr = entry.Replicas[i].DataAddr + info.CtrlAddr = entry.Replicas[i].CtrlAddr + entry.Replicas[i] = info + replaced = true + break + } + } + if !replaced { + entry.Replicas = append(entry.Replicas, info) + } return nil } -// ClearReplica removes replica info for a registered volume. +// ClearReplica removes all replica info for a registered volume. +// Deprecated: use RemoveReplica for new code. This method clears both scalar and Replicas[]. func (r *BlockVolumeRegistry) ClearReplica(name string) error { r.mu.Lock() defer r.mu.Unlock() @@ -306,12 +581,19 @@ func (r *BlockVolumeRegistry) ClearReplica(name string) error { if entry.ReplicaServer != "" { r.removeFromServer(entry.ReplicaServer, name) } + // Remove all replicas from byServer index. + for _, ri := range entry.Replicas { + if ri.Server != entry.ReplicaServer { + r.removeFromServer(ri.Server, name) + } + } entry.ReplicaServer = "" entry.ReplicaPath = "" entry.ReplicaISCSIAddr = "" entry.ReplicaIQN = "" entry.ReplicaDataAddr = "" entry.ReplicaCtrlAddr = "" + entry.Replicas = nil return nil } @@ -363,6 +645,198 @@ func (r *BlockVolumeRegistry) SwapPrimaryReplica(name string) (uint64, error) { return newEpoch, nil } +// AddReplica adds or replaces a replica in the Replicas slice (by server). +// Also updates the byServer index and deprecated scalar fields for backward compat. +func (r *BlockVolumeRegistry) AddReplica(name string, info ReplicaInfo) error { + r.mu.Lock() + defer r.mu.Unlock() + entry, ok := r.volumes[name] + if !ok { + return fmt.Errorf("block volume %q not found", name) + } + // Replace if same server already exists. + replaced := false + for i := range entry.Replicas { + if entry.Replicas[i].Server == info.Server { + entry.Replicas[i] = info + replaced = true + break + } + } + if !replaced { + entry.Replicas = append(entry.Replicas, info) + } + r.addToServer(info.Server, name) + + // Sync deprecated scalar fields (first replica → scalar). + if len(entry.Replicas) > 0 { + r0 := &entry.Replicas[0] + entry.ReplicaServer = r0.Server + entry.ReplicaPath = r0.Path + entry.ReplicaISCSIAddr = r0.ISCSIAddr + entry.ReplicaIQN = r0.IQN + entry.ReplicaDataAddr = r0.DataAddr + entry.ReplicaCtrlAddr = r0.CtrlAddr + } + return nil +} + +// RemoveReplica removes a replica by server address. +func (r *BlockVolumeRegistry) RemoveReplica(name, server string) error { + r.mu.Lock() + defer r.mu.Unlock() + entry, ok := r.volumes[name] + if !ok { + return fmt.Errorf("block volume %q not found", name) + } + found := false + newReplicas := make([]ReplicaInfo, 0, len(entry.Replicas)) + for _, ri := range entry.Replicas { + if ri.Server == server { + found = true + r.removeFromServer(server, name) + continue + } + newReplicas = append(newReplicas, ri) + } + if !found { + return fmt.Errorf("replica on %q not found for volume %q", server, name) + } + entry.Replicas = newReplicas + + // Sync deprecated scalar fields. + if len(entry.Replicas) > 0 { + r0 := &entry.Replicas[0] + entry.ReplicaServer = r0.Server + entry.ReplicaPath = r0.Path + entry.ReplicaISCSIAddr = r0.ISCSIAddr + entry.ReplicaIQN = r0.IQN + entry.ReplicaDataAddr = r0.DataAddr + entry.ReplicaCtrlAddr = r0.CtrlAddr + } else { + entry.ReplicaServer = "" + entry.ReplicaPath = "" + entry.ReplicaISCSIAddr = "" + entry.ReplicaIQN = "" + entry.ReplicaDataAddr = "" + entry.ReplicaCtrlAddr = "" + } + return nil +} + +// SetPromotionLSNTolerance configures the max WAL LSN lag for promotion eligibility. +func (r *BlockVolumeRegistry) SetPromotionLSNTolerance(tolerance uint64) { + r.mu.Lock() + defer r.mu.Unlock() + r.promotionLSNTolerance = tolerance +} + +// PromotionLSNTolerance returns the current promotion LSN tolerance. +func (r *BlockVolumeRegistry) PromotionLSNTolerance() uint64 { + r.mu.RLock() + defer r.mu.RUnlock() + return r.promotionLSNTolerance +} + +// PromoteBestReplica promotes the best eligible replica to primary. +// Eligibility: heartbeat fresh (within 2×LeaseTTL), WALHeadLSN within tolerance of primary, +// and role must be RoleReplica (not RoleRebuilding). +// The promoted replica is removed from Replicas[]. Other replicas stay. +// Old primary is NOT added to Replicas (needs rebuild). +// Returns the new epoch. +func (r *BlockVolumeRegistry) PromoteBestReplica(name string) (uint64, error) { + r.mu.Lock() + defer r.mu.Unlock() + entry, ok := r.volumes[name] + if !ok { + return 0, fmt.Errorf("block volume %q not found", name) + } + if len(entry.Replicas) == 0 { + return 0, fmt.Errorf("block volume %q has no replicas", name) + } + + // Filter eligible replicas. + now := time.Now() + freshnessCutoff := 2 * entry.LeaseTTL + if freshnessCutoff == 0 { + freshnessCutoff = 60 * time.Second // default if LeaseTTL not set + } + primaryLSN := entry.WALHeadLSN + + bestIdx := -1 + for i := range entry.Replicas { + ri := &entry.Replicas[i] + // Gate 1: heartbeat freshness. + if !ri.LastHeartbeat.IsZero() && now.Sub(ri.LastHeartbeat) > freshnessCutoff { + continue + } + // Gate 2: WAL LSN recency (skip if primary LSN is 0 — no data yet, all eligible). + if primaryLSN > 0 && ri.WALHeadLSN+r.promotionLSNTolerance < primaryLSN { + continue + } + // Gate 3: role must be RoleReplica (not rebuilding/stale). + if ri.Role != 0 && blockvol.RoleFromWire(ri.Role) != blockvol.RoleReplica { + continue + } + // Eligible — pick best by health score, tie-break by WALHeadLSN. + if bestIdx == -1 { + bestIdx = i + } else if ri.HealthScore > entry.Replicas[bestIdx].HealthScore { + bestIdx = i + } else if ri.HealthScore == entry.Replicas[bestIdx].HealthScore && + ri.WALHeadLSN > entry.Replicas[bestIdx].WALHeadLSN { + bestIdx = i + } + } + + if bestIdx == -1 { + return 0, fmt.Errorf("block volume %q: no eligible replicas for promotion", name) + } + + promoted := entry.Replicas[bestIdx] + + // Remove old primary from byServer index. + r.removeFromServer(entry.VolumeServer, name) + + // Bump epoch atomically. + newEpoch := entry.Epoch + 1 + + // Promote replica to primary. + entry.VolumeServer = promoted.Server + entry.Path = promoted.Path + entry.IQN = promoted.IQN + entry.ISCSIAddr = promoted.ISCSIAddr + entry.Epoch = newEpoch + entry.Role = blockvol.RoleToWire(blockvol.RolePrimary) + entry.LastLeaseGrant = time.Now() + + // Remove promoted from Replicas. Others stay. + entry.Replicas = append(entry.Replicas[:bestIdx], entry.Replicas[bestIdx+1:]...) + + // Sync deprecated scalar fields. + if len(entry.Replicas) > 0 { + r0 := &entry.Replicas[0] + entry.ReplicaServer = r0.Server + entry.ReplicaPath = r0.Path + entry.ReplicaISCSIAddr = r0.ISCSIAddr + entry.ReplicaIQN = r0.IQN + entry.ReplicaDataAddr = r0.DataAddr + entry.ReplicaCtrlAddr = r0.CtrlAddr + } else { + entry.ReplicaServer = "" + entry.ReplicaPath = "" + entry.ReplicaISCSIAddr = "" + entry.ReplicaIQN = "" + entry.ReplicaDataAddr = "" + entry.ReplicaCtrlAddr = "" + } + + // Update byServer index: new primary server now hosts this volume. + r.addToServer(entry.VolumeServer, name) + + return newEpoch, nil +} + // MarkBlockCapable records that the given server supports block volumes. func (r *BlockVolumeRegistry) MarkBlockCapable(server string) { r.mu.Lock() @@ -377,6 +851,45 @@ func (r *BlockVolumeRegistry) UnmarkBlockCapable(server string) { r.mu.Unlock() } +// ListAll returns all registered block volume entries, sorted by name. +func (r *BlockVolumeRegistry) ListAll() []*BlockVolumeEntry { + r.mu.RLock() + defer r.mu.RUnlock() + entries := make([]*BlockVolumeEntry, 0, len(r.volumes)) + for _, e := range r.volumes { + entries = append(entries, e) + } + sort.Slice(entries, func(i, j int) bool { return entries[i].Name < entries[j].Name }) + return entries +} + +// BlockServerSummary summarizes a block-capable volume server. +type BlockServerSummary struct { + Address string + VolumeCount int + BlockCapable bool +} + +// ServerSummaries returns a summary for each block-capable server. +func (r *BlockVolumeRegistry) ServerSummaries() []BlockServerSummary { + r.mu.RLock() + defer r.mu.RUnlock() + summaries := make([]BlockServerSummary, 0, len(r.blockServers)) + for addr := range r.blockServers { + count := 0 + if names, ok := r.byServer[addr]; ok { + count = len(names) + } + summaries = append(summaries, BlockServerSummary{ + Address: addr, + VolumeCount: count, + BlockCapable: true, + }) + } + sort.Slice(summaries, func(i, j int) bool { return summaries[i].Address < summaries[j].Address }) + return summaries +} + // BlockCapableServers returns the list of servers known to support block volumes. func (r *BlockVolumeRegistry) BlockCapableServers() []string { r.mu.RLock() @@ -387,3 +900,38 @@ func (r *BlockVolumeRegistry) BlockCapableServers() []string { } return servers } + +// MaxBarrierLagLSN returns the maximum WAL lag across all volumes and replicas. +// This is the primary durability-risk metric: primary WALHeadLSN minus +// replica's WALHeadLSN. SLO threshold: < 100 under normal load. +func (r *BlockVolumeRegistry) MaxBarrierLagLSN() uint64 { + r.mu.RLock() + defer r.mu.RUnlock() + var maxLag uint64 + for _, entry := range r.volumes { + for _, ri := range entry.Replicas { + if ri.WALLag > maxLag { + maxLag = ri.WALLag + } + } + } + return maxLag +} + +// AssignmentQueueDepth returns the total number of pending assignments across all servers. +func (r *BlockVolumeRegistry) AssignmentQueueDepth() int { + // Delegated to the assignment queue, not the registry. + // Placeholder: the queue tracks its own depth. + return 0 +} + +// nameFromPath extracts the volume name from a .blk file path. +// e.g. "/opt/data/block/my-volume.blk" -> "my-volume" +func nameFromPath(path string) string { + base := filepath.Base(path) + if strings.HasSuffix(base, ".blk") { + return strings.TrimSuffix(base, ".blk") + } + return base +} + diff --git a/weed/server/master_block_registry_test.go b/weed/server/master_block_registry_test.go index ec060980d6..aa28a1f112 100644 --- a/weed/server/master_block_registry_test.go +++ b/weed/server/master_block_registry_test.go @@ -4,8 +4,10 @@ import ( "fmt" "sync" "testing" + "time" "github.com/seaweedfs/seaweedfs/weed/pb/master_pb" + "github.com/seaweedfs/seaweedfs/weed/storage/blockvol" ) func TestRegistry_RegisterLookup(t *testing.T) { @@ -434,3 +436,558 @@ func TestFullHeartbeat_UpdatesReplicaAddrs(t *testing.T) { t.Fatalf("ReplicaCtrlAddr: got %q, want 10.0.0.2:14261", entry.ReplicaCtrlAddr) } } + +// --- CP8-2 new tests --- + +func TestRegistry_AddReplica(t *testing.T) { + r := NewBlockVolumeRegistry() + r.Register(&BlockVolumeEntry{Name: "vol1", VolumeServer: "s1", Path: "/v1.blk"}) + + err := r.AddReplica("vol1", ReplicaInfo{ + Server: "s2", + Path: "/replica/v1.blk", + ISCSIAddr: "10.0.0.2:3260", + IQN: "iqn:vol1-r1", + DataAddr: "s2:14260", + CtrlAddr: "s2:14261", + }) + if err != nil { + t.Fatalf("AddReplica: %v", err) + } + + e, _ := r.Lookup("vol1") + if len(e.Replicas) != 1 { + t.Fatalf("Replicas len: got %d, want 1", len(e.Replicas)) + } + if e.Replicas[0].Server != "s2" { + t.Fatalf("Replicas[0].Server: got %q", e.Replicas[0].Server) + } + // Deprecated scalar should be synced. + if e.ReplicaServer != "s2" { + t.Fatalf("ReplicaServer (deprecated): got %q", e.ReplicaServer) + } + // byServer index should include replica. + if len(r.ListByServer("s2")) != 1 { + t.Fatalf("ListByServer(s2): got %d, want 1", len(r.ListByServer("s2"))) + } +} + +func TestRegistry_AddReplica_TwoRF3(t *testing.T) { + r := NewBlockVolumeRegistry() + r.Register(&BlockVolumeEntry{Name: "vol1", VolumeServer: "s1", Path: "/v1.blk", ReplicaFactor: 3}) + + r.AddReplica("vol1", ReplicaInfo{Server: "s2", Path: "/r1.blk", IQN: "iqn:r1"}) + r.AddReplica("vol1", ReplicaInfo{Server: "s3", Path: "/r2.blk", IQN: "iqn:r2"}) + + e, _ := r.Lookup("vol1") + if len(e.Replicas) != 2 { + t.Fatalf("Replicas len: got %d, want 2", len(e.Replicas)) + } + if e.Replicas[0].Server != "s2" || e.Replicas[1].Server != "s3" { + t.Fatalf("Replicas: got %+v", e.Replicas) + } + // byServer index should include both. + if len(r.ListByServer("s2")) != 1 || len(r.ListByServer("s3")) != 1 { + t.Fatal("byServer should include both replica servers") + } +} + +func TestRegistry_AddReplica_Upsert(t *testing.T) { + r := NewBlockVolumeRegistry() + r.Register(&BlockVolumeEntry{Name: "vol1", VolumeServer: "s1", Path: "/v1.blk"}) + + r.AddReplica("vol1", ReplicaInfo{Server: "s2", Path: "/r1.blk"}) + r.AddReplica("vol1", ReplicaInfo{Server: "s2", Path: "/r1-new.blk"}) + + e, _ := r.Lookup("vol1") + if len(e.Replicas) != 1 { + t.Fatalf("Replicas len: got %d, want 1 (upsert, not duplicate)", len(e.Replicas)) + } + if e.Replicas[0].Path != "/r1-new.blk" { + t.Fatalf("Replicas[0].Path: got %q, want /r1-new.blk", e.Replicas[0].Path) + } +} + +func TestRegistry_RemoveReplica(t *testing.T) { + r := NewBlockVolumeRegistry() + r.Register(&BlockVolumeEntry{Name: "vol1", VolumeServer: "s1", Path: "/v1.blk"}) + r.AddReplica("vol1", ReplicaInfo{Server: "s2", Path: "/r1.blk"}) + r.AddReplica("vol1", ReplicaInfo{Server: "s3", Path: "/r2.blk"}) + + err := r.RemoveReplica("vol1", "s2") + if err != nil { + t.Fatalf("RemoveReplica: %v", err) + } + + e, _ := r.Lookup("vol1") + if len(e.Replicas) != 1 { + t.Fatalf("Replicas len: got %d, want 1", len(e.Replicas)) + } + if e.Replicas[0].Server != "s3" { + t.Fatalf("remaining replica should be s3, got %q", e.Replicas[0].Server) + } + // Deprecated scalar should sync to first remaining replica. + if e.ReplicaServer != "s3" { + t.Fatalf("ReplicaServer (deprecated): got %q, want s3", e.ReplicaServer) + } + // s2 should be removed from byServer. + if len(r.ListByServer("s2")) != 0 { + t.Fatalf("ListByServer(s2): got %d, want 0", len(r.ListByServer("s2"))) + } +} + +func TestRegistry_PromoteBestReplica_PicksHighest(t *testing.T) { + r := NewBlockVolumeRegistry() + r.Register(&BlockVolumeEntry{ + Name: "vol1", + VolumeServer: "s1", + Path: "/v1.blk", + Epoch: 5, + Role: 1, + Replicas: []ReplicaInfo{ + {Server: "s2", Path: "/r1.blk", IQN: "iqn:r1", ISCSIAddr: "s2:3260", HealthScore: 0.8, WALHeadLSN: 100}, + {Server: "s3", Path: "/r2.blk", IQN: "iqn:r2", ISCSIAddr: "s3:3260", HealthScore: 0.95, WALHeadLSN: 90}, + }, + }) + // Add to byServer for s2 and s3. + r.mu.Lock() + r.addToServer("s2", "vol1") + r.addToServer("s3", "vol1") + r.mu.Unlock() + + newEpoch, err := r.PromoteBestReplica("vol1") + if err != nil { + t.Fatalf("PromoteBestReplica: %v", err) + } + if newEpoch != 6 { + t.Fatalf("newEpoch: got %d, want 6", newEpoch) + } + + e, _ := r.Lookup("vol1") + // s3 had higher health score → promoted. + if e.VolumeServer != "s3" { + t.Fatalf("VolumeServer: got %q, want s3 (higher health)", e.VolumeServer) + } + if e.Path != "/r2.blk" { + t.Fatalf("Path: got %q", e.Path) + } + // s2 should remain in Replicas. + if len(e.Replicas) != 1 { + t.Fatalf("Replicas len: got %d, want 1 (s2 stays)", len(e.Replicas)) + } + if e.Replicas[0].Server != "s2" { + t.Fatalf("remaining replica: got %q, want s2", e.Replicas[0].Server) + } +} + +func TestRegistry_PromoteBestReplica_NoReplica(t *testing.T) { + r := NewBlockVolumeRegistry() + r.Register(&BlockVolumeEntry{Name: "vol1", VolumeServer: "s1", Path: "/v1.blk"}) + + _, err := r.PromoteBestReplica("vol1") + if err == nil { + t.Fatal("PromoteBestReplica with no replicas should return error") + } +} + +func TestRegistry_PromoteBestReplica_TiebreakByLSN(t *testing.T) { + r := NewBlockVolumeRegistry() + r.Register(&BlockVolumeEntry{ + Name: "vol1", + VolumeServer: "s1", + Path: "/v1.blk", + Epoch: 3, + Replicas: []ReplicaInfo{ + {Server: "s2", Path: "/r1.blk", IQN: "iqn:r1", ISCSIAddr: "s2:3260", HealthScore: 0.9, WALHeadLSN: 50}, + {Server: "s3", Path: "/r2.blk", IQN: "iqn:r2", ISCSIAddr: "s3:3260", HealthScore: 0.9, WALHeadLSN: 100}, + }, + }) + r.mu.Lock() + r.addToServer("s2", "vol1") + r.addToServer("s3", "vol1") + r.mu.Unlock() + + newEpoch, err := r.PromoteBestReplica("vol1") + if err != nil { + t.Fatalf("PromoteBestReplica: %v", err) + } + if newEpoch != 4 { + t.Fatalf("newEpoch: got %d, want 4", newEpoch) + } + + e, _ := r.Lookup("vol1") + // Same health → tie-break by WALHeadLSN → s3 wins. + if e.VolumeServer != "s3" { + t.Fatalf("VolumeServer: got %q, want s3 (higher LSN)", e.VolumeServer) + } + if len(e.Replicas) != 1 || e.Replicas[0].Server != "s2" { + t.Fatalf("remaining replica: got %+v, want [s2]", e.Replicas) + } +} + +func TestRegistry_PromoteBestReplica_KeepsOthers(t *testing.T) { + r := NewBlockVolumeRegistry() + r.Register(&BlockVolumeEntry{ + Name: "vol1", + VolumeServer: "s1", + Path: "/v1.blk", + Epoch: 1, + Replicas: []ReplicaInfo{ + {Server: "s2", Path: "/r1.blk", IQN: "iqn:r1", ISCSIAddr: "s2:3260", HealthScore: 1.0, WALHeadLSN: 100}, + {Server: "s3", Path: "/r2.blk", IQN: "iqn:r2", ISCSIAddr: "s3:3260", HealthScore: 0.5, WALHeadLSN: 100}, + }, + }) + r.mu.Lock() + r.addToServer("s2", "vol1") + r.addToServer("s3", "vol1") + r.mu.Unlock() + + r.PromoteBestReplica("vol1") + + e, _ := r.Lookup("vol1") + // s2 promoted, s3 stays. + if e.VolumeServer != "s2" { + t.Fatalf("VolumeServer: got %q, want s2", e.VolumeServer) + } + if len(e.Replicas) != 1 || e.Replicas[0].Server != "s3" { + t.Fatalf("remaining replicas: got %+v, want [s3]", e.Replicas) + } +} + +func TestRegistry_BackwardCompatAccessors(t *testing.T) { + r := NewBlockVolumeRegistry() + r.Register(&BlockVolumeEntry{Name: "vol1", VolumeServer: "s1", Path: "/v1.blk"}) + + e, _ := r.Lookup("vol1") + if e.HasReplica() { + t.Fatal("HasReplica should be false with no replicas") + } + if e.FirstReplica() != nil { + t.Fatal("FirstReplica should be nil") + } + if e.BestReplicaForPromotion() != nil { + t.Fatal("BestReplicaForPromotion should be nil") + } + + r.AddReplica("vol1", ReplicaInfo{Server: "s2", Path: "/r.blk", HealthScore: 0.9}) + + e, _ = r.Lookup("vol1") + if !e.HasReplica() { + t.Fatal("HasReplica should be true after AddReplica") + } + if e.FirstReplica() == nil || e.FirstReplica().Server != "s2" { + t.Fatal("FirstReplica should return s2") + } + if e.ReplicaByServer("s2") == nil { + t.Fatal("ReplicaByServer(s2) should not be nil") + } + if e.ReplicaByServer("s3") != nil { + t.Fatal("ReplicaByServer(s3) should be nil") + } +} + +func TestRegistry_ReplicaFactorDefault(t *testing.T) { + r := NewBlockVolumeRegistry() + r.Register(&BlockVolumeEntry{Name: "vol1", VolumeServer: "s1", Path: "/v1.blk"}) + + e, _ := r.Lookup("vol1") + // ReplicaFactor defaults to 0 (zero value). API handler defaults to 2. + if e.ReplicaFactor != 0 { + t.Fatalf("default ReplicaFactor: got %d, want 0", e.ReplicaFactor) + } + + // Explicit RF=3. + r.Register(&BlockVolumeEntry{Name: "vol2", VolumeServer: "s1", Path: "/v2.blk", ReplicaFactor: 3}) + e2, _ := r.Lookup("vol2") + if e2.ReplicaFactor != 3 { + t.Fatalf("ReplicaFactor: got %d, want 3", e2.ReplicaFactor) + } +} + +func TestRegistry_FullHeartbeat_UpdatesHealthScore(t *testing.T) { + r := NewBlockVolumeRegistry() + r.Register(&BlockVolumeEntry{ + Name: "vol1", + VolumeServer: "s1", + Path: "/v1.blk", + Status: StatusPending, + }) + + r.UpdateFullHeartbeat("s1", []*master_pb.BlockVolumeInfoMessage{ + { + Path: "/v1.blk", + VolumeSize: 1 << 30, + Epoch: 1, + Role: 1, + HealthScore: 0.85, + ScrubErrors: 2, + WalHeadLsn: 500, + }, + }) + + e, _ := r.Lookup("vol1") + if e.HealthScore != 0.85 { + t.Fatalf("HealthScore: got %f, want 0.85", e.HealthScore) + } + if e.WALHeadLSN != 500 { + t.Fatalf("WALHeadLSN: got %d, want 500", e.WALHeadLSN) + } +} + +// Fix #1: Replica heartbeat must NOT delete the volume. +func TestRegistry_ReplicaHeartbeat_DoesNotDeleteVolume(t *testing.T) { + r := NewBlockVolumeRegistry() + r.Register(&BlockVolumeEntry{ + Name: "vol1", + VolumeServer: "primary", + Path: "/data/vol1.blk", + Status: StatusActive, + Replicas: []ReplicaInfo{ + {Server: "replica1", Path: "/data/vol1.blk"}, + }, + }) + + // Replica sends heartbeat reporting its path. + r.UpdateFullHeartbeat("replica1", []*master_pb.BlockVolumeInfoMessage{ + {Path: "/data/vol1.blk", Epoch: 1, Role: 2}, + }) + + // Volume must still exist with primary intact. + e, ok := r.Lookup("vol1") + if !ok { + t.Fatal("vol1 should not be deleted when replica sends heartbeat") + } + if e.VolumeServer != "primary" { + t.Fatalf("primary should remain 'primary', got %q", e.VolumeServer) + } +} + +// Fix #1: Replica path NOT reported → replica removed, volume preserved. +func TestRegistry_ReplicaHeartbeat_StaleReplicaRemoved(t *testing.T) { + r := NewBlockVolumeRegistry() + r.Register(&BlockVolumeEntry{ + Name: "vol1", + VolumeServer: "primary", + Path: "/data/vol1.blk", + Status: StatusActive, + Replicas: []ReplicaInfo{ + {Server: "replica1", Path: "/data/vol1.blk"}, + {Server: "replica2", Path: "/data/vol1.blk"}, + }, + }) + + // replica1 heartbeat does NOT report vol1 path → stale replica. + r.UpdateFullHeartbeat("replica1", []*master_pb.BlockVolumeInfoMessage{}) + + // Volume still exists, but replica1 removed. + e, ok := r.Lookup("vol1") + if !ok { + t.Fatal("vol1 should exist (only replica removed)") + } + if len(e.Replicas) != 1 { + t.Fatalf("expected 1 replica after stale removal, got %d", len(e.Replicas)) + } + if e.Replicas[0].Server != "replica2" { + t.Fatalf("remaining replica should be replica2, got %q", e.Replicas[0].Server) + } +} + +// Fix #3: Replica heartbeat after master restart reconstructs ReplicaInfo. +func TestRegistry_ReplicaHeartbeat_ReconstructsAfterRestart(t *testing.T) { + r := NewBlockVolumeRegistry() + // Simulate master restart: primary heartbeat re-created entry without replicas. + r.Register(&BlockVolumeEntry{ + Name: "vol1", + VolumeServer: "primary", + Path: "/data/vol1.blk", + Status: StatusActive, + }) + + // Replica heartbeat arrives — vol1 exists but has no record of this server. + r.UpdateFullHeartbeat("replica1", []*master_pb.BlockVolumeInfoMessage{ + {Path: "/data/vol1.blk", Epoch: 1, Role: 2, HealthScore: 0.95, WalHeadLsn: 42}, + }) + + // vol1 should now have replica1 in Replicas[]. + e, ok := r.Lookup("vol1") + if !ok { + t.Fatal("vol1 should exist") + } + if len(e.Replicas) != 1 { + t.Fatalf("expected 1 replica after reconstruction, got %d", len(e.Replicas)) + } + ri := e.Replicas[0] + if ri.Server != "replica1" { + t.Fatalf("replica server: got %q, want replica1", ri.Server) + } + if ri.HealthScore != 0.95 { + t.Fatalf("replica health: got %f, want 0.95", ri.HealthScore) + } + if ri.WALHeadLSN != 42 { + t.Fatalf("replica WALHeadLSN: got %d, want 42", ri.WALHeadLSN) + } + // byServer index should include replica1. + entries := r.ListByServer("replica1") + if len(entries) != 1 || entries[0].Name != "vol1" { + t.Fatalf("ListByServer(replica1) should return vol1, got %+v", entries) + } +} + +// Fix #2: Stale replica (old heartbeat) not eligible for promotion. +func TestRegistry_PromoteBestReplica_StaleHeartbeatIneligible(t *testing.T) { + r := NewBlockVolumeRegistry() + r.Register(&BlockVolumeEntry{ + Name: "vol1", + VolumeServer: "primary", + Path: "/data/vol1.blk", + Epoch: 1, + LeaseTTL: 5 * time.Second, + WALHeadLSN: 100, + Replicas: []ReplicaInfo{ + { + Server: "stale-replica", + Path: "/data/vol1.blk", + HealthScore: 1.0, + WALHeadLSN: 100, + LastHeartbeat: time.Now().Add(-30 * time.Second), // stale (>2×5s) + }, + }, + }) + + _, err := r.PromoteBestReplica("vol1") + if err == nil { + t.Fatal("expected error: stale replica should not be eligible") + } +} + +// Fix #2: Replica with WAL lag too large is not eligible. +func TestRegistry_PromoteBestReplica_WALLagIneligible(t *testing.T) { + r := NewBlockVolumeRegistry() + r.Register(&BlockVolumeEntry{ + Name: "vol1", + VolumeServer: "primary", + Path: "/data/vol1.blk", + Epoch: 1, + LeaseTTL: 30 * time.Second, + WALHeadLSN: 1000, + Replicas: []ReplicaInfo{ + { + Server: "lagging", + Path: "/data/vol1.blk", + HealthScore: 1.0, + WALHeadLSN: 800, // lag=200, tolerance=100 + LastHeartbeat: time.Now(), + }, + }, + }) + + _, err := r.PromoteBestReplica("vol1") + if err == nil { + t.Fatal("expected error: lagging replica should not be eligible") + } +} + +// Fix #2: Rebuilding replica is not eligible for promotion. +func TestRegistry_PromoteBestReplica_RebuildingIneligible(t *testing.T) { + r := NewBlockVolumeRegistry() + r.Register(&BlockVolumeEntry{ + Name: "vol1", + VolumeServer: "primary", + Path: "/data/vol1.blk", + Epoch: 1, + LeaseTTL: 30 * time.Second, + WALHeadLSN: 100, + Replicas: []ReplicaInfo{ + { + Server: "rebuilding", + Path: "/data/vol1.blk", + HealthScore: 1.0, + WALHeadLSN: 100, + LastHeartbeat: time.Now(), + Role: blockvol.RoleToWire(blockvol.RoleRebuilding), + }, + }, + }) + + _, err := r.PromoteBestReplica("vol1") + if err == nil { + t.Fatal("expected error: rebuilding replica should not be eligible") + } +} + +// Fix #2: Among eligible replicas, best (health+LSN) wins. +func TestRegistry_PromoteBestReplica_EligibilityFiltersCorrectly(t *testing.T) { + r := NewBlockVolumeRegistry() + r.Register(&BlockVolumeEntry{ + Name: "vol1", + VolumeServer: "primary", + Path: "/data/vol1.blk", + Epoch: 1, + LeaseTTL: 30 * time.Second, + WALHeadLSN: 100, + Replicas: []ReplicaInfo{ + { + Server: "stale", // ineligible: old heartbeat + Path: "/data/vol1.blk", + HealthScore: 1.0, + WALHeadLSN: 100, + LastHeartbeat: time.Now().Add(-2 * time.Minute), + }, + { + Server: "good", // eligible + Path: "/data/vol1.blk", + HealthScore: 0.8, + WALHeadLSN: 95, + LastHeartbeat: time.Now(), + }, + }, + }) + + _, err := r.PromoteBestReplica("vol1") + if err != nil { + t.Fatalf("expected promotion to succeed: %v", err) + } + e, _ := r.Lookup("vol1") + if e.VolumeServer != "good" { + t.Fatalf("expected 'good' promoted (only eligible), got %q", e.VolumeServer) + } +} + +// Configurable tolerance: widen tolerance to allow lagging replicas. +func TestRegistry_PromoteBestReplica_ConfigurableTolerance(t *testing.T) { + r := NewBlockVolumeRegistry() + r.Register(&BlockVolumeEntry{ + Name: "vol1", + VolumeServer: "primary", + Path: "/data/vol1.blk", + Epoch: 1, + LeaseTTL: 30 * time.Second, + WALHeadLSN: 1000, + Replicas: []ReplicaInfo{ + { + Server: "lagging", + Path: "/data/vol1.blk", + HealthScore: 1.0, + WALHeadLSN: 800, // lag=200 + LastHeartbeat: time.Now(), + }, + }, + }) + + // Default tolerance (100): lag 200 > tolerance → ineligible. + _, err := r.PromoteBestReplica("vol1") + if err == nil { + t.Fatal("expected error with default tolerance") + } + + // Widen tolerance to 250: lag 200 < tolerance → eligible. + r.SetPromotionLSNTolerance(250) + _, err = r.PromoteBestReplica("vol1") + if err != nil { + t.Fatalf("expected success with widened tolerance: %v", err) + } + e, _ := r.Lookup("vol1") + if e.VolumeServer != "lagging" { + t.Fatalf("expected 'lagging' promoted, got %q", e.VolumeServer) + } +} diff --git a/weed/server/master_grpc_server_block.go b/weed/server/master_grpc_server_block.go index e74f55387b..bbf04df0b5 100644 --- a/weed/server/master_grpc_server_block.go +++ b/weed/server/master_grpc_server_block.go @@ -8,8 +8,21 @@ import ( "github.com/seaweedfs/seaweedfs/weed/glog" "github.com/seaweedfs/seaweedfs/weed/pb/master_pb" "github.com/seaweedfs/seaweedfs/weed/storage/blockvol" + "github.com/seaweedfs/seaweedfs/weed/util/request_id" ) +// blockReqID extracts a short request ID from context for log correlation. +func blockReqID(ctx context.Context) string { + id := request_id.Get(ctx) + if len(id) > 8 { + return id[:8] + } + if id == "" { + return "no-id" + } + return id +} + // CreateBlockVolume picks a volume server, delegates creation, and records // the mapping in the block volume registry. func (ms *MasterServer) CreateBlockVolume(ctx context.Context, req *master_pb.CreateBlockVolumeRequest) (*master_pb.CreateBlockVolumeResponse, error) { @@ -25,13 +38,7 @@ func (ms *MasterServer) CreateBlockVolume(ctx context.Context, req *master_pb.Cr if entry.SizeBytes < req.SizeBytes { return nil, fmt.Errorf("block volume %q exists with size %d (requested %d)", req.Name, entry.SizeBytes, req.SizeBytes) } - return &master_pb.CreateBlockVolumeResponse{ - VolumeId: entry.Name, - VolumeServer: entry.VolumeServer, - IscsiAddr: entry.ISCSIAddr, - Iqn: entry.IQN, - CapacityBytes: entry.SizeBytes, - }, nil + return ms.createBlockVolumeResponseFromEntry(entry), nil } // Per-name inflight lock prevents concurrent creates for the same name. @@ -42,13 +49,7 @@ func (ms *MasterServer) CreateBlockVolume(ctx context.Context, req *master_pb.Cr // Double-check after acquiring lock (another goroutine may have finished). if entry, ok := ms.blockRegistry.Lookup(req.Name); ok { - return &master_pb.CreateBlockVolumeResponse{ - VolumeId: entry.Name, - VolumeServer: entry.VolumeServer, - IscsiAddr: entry.ISCSIAddr, - Iqn: entry.IQN, - CapacityBytes: entry.SizeBytes, - }, nil + return ms.createBlockVolumeResponseFromEntry(entry), nil } // Get candidate servers. @@ -73,11 +74,17 @@ func (ms *MasterServer) CreateBlockVolume(ctx context.Context, req *master_pb.Cr result, err := ms.blockVSAllocate(ctx, server, req.Name, req.SizeBytes, req.DiskType) if err != nil { lastErr = fmt.Errorf("server %s: %w", server, err) - glog.V(0).Infof("CreateBlockVolume %q: attempt %d on %s failed: %v", req.Name, attempt+1, server, err) + glog.V(0).Infof("[reqID=%s] CreateBlockVolume %q: attempt %d on %s failed: %v", blockReqID(ctx), req.Name, attempt+1, server, err) servers = removeServer(servers, server) continue } + // CP8-2: determine replica factor from request (default 2). + replicaFactor := 2 + if req.ReplicaFactor > 0 && req.ReplicaFactor <= 3 { + replicaFactor = int(req.ReplicaFactor) + } + entry := &BlockVolumeEntry{ Name: req.Name, VolumeServer: server, @@ -88,66 +95,85 @@ func (ms *MasterServer) CreateBlockVolume(ctx context.Context, req *master_pb.Cr Epoch: 1, Role: blockvol.RoleToWire(blockvol.RolePrimary), Status: StatusActive, + ReplicaFactor: replicaFactor, LeaseTTL: 30 * time.Second, LastLeaseGrant: time.Now(), // R2-F1: set BEFORE Register to avoid stale-lease race } - // Try to create replica on a different server (F4: partial create OK). - var replicaServer string - remainingServers := removeServer(servers, server) - if len(remainingServers) > 0 { - replicaServer = ms.tryCreateReplica(ctx, req, entry, result, remainingServers) - } else { - glog.V(0).Infof("CreateBlockVolume %q: single-copy mode (only 1 server)", req.Name) + // Create replicaFactor-1 replicas on different servers (F4: partial create OK). + remaining := removeServer(servers, server) + for i := 0; i < replicaFactor-1 && len(remaining) > 0; i++ { + replicaServer := ms.tryCreateOneReplica(ctx, req, entry, result, remaining) + if replicaServer != "" { + remaining = removeServer(remaining, replicaServer) + } + } + if len(entry.Replicas) == 0 && replicaFactor > 1 { + glog.V(0).Infof("[reqID=%s] CreateBlockVolume %q: single-copy mode (replica allocation failed)", blockReqID(ctx), req.Name) + } + // Sync deprecated scalar fields from first replica. + if len(entry.Replicas) > 0 { + r0 := &entry.Replicas[0] + entry.ReplicaServer = r0.Server + entry.ReplicaPath = r0.Path + entry.ReplicaIQN = r0.IQN + entry.ReplicaISCSIAddr = r0.ISCSIAddr + entry.ReplicaDataAddr = r0.DataAddr + entry.ReplicaCtrlAddr = r0.CtrlAddr } // Register in registry as Active (VS confirmed creation). if err := ms.blockRegistry.Register(entry); err != nil { // Already registered (race condition) — return the existing entry. if existing, ok := ms.blockRegistry.Lookup(req.Name); ok { - return &master_pb.CreateBlockVolumeResponse{ - VolumeId: existing.Name, - VolumeServer: existing.VolumeServer, - IscsiAddr: existing.ISCSIAddr, - Iqn: existing.IQN, - CapacityBytes: existing.SizeBytes, - ReplicaServer: existing.ReplicaServer, - }, nil + return ms.createBlockVolumeResponseFromEntry(existing), nil } return nil, fmt.Errorf("register block volume: %w", err) } - // Enqueue assignments for primary (and replica if available). + // Enqueue assignments for primary (and replicas if available). leaseTTLMs := blockvol.LeaseTTLToWire(30 * time.Second) - ms.blockAssignmentQueue.Enqueue(server, blockvol.BlockVolumeAssignment{ - Path: result.Path, - Epoch: 1, - Role: blockvol.RoleToWire(blockvol.RolePrimary), - LeaseTtlMs: leaseTTLMs, - ReplicaDataAddr: entry.ReplicaDataAddr, - ReplicaCtrlAddr: entry.ReplicaCtrlAddr, - }) - if entry.ReplicaServer != "" { - ms.blockAssignmentQueue.Enqueue(entry.ReplicaServer, blockvol.BlockVolumeAssignment{ - Path: entry.ReplicaPath, + primaryAssignment := blockvol.BlockVolumeAssignment{ + Path: result.Path, + Epoch: 1, + Role: blockvol.RoleToWire(blockvol.RolePrimary), + LeaseTtlMs: leaseTTLMs, + } + // CP8-2: populate ReplicaAddrs for multi-replica. + for _, ri := range entry.Replicas { + primaryAssignment.ReplicaAddrs = append(primaryAssignment.ReplicaAddrs, blockvol.ReplicaAddr{ + DataAddr: ri.DataAddr, + CtrlAddr: ri.CtrlAddr, + }) + } + // Backward compat: also set scalar fields if exactly 1 replica. + if len(entry.Replicas) == 1 { + primaryAssignment.ReplicaDataAddr = entry.Replicas[0].DataAddr + primaryAssignment.ReplicaCtrlAddr = entry.Replicas[0].CtrlAddr + } + ms.blockAssignmentQueue.Enqueue(server, primaryAssignment) + + // Enqueue assignments for each replica. + for _, ri := range entry.Replicas { + ms.blockAssignmentQueue.Enqueue(ri.Server, blockvol.BlockVolumeAssignment{ + Path: ri.Path, Epoch: 1, Role: blockvol.RoleToWire(blockvol.RoleReplica), LeaseTtlMs: leaseTTLMs, - ReplicaDataAddr: entry.ReplicaDataAddr, - ReplicaCtrlAddr: entry.ReplicaCtrlAddr, + ReplicaDataAddr: ri.DataAddr, + ReplicaCtrlAddr: ri.CtrlAddr, }) } - glog.V(0).Infof("CreateBlockVolume %q: created on %s (path=%s, iqn=%s, replica=%s)", - req.Name, server, result.Path, result.IQN, replicaServer) - return &master_pb.CreateBlockVolumeResponse{ - VolumeId: req.Name, - VolumeServer: server, - IscsiAddr: result.ISCSIAddr, - Iqn: result.IQN, - CapacityBytes: req.SizeBytes, - ReplicaServer: replicaServer, - }, nil + // Collect replica server addresses for response. + var replicaServers []string + for _, ri := range entry.Replicas { + replicaServers = append(replicaServers, ri.Server) + } + + glog.V(0).Infof("[reqID=%s] CreateBlockVolume %q: created on %s (path=%s, iqn=%s, replicas=%v)", + blockReqID(ctx), req.Name, server, result.Path, result.IQN, replicaServers) + return ms.createBlockVolumeResponseFromEntry(entry), nil } return nil, fmt.Errorf("all volume servers failed for %q: %v", req.Name, lastErr) @@ -170,16 +196,16 @@ func (ms *MasterServer) DeleteBlockVolume(ctx context.Context, req *master_pb.De return nil, fmt.Errorf("delete block volume %q on %s: %w", req.Name, entry.VolumeServer, err) } - // R2-F4: Also delete replica (best-effort, don't fail if replica is down). - if entry.ReplicaServer != "" { - if err := ms.blockVSDelete(ctx, entry.ReplicaServer, req.Name); err != nil { - glog.Warningf("DeleteBlockVolume %q: replica delete on %s failed (best-effort): %v", - req.Name, entry.ReplicaServer, err) + // CP8-2: Delete ALL replicas (best-effort, don't fail if replica is down). + for _, ri := range entry.Replicas { + if err := ms.blockVSDelete(ctx, ri.Server, req.Name); err != nil { + glog.Warningf("[reqID=%s] DeleteBlockVolume %q: replica delete on %s failed (best-effort): %v", + blockReqID(ctx), req.Name, ri.Server, err) } } ms.blockRegistry.Unregister(req.Name) - glog.V(0).Infof("DeleteBlockVolume %q: removed from %s (replica=%s)", req.Name, entry.VolumeServer, entry.ReplicaServer) + glog.V(0).Infof("[reqID=%s] DeleteBlockVolume %q: removed from %s (replicas=%d)", blockReqID(ctx), req.Name, entry.VolumeServer, len(entry.Replicas)) return &master_pb.DeleteBlockVolumeResponse{}, nil } @@ -194,37 +220,180 @@ func (ms *MasterServer) LookupBlockVolume(ctx context.Context, req *master_pb.Lo return nil, fmt.Errorf("block volume %q not found", req.Name) } + replicaServers := replicaServerList(entry) + rf := entry.ReplicaFactor + if rf == 0 { + rf = 2 // default for pre-CP8-2 entries + } return &master_pb.LookupBlockVolumeResponse{ - VolumeServer: entry.VolumeServer, - IscsiAddr: entry.ISCSIAddr, - Iqn: entry.IQN, - CapacityBytes: entry.SizeBytes, - ReplicaServer: entry.ReplicaServer, + VolumeServer: entry.VolumeServer, + IscsiAddr: entry.ISCSIAddr, + Iqn: entry.IQN, + CapacityBytes: entry.SizeBytes, + ReplicaServer: entry.ReplicaServer, // backward compat + ReplicaFactor: uint32(rf), + ReplicaServers: replicaServers, }, nil } -// tryCreateReplica attempts to create a replica volume on a different server. +// tryCreateOneReplica attempts to create one replica volume on a different server. // Returns the replica server address on success, or empty string on failure (F4). -func (ms *MasterServer) tryCreateReplica(ctx context.Context, req *master_pb.CreateBlockVolumeRequest, entry *BlockVolumeEntry, primaryResult *blockAllocResult, candidates []string) string { +func (ms *MasterServer) tryCreateOneReplica(ctx context.Context, req *master_pb.CreateBlockVolumeRequest, entry *BlockVolumeEntry, primaryResult *blockAllocResult, candidates []string) string { for _, replicaServerStr := range candidates { replicaResult, err := ms.blockVSAllocate(ctx, replicaServerStr, req.Name, req.SizeBytes, req.DiskType) if err != nil { - glog.V(0).Infof("CreateBlockVolume %q: replica on %s failed: %v", req.Name, replicaServerStr, err) + glog.V(0).Infof("[reqID=%s] CreateBlockVolume %q: replica on %s failed: %v", blockReqID(ctx), req.Name, replicaServerStr, err) continue } - entry.ReplicaServer = replicaServerStr - entry.ReplicaPath = replicaResult.Path - entry.ReplicaIQN = replicaResult.IQN - entry.ReplicaISCSIAddr = replicaResult.ISCSIAddr - entry.ReplicaDataAddr = replicaResult.ReplicaDataAddr - entry.ReplicaCtrlAddr = replicaResult.ReplicaCtrlAddr entry.RebuildListenAddr = primaryResult.RebuildListenAddr + // CP8-2: populate Replicas[]. + entry.Replicas = append(entry.Replicas, ReplicaInfo{ + Server: replicaServerStr, + Path: replicaResult.Path, + ISCSIAddr: replicaResult.ISCSIAddr, + IQN: replicaResult.IQN, + DataAddr: replicaResult.ReplicaDataAddr, + CtrlAddr: replicaResult.ReplicaCtrlAddr, + }) return replicaServerStr } - glog.Warningf("CreateBlockVolume %q: created without replica (replica allocation failed)", req.Name) + glog.Warningf("[reqID=%s] CreateBlockVolume %q: created without replica (replica allocation failed)", blockReqID(ctx), req.Name) return "" } +// CreateBlockSnapshot creates a snapshot on a block volume via the volume server. +func (ms *MasterServer) CreateBlockSnapshot(ctx context.Context, req *master_pb.CreateBlockSnapshotRequest) (*master_pb.CreateBlockSnapshotResponse, error) { + if req.VolumeName == "" { + return nil, fmt.Errorf("volume_name is required") + } + + entry, ok := ms.blockRegistry.Lookup(req.VolumeName) + if !ok { + return nil, fmt.Errorf("block volume %q not found", req.VolumeName) + } + + createdAt, sizeBytes, err := ms.blockVSSnapshot(ctx, entry.VolumeServer, req.VolumeName, req.SnapshotId) + if err != nil { + return nil, fmt.Errorf("create snapshot on %s: %w", entry.VolumeServer, err) + } + + return &master_pb.CreateBlockSnapshotResponse{ + SnapshotId: req.SnapshotId, + CreatedAt: createdAt, + SizeBytes: sizeBytes, + }, nil +} + +// DeleteBlockSnapshot deletes a snapshot from a block volume via the volume server. +func (ms *MasterServer) DeleteBlockSnapshot(ctx context.Context, req *master_pb.DeleteBlockSnapshotRequest) (*master_pb.DeleteBlockSnapshotResponse, error) { + if req.VolumeName == "" { + return nil, fmt.Errorf("volume_name is required") + } + + entry, ok := ms.blockRegistry.Lookup(req.VolumeName) + if !ok { + // Idempotent: volume not found → snapshot doesn't exist either. + return &master_pb.DeleteBlockSnapshotResponse{}, nil + } + + if err := ms.blockVSDeleteSnap(ctx, entry.VolumeServer, req.VolumeName, req.SnapshotId); err != nil { + return nil, fmt.Errorf("delete snapshot on %s: %w", entry.VolumeServer, err) + } + + return &master_pb.DeleteBlockSnapshotResponse{}, nil +} + +// ListBlockSnapshots lists all snapshots on a block volume via the volume server. +func (ms *MasterServer) ListBlockSnapshots(ctx context.Context, req *master_pb.ListBlockSnapshotsRequest) (*master_pb.ListBlockSnapshotsResponse, error) { + if req.VolumeName == "" { + return nil, fmt.Errorf("volume_name is required") + } + + entry, ok := ms.blockRegistry.Lookup(req.VolumeName) + if !ok { + return nil, fmt.Errorf("block volume %q not found", req.VolumeName) + } + + vsInfos, err := ms.blockVSListSnaps(ctx, entry.VolumeServer, req.VolumeName) + if err != nil { + return nil, fmt.Errorf("list snapshots on %s: %w", entry.VolumeServer, err) + } + + resp := &master_pb.ListBlockSnapshotsResponse{} + for _, si := range vsInfos { + resp.Snapshots = append(resp.Snapshots, &master_pb.BlockSnapshotInfo{ + SnapshotId: si.SnapshotId, + CreatedAt: si.CreatedAt, + VolumeSizeBytes: si.VolumeSizeBytes, + }) + } + return resp, nil +} + +// ExpandBlockVolume expands a block volume via the volume server, then updates registry. +func (ms *MasterServer) ExpandBlockVolume(ctx context.Context, req *master_pb.ExpandBlockVolumeRequest) (*master_pb.ExpandBlockVolumeResponse, error) { + if req.Name == "" { + return nil, fmt.Errorf("name is required") + } + if req.NewSizeBytes == 0 { + return nil, fmt.Errorf("new_size_bytes must be > 0") + } + + entry, ok := ms.blockRegistry.Lookup(req.Name) + if !ok { + return nil, fmt.Errorf("block volume %q not found", req.Name) + } + + // Expand primary first; only update registry on success. + capacity, err := ms.blockVSExpand(ctx, entry.VolumeServer, req.Name, req.NewSizeBytes) + if err != nil { + return nil, fmt.Errorf("expand on %s: %w", entry.VolumeServer, err) + } + + // CP8-2: Expand ALL replicas (best-effort, log warning on failure). + for _, ri := range entry.Replicas { + if _, err := ms.blockVSExpand(ctx, ri.Server, req.Name, req.NewSizeBytes); err != nil { + glog.Warningf("[reqID=%s] ExpandBlockVolume %q: replica expand on %s failed (best-effort): %v", + blockReqID(ctx), req.Name, ri.Server, err) + } + } + + // Update registry with actual new size. + if uerr := ms.blockRegistry.UpdateSize(req.Name, capacity); uerr != nil { + glog.Warningf("[reqID=%s] ExpandBlockVolume %q: registry update failed (VS succeeded): %v", blockReqID(ctx), req.Name, uerr) + } + + return &master_pb.ExpandBlockVolumeResponse{ + CapacityBytes: capacity, + }, nil +} + +// createBlockVolumeResponseFromEntry builds a CreateBlockVolumeResponse from a registry entry. +func (ms *MasterServer) createBlockVolumeResponseFromEntry(entry *BlockVolumeEntry) *master_pb.CreateBlockVolumeResponse { + return &master_pb.CreateBlockVolumeResponse{ + VolumeId: entry.Name, + VolumeServer: entry.VolumeServer, + IscsiAddr: entry.ISCSIAddr, + Iqn: entry.IQN, + CapacityBytes: entry.SizeBytes, + ReplicaServer: entry.ReplicaServer, // backward compat + ReplicaServers: replicaServerList(entry), + } +} + +// replicaServerList returns the list of replica server addresses. +// Order matches Replicas[] (append-order), ensuring ReplicaServers[0] == ReplicaServer (legacy). +func replicaServerList(entry *BlockVolumeEntry) []string { + if len(entry.Replicas) == 0 { + return nil + } + servers := make([]string, len(entry.Replicas)) + for i, ri := range entry.Replicas { + servers[i] = ri.Server + } + return servers +} + // removeServer returns a new slice without the specified server. func removeServer(servers []string, server string) []string { result := make([]string, 0, len(servers)-1) diff --git a/weed/server/master_grpc_server_block_test.go b/weed/server/master_grpc_server_block_test.go index 2c932d7641..c7b4f750c2 100644 --- a/weed/server/master_grpc_server_block_test.go +++ b/weed/server/master_grpc_server_block_test.go @@ -8,6 +8,7 @@ import ( "testing" "github.com/seaweedfs/seaweedfs/weed/pb/master_pb" + "github.com/seaweedfs/seaweedfs/weed/pb/volume_server_pb" ) // testMasterServer creates a minimal MasterServer with mock VS calls for testing. @@ -492,6 +493,142 @@ func TestMaster_LookupReturnsReplicaServer(t *testing.T) { } } +func TestMaster_CreateBlockSnapshot(t *testing.T) { + ms := testMasterServer(t) + ms.blockRegistry.MarkBlockCapable("vs1:9333") + ms.blockVSSnapshot = func(ctx context.Context, server string, name string, snapID uint32) (int64, uint64, error) { + return 1709654400, 1 << 30, nil + } + + // Create volume first. + ms.CreateBlockVolume(context.Background(), &master_pb.CreateBlockVolumeRequest{ + Name: "snap-vol", SizeBytes: 1 << 30, + }) + + resp, err := ms.CreateBlockSnapshot(context.Background(), &master_pb.CreateBlockSnapshotRequest{ + VolumeName: "snap-vol", SnapshotId: 42, + }) + if err != nil { + t.Fatalf("CreateBlockSnapshot: %v", err) + } + if resp.SnapshotId != 42 { + t.Fatalf("SnapshotId: got %d, want 42", resp.SnapshotId) + } + if resp.CreatedAt != 1709654400 { + t.Fatalf("CreatedAt: got %d, want 1709654400", resp.CreatedAt) + } +} + +func TestMaster_CreateBlockSnapshot_VolumeNotFound(t *testing.T) { + ms := testMasterServer(t) + _, err := ms.CreateBlockSnapshot(context.Background(), &master_pb.CreateBlockSnapshotRequest{ + VolumeName: "nonexistent", SnapshotId: 1, + }) + if err == nil { + t.Fatal("expected error for nonexistent volume") + } +} + +func TestMaster_DeleteBlockSnapshot(t *testing.T) { + ms := testMasterServer(t) + ms.blockRegistry.MarkBlockCapable("vs1:9333") + ms.blockVSDeleteSnap = func(ctx context.Context, server string, name string, snapID uint32) error { + return nil + } + + ms.CreateBlockVolume(context.Background(), &master_pb.CreateBlockVolumeRequest{ + Name: "snap-vol", SizeBytes: 1 << 30, + }) + + _, err := ms.DeleteBlockSnapshot(context.Background(), &master_pb.DeleteBlockSnapshotRequest{ + VolumeName: "snap-vol", SnapshotId: 42, + }) + if err != nil { + t.Fatalf("DeleteBlockSnapshot: %v", err) + } +} + +func TestMaster_ListBlockSnapshots(t *testing.T) { + ms := testMasterServer(t) + ms.blockRegistry.MarkBlockCapable("vs1:9333") + ms.blockVSListSnaps = func(ctx context.Context, server string, name string) ([]*volume_server_pb.BlockSnapshotInfo, error) { + return []*volume_server_pb.BlockSnapshotInfo{ + {SnapshotId: 1, CreatedAt: 100}, + {SnapshotId: 2, CreatedAt: 200}, + }, nil + } + + ms.CreateBlockVolume(context.Background(), &master_pb.CreateBlockVolumeRequest{ + Name: "snap-vol", SizeBytes: 1 << 30, + }) + + resp, err := ms.ListBlockSnapshots(context.Background(), &master_pb.ListBlockSnapshotsRequest{ + VolumeName: "snap-vol", + }) + if err != nil { + t.Fatalf("ListBlockSnapshots: %v", err) + } + if len(resp.Snapshots) != 2 { + t.Fatalf("expected 2 snapshots, got %d", len(resp.Snapshots)) + } +} + +func TestMaster_ExpandBlockVolume(t *testing.T) { + ms := testMasterServer(t) + ms.blockRegistry.MarkBlockCapable("vs1:9333") + ms.blockVSExpand = func(ctx context.Context, server string, name string, newSize uint64) (uint64, error) { + return newSize, nil + } + + ms.CreateBlockVolume(context.Background(), &master_pb.CreateBlockVolumeRequest{ + Name: "expand-vol", SizeBytes: 1 << 30, + }) + + resp, err := ms.ExpandBlockVolume(context.Background(), &master_pb.ExpandBlockVolumeRequest{ + Name: "expand-vol", NewSizeBytes: 2 << 30, + }) + if err != nil { + t.Fatalf("ExpandBlockVolume: %v", err) + } + if resp.CapacityBytes != 2<<30 { + t.Fatalf("CapacityBytes: got %d, want %d", resp.CapacityBytes, 2<<30) + } + + // Verify registry was updated. + entry, ok := ms.blockRegistry.Lookup("expand-vol") + if !ok { + t.Fatal("volume not found in registry") + } + if entry.SizeBytes != 2<<30 { + t.Fatalf("registry size: got %d, want %d", entry.SizeBytes, 2<<30) + } +} + +func TestMaster_ExpandBlockVolume_VSFailure(t *testing.T) { + ms := testMasterServer(t) + ms.blockRegistry.MarkBlockCapable("vs1:9333") + ms.blockVSExpand = func(ctx context.Context, server string, name string, newSize uint64) (uint64, error) { + return 0, fmt.Errorf("disk full") + } + + ms.CreateBlockVolume(context.Background(), &master_pb.CreateBlockVolumeRequest{ + Name: "expand-vol", SizeBytes: 1 << 30, + }) + + _, err := ms.ExpandBlockVolume(context.Background(), &master_pb.ExpandBlockVolumeRequest{ + Name: "expand-vol", NewSizeBytes: 2 << 30, + }) + if err == nil { + t.Fatal("expected error when VS expand fails") + } + + // Registry should NOT have been updated. + entry, _ := ms.blockRegistry.Lookup("expand-vol") + if entry.SizeBytes != 1<<30 { + t.Fatalf("registry size should be unchanged: got %d, want %d", entry.SizeBytes, 1<<30) + } +} + func TestMaster_LookupBlockVolume(t *testing.T) { ms := testMasterServer(t) ms.blockRegistry.MarkBlockCapable("vs1:9333") @@ -525,3 +662,354 @@ func TestMaster_LookupBlockVolume(t *testing.T) { t.Fatal("lookup nonexistent should return error") } } + +// ============================================================ +// CP8-2 T9: Multi-Replica Create/Delete/Assign Tests +// ============================================================ + +func testMasterServerRF3(t *testing.T) *MasterServer { + t.Helper() + ms := testMasterServer(t) + ms.blockVSAllocate = func(ctx context.Context, server string, name string, sizeBytes uint64, diskType string) (*blockAllocResult, error) { + return &blockAllocResult{ + Path: fmt.Sprintf("/data/%s.blk", name), + IQN: fmt.Sprintf("iqn.2024.test:%s", name), + ISCSIAddr: server + ":3260", + ReplicaDataAddr: server + ":14260", + ReplicaCtrlAddr: server + ":14261", + }, nil + } + ms.blockRegistry.MarkBlockCapable("vs1:9333") + ms.blockRegistry.MarkBlockCapable("vs2:9333") + ms.blockRegistry.MarkBlockCapable("vs3:9333") + return ms +} + +// RF=3 with 3 servers: should create 2 replicas. +func TestMaster_CreateRF3_ThreeServers(t *testing.T) { + ms := testMasterServerRF3(t) + + resp, err := ms.CreateBlockVolume(context.Background(), &master_pb.CreateBlockVolumeRequest{ + Name: "vol1", + SizeBytes: 1 << 30, + ReplicaFactor: 3, + }) + if err != nil { + t.Fatalf("CreateBlockVolume: %v", err) + } + + entry, ok := ms.blockRegistry.Lookup("vol1") + if !ok { + t.Fatal("vol1 not in registry") + } + if entry.ReplicaFactor != 3 { + t.Fatalf("ReplicaFactor: got %d, want 3", entry.ReplicaFactor) + } + if len(entry.Replicas) != 2 { + t.Fatalf("expected 2 replicas, got %d", len(entry.Replicas)) + } + + // All 3 servers should be different. + servers := map[string]bool{resp.VolumeServer: true} + for _, ri := range entry.Replicas { + if servers[ri.Server] { + t.Fatalf("duplicate server: %q", ri.Server) + } + servers[ri.Server] = true + } + if len(servers) != 3 { + t.Fatalf("expected 3 distinct servers, got %d", len(servers)) + } +} + +// RF=3 with only 2 servers: should create 1 replica (partial). +func TestMaster_CreateRF3_TwoServers(t *testing.T) { + ms := testMasterServer(t) + ms.blockVSAllocate = func(ctx context.Context, server string, name string, sizeBytes uint64, diskType string) (*blockAllocResult, error) { + return &blockAllocResult{ + Path: fmt.Sprintf("/data/%s.blk", name), + IQN: fmt.Sprintf("iqn.2024.test:%s", name), + ISCSIAddr: server + ":3260", + ReplicaDataAddr: server + ":14260", + ReplicaCtrlAddr: server + ":14261", + }, nil + } + ms.blockRegistry.MarkBlockCapable("vs1:9333") + ms.blockRegistry.MarkBlockCapable("vs2:9333") + + _, err := ms.CreateBlockVolume(context.Background(), &master_pb.CreateBlockVolumeRequest{ + Name: "vol1", + SizeBytes: 1 << 30, + ReplicaFactor: 3, + }) + if err != nil { + t.Fatalf("CreateBlockVolume: %v", err) + } + + entry, _ := ms.blockRegistry.Lookup("vol1") + // Only 2 servers available, so only 1 replica (not 2). + if len(entry.Replicas) != 1 { + t.Fatalf("expected 1 replica (only 2 servers), got %d", len(entry.Replicas)) + } +} + +// RF=2 unchanged after CP8-2 changes. +func TestMaster_CreateRF2_Unchanged(t *testing.T) { + ms := testMasterServerRF3(t) + + _, err := ms.CreateBlockVolume(context.Background(), &master_pb.CreateBlockVolumeRequest{ + Name: "vol1", + SizeBytes: 1 << 30, + // No ReplicaFactor → defaults to 2. + }) + if err != nil { + t.Fatalf("CreateBlockVolume: %v", err) + } + + entry, _ := ms.blockRegistry.Lookup("vol1") + if entry.ReplicaFactor != 2 { + t.Fatalf("ReplicaFactor: got %d, want 2 (default)", entry.ReplicaFactor) + } + if len(entry.Replicas) != 1 { + t.Fatalf("expected 1 replica for RF=2, got %d", len(entry.Replicas)) + } + // Backward compat scalar fields should be set. + if entry.ReplicaServer == "" { + t.Fatal("ReplicaServer (deprecated) should still be set for backward compat") + } +} + +// DeleteBlockVolume RF=3 deletes all replicas. +func TestMaster_DeleteRF3_DeletesAllReplicas(t *testing.T) { + ms := testMasterServerRF3(t) + + var deletedServers []string + ms.blockVSDelete = func(ctx context.Context, server string, name string) error { + deletedServers = append(deletedServers, server) + return nil + } + + _, err := ms.CreateBlockVolume(context.Background(), &master_pb.CreateBlockVolumeRequest{ + Name: "vol1", + SizeBytes: 1 << 30, + ReplicaFactor: 3, + }) + if err != nil { + t.Fatalf("create: %v", err) + } + + _, err = ms.DeleteBlockVolume(context.Background(), &master_pb.DeleteBlockVolumeRequest{ + Name: "vol1", + }) + if err != nil { + t.Fatalf("delete: %v", err) + } + + // Should have deleted on primary + 2 replicas = 3 servers. + if len(deletedServers) != 3 { + t.Fatalf("expected 3 delete calls (primary + 2 replicas), got %d: %v", + len(deletedServers), deletedServers) + } +} + +// ExpandBlockVolume RF=3 expands all replicas. +func TestMaster_ExpandRF3_ExpandsAllReplicas(t *testing.T) { + ms := testMasterServerRF3(t) + + var expandedServers []string + ms.blockVSExpand = func(ctx context.Context, server string, name string, newSize uint64) (uint64, error) { + expandedServers = append(expandedServers, server) + return newSize, nil + } + + _, err := ms.CreateBlockVolume(context.Background(), &master_pb.CreateBlockVolumeRequest{ + Name: "vol1", + SizeBytes: 1 << 30, + ReplicaFactor: 3, + }) + if err != nil { + t.Fatalf("create: %v", err) + } + + _, err = ms.ExpandBlockVolume(context.Background(), &master_pb.ExpandBlockVolumeRequest{ + Name: "vol1", NewSizeBytes: 2 << 30, + }) + if err != nil { + t.Fatalf("expand: %v", err) + } + + // Should have expanded on primary + 2 replicas = 3 servers. + if len(expandedServers) != 3 { + t.Fatalf("expected 3 expand calls (primary + 2 replicas), got %d: %v", + len(expandedServers), expandedServers) + } +} + +// ProcessAssignments with multi-replica addrs. +func TestMaster_CreateRF3_AssignmentsIncludeReplicaAddrs(t *testing.T) { + ms := testMasterServerRF3(t) + + resp, err := ms.CreateBlockVolume(context.Background(), &master_pb.CreateBlockVolumeRequest{ + Name: "vol1", + SizeBytes: 1 << 30, + ReplicaFactor: 3, + }) + if err != nil { + t.Fatalf("create: %v", err) + } + + // Primary assignment should have ReplicaAddrs with 2 entries. + assignments := ms.blockAssignmentQueue.Peek(resp.VolumeServer) + if len(assignments) != 1 { + t.Fatalf("expected 1 primary assignment, got %d", len(assignments)) + } + pa := assignments[0] + if len(pa.ReplicaAddrs) != 2 { + t.Fatalf("primary assignment ReplicaAddrs: got %d, want 2", len(pa.ReplicaAddrs)) + } + // Each replica addr should have non-empty data/ctrl. + for i, ra := range pa.ReplicaAddrs { + if ra.DataAddr == "" || ra.CtrlAddr == "" { + t.Fatalf("ReplicaAddrs[%d] has empty addr: %+v", i, ra) + } + } +} + +// Fix #4: CreateBlockVolumeResponse includes ReplicaServers. +func TestMaster_CreateResponse_IncludesReplicaServers(t *testing.T) { + ms := testMasterServerRF3(t) + resp, err := ms.CreateBlockVolume(context.Background(), &master_pb.CreateBlockVolumeRequest{ + Name: "vol1", + SizeBytes: 1 << 30, + ReplicaFactor: 3, + }) + if err != nil { + t.Fatalf("CreateBlockVolume: %v", err) + } + if len(resp.ReplicaServers) != 2 { + t.Fatalf("ReplicaServers: got %d, want 2", len(resp.ReplicaServers)) + } + // ReplicaServers should match registry entry Replicas. + entry, _ := ms.blockRegistry.Lookup("vol1") + for i, ri := range entry.Replicas { + if resp.ReplicaServers[i] != ri.Server { + t.Errorf("ReplicaServers[%d]: got %q, want %q", i, resp.ReplicaServers[i], ri.Server) + } + } + // Backward compat: ReplicaServer (scalar) should be first replica. + if resp.ReplicaServer != entry.ReplicaServer { + t.Errorf("ReplicaServer: got %q, want %q", resp.ReplicaServer, entry.ReplicaServer) + } +} + +// Fix #4: LookupBlockVolumeResponse includes ReplicaFactor and ReplicaServers. +func TestMaster_LookupResponse_IncludesReplicaFields(t *testing.T) { + ms := testMasterServerRF3(t) + _, err := ms.CreateBlockVolume(context.Background(), &master_pb.CreateBlockVolumeRequest{ + Name: "vol1", + SizeBytes: 1 << 30, + ReplicaFactor: 3, + }) + if err != nil { + t.Fatalf("CreateBlockVolume: %v", err) + } + resp, err := ms.LookupBlockVolume(context.Background(), &master_pb.LookupBlockVolumeRequest{ + Name: "vol1", + }) + if err != nil { + t.Fatalf("LookupBlockVolume: %v", err) + } + if resp.ReplicaFactor != 3 { + t.Fatalf("ReplicaFactor: got %d, want 3", resp.ReplicaFactor) + } + if len(resp.ReplicaServers) != 2 { + t.Fatalf("ReplicaServers: got %d, want 2", len(resp.ReplicaServers)) + } + // Backward compat: ReplicaServer (scalar). + if resp.ReplicaServer == "" { + t.Error("ReplicaServer (scalar) should be non-empty for backward compat") + } +} + +// Fix #4: Idempotent create returns ReplicaServers. +func TestMaster_CreateIdempotent_IncludesReplicaServers(t *testing.T) { + ms := testMasterServerRF3(t) + _, err := ms.CreateBlockVolume(context.Background(), &master_pb.CreateBlockVolumeRequest{ + Name: "vol1", + SizeBytes: 1 << 30, + ReplicaFactor: 3, + }) + if err != nil { + t.Fatalf("first create: %v", err) + } + // Second create (idempotent) should also include ReplicaServers. + resp, err := ms.CreateBlockVolume(context.Background(), &master_pb.CreateBlockVolumeRequest{ + Name: "vol1", + SizeBytes: 1 << 30, + ReplicaFactor: 3, + }) + if err != nil { + t.Fatalf("idempotent create: %v", err) + } + if len(resp.ReplicaServers) != 2 { + t.Fatalf("idempotent ReplicaServers: got %d, want 2", len(resp.ReplicaServers)) + } +} + +// Lookup returns ReplicaFactor=2 for pre-CP8-2 entries where ReplicaFactor is 0. +func TestMaster_LookupResponse_ReplicaFactorDefault(t *testing.T) { + ms := testMasterServer(t) + ms.blockRegistry.MarkBlockCapable("vs1:9333") + + // Manually register entry with ReplicaFactor=0 (pre-CP8-2 legacy). + ms.blockRegistry.Register(&BlockVolumeEntry{ + Name: "legacy-vol", + VolumeServer: "vs1:9333", + Path: "/data/legacy-vol.blk", + SizeBytes: 1 << 30, + Status: StatusActive, + // ReplicaFactor intentionally 0 (zero value). + }) + + resp, err := ms.LookupBlockVolume(context.Background(), &master_pb.LookupBlockVolumeRequest{ + Name: "legacy-vol", + }) + if err != nil { + t.Fatalf("LookupBlockVolume: %v", err) + } + if resp.ReplicaFactor != 2 { + t.Fatalf("ReplicaFactor: got %d, want 2 (default for pre-CP8-2)", resp.ReplicaFactor) + } +} + +// ReplicaServers[0] matches ReplicaServer (legacy scalar). +func TestMaster_ResponseConsistency_ReplicaServerVsReplicaServers(t *testing.T) { + ms := testMasterServerRF3(t) + resp, err := ms.CreateBlockVolume(context.Background(), &master_pb.CreateBlockVolumeRequest{ + Name: "vol1", + SizeBytes: 1 << 30, + ReplicaFactor: 3, + }) + if err != nil { + t.Fatalf("CreateBlockVolume: %v", err) + } + if len(resp.ReplicaServers) < 1 { + t.Fatal("expected at least 1 replica server") + } + if resp.ReplicaServer != resp.ReplicaServers[0] { + t.Fatalf("ReplicaServer=%q != ReplicaServers[0]=%q", + resp.ReplicaServer, resp.ReplicaServers[0]) + } + + // Same for Lookup. + lresp, err := ms.LookupBlockVolume(context.Background(), &master_pb.LookupBlockVolumeRequest{ + Name: "vol1", + }) + if err != nil { + t.Fatalf("LookupBlockVolume: %v", err) + } + if lresp.ReplicaServer != lresp.ReplicaServers[0] { + t.Fatalf("Lookup: ReplicaServer=%q != ReplicaServers[0]=%q", + lresp.ReplicaServer, lresp.ReplicaServers[0]) + } +} diff --git a/weed/server/master_server.go b/weed/server/master_server.go index 88c67ae996..35d1766e45 100644 --- a/weed/server/master_server.go +++ b/weed/server/master_server.go @@ -60,9 +60,10 @@ type MasterOption struct { MetricsAddress string MetricsIntervalSec int IsFollower bool - TelemetryUrl string - TelemetryEnabled bool - VolumeGrowthDisabled bool + TelemetryUrl string + TelemetryEnabled bool + VolumeGrowthDisabled bool + BlockPromotionLSNTolerance int } type MasterServer struct { @@ -97,8 +98,12 @@ type MasterServer struct { blockRegistry *BlockVolumeRegistry blockAssignmentQueue *BlockAssignmentQueue blockFailover *blockFailoverState - blockVSAllocate func(ctx context.Context, server string, name string, sizeBytes uint64, diskType string) (*blockAllocResult, error) - blockVSDelete func(ctx context.Context, server string, name string) error + blockVSAllocate func(ctx context.Context, server string, name string, sizeBytes uint64, diskType string) (*blockAllocResult, error) + blockVSDelete func(ctx context.Context, server string, name string) error + blockVSSnapshot func(ctx context.Context, server string, name string, snapID uint32) (int64, uint64, error) + blockVSDeleteSnap func(ctx context.Context, server string, name string, snapID uint32) error + blockVSListSnaps func(ctx context.Context, server string, name string) ([]*volume_server_pb.BlockSnapshotInfo, error) + blockVSExpand func(ctx context.Context, server string, name string, newSize uint64) (uint64, error) } func NewMasterServer(r *mux.Router, option *MasterOption, peers map[string]pb.ServerAddress) *MasterServer { @@ -148,10 +153,17 @@ func NewMasterServer(r *mux.Router, option *MasterOption, peers map[string]pb.Se } ms.blockRegistry = NewBlockVolumeRegistry() + if option.BlockPromotionLSNTolerance > 0 { + ms.blockRegistry.SetPromotionLSNTolerance(uint64(option.BlockPromotionLSNTolerance)) + } ms.blockAssignmentQueue = NewBlockAssignmentQueue() ms.blockFailover = newBlockFailoverState() ms.blockVSAllocate = ms.defaultBlockVSAllocate ms.blockVSDelete = ms.defaultBlockVSDelete + ms.blockVSSnapshot = ms.defaultBlockVSSnapshot + ms.blockVSDeleteSnap = ms.defaultBlockVSDeleteSnap + ms.blockVSListSnaps = ms.defaultBlockVSListSnaps + ms.blockVSExpand = ms.defaultBlockVSExpand ms.MasterClient.SetOnPeerUpdateFn(ms.OnPeerUpdate) @@ -197,6 +209,18 @@ func NewMasterServer(r *mux.Router, option *MasterOption, peers map[string]pb.Se r.HandleFunc("/stats/counter", ms.guard.WhiteList(statsCounterHandler)) r.HandleFunc("/stats/memory", ms.guard.WhiteList(statsMemoryHandler)) */ + + // Block volume management routes. + r.HandleFunc("/block/volume", ms.proxyToLeader(ms.guard.WhiteList(requestIDMiddleware(ms.blockVolumeCreateHandler)))).Methods("POST") + r.HandleFunc("/block/volume/{name}", ms.proxyToLeader(ms.guard.WhiteList(requestIDMiddleware(ms.blockVolumeDeleteHandler)))).Methods("DELETE") + r.HandleFunc("/block/volume/{name}", ms.guard.WhiteList(requestIDMiddleware(ms.blockVolumeLookupHandler))).Methods("GET") + r.HandleFunc("/block/volumes", ms.guard.WhiteList(requestIDMiddleware(ms.blockVolumeListHandler))).Methods("GET") + r.HandleFunc("/block/assign", ms.proxyToLeader(ms.guard.WhiteList(requestIDMiddleware(ms.blockAssignHandler)))).Methods("POST") + r.HandleFunc("/block/servers", ms.guard.WhiteList(requestIDMiddleware(ms.blockServersHandler))).Methods("GET") + r.HandleFunc("/block/status", ms.guard.WhiteList(requestIDMiddleware(ms.blockStatusHandler))).Methods("GET") + r.HandleFunc("/block/ops", requestIDMiddleware(ms.blockOpsHandler)) + r.HandleFunc("/block/", requestIDMiddleware(ms.blockUIHandler)) + r.HandleFunc("/{fileId}", requestIDMiddleware(ms.redirectHandler)) } @@ -560,3 +584,62 @@ func (ms *MasterServer) defaultBlockVSDelete(ctx context.Context, server string, return err }) } + +func (ms *MasterServer) defaultBlockVSSnapshot(ctx context.Context, server string, name string, snapID uint32) (int64, uint64, error) { + var createdAt int64 + var sizeBytes uint64 + err := operation.WithVolumeServerClient(false, pb.ServerAddress(server), ms.grpcDialOption, func(client volume_server_pb.VolumeServerClient) error { + resp, rerr := client.SnapshotBlockVolume(ctx, &volume_server_pb.SnapshotBlockVolumeRequest{ + Name: name, + SnapshotId: snapID, + }) + if rerr != nil { + return rerr + } + createdAt = resp.CreatedAt + sizeBytes = resp.SizeBytes + return nil + }) + return createdAt, sizeBytes, err +} + +func (ms *MasterServer) defaultBlockVSDeleteSnap(ctx context.Context, server string, name string, snapID uint32) error { + return operation.WithVolumeServerClient(false, pb.ServerAddress(server), ms.grpcDialOption, func(client volume_server_pb.VolumeServerClient) error { + _, err := client.DeleteBlockSnapshot(ctx, &volume_server_pb.DeleteBlockSnapshotRequest{ + Name: name, + SnapshotId: snapID, + }) + return err + }) +} + +func (ms *MasterServer) defaultBlockVSListSnaps(ctx context.Context, server string, name string) ([]*volume_server_pb.BlockSnapshotInfo, error) { + var infos []*volume_server_pb.BlockSnapshotInfo + err := operation.WithVolumeServerClient(false, pb.ServerAddress(server), ms.grpcDialOption, func(client volume_server_pb.VolumeServerClient) error { + resp, rerr := client.ListBlockSnapshots(ctx, &volume_server_pb.ListBlockSnapshotsRequest{ + Name: name, + }) + if rerr != nil { + return rerr + } + infos = resp.Snapshots + return nil + }) + return infos, err +} + +func (ms *MasterServer) defaultBlockVSExpand(ctx context.Context, server string, name string, newSize uint64) (uint64, error) { + var capacity uint64 + err := operation.WithVolumeServerClient(false, pb.ServerAddress(server), ms.grpcDialOption, func(client volume_server_pb.VolumeServerClient) error { + resp, rerr := client.ExpandBlockVolume(ctx, &volume_server_pb.ExpandBlockVolumeRequest{ + Name: name, + NewSizeBytes: newSize, + }) + if rerr != nil { + return rerr + } + capacity = resp.CapacityBytes + return nil + }) + return capacity, err +} diff --git a/weed/server/master_server_handlers_block.go b/weed/server/master_server_handlers_block.go new file mode 100644 index 0000000000..7d1285724a --- /dev/null +++ b/weed/server/master_server_handlers_block.go @@ -0,0 +1,209 @@ +package weed_server + +import ( + "encoding/json" + "fmt" + "net/http" + + "github.com/gorilla/mux" + + "github.com/seaweedfs/seaweedfs/weed/pb/master_pb" + "github.com/seaweedfs/seaweedfs/weed/storage/blockvol" + "github.com/seaweedfs/seaweedfs/weed/storage/blockvol/blockapi" +) + +// blockVolumeCreateHandler handles POST /block/volume. +func (ms *MasterServer) blockVolumeCreateHandler(w http.ResponseWriter, r *http.Request) { + var req blockapi.CreateVolumeRequest + if err := json.NewDecoder(r.Body).Decode(&req); err != nil { + writeJsonError(w, r, http.StatusBadRequest, fmt.Errorf("invalid request body: %w", err)) + return + } + + // Store replica_placement in registry after creation. + replicaPlacement := req.ReplicaPlacement + if replicaPlacement == "" { + replicaPlacement = "000" + } + + resp, err := ms.CreateBlockVolume(r.Context(), &master_pb.CreateBlockVolumeRequest{ + Name: req.Name, + SizeBytes: req.SizeBytes, + DiskType: req.DiskType, + }) + if err != nil { + writeJsonError(w, r, http.StatusInternalServerError, err) + return + } + + // Store replica_placement on the registry entry. + if entry, ok := ms.blockRegistry.Lookup(resp.VolumeId); ok { + entry.ReplicaPlacement = replicaPlacement + } + + // Look up the full entry to populate all fields. + info := blockapi.VolumeInfo{ + Name: resp.VolumeId, + VolumeServer: resp.VolumeServer, + SizeBytes: resp.CapacityBytes, + ReplicaPlacement: replicaPlacement, + ISCSIAddr: resp.IscsiAddr, + IQN: resp.Iqn, + } + if entry, ok := ms.blockRegistry.Lookup(resp.VolumeId); ok { + info = entryToVolumeInfo(entry) + } + writeJsonQuiet(w, r, http.StatusOK, info) +} + +// blockVolumeDeleteHandler handles DELETE /block/volume/{name}. +func (ms *MasterServer) blockVolumeDeleteHandler(w http.ResponseWriter, r *http.Request) { + name := mux.Vars(r)["name"] + if name == "" { + writeJsonError(w, r, http.StatusBadRequest, fmt.Errorf("name is required")) + return + } + + _, err := ms.DeleteBlockVolume(r.Context(), &master_pb.DeleteBlockVolumeRequest{ + Name: name, + }) + if err != nil { + writeJsonError(w, r, http.StatusInternalServerError, err) + return + } + writeJsonQuiet(w, r, http.StatusOK, map[string]string{"status": "deleted"}) +} + +// blockVolumeLookupHandler handles GET /block/volume/{name}. +func (ms *MasterServer) blockVolumeLookupHandler(w http.ResponseWriter, r *http.Request) { + name := mux.Vars(r)["name"] + if name == "" { + writeJsonError(w, r, http.StatusBadRequest, fmt.Errorf("name is required")) + return + } + + entry, ok := ms.blockRegistry.Lookup(name) + if !ok { + writeJsonError(w, r, http.StatusNotFound, fmt.Errorf("block volume %q not found", name)) + return + } + writeJsonQuiet(w, r, http.StatusOK, entryToVolumeInfo(entry)) +} + +// blockVolumeListHandler handles GET /block/volumes. +func (ms *MasterServer) blockVolumeListHandler(w http.ResponseWriter, r *http.Request) { + entries := ms.blockRegistry.ListAll() + infos := make([]blockapi.VolumeInfo, len(entries)) + for i, e := range entries { + infos[i] = entryToVolumeInfo(e) + } + writeJsonQuiet(w, r, http.StatusOK, infos) +} + +// blockAssignHandler handles POST /block/assign. +func (ms *MasterServer) blockAssignHandler(w http.ResponseWriter, r *http.Request) { + var req blockapi.AssignRequest + if err := json.NewDecoder(r.Body).Decode(&req); err != nil { + writeJsonError(w, r, http.StatusBadRequest, fmt.Errorf("invalid request body: %w", err)) + return + } + if req.Name == "" { + writeJsonError(w, r, http.StatusBadRequest, fmt.Errorf("name is required")) + return + } + + // Resolve name → registry entry. + entry, ok := ms.blockRegistry.Lookup(req.Name) + if !ok { + writeJsonError(w, r, http.StatusNotFound, fmt.Errorf("block volume %q not found", req.Name)) + return + } + + // Determine target server + path based on role. + server, path := entry.VolumeServer, entry.Path + if req.Role == "replica" { + if entry.ReplicaServer == "" { + writeJsonError(w, r, http.StatusBadRequest, fmt.Errorf("block volume %q has no replica", req.Name)) + return + } + server, path = entry.ReplicaServer, entry.ReplicaPath + } + + ms.blockAssignmentQueue.Enqueue(server, blockvol.BlockVolumeAssignment{ + Path: path, + Epoch: req.Epoch, + Role: blockapi.RoleFromString(req.Role), + LeaseTtlMs: uint32(req.LeaseTTLMs), + }) + writeJsonQuiet(w, r, http.StatusOK, map[string]string{"status": "queued"}) +} + +// blockServersHandler handles GET /block/servers. +func (ms *MasterServer) blockServersHandler(w http.ResponseWriter, r *http.Request) { + summaries := ms.blockRegistry.ServerSummaries() + infos := make([]blockapi.ServerInfo, len(summaries)) + for i, s := range summaries { + infos[i] = blockapi.ServerInfo{ + Address: s.Address, + VolumeCount: s.VolumeCount, + BlockCapable: s.BlockCapable, + } + } + writeJsonQuiet(w, r, http.StatusOK, infos) +} + +// blockStatusHandler handles GET /block/status — returns registry configuration for debugging. +func (ms *MasterServer) blockStatusHandler(w http.ResponseWriter, r *http.Request) { + status := map[string]interface{}{ + "promotion_lsn_tolerance": ms.blockRegistry.PromotionLSNTolerance(), + "volume_count": len(ms.blockRegistry.ListAll()), + "server_count": len(ms.blockRegistry.BlockCapableServers()), + "barrier_lag_lsn": ms.blockRegistry.MaxBarrierLagLSN(), + "promotions_total": ms.blockRegistry.PromotionsTotal.Load(), + "failovers_total": ms.blockRegistry.FailoversTotal.Load(), + "rebuilds_total": ms.blockRegistry.RebuildsTotal.Load(), + "assignment_queue_depth": ms.blockAssignmentQueue.TotalPending(), + } + writeJsonQuiet(w, r, http.StatusOK, status) +} + +// entryToVolumeInfo converts a BlockVolumeEntry to a blockapi.VolumeInfo. +func entryToVolumeInfo(e *BlockVolumeEntry) blockapi.VolumeInfo { + status := "pending" + if e.Status == StatusActive { + status = "active" + } + rf := e.ReplicaFactor + if rf == 0 { + rf = 2 // default + } + info := blockapi.VolumeInfo{ + Name: e.Name, + VolumeServer: e.VolumeServer, + SizeBytes: e.SizeBytes, + ReplicaPlacement: e.ReplicaPlacement, + Epoch: e.Epoch, + Role: blockvol.RoleFromWire(e.Role).String(), + Status: status, + ISCSIAddr: e.ISCSIAddr, + IQN: e.IQN, + ReplicaServer: e.ReplicaServer, + ReplicaISCSIAddr: e.ReplicaISCSIAddr, + ReplicaIQN: e.ReplicaIQN, + ReplicaDataAddr: e.ReplicaDataAddr, + ReplicaCtrlAddr: e.ReplicaCtrlAddr, + ReplicaFactor: rf, + HealthScore: e.HealthScore, + ReplicaDegraded: e.ReplicaDegraded, + } + for _, ri := range e.Replicas { + info.Replicas = append(info.Replicas, blockapi.ReplicaDetail{ + Server: ri.Server, + ISCSIAddr: ri.ISCSIAddr, + IQN: ri.IQN, + HealthScore: ri.HealthScore, + WALLag: ri.WALLag, + }) + } + return info +} diff --git a/weed/server/master_server_handlers_block_test.go b/weed/server/master_server_handlers_block_test.go new file mode 100644 index 0000000000..c2f3096506 --- /dev/null +++ b/weed/server/master_server_handlers_block_test.go @@ -0,0 +1,288 @@ +package weed_server + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "net/http" + "net/http/httptest" + "testing" + + "github.com/gorilla/mux" + "github.com/seaweedfs/seaweedfs/weed/storage/blockvol/blockapi" +) + +// blockTestServer creates a MasterServer + httptest.Server with block routes registered. +func blockTestServer(t *testing.T) (*MasterServer, *httptest.Server) { + t.Helper() + ms := &MasterServer{ + blockRegistry: NewBlockVolumeRegistry(), + blockAssignmentQueue: NewBlockAssignmentQueue(), + } + ms.blockVSAllocate = func(ctx context.Context, server string, name string, sizeBytes uint64, diskType string) (*blockAllocResult, error) { + return &blockAllocResult{ + Path: fmt.Sprintf("/data/%s.blk", name), + IQN: fmt.Sprintf("iqn.2024.test:%s", name), + ISCSIAddr: server + ":3260", + }, nil + } + ms.blockVSDelete = func(ctx context.Context, server string, name string) error { + return nil + } + ms.blockRegistry.MarkBlockCapable("vs1:9333") + + r := mux.NewRouter() + r.HandleFunc("/block/volume", ms.blockVolumeCreateHandler).Methods("POST") + r.HandleFunc("/block/volume/{name}", ms.blockVolumeDeleteHandler).Methods("DELETE") + r.HandleFunc("/block/volume/{name}", ms.blockVolumeLookupHandler).Methods("GET") + r.HandleFunc("/block/volumes", ms.blockVolumeListHandler).Methods("GET") + r.HandleFunc("/block/assign", ms.blockAssignHandler).Methods("POST") + r.HandleFunc("/block/servers", ms.blockServersHandler).Methods("GET") + + ts := httptest.NewServer(r) + t.Cleanup(ts.Close) + return ms, ts +} + +func TestBlockVolumeCreateHandler(t *testing.T) { + _, ts := blockTestServer(t) + + body, _ := json.Marshal(blockapi.CreateVolumeRequest{ + Name: "vol1", + SizeBytes: 1 << 30, + }) + resp, err := http.Post(ts.URL+"/block/volume", "application/json", bytes.NewReader(body)) + if err != nil { + t.Fatal(err) + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusOK { + t.Fatalf("expected 200, got %d", resp.StatusCode) + } + + var info blockapi.VolumeInfo + if err := json.NewDecoder(resp.Body).Decode(&info); err != nil { + t.Fatal(err) + } + if info.Name != "vol1" { + t.Errorf("expected name vol1, got %s", info.Name) + } + if info.VolumeServer == "" { + t.Error("expected non-empty volume server") + } +} + +func TestBlockVolumeListHandler(t *testing.T) { + ms, ts := blockTestServer(t) + + // Create two volumes via gRPC. + for _, name := range []string{"alpha", "beta"} { + ms.blockRegistry.Register(&BlockVolumeEntry{ + Name: name, + VolumeServer: "vs1:9333", + SizeBytes: 1 << 20, + Status: StatusActive, + }) + } + + resp, err := http.Get(ts.URL + "/block/volumes") + if err != nil { + t.Fatal(err) + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusOK { + t.Fatalf("expected 200, got %d", resp.StatusCode) + } + + var infos []blockapi.VolumeInfo + if err := json.NewDecoder(resp.Body).Decode(&infos); err != nil { + t.Fatal(err) + } + if len(infos) != 2 { + t.Fatalf("expected 2 volumes, got %d", len(infos)) + } + if infos[0].Name != "alpha" || infos[1].Name != "beta" { + t.Errorf("expected sorted [alpha, beta], got [%s, %s]", infos[0].Name, infos[1].Name) + } +} + +func TestBlockVolumeLookupHandler(t *testing.T) { + ms, ts := blockTestServer(t) + + ms.blockRegistry.Register(&BlockVolumeEntry{ + Name: "vol1", + VolumeServer: "vs1:9333", + SizeBytes: 1 << 30, + Epoch: 3, + Role: 1, // RolePrimary + Status: StatusActive, + ReplicaServer: "vs2:9333", + }) + + resp, err := http.Get(ts.URL + "/block/volume/vol1") + if err != nil { + t.Fatal(err) + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusOK { + t.Fatalf("expected 200, got %d", resp.StatusCode) + } + + var info blockapi.VolumeInfo + if err := json.NewDecoder(resp.Body).Decode(&info); err != nil { + t.Fatal(err) + } + if info.Epoch != 3 { + t.Errorf("expected epoch 3, got %d", info.Epoch) + } + if info.Role != "primary" { + t.Errorf("expected role primary, got %s", info.Role) + } + if info.ReplicaServer != "vs2:9333" { + t.Errorf("expected replica vs2:9333, got %s", info.ReplicaServer) + } + + // Not found case. + resp2, err := http.Get(ts.URL + "/block/volume/nonexistent") + if err != nil { + t.Fatal(err) + } + defer resp2.Body.Close() + if resp2.StatusCode != http.StatusNotFound { + t.Fatalf("expected 404, got %d", resp2.StatusCode) + } +} + +func TestBlockVolumeDeleteHandler(t *testing.T) { + ms, ts := blockTestServer(t) + + ms.blockRegistry.Register(&BlockVolumeEntry{ + Name: "vol1", + VolumeServer: "vs1:9333", + SizeBytes: 1 << 30, + Status: StatusActive, + }) + + req, _ := http.NewRequest("DELETE", ts.URL+"/block/volume/vol1", nil) + resp, err := http.DefaultClient.Do(req) + if err != nil { + t.Fatal(err) + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusOK { + t.Fatalf("expected 200, got %d", resp.StatusCode) + } + + // Verify it's gone. + if _, ok := ms.blockRegistry.Lookup("vol1"); ok { + t.Error("expected vol1 to be unregistered") + } +} + +func TestBlockAssignHandler(t *testing.T) { + ms, ts := blockTestServer(t) + + ms.blockRegistry.Register(&BlockVolumeEntry{ + Name: "vol1", + VolumeServer: "vs1:9333", + Path: "/data/vol1.blk", + SizeBytes: 1 << 30, + Status: StatusActive, + }) + + body, _ := json.Marshal(blockapi.AssignRequest{ + Name: "vol1", + Epoch: 5, + Role: "primary", + LeaseTTLMs: 30000, + }) + resp, err := http.Post(ts.URL+"/block/assign", "application/json", bytes.NewReader(body)) + if err != nil { + t.Fatal(err) + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusOK { + t.Fatalf("expected 200, got %d", resp.StatusCode) + } + + // Verify assignment was queued. + pending := ms.blockAssignmentQueue.Peek("vs1:9333") + if len(pending) != 1 { + t.Fatalf("expected 1 pending assignment, got %d", len(pending)) + } + if pending[0].Epoch != 5 { + t.Errorf("expected epoch 5, got %d", pending[0].Epoch) + } + + // Not-found case. + body2, _ := json.Marshal(blockapi.AssignRequest{Name: "missing", Epoch: 1, Role: "primary"}) + resp2, err := http.Post(ts.URL+"/block/assign", "application/json", bytes.NewReader(body2)) + if err != nil { + t.Fatal(err) + } + defer resp2.Body.Close() + if resp2.StatusCode != http.StatusNotFound { + t.Fatalf("expected 404, got %d", resp2.StatusCode) + } +} + +func TestBlockServersHandler(t *testing.T) { + _, ts := blockTestServer(t) + + resp, err := http.Get(ts.URL + "/block/servers") + if err != nil { + t.Fatal(err) + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusOK { + t.Fatalf("expected 200, got %d", resp.StatusCode) + } + + var servers []blockapi.ServerInfo + if err := json.NewDecoder(resp.Body).Decode(&servers); err != nil { + t.Fatal(err) + } + if len(servers) != 1 { + t.Fatalf("expected 1 server, got %d", len(servers)) + } + if servers[0].Address != "vs1:9333" { + t.Errorf("expected vs1:9333, got %s", servers[0].Address) + } +} + +func TestListAll(t *testing.T) { + r := NewBlockVolumeRegistry() + r.Register(&BlockVolumeEntry{Name: "charlie", VolumeServer: "vs1:9333"}) + r.Register(&BlockVolumeEntry{Name: "alpha", VolumeServer: "vs1:9333"}) + r.Register(&BlockVolumeEntry{Name: "bravo", VolumeServer: "vs1:9333"}) + + entries := r.ListAll() + if len(entries) != 3 { + t.Fatalf("expected 3 entries, got %d", len(entries)) + } + if entries[0].Name != "alpha" || entries[1].Name != "bravo" || entries[2].Name != "charlie" { + t.Errorf("expected sorted order, got %s %s %s", entries[0].Name, entries[1].Name, entries[2].Name) + } +} + +func TestServerSummaries(t *testing.T) { + r := NewBlockVolumeRegistry() + r.MarkBlockCapable("vs1:9333") + r.MarkBlockCapable("vs2:9333") + r.Register(&BlockVolumeEntry{Name: "vol1", VolumeServer: "vs1:9333"}) + r.Register(&BlockVolumeEntry{Name: "vol2", VolumeServer: "vs1:9333"}) + r.Register(&BlockVolumeEntry{Name: "vol3", VolumeServer: "vs2:9333"}) + + summaries := r.ServerSummaries() + if len(summaries) != 2 { + t.Fatalf("expected 2 summaries, got %d", len(summaries)) + } + // Sorted by address. + if summaries[0].Address != "vs1:9333" || summaries[0].VolumeCount != 2 { + t.Errorf("vs1: got %+v", summaries[0]) + } + if summaries[1].Address != "vs2:9333" || summaries[1].VolumeCount != 1 { + t.Errorf("vs2: got %+v", summaries[1]) + } +} diff --git a/weed/server/master_server_handlers_block_ui.go b/weed/server/master_server_handlers_block_ui.go new file mode 100644 index 0000000000..be0ca0ae58 --- /dev/null +++ b/weed/server/master_server_handlers_block_ui.go @@ -0,0 +1,304 @@ +package weed_server + +import ( + "html/template" + "net/http" + + "github.com/seaweedfs/seaweedfs/weed/glog" + "github.com/seaweedfs/seaweedfs/weed/storage/blockvol/blockapi" +) + +var blockDashTemplate = template.Must(template.New("blockDash").Parse(blockLayoutHTML + blockDashContentHTML)) +var blockOpsTemplate = template.Must(template.New("blockOps").Parse(blockLayoutHTML + blockOpsContentHTML)) + +type blockUIVolume struct { + blockapi.VolumeInfo + SizeMB uint64 +} + +type blockUIData struct { + Tab string // "dashboard" or "ops" + Volumes []blockUIVolume + Servers []blockapi.ServerInfo + // Dashboard stats + TotalVolumes int + ActiveCount int + PendingCount int + TotalSizeMB uint64 +} + +func (ms *MasterServer) buildBlockUIData(tab string) blockUIData { + entries := ms.blockRegistry.ListAll() + volumes := make([]blockUIVolume, len(entries)) + var totalSizeMB uint64 + var activeCount, pendingCount int + for i, e := range entries { + info := entryToVolumeInfo(e) + mb := info.SizeBytes / (1024 * 1024) + volumes[i] = blockUIVolume{ + VolumeInfo: info, + SizeMB: mb, + } + totalSizeMB += mb + if e.Status == StatusActive { + activeCount++ + } else { + pendingCount++ + } + } + + summaries := ms.blockRegistry.ServerSummaries() + servers := make([]blockapi.ServerInfo, len(summaries)) + for i, s := range summaries { + servers[i] = blockapi.ServerInfo{ + Address: s.Address, + VolumeCount: s.VolumeCount, + BlockCapable: s.BlockCapable, + } + } + + return blockUIData{ + Tab: tab, + Volumes: volumes, + Servers: servers, + TotalVolumes: len(entries), + ActiveCount: activeCount, + PendingCount: pendingCount, + TotalSizeMB: totalSizeMB, + } +} + +// blockUIHandler serves the /block/ dashboard page. +func (ms *MasterServer) blockUIHandler(w http.ResponseWriter, r *http.Request) { + data := ms.buildBlockUIData("dashboard") + w.Header().Set("Content-Type", "text/html; charset=utf-8") + if err := blockDashTemplate.Execute(w, data); err != nil { + glog.V(0).Infof("block UI template error: %v", err) + } +} + +// blockOpsHandler serves the /block/ops operations page. +func (ms *MasterServer) blockOpsHandler(w http.ResponseWriter, r *http.Request) { + data := ms.buildBlockUIData("ops") + w.Header().Set("Content-Type", "text/html; charset=utf-8") + if err := blockOpsTemplate.Execute(w, data); err != nil { + glog.V(0).Infof("block UI template error: %v", err) + } +} + +// blockLayoutHTML is the shared HTML layout with tab navigation. +const blockLayoutHTML = ` + + +SeaweedFS Block Storage + + + +

SeaweedFS Block Storage

+
+ Dashboard + Operations +
+
+` + +// blockDashContentHTML is the Dashboard tab content. +const blockDashContentHTML = ` +
+
+
Block Servers
+
{{len .Servers}}
+
+
+
Total Volumes
+
{{.TotalVolumes}}
+
+
+
Active
+
{{.ActiveCount}}
+
+
+
Pending
+
{{.PendingCount}}
+
+
+
Total Capacity
+
{{.TotalSizeMB}} MB
+
+
+ +

Servers

+{{if .Servers}} + + + {{range .Servers}} + + + + + + {{end}} +
AddressVolume CountBlock Capable
{{.Address}}{{.VolumeCount}}{{if .BlockCapable}}Yes{{else}}No{{end}}
+{{else}} +
No block servers connected
+{{end}} + +

Volumes

+{{if .Volumes}} + + + + + + {{range .Volumes}} + + + + + + + + + + + + {{end}} +
NameServerSizePlacementEpochRoleStatusiSCSIReplica
{{.Name}}{{.VolumeServer}}{{.SizeMB}} MB{{.ReplicaPlacement}}{{.Epoch}}{{if eq .Role "primary"}}primary{{else if eq .Role "replica"}}replica{{else}}{{.Role}}{{end}}{{if eq .Status "active"}}active{{else}}{{.Status}}{{end}}{{.ISCSIAddr}}{{.ReplicaServer}}
+{{else}} +
No block volumes
+{{end}} + +
` + +// blockOpsContentHTML is the Operations tab content. +const blockOpsContentHTML = ` +
+

Create Volume

+
+
+ + +
+
+ + +
+
+ + +
+
+ + +
+
+ + +
+
+
+ +

Volumes

+{{if .Volumes}} + + + + + + {{range .Volumes}} + + + + + + + + + + + + + {{end}} +
NameServerSizePlacementEpochRoleStatusiSCSIReplicaAction
{{.Name}}{{.VolumeServer}}{{.SizeMB}} MB{{.ReplicaPlacement}}{{.Epoch}}{{if eq .Role "primary"}}primary{{else if eq .Role "replica"}}replica{{else}}{{.Role}}{{end}}{{if eq .Status "active"}}active{{else}}{{.Status}}{{end}}{{.ISCSIAddr}}{{.ReplicaServer}}
+{{else}} +
No block volumes
+{{end}} + + +` diff --git a/weed/server/qa_block_cp63_test.go b/weed/server/qa_block_cp63_test.go index c89fc505be..28176602be 100644 --- a/weed/server/qa_block_cp63_test.go +++ b/weed/server/qa_block_cp63_test.go @@ -62,6 +62,16 @@ func registerQAVolume(t *testing.T, ms *MasterServer, name, primary, replica str entry.ReplicaPath = fmt.Sprintf("/data/%s.blk", name) entry.ReplicaIQN = fmt.Sprintf("iqn.2024.test:%s-r", name) entry.ReplicaISCSIAddr = replica + ":3260" + // CP8-2: also populate Replicas[]. + entry.Replicas = []ReplicaInfo{ + { + Server: replica, + Path: fmt.Sprintf("/data/%s.blk", name), + IQN: fmt.Sprintf("iqn.2024.test:%s-r", name), + ISCSIAddr: replica + ":3260", + HealthScore: 1.0, + }, + } } if err := ms.blockRegistry.Register(entry); err != nil { t.Fatalf("register %s: %v", name, err) @@ -382,11 +392,12 @@ func TestQA_Failover_PromoteIdempotent_NoReplicaAfterFirstSwap(t *testing.T) { ms := testMSForQA(t) registerQAVolume(t, ms, "vol1", "vs1", "vs2", 1, 5*time.Second, true) - ms.failoverBlockVolumes("vs1") // promotes vs2, vs1 becomes replica + ms.failoverBlockVolumes("vs1") // promotes vs2 + + // CP8-2: PromoteBestReplica does NOT add old primary back as replica. + // Reconnect vs1 first so it becomes a replica. + ms.recoverBlockVolumes("vs1") - // Now if vs2 also disconnects, it should try to failover. - // After first failover: primary=vs2, replica=vs1. - // vs2 disconnects: primary IS vs2, replica=vs1 — should swap back. e, _ := ms.blockRegistry.Lookup("vol1") e.LastLeaseGrant = time.Now().Add(-1 * time.Minute) // expire the new lease ms.failoverBlockVolumes("vs2") @@ -555,17 +566,20 @@ func TestQA_Create_ReplicaDeleteFailure_PrimaryStillDeleted(t *testing.T) { ms.blockRegistry.MarkBlockCapable("vs1") ms.blockRegistry.MarkBlockCapable("vs2") + ms.CreateBlockVolume(context.Background(), &master_pb.CreateBlockVolumeRequest{ + Name: "vol1", SizeBytes: 1 << 30, + }) + + // Find the replica server and make its delete fail. + entry, _ := ms.blockRegistry.Lookup("vol1") + replicaServer := entry.ReplicaServer ms.blockVSDelete = func(ctx context.Context, server string, name string) error { - if server == "vs2" { + if server == replicaServer { return fmt.Errorf("replica down") } return nil } - ms.CreateBlockVolume(context.Background(), &master_pb.CreateBlockVolumeRequest{ - Name: "vol1", SizeBytes: 1 << 30, - }) - // Delete should succeed even if replica delete fails (best-effort). _, err := ms.DeleteBlockVolume(context.Background(), &master_pb.DeleteBlockVolumeRequest{Name: "vol1"}) if err != nil { diff --git a/weed/server/qa_block_cp82_adversarial_test.go b/weed/server/qa_block_cp82_adversarial_test.go new file mode 100644 index 0000000000..f29c6ca480 --- /dev/null +++ b/weed/server/qa_block_cp82_adversarial_test.go @@ -0,0 +1,849 @@ +package weed_server + +import ( + "context" + "fmt" + "sync" + "testing" + "time" + + "github.com/seaweedfs/seaweedfs/weed/pb/master_pb" + "github.com/seaweedfs/seaweedfs/weed/storage/blockvol" +) + +// ============================================================ +// CP8-2 Adversarial Test Suite +// +// 12 scenarios covering multi-replica, scrub, promotion gating, +// heartbeat reconciliation, and partial-failure edge cases. +// ============================================================ + +// qaCP82Master creates a MasterServer with 3 block-capable servers +// and configurable mocks for adversarial testing. +func qaCP82Master(t *testing.T) *MasterServer { + t.Helper() + ms := &MasterServer{ + blockRegistry: NewBlockVolumeRegistry(), + blockAssignmentQueue: NewBlockAssignmentQueue(), + blockFailover: newBlockFailoverState(), + } + ms.blockVSAllocate = func(ctx context.Context, server string, name string, sizeBytes uint64, diskType string) (*blockAllocResult, error) { + return &blockAllocResult{ + Path: fmt.Sprintf("/data/%s.blk", name), + IQN: fmt.Sprintf("iqn.2024.test:%s", name), + ISCSIAddr: server + ":3260", + ReplicaDataAddr: server + ":14260", + ReplicaCtrlAddr: server + ":14261", + RebuildListenAddr: server + ":15000", + }, nil + } + ms.blockVSDelete = func(ctx context.Context, server string, name string) error { + return nil + } + ms.blockVSExpand = func(ctx context.Context, server string, name string, newSize uint64) (uint64, error) { + return newSize, nil + } + ms.blockRegistry.MarkBlockCapable("vs1:9333") + ms.blockRegistry.MarkBlockCapable("vs2:9333") + ms.blockRegistry.MarkBlockCapable("vs3:9333") + return ms +} + +// qaRegisterRF2WithLSN creates an RF=2 volume with explicit WAL LSN state. +func qaRegisterRF2WithLSN(t *testing.T, ms *MasterServer, name, primary, replica string, epoch, primaryLSN, replicaLSN uint64) { + t.Helper() + entry := &BlockVolumeEntry{ + Name: name, + VolumeServer: primary, + Path: fmt.Sprintf("/data/%s.blk", name), + IQN: fmt.Sprintf("iqn.2024.test:%s", name), + ISCSIAddr: primary + ":3260", + SizeBytes: 1 << 30, + Epoch: epoch, + Role: blockvol.RoleToWire(blockvol.RolePrimary), + Status: StatusActive, + WALHeadLSN: primaryLSN, + LeaseTTL: 5 * time.Second, + LastLeaseGrant: time.Now().Add(-10 * time.Second), // expired + ReplicaServer: replica, + ReplicaPath: fmt.Sprintf("/data/%s.blk", name), + ReplicaIQN: fmt.Sprintf("iqn.2024.test:%s-r", name), + ReplicaISCSIAddr: replica + ":3260", + Replicas: []ReplicaInfo{ + { + Server: replica, + Path: fmt.Sprintf("/data/%s.blk", name), + IQN: fmt.Sprintf("iqn.2024.test:%s-r", name), + ISCSIAddr: replica + ":3260", + HealthScore: 1.0, + WALHeadLSN: replicaLSN, + LastHeartbeat: time.Now(), // fresh + Role: blockvol.RoleToWire(blockvol.RoleReplica), + }, + }, + } + if err := ms.blockRegistry.Register(entry); err != nil { + t.Fatalf("register %s: %v", name, err) + } +} + +// qaRegisterRF3WithState creates an RF=3 volume with per-replica state control. +func qaRegisterRF3WithState(t *testing.T, ms *MasterServer, name, primary string, + replicas []struct { + Server string + HealthScore float64 + WALHeadLSN uint64 + Role blockvol.Role + Heartbeat time.Time + }, + primaryLSN uint64, +) { + t.Helper() + entry := &BlockVolumeEntry{ + Name: name, + VolumeServer: primary, + Path: fmt.Sprintf("/data/%s.blk", name), + IQN: fmt.Sprintf("iqn.2024.test:%s", name), + ISCSIAddr: primary + ":3260", + SizeBytes: 1 << 30, + Epoch: 1, + Role: blockvol.RoleToWire(blockvol.RolePrimary), + Status: StatusActive, + WALHeadLSN: primaryLSN, + ReplicaFactor: len(replicas) + 1, + LeaseTTL: 5 * time.Second, + LastLeaseGrant: time.Now().Add(-10 * time.Second), // expired + } + for i, r := range replicas { + ri := ReplicaInfo{ + Server: r.Server, + Path: fmt.Sprintf("/data/%s.blk", name), + IQN: fmt.Sprintf("iqn.2024.test:%s-r%d", name, i+1), + ISCSIAddr: r.Server + ":3260", + HealthScore: r.HealthScore, + WALHeadLSN: r.WALHeadLSN, + Role: blockvol.RoleToWire(r.Role), + LastHeartbeat: r.Heartbeat, + } + entry.Replicas = append(entry.Replicas, ri) + } + // Sync scalar fields from first replica. + if len(entry.Replicas) > 0 { + entry.ReplicaServer = entry.Replicas[0].Server + entry.ReplicaPath = entry.Replicas[0].Path + entry.ReplicaIQN = entry.Replicas[0].IQN + entry.ReplicaISCSIAddr = entry.Replicas[0].ISCSIAddr + } + if err := ms.blockRegistry.Register(entry); err != nil { + t.Fatalf("register %s: %v", name, err) + } +} + +// ──────────────────────────────────────────────────────────── +// QA-1: PrimaryCrash_AfterAck_BeforeReplicaBarrier +// +// Verify failover data loss window is bounded by barrier_lag_lsn. +// Simulates primary at LSN 200, replica at barrier LSN 190. +// After failover, new primary should be at replica's LSN (190). +// The gap (10 LSN entries) is the bounded loss window. +// ──────────────────────────────────────────────────────────── +func TestQA_CP82_PrimaryCrash_AfterAck_BeforeReplicaBarrier(t *testing.T) { + ms := qaCP82Master(t) + qaRegisterRF2WithLSN(t, ms, "vol-lag", "vs1:9333", "vs2:9333", 1, 200, 190) + + // Kill primary. + ms.failoverBlockVolumes("vs1:9333") + + entry, ok := ms.blockRegistry.Lookup("vol-lag") + if !ok { + t.Fatal("volume should still exist after failover") + } + if entry.VolumeServer != "vs2:9333" { + t.Fatalf("replica should be promoted, got %q", entry.VolumeServer) + } + // The promoted replica was at LSN 190. The 10-LSN gap represents + // acknowledged-but-unbarriered writes (bounded data loss). + // New epoch should be 2. + if entry.Epoch != 2 { + t.Fatalf("epoch: got %d, want 2", entry.Epoch) + } +} + +// ──────────────────────────────────────────────────────────── +// QA-2: ReplicaHeartbeatSpoof_DoesNotDeletePrimary +// +// Regression test for Fix #1. A replica heartbeat that does NOT +// include the primary path must NOT delete the volume entry. +// ──────────────────────────────────────────────────────────── +func TestQA_CP82_ReplicaHeartbeatSpoof_DoesNotDeletePrimary(t *testing.T) { + ms := qaCP82Master(t) + qaRegisterRF2WithLSN(t, ms, "vol-spoof", "vs1:9333", "vs2:9333", 1, 100, 100) + + // Simulate replica heartbeat from vs2 with ONLY its replica path. + // This must NOT remove the volume (vs1 is the primary, not vs2). + ms.blockRegistry.UpdateFullHeartbeat("vs2:9333", []*master_pb.BlockVolumeInfoMessage{ + { + Path: "/data/vol-spoof.blk", + VolumeSize: 1 << 30, + Epoch: 1, + Role: blockvol.RoleToWire(blockvol.RoleReplica), + }, + }) + + entry, ok := ms.blockRegistry.Lookup("vol-spoof") + if !ok { + t.Fatal("volume must NOT be deleted by replica heartbeat (Fix #1 regression)") + } + if entry.VolumeServer != "vs1:9333" { + t.Fatalf("primary must remain vs1:9333, got %q", entry.VolumeServer) + } + + // Now simulate a full heartbeat from vs2 with NO volumes at all. + // This should remove vs2 as replica but NOT delete the volume. + ms.blockRegistry.UpdateFullHeartbeat("vs2:9333", []*master_pb.BlockVolumeInfoMessage{}) + + entry, ok = ms.blockRegistry.Lookup("vol-spoof") + if !ok { + t.Fatal("volume must still exist after empty replica heartbeat") + } + if entry.VolumeServer != "vs1:9333" { + t.Fatal("primary must still be vs1:9333") + } +} + +// ──────────────────────────────────────────────────────────── +// QA-3: PromotionRejects_StaleButHealthyReplica +// +// Replica has perfect health (1.0) but stale heartbeat. +// Must NOT be promoted (Gate 1: heartbeat freshness). +// ──────────────────────────────────────────────────────────── +func TestQA_CP82_PromotionRejects_StaleButHealthyReplica(t *testing.T) { + ms := qaCP82Master(t) + staleTime := time.Now().Add(-5 * time.Minute) // way beyond 2×LeaseTTL + qaRegisterRF3WithState(t, ms, "vol-stale", "vs1:9333", + []struct { + Server string + HealthScore float64 + WALHeadLSN uint64 + Role blockvol.Role + Heartbeat time.Time + }{ + {"vs2:9333", 1.0, 100, blockvol.RoleReplica, staleTime}, // healthy but stale + {"vs3:9333", 0.5, 100, blockvol.RoleReplica, staleTime}, // also stale + }, + 100, + ) + + _, err := ms.blockRegistry.PromoteBestReplica("vol-stale") + if err == nil { + t.Fatal("expected promotion to fail — all replicas have stale heartbeats") + } + // Volume should remain unchanged. + entry, _ := ms.blockRegistry.Lookup("vol-stale") + if entry.VolumeServer != "vs1:9333" { + t.Fatalf("primary must not change, got %q", entry.VolumeServer) + } +} + +// ──────────────────────────────────────────────────────────── +// QA-4: PromotionRejects_RebuildingReplica +// +// Replica is fresh and healthy but in RoleRebuilding. +// Must NOT be promoted (Gate 3: role check). +// ──────────────────────────────────────────────────────────── +func TestQA_CP82_PromotionRejects_RebuildingReplica(t *testing.T) { + ms := qaCP82Master(t) + now := time.Now() + qaRegisterRF3WithState(t, ms, "vol-rebuild", "vs1:9333", + []struct { + Server string + HealthScore float64 + WALHeadLSN uint64 + Role blockvol.Role + Heartbeat time.Time + }{ + {"vs2:9333", 1.0, 100, blockvol.RoleRebuilding, now}, // rebuilding + {"vs3:9333", 1.0, 100, blockvol.RoleRebuilding, now}, // also rebuilding + }, + 100, + ) + + _, err := ms.blockRegistry.PromoteBestReplica("vol-rebuild") + if err == nil { + t.Fatal("expected promotion to fail — all replicas are rebuilding") + } + + // One rebuilding, one ready: only the ready one should be promoted. + ms2 := qaCP82Master(t) + qaRegisterRF3WithState(t, ms2, "vol-mixed", "vs1:9333", + []struct { + Server string + HealthScore float64 + WALHeadLSN uint64 + Role blockvol.Role + Heartbeat time.Time + }{ + {"vs2:9333", 1.0, 100, blockvol.RoleRebuilding, now}, + {"vs3:9333", 0.7, 100, blockvol.RoleReplica, now}, // eligible despite lower health + }, + 100, + ) + + _, err = ms2.blockRegistry.PromoteBestReplica("vol-mixed") + if err != nil { + t.Fatalf("expected promotion to succeed: %v", err) + } + entry, _ := ms2.blockRegistry.Lookup("vol-mixed") + if entry.VolumeServer != "vs3:9333" { + t.Fatalf("vs3 should be promoted (only eligible), got %q", entry.VolumeServer) + } +} + +// ──────────────────────────────────────────────────────────── +// QA-5: PromotionToleranceBoundary_ExactLSN +// +// Tests promotion_lsn_tolerance boundary: tolerance-1, exact, tolerance+1. +// Catches off-by-one in the eligibility gate. +// ──────────────────────────────────────────────────────────── +func TestQA_CP82_PromotionToleranceBoundary_ExactLSN(t *testing.T) { + const tolerance uint64 = 50 + const primaryLSN uint64 = 200 + + cases := []struct { + name string + replicaLSN uint64 + wantErr bool + }{ + // replicaLSN + tolerance < primaryLSN → ineligible + // 149 + 50 = 199 < 200 → INELIGIBLE + {"below_tolerance", primaryLSN - tolerance - 1, true}, + // 150 + 50 = 200 == 200 → ELIGIBLE (not strict less-than) + {"at_tolerance", primaryLSN - tolerance, false}, + // 151 + 50 = 201 > 200 → ELIGIBLE + {"above_tolerance", primaryLSN - tolerance + 1, false}, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + ms := qaCP82Master(t) + ms.blockRegistry.SetPromotionLSNTolerance(tolerance) + + now := time.Now() + qaRegisterRF3WithState(t, ms, "vol-tol", "vs1:9333", + []struct { + Server string + HealthScore float64 + WALHeadLSN uint64 + Role blockvol.Role + Heartbeat time.Time + }{ + {"vs2:9333", 1.0, tc.replicaLSN, blockvol.RoleReplica, now}, + }, + primaryLSN, + ) + + _, err := ms.blockRegistry.PromoteBestReplica("vol-tol") + if tc.wantErr && err == nil { + t.Fatalf("replicaLSN=%d: expected promotion to fail", tc.replicaLSN) + } + if !tc.wantErr && err != nil { + t.Fatalf("replicaLSN=%d: expected promotion to succeed: %v", tc.replicaLSN, err) + } + }) + } +} + +// ──────────────────────────────────────────────────────────── +// QA-6: MasterRestart_ReconstructReplicas_ThenFailover +// +// Simulates master restart: empty registry, full heartbeats +// from primary and replica servers reconstruct state, then +// failover correctly promotes the replica. +// ──────────────────────────────────────────────────────────── +func TestQA_CP82_MasterRestart_ReconstructReplicas_ThenFailover(t *testing.T) { + ms := qaCP82Master(t) + + // Phase 1: Primary heartbeat — auto-registers volume from heartbeat. + ms.blockRegistry.UpdateFullHeartbeat("vs1:9333", []*master_pb.BlockVolumeInfoMessage{ + { + Path: "/data/vol-restart.blk", + VolumeSize: 1 << 30, + Epoch: 5, + Role: blockvol.RoleToWire(blockvol.RolePrimary), + WalHeadLsn: 500, + }, + }) + + entry, ok := ms.blockRegistry.Lookup("vol-restart") + if !ok { + t.Fatal("volume should be auto-registered from primary heartbeat") + } + if entry.VolumeServer != "vs1:9333" { + t.Fatalf("primary: got %q, want vs1:9333", entry.VolumeServer) + } + + // Phase 2: Replica heartbeat — should reconstruct ReplicaInfo. + ms.blockRegistry.UpdateFullHeartbeat("vs2:9333", []*master_pb.BlockVolumeInfoMessage{ + { + Path: "/data/vol-restart.blk", + VolumeSize: 1 << 30, + Epoch: 5, + Role: blockvol.RoleToWire(blockvol.RoleReplica), + WalHeadLsn: 498, + }, + }) + + entry, _ = ms.blockRegistry.Lookup("vol-restart") + if len(entry.Replicas) == 0 { + t.Fatal("replica should be reconstructed from heartbeat (Fix #3)") + } + if entry.Replicas[0].Server != "vs2:9333" { + t.Fatalf("replica server: got %q, want vs2:9333", entry.Replicas[0].Server) + } + + // Phase 3: Set lease expired and trigger failover. + entry.LeaseTTL = 5 * time.Second + entry.LastLeaseGrant = time.Now().Add(-1 * time.Minute) + // Ensure replica is eligible for promotion. + entry.Replicas[0].LastHeartbeat = time.Now() + entry.Replicas[0].Role = blockvol.RoleToWire(blockvol.RoleReplica) + + ms.failoverBlockVolumes("vs1:9333") + + entry, ok = ms.blockRegistry.Lookup("vol-restart") + if !ok { + t.Fatal("volume should survive failover") + } + if entry.VolumeServer != "vs2:9333" { + t.Fatalf("after failover: primary should be vs2:9333, got %q", entry.VolumeServer) + } + if entry.Epoch <= 5 { + t.Fatalf("epoch should be bumped, got %d", entry.Epoch) + } +} + +// ──────────────────────────────────────────────────────────── +// QA-7: RF3_OneReplicaFlaps_UnderWriteLoad +// +// One replica rapidly disconnects/reconnects while concurrent +// failover checks run. Must not cause split-brain or panic. +// Simulates via concurrent UpdateFullHeartbeat + failoverBlockVolumes. +// ──────────────────────────────────────────────────────────── +func TestQA_CP82_RF3_OneReplicaFlaps_UnderWriteLoad(t *testing.T) { + ms := qaCP82Master(t) + now := time.Now() + qaRegisterRF3WithState(t, ms, "vol-flap", "vs1:9333", + []struct { + Server string + HealthScore float64 + WALHeadLSN uint64 + Role blockvol.Role + Heartbeat time.Time + }{ + {"vs2:9333", 1.0, 100, blockvol.RoleReplica, now}, + {"vs3:9333", 1.0, 100, blockvol.RoleReplica, now}, + }, + 100, + ) + + var wg sync.WaitGroup + const rounds = 50 + + // Goroutine 1: flapping replica vs3 (heartbeat with/without volume). + wg.Add(1) + go func() { + defer wg.Done() + for i := 0; i < rounds; i++ { + if i%2 == 0 { + // Replica reports volume. + ms.blockRegistry.UpdateFullHeartbeat("vs3:9333", []*master_pb.BlockVolumeInfoMessage{ + { + Path: "/data/vol-flap.blk", + VolumeSize: 1 << 30, + Epoch: 1, + Role: blockvol.RoleToWire(blockvol.RoleReplica), + WalHeadLsn: 100, + }, + }) + } else { + // Replica reports no volumes (simulates disconnect). + ms.blockRegistry.UpdateFullHeartbeat("vs3:9333", []*master_pb.BlockVolumeInfoMessage{}) + } + } + }() + + // Goroutine 2: concurrent heartbeats from primary. + wg.Add(1) + go func() { + defer wg.Done() + for i := 0; i < rounds; i++ { + ms.blockRegistry.UpdateFullHeartbeat("vs1:9333", []*master_pb.BlockVolumeInfoMessage{ + { + Path: "/data/vol-flap.blk", + VolumeSize: 1 << 30, + Epoch: 1, + Role: blockvol.RoleToWire(blockvol.RolePrimary), + WalHeadLsn: uint64(100 + i), + }, + }) + } + }() + + // Goroutine 3: concurrent heartbeats from stable replica vs2. + wg.Add(1) + go func() { + defer wg.Done() + for i := 0; i < rounds; i++ { + ms.blockRegistry.UpdateFullHeartbeat("vs2:9333", []*master_pb.BlockVolumeInfoMessage{ + { + Path: "/data/vol-flap.blk", + VolumeSize: 1 << 30, + Epoch: 1, + Role: blockvol.RoleToWire(blockvol.RoleReplica), + WalHeadLsn: uint64(100 + i), + }, + }) + } + }() + + wg.Wait() + + // Invariant: volume must still exist, primary must be vs1. + entry, ok := ms.blockRegistry.Lookup("vol-flap") + if !ok { + t.Fatal("volume must survive flapping replica") + } + if entry.VolumeServer != "vs1:9333" { + t.Fatalf("primary must remain vs1:9333, got %q (split-brain!)", entry.VolumeServer) + } +} + +// ──────────────────────────────────────────────────────────── +// QA-8: AssignmentPrecedence_ReplicaAddrsVsScalar +// +// Creates RF=3 volume and verifies primary assignment includes +// ReplicaAddrs (not just scalar fields). VS should use +// ReplicaAddrs when non-empty. +// ──────────────────────────────────────────────────────────── +func TestQA_CP82_AssignmentPrecedence_ReplicaAddrsVsScalar(t *testing.T) { + ms := qaCP82Master(t) + ctx := context.Background() + + resp, err := ms.CreateBlockVolume(ctx, &master_pb.CreateBlockVolumeRequest{ + Name: "vol-addrs", + SizeBytes: 1 << 30, + ReplicaFactor: 3, + }) + if err != nil { + t.Fatalf("create: %v", err) + } + + primary := resp.VolumeServer + assignments := ms.blockAssignmentQueue.Peek(primary) + + // Find the primary assignment. + var primaryAssignment *blockvol.BlockVolumeAssignment + for i := range assignments { + if blockvol.RoleFromWire(assignments[i].Role) == blockvol.RolePrimary { + primaryAssignment = &assignments[i] + break + } + } + + if primaryAssignment == nil { + t.Fatal("no primary assignment found") + } + + // RF=3 → 2 replicas → ReplicaAddrs should have 2 entries. + if len(primaryAssignment.ReplicaAddrs) != 2 { + t.Fatalf("primary assignment ReplicaAddrs: got %d, want 2", len(primaryAssignment.ReplicaAddrs)) + } + + // Verify each ReplicaAddr has data+ctrl. + for i, ra := range primaryAssignment.ReplicaAddrs { + if ra.DataAddr == "" || ra.CtrlAddr == "" { + t.Errorf("ReplicaAddrs[%d]: data=%q ctrl=%q — both must be non-empty", i, ra.DataAddr, ra.CtrlAddr) + } + } + + // For RF=3, legacy scalar ReplicaDataAddr is NOT set on the primary + // assignment (only ReplicaAddrs is used). Verify precedence: ReplicaAddrs + // is the authoritative source when non-empty. + if len(primaryAssignment.ReplicaAddrs) > 0 && primaryAssignment.ReplicaDataAddr != "" { + t.Error("legacy ReplicaDataAddr should be empty when ReplicaAddrs is populated (RF=3)") + } +} + +// ──────────────────────────────────────────────────────────── +// QA-9: ScrubConcurrentWrites_NoFalseCorruption +// +// Heavy writes while scrub runs. Expect zero false positives. +// Tests at the HealthScore level (engine scrub tested separately). +// ──────────────────────────────────────────────────────────── +func TestQA_CP82_ScrubConcurrentWrites_NoFalseCorruption(t *testing.T) { + ms := qaCP82Master(t) + now := time.Now() + qaRegisterRF3WithState(t, ms, "vol-scrub", "vs1:9333", + []struct { + Server string + HealthScore float64 + WALHeadLSN uint64 + Role blockvol.Role + Heartbeat time.Time + }{ + {"vs2:9333", 1.0, 500, blockvol.RoleReplica, now}, + {"vs3:9333", 1.0, 500, blockvol.RoleReplica, now}, + }, + 500, + ) + + var wg sync.WaitGroup + const rounds = 100 + + // Simulate heartbeats with varying health scores + // (as would happen during scrub with concurrent writes). + wg.Add(1) + go func() { + defer wg.Done() + for i := 0; i < rounds; i++ { + // Health stays 1.0 — no corruption detected (correct behavior + // when writes are excluded from scrub checks). + ms.blockRegistry.UpdateFullHeartbeat("vs1:9333", []*master_pb.BlockVolumeInfoMessage{ + { + Path: "/data/vol-scrub.blk", + VolumeSize: 1 << 30, + Epoch: 1, + Role: blockvol.RoleToWire(blockvol.RolePrimary), + WalHeadLsn: uint64(500 + i), + HealthScore: 1.0, // Clean — no false positives + }, + }) + } + }() + + // Concurrent replica heartbeats. + wg.Add(1) + go func() { + defer wg.Done() + for i := 0; i < rounds; i++ { + ms.blockRegistry.UpdateFullHeartbeat("vs2:9333", []*master_pb.BlockVolumeInfoMessage{ + { + Path: "/data/vol-scrub.blk", + VolumeSize: 1 << 30, + Epoch: 1, + Role: blockvol.RoleToWire(blockvol.RoleReplica), + WalHeadLsn: uint64(500 + i), + HealthScore: 1.0, + }, + }) + } + }() + + wg.Wait() + + entry, _ := ms.blockRegistry.Lookup("vol-scrub") + if entry.HealthScore < 1.0 { + t.Fatalf("health score should be 1.0 (no corruption), got %f", entry.HealthScore) + } +} + +// ──────────────────────────────────────────────────────────── +// QA-10: ScrubDetectsCorruption_HealthDrops_PromotionAvoids +// +// One replica has corruption (low health), the other is clean. +// Promotion must prefer the healthy replica. +// ──────────────────────────────────────────────────────────── +func TestQA_CP82_ScrubDetectsCorruption_HealthDrops_PromotionAvoids(t *testing.T) { + ms := qaCP82Master(t) + now := time.Now() + qaRegisterRF3WithState(t, ms, "vol-corrupt", "vs1:9333", + []struct { + Server string + HealthScore float64 + WALHeadLSN uint64 + Role blockvol.Role + Heartbeat time.Time + }{ + {"vs2:9333", 0.3, 100, blockvol.RoleReplica, now}, // corrupted (low health) + {"vs3:9333", 1.0, 100, blockvol.RoleReplica, now}, // clean + }, + 100, + ) + + // Simulate heartbeat reporting low health on vs2 (scrub found errors). + ms.blockRegistry.UpdateFullHeartbeat("vs2:9333", []*master_pb.BlockVolumeInfoMessage{ + { + Path: "/data/vol-corrupt.blk", + VolumeSize: 1 << 30, + Epoch: 1, + Role: blockvol.RoleToWire(blockvol.RoleReplica), + WalHeadLsn: 100, + HealthScore: 0.3, + ScrubErrors: 5, + }, + }) + + // Trigger failover — vs3 (healthy) should be promoted, not vs2. + newEpoch, err := ms.blockRegistry.PromoteBestReplica("vol-corrupt") + if err != nil { + t.Fatalf("promotion should succeed: %v", err) + } + if newEpoch != 2 { + t.Fatalf("epoch: got %d, want 2", newEpoch) + } + + entry, _ := ms.blockRegistry.Lookup("vol-corrupt") + if entry.VolumeServer != "vs3:9333" { + t.Fatalf("vs3 (healthy) should be promoted, got %q", entry.VolumeServer) + } + // vs2 (corrupted) should remain as replica. + if len(entry.Replicas) != 1 || entry.Replicas[0].Server != "vs2:9333" { + t.Fatalf("vs2 should remain as replica, got %+v", entry.Replicas) + } +} + +// ──────────────────────────────────────────────────────────── +// QA-11: ExpandRF3_PartialReplicaFailure +// +// Primary expand succeeds, one replica expand fails. +// Verify: operation succeeds (best-effort), registry updated, +// failed replica state is degraded. +// ──────────────────────────────────────────────────────────── +func TestQA_CP82_ExpandRF3_PartialReplicaFailure(t *testing.T) { + ms := qaCP82Master(t) + ctx := context.Background() + + // Create RF=3 volume first. + _, err := ms.CreateBlockVolume(ctx, &master_pb.CreateBlockVolumeRequest{ + Name: "vol-expand", + SizeBytes: 1 << 30, + ReplicaFactor: 3, + }) + if err != nil { + t.Fatalf("create: %v", err) + } + + entry, _ := ms.blockRegistry.Lookup("vol-expand") + failServer := entry.Replicas[1].Server + + // Override expand mock: one replica fails. + ms.blockVSExpand = func(ctx context.Context, server string, name string, newSize uint64) (uint64, error) { + if server == failServer { + return 0, fmt.Errorf("disk full on %s", server) + } + return newSize, nil + } + + // Expand should succeed (primary + one replica succeed, one fails best-effort). + resp, err := ms.ExpandBlockVolume(ctx, &master_pb.ExpandBlockVolumeRequest{ + Name: "vol-expand", + NewSizeBytes: 2 << 30, + }) + if err != nil { + t.Fatalf("expand should succeed despite partial replica failure: %v", err) + } + if resp.CapacityBytes != 2<<30 { + t.Fatalf("capacity: got %d, want %d", resp.CapacityBytes, uint64(2<<30)) + } + + // Registry should reflect new size. + entry, _ = ms.blockRegistry.Lookup("vol-expand") + if entry.SizeBytes != 2<<30 { + t.Fatalf("registry size: got %d, want %d", entry.SizeBytes, uint64(2<<30)) + } +} + +// ──────────────────────────────────────────────────────────── +// QA-12: ByServerIndexConsistency_AfterReplicaMove +// +// Moves replica between servers repeatedly. ListByServer and +// byServer counts must stay exact — no dangling references. +// ──────────────────────────────────────────────────────────── +func TestQA_CP82_ByServerIndexConsistency_AfterReplicaMove(t *testing.T) { + ms := qaCP82Master(t) + now := time.Now() + qaRegisterRF3WithState(t, ms, "vol-move", "vs1:9333", + []struct { + Server string + HealthScore float64 + WALHeadLSN uint64 + Role blockvol.Role + Heartbeat time.Time + }{ + {"vs2:9333", 1.0, 100, blockvol.RoleReplica, now}, + }, + 100, + ) + + // Verify initial state: vs1 and vs2 both have vol-move. + assertServerHasVolume(t, ms, "vs1:9333", "vol-move") + assertServerHasVolume(t, ms, "vs2:9333", "vol-move") + + // Move replica from vs2 to vs3 repeatedly. + servers := []string{"vs3:9333", "vs2:9333", "vs3:9333", "vs2:9333", "vs3:9333"} + for _, newServer := range servers { + // Remove old replica. + entry, _ := ms.blockRegistry.Lookup("vol-move") + oldServer := "" + if len(entry.Replicas) > 0 { + oldServer = entry.Replicas[0].Server + } + if oldServer != "" { + if err := ms.blockRegistry.RemoveReplica("vol-move", oldServer); err != nil { + t.Fatalf("remove replica from %s: %v", oldServer, err) + } + } + + // Add new replica. + if err := ms.blockRegistry.AddReplica("vol-move", ReplicaInfo{ + Server: newServer, + Path: "/data/vol-move.blk", + IQN: fmt.Sprintf("iqn.2024.test:vol-move-r"), + ISCSIAddr: newServer + ":3260", + HealthScore: 1.0, + }); err != nil { + t.Fatalf("add replica on %s: %v", newServer, err) + } + + // Invariant: primary always indexed. + assertServerHasVolume(t, ms, "vs1:9333", "vol-move") + // New replica indexed. + assertServerHasVolume(t, ms, newServer, "vol-move") + // Old server should NOT have it (unless it's the primary). + if oldServer != "" && oldServer != "vs1:9333" && oldServer != newServer { + assertServerDoesNotHaveVolume(t, ms, oldServer, "vol-move") + } + } + + // Final check: ListAll should have exactly 1 volume. + all := ms.blockRegistry.ListAll() + if len(all) != 1 { + t.Fatalf("expected 1 volume, got %d", len(all)) + } +} + +// ──────────────────────────────────────────────────────────── +// Helpers +// ──────────────────────────────────────────────────────────── + +func assertServerHasVolume(t *testing.T, ms *MasterServer, server, volName string) { + t.Helper() + entries := ms.blockRegistry.ListByServer(server) + for _, e := range entries { + if e.Name == volName { + return + } + } + t.Errorf("server %q should have volume %q in byServer index", server, volName) +} + +func assertServerDoesNotHaveVolume(t *testing.T, ms *MasterServer, server, volName string) { + t.Helper() + entries := ms.blockRegistry.ListByServer(server) + for _, e := range entries { + if e.Name == volName { + t.Errorf("server %q should NOT have volume %q in byServer index", server, volName) + return + } + } +} diff --git a/weed/server/qa_block_rf3_test.go b/weed/server/qa_block_rf3_test.go new file mode 100644 index 0000000000..6ad0226036 --- /dev/null +++ b/weed/server/qa_block_rf3_test.go @@ -0,0 +1,430 @@ +package weed_server + +import ( + "context" + "fmt" + "testing" + "time" + + "github.com/seaweedfs/seaweedfs/weed/pb/master_pb" + "github.com/seaweedfs/seaweedfs/weed/storage/blockvol" +) + +// ============================================================ +// CP8-2 T10: RF=3 QA + Integration Tests +// +// Adversarial and integration tests for multi-replica (N>2) +// support introduced in CP8-2. +// ============================================================ + +// Helper: create master with 3 block-capable servers and RF=3-aware alloc mock. +func qaRF3Master(t *testing.T) *MasterServer { + t.Helper() + ms := &MasterServer{ + blockRegistry: NewBlockVolumeRegistry(), + blockAssignmentQueue: NewBlockAssignmentQueue(), + blockFailover: newBlockFailoverState(), + } + ms.blockVSAllocate = func(ctx context.Context, server string, name string, sizeBytes uint64, diskType string) (*blockAllocResult, error) { + return &blockAllocResult{ + Path: fmt.Sprintf("/data/%s.blk", name), + IQN: fmt.Sprintf("iqn.2024.test:%s", name), + ISCSIAddr: server + ":3260", + ReplicaDataAddr: server + ":14260", + ReplicaCtrlAddr: server + ":14261", + RebuildListenAddr: server + ":15000", + }, nil + } + ms.blockVSDelete = func(ctx context.Context, server string, name string) error { + return nil + } + ms.blockVSExpand = func(ctx context.Context, server string, name string, newSize uint64) (uint64, error) { + return newSize, nil + } + ms.blockRegistry.MarkBlockCapable("vs1:9333") + ms.blockRegistry.MarkBlockCapable("vs2:9333") + ms.blockRegistry.MarkBlockCapable("vs3:9333") + return ms +} + +// 1. Create RF=3, verify 2 replicas in registry with correct data. +func TestQA_RF3_CreateAndVerifyReplicas(t *testing.T) { + ms := qaRF3Master(t) + ctx := context.Background() + + resp, err := ms.CreateBlockVolume(ctx, &master_pb.CreateBlockVolumeRequest{ + Name: "pvc-rf3-1", + SizeBytes: 1 << 30, + ReplicaFactor: 3, + }) + if err != nil { + t.Fatalf("create: %v", err) + } + + entry, ok := ms.blockRegistry.Lookup("pvc-rf3-1") + if !ok { + t.Fatal("volume not in registry") + } + + // Verify ReplicaFactor. + if entry.ReplicaFactor != 3 { + t.Fatalf("ReplicaFactor: got %d, want 3", entry.ReplicaFactor) + } + + // Verify 2 replicas. + if len(entry.Replicas) != 2 { + t.Fatalf("expected 2 replicas, got %d", len(entry.Replicas)) + } + + // All 3 servers are distinct. + servers := map[string]bool{resp.VolumeServer: true} + for _, ri := range entry.Replicas { + if servers[ri.Server] { + t.Fatalf("duplicate server in RF=3 volume: %q", ri.Server) + } + servers[ri.Server] = true + // Each replica should have path, IQN, ISCSIAddr set. + if ri.Path == "" || ri.IQN == "" || ri.ISCSIAddr == "" { + t.Fatalf("replica %q missing fields: path=%q iqn=%q iscsi=%q", + ri.Server, ri.Path, ri.IQN, ri.ISCSIAddr) + } + } + + // Backward compat: ReplicaServer (deprecated) should point to first replica. + if entry.ReplicaServer == "" { + t.Fatal("deprecated ReplicaServer should be set for backward compat") + } + if entry.ReplicaServer != entry.Replicas[0].Server { + t.Fatalf("deprecated ReplicaServer %q != Replicas[0].Server %q", + entry.ReplicaServer, entry.Replicas[0].Server) + } + + // Assignments: primary should have 2 ReplicaAddrs. + primaryAssignments := ms.blockAssignmentQueue.Peek(resp.VolumeServer) + if len(primaryAssignments) != 1 { + t.Fatalf("expected 1 primary assignment, got %d", len(primaryAssignments)) + } + if len(primaryAssignments[0].ReplicaAddrs) != 2 { + t.Fatalf("primary assignment ReplicaAddrs: got %d, want 2", + len(primaryAssignments[0].ReplicaAddrs)) + } + + // Each replica server should have 1 assignment. + for _, ri := range entry.Replicas { + ra := ms.blockAssignmentQueue.Peek(ri.Server) + if len(ra) != 1 { + t.Fatalf("replica %q: expected 1 assignment, got %d", ri.Server, len(ra)) + } + if blockvol.RoleFromWire(ra[0].Role) != blockvol.RoleReplica { + t.Fatalf("replica %q: expected RoleReplica, got %d", ri.Server, ra[0].Role) + } + } +} + +// 2. Kill primary → best replica (highest health score) promoted. +func TestQA_RF3_PrimaryDies_BestReplicaPromoted(t *testing.T) { + ms := qaRF3Master(t) + ctx := context.Background() + + resp, err := ms.CreateBlockVolume(ctx, &master_pb.CreateBlockVolumeRequest{ + Name: "pvc-rf3-promo", + SizeBytes: 1 << 30, + ReplicaFactor: 3, + }) + if err != nil { + t.Fatalf("create: %v", err) + } + + entry, _ := ms.blockRegistry.Lookup("pvc-rf3-promo") + primary := resp.VolumeServer + + // Set different health scores on replicas. + for i := range entry.Replicas { + if i == 0 { + entry.Replicas[i].HealthScore = 0.7 + } else { + entry.Replicas[i].HealthScore = 1.0 + } + } + expectedPromoted := entry.Replicas[1].Server // higher health score + entry.LastLeaseGrant = time.Now().Add(-1 * time.Minute) // expired + + // Kill primary. + ms.failoverBlockVolumes(primary) + + entry, _ = ms.blockRegistry.Lookup("pvc-rf3-promo") + if entry.VolumeServer != expectedPromoted { + t.Fatalf("expected %q promoted (highest health), got %q", expectedPromoted, entry.VolumeServer) + } + // 1 replica should remain (the one not promoted). + if len(entry.Replicas) != 1 { + t.Fatalf("expected 1 remaining replica, got %d", len(entry.Replicas)) + } + // Pending rebuild for old primary. + ms.blockFailover.mu.Lock() + rebuilds := ms.blockFailover.pendingRebuilds[primary] + ms.blockFailover.mu.Unlock() + if len(rebuilds) != 1 || rebuilds[0].VolumeName != "pvc-rf3-promo" { + t.Fatalf("expected 1 pending rebuild for %q, got %+v", primary, rebuilds) + } +} + +// 3. Kill one replica → writes unaffected (primary unchanged, dead replica removed). +func TestQA_RF3_OneReplicaDies_WritesUnaffected(t *testing.T) { + ms := qaRF3Master(t) + ctx := context.Background() + + _, err := ms.CreateBlockVolume(ctx, &master_pb.CreateBlockVolumeRequest{ + Name: "pvc-rf3-repdeath", + SizeBytes: 1 << 30, + ReplicaFactor: 3, + }) + if err != nil { + t.Fatalf("create: %v", err) + } + + entry, _ := ms.blockRegistry.Lookup("pvc-rf3-repdeath") + originalPrimary := entry.VolumeServer + originalEpoch := entry.Epoch + deadReplica := entry.Replicas[1].Server // kill second replica + + ms.failoverBlockVolumes(deadReplica) + + entry, _ = ms.blockRegistry.Lookup("pvc-rf3-repdeath") + // Primary unchanged. + if entry.VolumeServer != originalPrimary { + t.Fatalf("primary should remain %q, got %q", originalPrimary, entry.VolumeServer) + } + // Epoch unchanged (no promotion). + if entry.Epoch != originalEpoch { + t.Fatalf("epoch should remain %d, got %d", originalEpoch, entry.Epoch) + } + // Dead replica removed, 1 replica remains. + if len(entry.Replicas) != 1 { + t.Fatalf("expected 1 replica after death, got %d", len(entry.Replicas)) + } + if entry.Replicas[0].Server == deadReplica { + t.Fatalf("dead replica %q should be removed", deadReplica) + } +} + +// 4. Dead replica reconnects → rebuilds and rejoins. +func TestQA_RF3_Rebuild_DeadReplicaCatchesUp(t *testing.T) { + ms := qaRF3Master(t) + ctx := context.Background() + + _, err := ms.CreateBlockVolume(ctx, &master_pb.CreateBlockVolumeRequest{ + Name: "pvc-rf3-rebuild", + SizeBytes: 1 << 30, + ReplicaFactor: 3, + }) + if err != nil { + t.Fatalf("create: %v", err) + } + + entry, _ := ms.blockRegistry.Lookup("pvc-rf3-rebuild") + deadReplica := entry.Replicas[0].Server + + // Kill one replica. + ms.failoverBlockVolumes(deadReplica) + + // Verify it's removed. + entry, _ = ms.blockRegistry.Lookup("pvc-rf3-rebuild") + if len(entry.Replicas) != 1 { + t.Fatalf("after kill: expected 1 replica, got %d", len(entry.Replicas)) + } + + // Dead replica reconnects. + ms.recoverBlockVolumes(deadReplica) + + // Verify it's back. + entry, _ = ms.blockRegistry.Lookup("pvc-rf3-rebuild") + if len(entry.Replicas) != 2 { + t.Fatalf("after reconnect: expected 2 replicas, got %d", len(entry.Replicas)) + } + + // Rebuild assignment should be queued. + assignments := ms.blockAssignmentQueue.Peek(deadReplica) + foundRebuild := false + for _, a := range assignments { + if blockvol.RoleFromWire(a.Role) == blockvol.RoleRebuilding { + foundRebuild = true + if a.RebuildAddr == "" { + t.Fatal("rebuild assignment missing RebuildAddr") + } + } + } + if !foundRebuild { + t.Fatalf("expected rebuild assignment for %q", deadReplica) + } +} + +// 5. Health score influences failover preference. +func TestQA_RF3_HealthScore_FailoverPreference(t *testing.T) { + ms := qaRF3Master(t) + ctx := context.Background() + + _, err := ms.CreateBlockVolume(ctx, &master_pb.CreateBlockVolumeRequest{ + Name: "pvc-rf3-health", + SizeBytes: 1 << 30, + ReplicaFactor: 3, + }) + if err != nil { + t.Fatalf("create: %v", err) + } + + entry, _ := ms.blockRegistry.Lookup("pvc-rf3-health") + primary := entry.VolumeServer + + // Set health scores: replica[0] = 0.3 (low), replica[1] = 0.9 (high). + entry.Replicas[0].HealthScore = 0.3 + entry.Replicas[0].WALHeadLSN = 100 + entry.Replicas[1].HealthScore = 0.9 + entry.Replicas[1].WALHeadLSN = 100 + expectedWinner := entry.Replicas[1].Server + entry.LastLeaseGrant = time.Now().Add(-1 * time.Minute) // expired + + ms.failoverBlockVolumes(primary) + + entry, _ = ms.blockRegistry.Lookup("pvc-rf3-health") + if entry.VolumeServer != expectedWinner { + t.Fatalf("expected %q (health=0.9) promoted, got %q", expectedWinner, entry.VolumeServer) + } + + // The low-health replica should remain in the list. + if len(entry.Replicas) != 1 { + t.Fatalf("expected 1 remaining replica, got %d", len(entry.Replicas)) + } + if entry.Replicas[0].HealthScore != 0.3 { + t.Fatalf("remaining replica health: got %.1f, want 0.3", entry.Replicas[0].HealthScore) + } +} + +// 6. RF=2 backward compat: identical behavior to pre-CP8-2. +func TestQA_RF3_BackwardCompat_RF2_Unchanged(t *testing.T) { + ms := qaRF3Master(t) + ctx := context.Background() + + // Create with default RF (should be 2). + resp, err := ms.CreateBlockVolume(ctx, &master_pb.CreateBlockVolumeRequest{ + Name: "pvc-rf2-compat", + SizeBytes: 1 << 30, + }) + if err != nil { + t.Fatalf("create: %v", err) + } + + entry, _ := ms.blockRegistry.Lookup("pvc-rf2-compat") + + // RF should default to 2. + if entry.ReplicaFactor != 2 { + t.Fatalf("RF default: got %d, want 2", entry.ReplicaFactor) + } + // Exactly 1 replica. + if len(entry.Replicas) != 1 { + t.Fatalf("expected 1 replica for RF=2, got %d", len(entry.Replicas)) + } + + // Failover should work identically. + primary := resp.VolumeServer + entry.LastLeaseGrant = time.Now().Add(-1 * time.Minute) + ms.failoverBlockVolumes(primary) + + entry, _ = ms.blockRegistry.Lookup("pvc-rf2-compat") + if entry.VolumeServer == primary { + t.Fatalf("failover failed: primary should have changed from %q", primary) + } + // No replicas left after RF=2 promotion. + if len(entry.Replicas) != 0 { + t.Fatalf("expected 0 replicas after RF=2 promotion, got %d", len(entry.Replicas)) + } + + // Recover → rebuild. + ms.recoverBlockVolumes(primary) + entry, _ = ms.blockRegistry.Lookup("pvc-rf2-compat") + if len(entry.Replicas) != 1 || entry.Replicas[0].Server != primary { + t.Fatalf("after recovery: expected %q as replica, got %+v", primary, entry.Replicas) + } +} + +// 7. Full lifecycle: create RF=3 → kill primary → promote → kill another → promote → recover all. +func TestQA_RF3_FullLifecycle(t *testing.T) { + ms := qaRF3Master(t) + ctx := context.Background() + + resp, err := ms.CreateBlockVolume(ctx, &master_pb.CreateBlockVolumeRequest{ + Name: "pvc-rf3-lifecycle", + SizeBytes: 1 << 30, + ReplicaFactor: 3, + }) + if err != nil { + t.Fatalf("create: %v", err) + } + + entry, _ := ms.blockRegistry.Lookup("pvc-rf3-lifecycle") + vs1 := resp.VolumeServer + vs2 := entry.Replicas[0].Server + vs3 := entry.Replicas[1].Server + entry.LastLeaseGrant = time.Now().Add(-1 * time.Minute) + + // Step 1: Kill vs1 (primary). One of vs2/vs3 promoted. + ms.failoverBlockVolumes(vs1) + entry, _ = ms.blockRegistry.Lookup("pvc-rf3-lifecycle") + if entry.VolumeServer == vs1 { + t.Fatal("step1: primary should not be vs1 after failover") + } + step1Primary := entry.VolumeServer + if len(entry.Replicas) != 1 { + t.Fatalf("step1: expected 1 replica, got %d", len(entry.Replicas)) + } + step1Replica := entry.Replicas[0].Server + + // Step 2: Recover vs1 → becomes replica (rebuilding). + ms.recoverBlockVolumes(vs1) + entry, _ = ms.blockRegistry.Lookup("pvc-rf3-lifecycle") + if len(entry.Replicas) != 2 { + t.Fatalf("step2: expected 2 replicas after vs1 recovery, got %d", len(entry.Replicas)) + } + + // Step 3: Kill step1Primary. The surviving replica (step1Replica or vs1) should be promoted. + entry.LastLeaseGrant = time.Now().Add(-1 * time.Minute) + // Set health scores: vs1 low (just rebuilt), step1Replica high. + for i := range entry.Replicas { + if entry.Replicas[i].Server == vs1 { + entry.Replicas[i].HealthScore = 0.5 + } else { + entry.Replicas[i].HealthScore = 1.0 + } + } + ms.failoverBlockVolumes(step1Primary) + + entry, _ = ms.blockRegistry.Lookup("pvc-rf3-lifecycle") + if entry.VolumeServer == step1Primary { + t.Fatal("step3: primary should not be step1Primary after second failover") + } + // The higher health replica should win. + if entry.VolumeServer != step1Replica { + t.Fatalf("step3: expected %q promoted (health=1.0), got %q", step1Replica, entry.VolumeServer) + } + + // Step 4: Recover step1Primary → becomes replica. + ms.recoverBlockVolumes(step1Primary) + entry, _ = ms.blockRegistry.Lookup("pvc-rf3-lifecycle") + // Should have 2 replicas now (vs1 from step2 still there + step1Primary recovered). + // Actually vs1's health was 0.5 and was NOT promoted, so vs1 stays as replica. + if len(entry.Replicas) < 1 { + t.Fatalf("step4: expected at least 1 replica after recovery, got %d", len(entry.Replicas)) + } + + // Step 5: Delete. + _, err = ms.DeleteBlockVolume(ctx, &master_pb.DeleteBlockVolumeRequest{Name: "pvc-rf3-lifecycle"}) + if err != nil { + t.Fatalf("delete: %v", err) + } + if _, ok := ms.blockRegistry.Lookup("pvc-rf3-lifecycle"); ok { + t.Fatal("volume should be deleted") + } + + _ = vs2 + _ = vs3 +} diff --git a/weed/server/volume_grpc_block.go b/weed/server/volume_grpc_block.go index 97f858affd..a3aaa553e9 100644 --- a/weed/server/volume_grpc_block.go +++ b/weed/server/volume_grpc_block.go @@ -57,3 +57,87 @@ func (vs *VolumeServer) VolumeServerDeleteBlockVolume(_ context.Context, req *vo return &volume_server_pb.VolumeServerDeleteBlockVolumeResponse{}, nil } + +// SnapshotBlockVolume creates a snapshot on a block volume. +func (vs *VolumeServer) SnapshotBlockVolume(_ context.Context, req *volume_server_pb.SnapshotBlockVolumeRequest) (*volume_server_pb.SnapshotBlockVolumeResponse, error) { + if vs.blockService == nil { + return nil, fmt.Errorf("block service not enabled on this volume server") + } + if req.Name == "" { + return nil, fmt.Errorf("name is required") + } + + createdAt, sizeBytes, err := vs.blockService.SnapshotBlockVol(req.Name, req.SnapshotId) + if err != nil { + return nil, fmt.Errorf("snapshot block volume %q: %w", req.Name, err) + } + + return &volume_server_pb.SnapshotBlockVolumeResponse{ + SnapshotId: req.SnapshotId, + CreatedAt: createdAt, + SizeBytes: sizeBytes, + }, nil +} + +// DeleteBlockSnapshot deletes a snapshot from a block volume. +func (vs *VolumeServer) DeleteBlockSnapshot(_ context.Context, req *volume_server_pb.DeleteBlockSnapshotRequest) (*volume_server_pb.DeleteBlockSnapshotResponse, error) { + if vs.blockService == nil { + return nil, fmt.Errorf("block service not enabled on this volume server") + } + if req.Name == "" { + return nil, fmt.Errorf("name is required") + } + + if err := vs.blockService.DeleteBlockSnapshot(req.Name, req.SnapshotId); err != nil { + return nil, fmt.Errorf("delete block snapshot %q/%d: %w", req.Name, req.SnapshotId, err) + } + + return &volume_server_pb.DeleteBlockSnapshotResponse{}, nil +} + +// ListBlockSnapshots lists all snapshots on a block volume. +func (vs *VolumeServer) ListBlockSnapshots(_ context.Context, req *volume_server_pb.ListBlockSnapshotsRequest) (*volume_server_pb.ListBlockSnapshotsResponse, error) { + if vs.blockService == nil { + return nil, fmt.Errorf("block service not enabled on this volume server") + } + if req.Name == "" { + return nil, fmt.Errorf("name is required") + } + + infos, volSize, err := vs.blockService.ListBlockSnapshots(req.Name) + if err != nil { + return nil, fmt.Errorf("list block snapshots %q: %w", req.Name, err) + } + + resp := &volume_server_pb.ListBlockSnapshotsResponse{} + for _, s := range infos { + resp.Snapshots = append(resp.Snapshots, &volume_server_pb.BlockSnapshotInfo{ + SnapshotId: s.ID, + CreatedAt: s.CreatedAt.Unix(), + VolumeSizeBytes: volSize, + }) + } + return resp, nil +} + +// ExpandBlockVolume expands a block volume to a new size. +func (vs *VolumeServer) ExpandBlockVolume(_ context.Context, req *volume_server_pb.ExpandBlockVolumeRequest) (*volume_server_pb.ExpandBlockVolumeResponse, error) { + if vs.blockService == nil { + return nil, fmt.Errorf("block service not enabled on this volume server") + } + if req.Name == "" { + return nil, fmt.Errorf("name is required") + } + if req.NewSizeBytes == 0 { + return nil, fmt.Errorf("new_size_bytes must be > 0") + } + + actualSize, err := vs.blockService.ExpandBlockVol(req.Name, req.NewSizeBytes) + if err != nil { + return nil, fmt.Errorf("expand block volume %q: %w", req.Name, err) + } + + return &volume_server_pb.ExpandBlockVolumeResponse{ + CapacityBytes: actualSize, + }, nil +} diff --git a/weed/server/volume_grpc_block_test.go b/weed/server/volume_grpc_block_test.go index b2bab8d984..53942a3212 100644 --- a/weed/server/volume_grpc_block_test.go +++ b/weed/server/volume_grpc_block_test.go @@ -108,3 +108,96 @@ func TestVS_DeleteNotFound(t *testing.T) { t.Fatalf("delete non-existent should not error: %v", err) } } + +func TestVS_SnapshotBlockVol(t *testing.T) { + bs, _ := newTestBlockServiceWithDir(t) + + bs.CreateBlockVol("snap-vol", 4*1024*1024, "") + + createdAt, sizeBytes, err := bs.SnapshotBlockVol("snap-vol", 1) + if err != nil { + t.Fatalf("SnapshotBlockVol: %v", err) + } + if createdAt == 0 { + t.Fatal("createdAt should not be zero") + } + if sizeBytes == 0 { + t.Fatal("sizeBytes should not be zero") + } +} + +func TestVS_SnapshotVolumeNotFound(t *testing.T) { + bs, _ := newTestBlockServiceWithDir(t) + + _, _, err := bs.SnapshotBlockVol("nonexistent", 1) + if err == nil { + t.Fatal("expected error for nonexistent volume") + } +} + +func TestVS_DeleteBlockSnapshot(t *testing.T) { + bs, _ := newTestBlockServiceWithDir(t) + + bs.CreateBlockVol("snap-vol", 4*1024*1024, "") + bs.SnapshotBlockVol("snap-vol", 1) + + if err := bs.DeleteBlockSnapshot("snap-vol", 1); err != nil { + t.Fatalf("DeleteBlockSnapshot: %v", err) + } + + // Delete again should be idempotent. + if err := bs.DeleteBlockSnapshot("snap-vol", 1); err != nil { + t.Fatalf("idempotent delete: %v", err) + } +} + +func TestVS_ListBlockSnapshots(t *testing.T) { + bs, _ := newTestBlockServiceWithDir(t) + + bs.CreateBlockVol("snap-vol", 4*1024*1024, "") + bs.SnapshotBlockVol("snap-vol", 1) + bs.SnapshotBlockVol("snap-vol", 2) + + infos, volSize, err := bs.ListBlockSnapshots("snap-vol") + if err != nil { + t.Fatalf("ListBlockSnapshots: %v", err) + } + if len(infos) != 2 { + t.Fatalf("expected 2 snapshots, got %d", len(infos)) + } + if volSize != 4*1024*1024 { + t.Fatalf("volSize: got %d, want %d", volSize, 4*1024*1024) + } +} + +func TestVS_ListSnapshotsVolumeNotFound(t *testing.T) { + bs, _ := newTestBlockServiceWithDir(t) + + _, _, err := bs.ListBlockSnapshots("nonexistent") + if err == nil { + t.Fatal("expected error for nonexistent volume") + } +} + +func TestVS_ExpandBlockVol(t *testing.T) { + bs, _ := newTestBlockServiceWithDir(t) + + bs.CreateBlockVol("expand-vol", 4*1024*1024, "") + + actualSize, err := bs.ExpandBlockVol("expand-vol", 8*1024*1024) + if err != nil { + t.Fatalf("ExpandBlockVol: %v", err) + } + if actualSize != 8*1024*1024 { + t.Fatalf("expected 8MiB, got %d", actualSize) + } +} + +func TestVS_ExpandVolumeNotFound(t *testing.T) { + bs, _ := newTestBlockServiceWithDir(t) + + _, err := bs.ExpandBlockVol("nonexistent", 8*1024*1024) + if err == nil { + t.Fatal("expected error for nonexistent volume") + } +} diff --git a/weed/server/volume_server_block.go b/weed/server/volume_server_block.go index 977ed2280a..543e177f0d 100644 --- a/weed/server/volume_server_block.go +++ b/weed/server/volume_server_block.go @@ -228,7 +228,10 @@ func (bs *BlockService) ProcessAssignments(assignments []blockvol.BlockVolumeAss // 2. Replication setup based on role + addresses. switch role { case blockvol.RolePrimary: - if a.ReplicaDataAddr != "" && a.ReplicaCtrlAddr != "" { + // CP8-2: ReplicaAddrs (multi-replica) takes precedence over scalar fields. + if len(a.ReplicaAddrs) > 0 { + bs.setupPrimaryReplicationMulti(a.Path, a.ReplicaAddrs) + } else if a.ReplicaDataAddr != "" && a.ReplicaCtrlAddr != "" { bs.setupPrimaryReplication(a.Path, a.ReplicaDataAddr, a.ReplicaCtrlAddr) } case blockvol.RoleReplica: @@ -279,6 +282,43 @@ func (bs *BlockService) setupPrimaryReplication(path, replicaDataAddr, replicaCt glog.V(0).Infof("block service: primary %s shipping WAL to %s/%s (rebuild=%s)", path, replicaDataAddr, replicaCtrlAddr, rebuildAddr) } +// setupPrimaryReplicationMulti configures WAL shipping from primary to N replicas +// using SetReplicaAddrs (CP8-2: multi-replica support). +func (bs *BlockService) setupPrimaryReplicationMulti(path string, addrs []blockvol.ReplicaAddr) { + // Compute deterministic rebuild listen address. + _, _, rebuildPort := bs.ReplicationPorts(path) + host := bs.listenAddr + if idx := strings.LastIndex(host, ":"); idx >= 0 { + host = host[:idx] + } + rebuildAddr := fmt.Sprintf("%s:%d", host, rebuildPort) + + if err := bs.blockStore.WithVolume(path, func(vol *blockvol.BlockVol) error { + vol.SetReplicaAddrs(addrs) + if err := vol.StartRebuildServer(rebuildAddr); err != nil { + glog.Warningf("block service: start rebuild server %s on %s: %v", path, rebuildAddr, err) + } + return nil + }); err != nil { + glog.Warningf("block service: setup primary replication (multi) %s: %v", path, err) + return + } + // Track replication state for heartbeat reporting. + bs.replMu.Lock() + if bs.replStates == nil { + bs.replStates = make(map[string]*volReplState) + } + // Store first replica in replState for backward compat heartbeat reporting. + if len(addrs) > 0 { + bs.replStates[path] = &volReplState{ + replicaDataAddr: addrs[0].DataAddr, + replicaCtrlAddr: addrs[0].CtrlAddr, + } + } + bs.replMu.Unlock() + glog.V(0).Infof("block service: primary %s shipping WAL to %d replicas (rebuild=%s)", path, len(addrs), rebuildAddr) +} + // setupReplicaReceiver starts the replica WAL receiver. func (bs *BlockService) setupReplicaReceiver(path, dataAddr, ctrlAddr string) { if err := bs.blockStore.WithVolume(path, func(vol *blockvol.BlockVol) error { @@ -318,6 +358,72 @@ func (bs *BlockService) startRebuild(path, rebuildAddr string, epoch uint64) { }() } +// SnapshotBlockVol creates a snapshot on the named volume. +func (bs *BlockService) SnapshotBlockVol(name string, snapID uint32) (createdAt int64, sizeBytes uint64, err error) { + path := bs.volumePath(name) + err = bs.blockStore.WithVolume(path, func(vol *blockvol.BlockVol) error { + if serr := vol.CreateSnapshot(snapID); serr != nil { + return serr + } + // Find the snapshot we just created for its metadata. + for _, s := range vol.ListSnapshots() { + if s.ID == snapID { + createdAt = s.CreatedAt.Unix() + sizeBytes = vol.Info().VolumeSize + return nil + } + } + return fmt.Errorf("snapshot %d created but not found in list", snapID) + }) + return +} + +// DeleteBlockSnapshot deletes a snapshot on the named volume. +// Idempotent: returns nil if snapshot does not exist. +func (bs *BlockService) DeleteBlockSnapshot(name string, snapID uint32) error { + path := bs.volumePath(name) + return bs.blockStore.WithVolume(path, func(vol *blockvol.BlockVol) error { + err := vol.DeleteSnapshot(snapID) + if err != nil && err.Error() == "blockvol: snapshot not found" { + return nil // idempotent + } + return err + }) +} + +// ListBlockSnapshots lists all snapshots on the named volume. +func (bs *BlockService) ListBlockSnapshots(name string) ([]blockvol.SnapshotInfo, uint64, error) { + path := bs.volumePath(name) + var infos []blockvol.SnapshotInfo + var volSize uint64 + err := bs.blockStore.WithVolume(path, func(vol *blockvol.BlockVol) error { + infos = vol.ListSnapshots() + volSize = vol.Info().VolumeSize + return nil + }) + return infos, volSize, err +} + +// ExpandBlockVol expands the named volume to newSize bytes. +func (bs *BlockService) ExpandBlockVol(name string, newSize uint64) (uint64, error) { + path := bs.volumePath(name) + var actualSize uint64 + err := bs.blockStore.WithVolume(path, func(vol *blockvol.BlockVol) error { + if eerr := vol.Expand(newSize); eerr != nil { + return eerr + } + actualSize = vol.Info().VolumeSize + return nil + }) + return actualSize, err +} + +// volumePath converts a volume name to its .blk file path. +func (bs *BlockService) volumePath(name string) string { + sanitized := blockvol.SanitizeFilename(name) + return filepath.Join(bs.blockDir, sanitized+".blk") +} + // GetReplState returns the replication state for a volume path. func (bs *BlockService) GetReplState(path string) (dataAddr, ctrlAddr string) { bs.replMu.RLock() @@ -374,3 +480,9 @@ func (bs *BlockService) Shutdown() { bs.blockStore.Close() glog.V(0).Infof("block service: shut down") } + +// SetBlockService wires a BlockService into the VolumeServer so that +// heartbeats include block volume info and the server is marked block-capable. +func (vs *VolumeServer) SetBlockService(bs *BlockService) { + vs.blockService = bs +} diff --git a/weed/shell/command_block_volume.go b/weed/shell/command_block_volume.go new file mode 100644 index 0000000000..59a7990614 --- /dev/null +++ b/weed/shell/command_block_volume.go @@ -0,0 +1,220 @@ +package shell + +import ( + "context" + "flag" + "fmt" + "io" + "strings" + + "github.com/seaweedfs/seaweedfs/weed/storage/blockvol/blockapi" +) + +func init() { + Commands = append(Commands, &commandBlockCreate{}) + Commands = append(Commands, &commandBlockList{}) + Commands = append(Commands, &commandBlockStatus{}) + Commands = append(Commands, &commandBlockDelete{}) + Commands = append(Commands, &commandBlockAssign{}) +} + +// mastersHTTP builds a comma-separated list of "http://host:port" URLs from +// the shell environment's master addresses. +func mastersHTTP(commandEnv *CommandEnv) string { + addrs := commandEnv.MasterClient.GetMasters(context.Background()) + parts := make([]string, len(addrs)) + for i, a := range addrs { + s := string(a) + if !strings.HasPrefix(s, "http://") && !strings.HasPrefix(s, "https://") { + s = "http://" + s + } + parts[i] = s + } + return strings.Join(parts, ",") +} + +// ---- block.create ---- + +type commandBlockCreate struct{} + +func (c *commandBlockCreate) Name() string { return "block.create" } +func (c *commandBlockCreate) Help() string { + return `create a block volume + + block.create -name -size [-replicaPlacement ] [-disk ] +` +} +func (c *commandBlockCreate) HasTag(CommandTag) bool { return false } + +func (c *commandBlockCreate) Do(args []string, commandEnv *CommandEnv, writer io.Writer) error { + f := flag.NewFlagSet(c.Name(), flag.ContinueOnError) + name := f.String("name", "", "volume name") + size := f.Uint64("size", 0, "volume size in bytes") + placement := f.String("replicaPlacement", "000", "placement string: 000, 001, 010, 100") + disk := f.String("disk", "", "disk type (e.g. ssd, hdd)") + if err := f.Parse(args); err != nil { + return nil + } + if *name == "" || *size == 0 { + return fmt.Errorf("both -name and -size are required") + } + + client := blockapi.NewClient(mastersHTTP(commandEnv)) + info, err := client.CreateVolume(context.Background(), blockapi.CreateVolumeRequest{ + Name: *name, + SizeBytes: *size, + ReplicaPlacement: *placement, + DiskType: *disk, + }) + if err != nil { + return err + } + fmt.Fprintf(writer, "created block volume %q on %s (iqn=%s, iscsi=%s)\n", + info.Name, info.VolumeServer, info.IQN, info.ISCSIAddr) + if info.ReplicaServer != "" { + fmt.Fprintf(writer, " replica on %s (iqn=%s, iscsi=%s)\n", + info.ReplicaServer, info.ReplicaIQN, info.ReplicaISCSIAddr) + } + return nil +} + +// ---- block.list ---- + +type commandBlockList struct{} + +func (c *commandBlockList) Name() string { return "block.list" } +func (c *commandBlockList) Help() string { + return `list all block volumes + + block.list +` +} +func (c *commandBlockList) HasTag(CommandTag) bool { return false } + +func (c *commandBlockList) Do(args []string, commandEnv *CommandEnv, writer io.Writer) error { + client := blockapi.NewClient(mastersHTTP(commandEnv)) + vols, err := client.ListVolumes(context.Background()) + if err != nil { + return err + } + if len(vols) == 0 { + fmt.Fprintln(writer, "no block volumes") + return nil + } + fmt.Fprintf(writer, "%-20s %-20s %-12s %-8s %-8s %-20s\n", + "NAME", "SERVER", "SIZE", "EPOCH", "ROLE", "STATUS") + for _, v := range vols { + fmt.Fprintf(writer, "%-20s %-20s %-12d %-8d %-8s %-20s\n", + v.Name, v.VolumeServer, v.SizeBytes, v.Epoch, v.Role, v.Status) + } + return nil +} + +// ---- block.status ---- + +type commandBlockStatus struct{} + +func (c *commandBlockStatus) Name() string { return "block.status" } +func (c *commandBlockStatus) Help() string { + return `show status of a block volume + + block.status +` +} +func (c *commandBlockStatus) HasTag(CommandTag) bool { return false } + +func (c *commandBlockStatus) Do(args []string, commandEnv *CommandEnv, writer io.Writer) error { + if len(args) == 0 { + return fmt.Errorf("usage: block.status ") + } + name := args[0] + + client := blockapi.NewClient(mastersHTTP(commandEnv)) + info, err := client.LookupVolume(context.Background(), name) + if err != nil { + return err + } + fmt.Fprintf(writer, "Name: %s\n", info.Name) + fmt.Fprintf(writer, "VolumeServer: %s\n", info.VolumeServer) + fmt.Fprintf(writer, "SizeBytes: %d\n", info.SizeBytes) + fmt.Fprintf(writer, "Epoch: %d\n", info.Epoch) + fmt.Fprintf(writer, "Role: %s\n", info.Role) + fmt.Fprintf(writer, "Status: %s\n", info.Status) + fmt.Fprintf(writer, "ISCSIAddr: %s\n", info.ISCSIAddr) + fmt.Fprintf(writer, "IQN: %s\n", info.IQN) + if info.ReplicaServer != "" { + fmt.Fprintf(writer, "ReplicaServer: %s\n", info.ReplicaServer) + fmt.Fprintf(writer, "ReplicaISCSI: %s\n", info.ReplicaISCSIAddr) + fmt.Fprintf(writer, "ReplicaIQN: %s\n", info.ReplicaIQN) + fmt.Fprintf(writer, "ReplicaData: %s\n", info.ReplicaDataAddr) + fmt.Fprintf(writer, "ReplicaCtrl: %s\n", info.ReplicaCtrlAddr) + } + return nil +} + +// ---- block.delete ---- + +type commandBlockDelete struct{} + +func (c *commandBlockDelete) Name() string { return "block.delete" } +func (c *commandBlockDelete) Help() string { + return `delete a block volume + + block.delete +` +} +func (c *commandBlockDelete) HasTag(CommandTag) bool { return false } + +func (c *commandBlockDelete) Do(args []string, commandEnv *CommandEnv, writer io.Writer) error { + if len(args) == 0 { + return fmt.Errorf("usage: block.delete ") + } + name := args[0] + + client := blockapi.NewClient(mastersHTTP(commandEnv)) + if err := client.DeleteVolume(context.Background(), name); err != nil { + return err + } + fmt.Fprintf(writer, "deleted block volume %q\n", name) + return nil +} + +// ---- block.assign ---- + +type commandBlockAssign struct{} + +func (c *commandBlockAssign) Name() string { return "block.assign" } +func (c *commandBlockAssign) Help() string { + return `assign a role to a block volume + + block.assign -name -epoch -role [-lease ] +` +} +func (c *commandBlockAssign) HasTag(CommandTag) bool { return false } + +func (c *commandBlockAssign) Do(args []string, commandEnv *CommandEnv, writer io.Writer) error { + f := flag.NewFlagSet(c.Name(), flag.ContinueOnError) + name := f.String("name", "", "volume name") + epoch := f.Uint64("epoch", 0, "epoch number") + role := f.String("role", "", "role: primary or replica") + leaseTTLMs := f.Uint64("lease", 30000, "lease TTL in milliseconds") + if err := f.Parse(args); err != nil { + return nil + } + if *name == "" || *role == "" { + return fmt.Errorf("both -name and -role are required") + } + + client := blockapi.NewClient(mastersHTTP(commandEnv)) + if err := client.AssignRole(context.Background(), blockapi.AssignRequest{ + Name: *name, + Epoch: *epoch, + Role: *role, + LeaseTTLMs: *leaseTTLMs, + }); err != nil { + return err + } + fmt.Fprintf(writer, "assignment queued: %s epoch=%d role=%s lease=%dms\n", + *name, *epoch, *role, *leaseTTLMs) + return nil +} diff --git a/weed/storage/blockvol/block_heartbeat.go b/weed/storage/blockvol/block_heartbeat.go index c5698bcd9e..c062ec85b7 100644 --- a/weed/storage/blockvol/block_heartbeat.go +++ b/weed/storage/blockvol/block_heartbeat.go @@ -8,17 +8,21 @@ import ( // BlockVolumeInfoMessage is the heartbeat status for one block volume. // Mirrors the proto message that will be generated from master.proto. type BlockVolumeInfoMessage struct { - Path string // volume file path (unique ID on this server) - VolumeSize uint64 // logical size in bytes - BlockSize uint32 // block size in bytes - Epoch uint64 // current fencing epoch - Role uint32 // blockvol.Role as uint32 for wire compat - WalHeadLsn uint64 // WAL head LSN - CheckpointLsn uint64 // last flushed LSN - HasLease bool // whether volume holds a valid lease - DiskType string // e.g., "ssd", "hdd" - ReplicaDataAddr string // receiver data listen addr (VS reports in heartbeat) - ReplicaCtrlAddr string // receiver ctrl listen addr + Path string // volume file path (unique ID on this server) + VolumeSize uint64 // logical size in bytes + BlockSize uint32 // block size in bytes + Epoch uint64 // current fencing epoch + Role uint32 // blockvol.Role as uint32 for wire compat + WalHeadLsn uint64 // WAL head LSN + CheckpointLsn uint64 // last flushed LSN + HasLease bool // whether volume holds a valid lease + DiskType string // e.g., "ssd", "hdd" + ReplicaDataAddr string // receiver data listen addr (VS reports in heartbeat) + ReplicaCtrlAddr string // receiver ctrl listen addr + HealthScore float64 // CP8-2: 0.0-1.0 + ScrubErrors int64 // CP8-2: lifetime scrub error count + LastScrubTime int64 // CP8-2: unix seconds + ReplicaDegraded bool // CP8-2: true if any replica shipper degraded } // BlockVolumeShortInfoMessage is used for delta heartbeats @@ -33,13 +37,14 @@ type BlockVolumeShortInfoMessage struct { // BlockVolumeAssignment carries a role/epoch/lease assignment // from master to volume server for one block volume. type BlockVolumeAssignment struct { - Path string // which block volume - Epoch uint64 // new epoch - Role uint32 // target role (blockvol.Role as uint32) - LeaseTtlMs uint32 // lease TTL in milliseconds (0 = no lease) - ReplicaDataAddr string // where primary ships WAL data - ReplicaCtrlAddr string // where primary sends barriers - RebuildAddr string // where rebuild server listens + Path string // which block volume + Epoch uint64 // new epoch + Role uint32 // target role (blockvol.Role as uint32) + LeaseTtlMs uint32 // lease TTL in milliseconds (0 = no lease) + ReplicaDataAddr string // where primary ships WAL data (scalar, RF=2 compat) + ReplicaCtrlAddr string // where primary sends barriers (scalar, RF=2 compat) + RebuildAddr string // where rebuild server listens + ReplicaAddrs []ReplicaAddr // CP8-2: multi-replica addrs (precedence over scalar) } // ToBlockVolumeInfoMessage converts a BlockVol's current state @@ -48,16 +53,21 @@ type BlockVolumeAssignment struct { func ToBlockVolumeInfoMessage(path, diskType string, vol *BlockVol) BlockVolumeInfoMessage { info := vol.Info() status := vol.Status() + hs := vol.HealthStats() return BlockVolumeInfoMessage{ - Path: path, - VolumeSize: info.VolumeSize, - BlockSize: info.BlockSize, - Epoch: status.Epoch, - Role: RoleToWire(status.Role), - WalHeadLsn: status.WALHeadLSN, - CheckpointLsn: status.CheckpointLSN, - HasLease: status.HasLease, - DiskType: diskType, + Path: path, + VolumeSize: info.VolumeSize, + BlockSize: info.BlockSize, + Epoch: status.Epoch, + Role: RoleToWire(status.Role), + WalHeadLsn: status.WALHeadLSN, + CheckpointLsn: status.CheckpointLSN, + HasLease: status.HasLease, + DiskType: diskType, + HealthScore: status.HealthScore, + ScrubErrors: hs.ScrubErrors, + LastScrubTime: hs.LastScrubTime, + ReplicaDegraded: status.ReplicaDegraded, } } diff --git a/weed/storage/blockvol/block_heartbeat_proto.go b/weed/storage/blockvol/block_heartbeat_proto.go index 1f015ad617..d4f1bd6941 100644 --- a/weed/storage/blockvol/block_heartbeat_proto.go +++ b/weed/storage/blockvol/block_heartbeat_proto.go @@ -18,6 +18,10 @@ func InfoMessageToProto(m BlockVolumeInfoMessage) *master_pb.BlockVolumeInfoMess DiskType: m.DiskType, ReplicaDataAddr: m.ReplicaDataAddr, ReplicaCtrlAddr: m.ReplicaCtrlAddr, + HealthScore: m.HealthScore, + ScrubErrors: m.ScrubErrors, + LastScrubTime: m.LastScrubTime, + ReplicaDegraded: m.ReplicaDegraded, } } @@ -38,6 +42,10 @@ func InfoMessageFromProto(p *master_pb.BlockVolumeInfoMessage) BlockVolumeInfoMe DiskType: p.DiskType, ReplicaDataAddr: p.ReplicaDataAddr, ReplicaCtrlAddr: p.ReplicaCtrlAddr, + HealthScore: p.HealthScore, + ScrubErrors: p.ScrubErrors, + LastScrubTime: p.LastScrubTime, + ReplicaDegraded: p.ReplicaDegraded, } } @@ -84,7 +92,7 @@ func ShortInfoFromProto(p *master_pb.BlockVolumeShortInfoMessage) BlockVolumeSho // AssignmentToProto converts a Go assignment to proto. func AssignmentToProto(a BlockVolumeAssignment) *master_pb.BlockVolumeAssignment { - return &master_pb.BlockVolumeAssignment{ + pb := &master_pb.BlockVolumeAssignment{ Path: a.Path, Epoch: a.Epoch, Role: a.Role, @@ -93,22 +101,42 @@ func AssignmentToProto(a BlockVolumeAssignment) *master_pb.BlockVolumeAssignment ReplicaCtrlAddr: a.ReplicaCtrlAddr, RebuildAddr: a.RebuildAddr, } + for _, ra := range a.ReplicaAddrs { + pb.ReplicaAddrs = append(pb.ReplicaAddrs, &master_pb.ReplicaAddrMessage{ + DataAddr: ra.DataAddr, + CtrlAddr: ra.CtrlAddr, + }) + } + return pb } // AssignmentFromProto converts a proto assignment to Go wire type. +// Precedence: if ReplicaAddrs is non-empty, use it and ignore scalar fields. func AssignmentFromProto(p *master_pb.BlockVolumeAssignment) BlockVolumeAssignment { if p == nil { return BlockVolumeAssignment{} } - return BlockVolumeAssignment{ - Path: p.Path, - Epoch: p.Epoch, - Role: p.Role, - LeaseTtlMs: p.LeaseTtlMs, - ReplicaDataAddr: p.ReplicaDataAddr, - ReplicaCtrlAddr: p.ReplicaCtrlAddr, - RebuildAddr: p.RebuildAddr, + a := BlockVolumeAssignment{ + Path: p.Path, + Epoch: p.Epoch, + Role: p.Role, + LeaseTtlMs: p.LeaseTtlMs, + RebuildAddr: p.RebuildAddr, } + if len(p.ReplicaAddrs) > 0 { + // Multi-replica: populate ReplicaAddrs, leave scalar fields empty. + for _, ra := range p.ReplicaAddrs { + a.ReplicaAddrs = append(a.ReplicaAddrs, ReplicaAddr{ + DataAddr: ra.DataAddr, + CtrlAddr: ra.CtrlAddr, + }) + } + } else { + // Backward compat: use scalar fields. + a.ReplicaDataAddr = p.ReplicaDataAddr + a.ReplicaCtrlAddr = p.ReplicaCtrlAddr + } + return a } // AssignmentsToProto converts a slice of Go assignments to proto. diff --git a/weed/storage/blockvol/block_heartbeat_proto_test.go b/weed/storage/blockvol/block_heartbeat_proto_test.go index 237cad0d97..88ced4e122 100644 --- a/weed/storage/blockvol/block_heartbeat_proto_test.go +++ b/weed/storage/blockvol/block_heartbeat_proto_test.go @@ -1,6 +1,7 @@ package blockvol import ( + "reflect" "testing" ) @@ -46,7 +47,7 @@ func TestAssignmentRoundTrip(t *testing.T) { } pb := AssignmentToProto(orig) back := AssignmentFromProto(pb) - if back != orig { + if !reflect.DeepEqual(back, orig) { t.Fatalf("round-trip mismatch:\n got %+v\n want %+v", back, orig) } } @@ -80,7 +81,7 @@ func TestAssignmentRoundTripWithReplicaAddrs(t *testing.T) { } pb := AssignmentToProto(orig) back := AssignmentFromProto(pb) - if back != orig { + if !reflect.DeepEqual(back, orig) { t.Fatalf("round-trip mismatch:\n got %+v\n want %+v", back, orig) } } @@ -195,7 +196,199 @@ func TestNilProtoConversions(t *testing.T) { t.Fatalf("nil short proto should yield zero value, got %+v", short) } assign := AssignmentFromProto(nil) - if assign != (BlockVolumeAssignment{}) { + if !reflect.DeepEqual(assign, BlockVolumeAssignment{}) { t.Fatalf("nil assignment proto should yield zero value, got %+v", assign) } } + +// --- CP8-2 new tests --- + +func TestInfoMessage_HealthScoreRoundTrip(t *testing.T) { + orig := BlockVolumeInfoMessage{ + Path: "/data/vol.blk", + VolumeSize: 1 << 30, + BlockSize: 4096, + Epoch: 5, + HealthScore: 0.85, + ScrubErrors: 3, + LastScrubTime: 1709000000, + } + pb := InfoMessageToProto(orig) + back := InfoMessageFromProto(pb) + if back.HealthScore != 0.85 { + t.Fatalf("HealthScore: got %f, want 0.85", back.HealthScore) + } + if back.ScrubErrors != 3 { + t.Fatalf("ScrubErrors: got %d, want 3", back.ScrubErrors) + } + if back.LastScrubTime != 1709000000 { + t.Fatalf("LastScrubTime: got %d, want 1709000000", back.LastScrubTime) + } +} + +func TestInfoMessage_ReplicaDegradedRoundTrip(t *testing.T) { + orig := BlockVolumeInfoMessage{ + Path: "/data/vol.blk", + Epoch: 1, + ReplicaDegraded: true, + } + pb := InfoMessageToProto(orig) + back := InfoMessageFromProto(pb) + if !back.ReplicaDegraded { + t.Fatal("ReplicaDegraded should be true after round-trip") + } + + // Also verify false round-trips. + orig.ReplicaDegraded = false + pb = InfoMessageToProto(orig) + back = InfoMessageFromProto(pb) + if back.ReplicaDegraded { + t.Fatal("ReplicaDegraded should be false after round-trip") + } +} + +func TestAssignment_MultiReplicaRoundTrip(t *testing.T) { + orig := BlockVolumeAssignment{ + Path: "/data/vol.blk", + Epoch: 10, + Role: RoleToWire(RolePrimary), + LeaseTtlMs: 30000, + ReplicaAddrs: []ReplicaAddr{ + {DataAddr: "10.0.0.2:14260", CtrlAddr: "10.0.0.2:14261"}, + {DataAddr: "10.0.0.3:14260", CtrlAddr: "10.0.0.3:14261"}, + }, + } + pb := AssignmentToProto(orig) + back := AssignmentFromProto(pb) + + if len(back.ReplicaAddrs) != 2 { + t.Fatalf("ReplicaAddrs len: got %d, want 2", len(back.ReplicaAddrs)) + } + if back.ReplicaAddrs[0].DataAddr != "10.0.0.2:14260" { + t.Fatalf("ReplicaAddrs[0].DataAddr: got %q", back.ReplicaAddrs[0].DataAddr) + } + if back.ReplicaAddrs[1].CtrlAddr != "10.0.0.3:14261" { + t.Fatalf("ReplicaAddrs[1].CtrlAddr: got %q", back.ReplicaAddrs[1].CtrlAddr) + } + // Scalar fields should be empty when ReplicaAddrs is populated. + if back.ReplicaDataAddr != "" || back.ReplicaCtrlAddr != "" { + t.Fatalf("scalar addrs should be empty when ReplicaAddrs set, got data=%q ctrl=%q", + back.ReplicaDataAddr, back.ReplicaCtrlAddr) + } +} + +func TestAssignment_PrecedenceRule(t *testing.T) { + // When proto has BOTH ReplicaAddrs AND scalar fields, + // ReplicaAddrs takes precedence and scalar fields are ignored. + orig := BlockVolumeAssignment{ + Path: "/data/vol.blk", + Epoch: 5, + Role: RoleToWire(RolePrimary), + ReplicaDataAddr: "scalar-data:1234", + ReplicaCtrlAddr: "scalar-ctrl:1235", + ReplicaAddrs: []ReplicaAddr{ + {DataAddr: "multi-data:4260", CtrlAddr: "multi-ctrl:4261"}, + }, + } + pb := AssignmentToProto(orig) + // Proto will have both scalar and repeated fields set. + if pb.ReplicaDataAddr != "scalar-data:1234" { + t.Fatalf("proto should preserve scalar, got %q", pb.ReplicaDataAddr) + } + if len(pb.ReplicaAddrs) != 1 { + t.Fatalf("proto should have 1 ReplicaAddr, got %d", len(pb.ReplicaAddrs)) + } + + // FromProto: ReplicaAddrs non-empty → use it, ignore scalar. + back := AssignmentFromProto(pb) + if len(back.ReplicaAddrs) != 1 { + t.Fatalf("back.ReplicaAddrs len: got %d, want 1", len(back.ReplicaAddrs)) + } + if back.ReplicaDataAddr != "" { + t.Fatalf("scalar ReplicaDataAddr should be empty due to precedence, got %q", back.ReplicaDataAddr) + } + if back.ReplicaCtrlAddr != "" { + t.Fatalf("scalar ReplicaCtrlAddr should be empty due to precedence, got %q", back.ReplicaCtrlAddr) + } +} + +func TestAssignment_BackwardCompatScalar(t *testing.T) { + // When proto has only scalar fields (no ReplicaAddrs), backward compat path. + orig := BlockVolumeAssignment{ + Path: "/data/vol.blk", + Epoch: 3, + Role: RoleToWire(RolePrimary), + LeaseTtlMs: 5000, + ReplicaDataAddr: "host:4260", + ReplicaCtrlAddr: "host:4261", + } + pb := AssignmentToProto(orig) + back := AssignmentFromProto(pb) + + if back.ReplicaDataAddr != "host:4260" { + t.Fatalf("scalar ReplicaDataAddr: got %q, want %q", back.ReplicaDataAddr, "host:4260") + } + if back.ReplicaCtrlAddr != "host:4261" { + t.Fatalf("scalar ReplicaCtrlAddr: got %q, want %q", back.ReplicaCtrlAddr, "host:4261") + } + if len(back.ReplicaAddrs) != 0 { + t.Fatalf("ReplicaAddrs should be empty for backward compat, got %d", len(back.ReplicaAddrs)) + } +} + +func TestAssignmentsSlice_MultiReplicaRoundTrip(t *testing.T) { + as := []BlockVolumeAssignment{ + { + Path: "/a.blk", + Epoch: 1, + ReplicaAddrs: []ReplicaAddr{ + {DataAddr: "h1:1", CtrlAddr: "h1:2"}, + {DataAddr: "h2:1", CtrlAddr: "h2:2"}, + }, + }, + { + Path: "/b.blk", + Epoch: 2, + ReplicaDataAddr: "scalar:1", + }, + } + pbs := AssignmentsToProto(as) + back := AssignmentsFromProto(pbs) + if len(back) != 2 { + t.Fatalf("len: got %d, want 2", len(back)) + } + // First: multi-replica path. + if len(back[0].ReplicaAddrs) != 2 { + t.Fatalf("back[0].ReplicaAddrs len: got %d, want 2", len(back[0].ReplicaAddrs)) + } + // Second: scalar path. + if back[1].ReplicaDataAddr != "scalar:1" { + t.Fatalf("back[1].ReplicaDataAddr: got %q", back[1].ReplicaDataAddr) + } + if len(back[1].ReplicaAddrs) != 0 { + t.Fatalf("back[1].ReplicaAddrs should be empty, got %d", len(back[1].ReplicaAddrs)) + } +} + +func TestInfoMessage_HealthFieldsZeroDefault(t *testing.T) { + // Verify zero-valued health fields round-trip correctly (backward compat). + orig := BlockVolumeInfoMessage{ + Path: "/data/old.blk", + Epoch: 1, + // All CP8-2 fields at zero default. + } + pb := InfoMessageToProto(orig) + back := InfoMessageFromProto(pb) + if back.HealthScore != 0 { + t.Fatalf("HealthScore zero default: got %f", back.HealthScore) + } + if back.ScrubErrors != 0 { + t.Fatalf("ScrubErrors zero default: got %d", back.ScrubErrors) + } + if back.LastScrubTime != 0 { + t.Fatalf("LastScrubTime zero default: got %d", back.LastScrubTime) + } + if back.ReplicaDegraded { + t.Fatal("ReplicaDegraded zero default should be false") + } +} diff --git a/weed/storage/blockvol/blockapi/client.go b/weed/storage/blockvol/blockapi/client.go new file mode 100644 index 0000000000..bf2e6739e1 --- /dev/null +++ b/weed/storage/blockvol/blockapi/client.go @@ -0,0 +1,189 @@ +package blockapi + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "io" + "net/http" + "strings" + "time" +) + +// Client is a Go HTTP client for the master's block volume REST API. +// It supports multiple master addresses for failover. +type Client struct { + Masters []string + HTTPClient *http.Client +} + +// NewClient creates a Client from a comma-separated list of master URLs. +// Example: "http://m1:9333,http://m2:9333" +func NewClient(masters string) *Client { + var addrs []string + for _, m := range strings.Split(masters, ",") { + m = strings.TrimSpace(m) + if m != "" { + addrs = append(addrs, m) + } + } + return &Client{ + Masters: addrs, + HTTPClient: &http.Client{ + Timeout: 30 * time.Second, + }, + } +} + +// CreateVolume creates a new block volume. +func (c *Client) CreateVolume(ctx context.Context, req CreateVolumeRequest) (*VolumeInfo, error) { + body, err := json.Marshal(req) + if err != nil { + return nil, fmt.Errorf("marshal request: %w", err) + } + resp, err := c.doRequest(ctx, http.MethodPost, "/block/volume", bytes.NewReader(body)) + if err != nil { + return nil, err + } + defer resp.Body.Close() + if err := checkStatus(resp, http.StatusOK, http.StatusCreated); err != nil { + return nil, err + } + var info VolumeInfo + if err := json.NewDecoder(resp.Body).Decode(&info); err != nil { + return nil, fmt.Errorf("decode response: %w", err) + } + return &info, nil +} + +// DeleteVolume deletes a block volume by name. +func (c *Client) DeleteVolume(ctx context.Context, name string) error { + resp, err := c.doRequest(ctx, http.MethodDelete, "/block/volume/"+name, nil) + if err != nil { + return err + } + defer resp.Body.Close() + return checkStatus(resp, http.StatusOK) +} + +// LookupVolume looks up a single block volume by name. +func (c *Client) LookupVolume(ctx context.Context, name string) (*VolumeInfo, error) { + resp, err := c.doRequest(ctx, http.MethodGet, "/block/volume/"+name, nil) + if err != nil { + return nil, err + } + defer resp.Body.Close() + if err := checkStatus(resp, http.StatusOK); err != nil { + return nil, err + } + var info VolumeInfo + if err := json.NewDecoder(resp.Body).Decode(&info); err != nil { + return nil, fmt.Errorf("decode response: %w", err) + } + return &info, nil +} + +// ListVolumes lists all block volumes. +func (c *Client) ListVolumes(ctx context.Context) ([]VolumeInfo, error) { + resp, err := c.doRequest(ctx, http.MethodGet, "/block/volumes", nil) + if err != nil { + return nil, err + } + defer resp.Body.Close() + if err := checkStatus(resp, http.StatusOK); err != nil { + return nil, err + } + var infos []VolumeInfo + if err := json.NewDecoder(resp.Body).Decode(&infos); err != nil { + return nil, fmt.Errorf("decode response: %w", err) + } + return infos, nil +} + +// AssignRole enqueues a role assignment for a block volume. +func (c *Client) AssignRole(ctx context.Context, req AssignRequest) error { + body, err := json.Marshal(req) + if err != nil { + return fmt.Errorf("marshal request: %w", err) + } + resp, err := c.doRequest(ctx, http.MethodPost, "/block/assign", bytes.NewReader(body)) + if err != nil { + return err + } + defer resp.Body.Close() + return checkStatus(resp, http.StatusOK) +} + +// ListServers lists all block-capable volume servers. +func (c *Client) ListServers(ctx context.Context) ([]ServerInfo, error) { + resp, err := c.doRequest(ctx, http.MethodGet, "/block/servers", nil) + if err != nil { + return nil, err + } + defer resp.Body.Close() + if err := checkStatus(resp, http.StatusOK); err != nil { + return nil, err + } + var infos []ServerInfo + if err := json.NewDecoder(resp.Body).Decode(&infos); err != nil { + return nil, fmt.Errorf("decode response: %w", err) + } + return infos, nil +} + +// doRequest tries each master in order until one responds. +func (c *Client) doRequest(ctx context.Context, method, path string, body io.Reader) (*http.Response, error) { + var lastErr error + for _, master := range c.Masters { + url := strings.TrimRight(master, "/") + path + var bodyReader io.Reader + if body != nil { + // Re-read body for retries: buffer it + if lastErr != nil { + // body was already consumed on previous attempt + // We need a seekable body — caller should pass bytes.Reader + if seeker, ok := body.(io.Seeker); ok { + seeker.Seek(0, io.SeekStart) + } + } + bodyReader = body + } + req, err := http.NewRequestWithContext(ctx, method, url, bodyReader) + if err != nil { + lastErr = fmt.Errorf("master %s: %w", master, err) + continue + } + if method == http.MethodPost || method == http.MethodPut { + req.Header.Set("Content-Type", "application/json") + } + resp, err := c.HTTPClient.Do(req) + if err != nil { + lastErr = fmt.Errorf("master %s: %w", master, err) + continue + } + return resp, nil + } + if lastErr != nil { + return nil, lastErr + } + return nil, fmt.Errorf("no master addresses configured") +} + +// checkStatus returns an error if the response status is not in the accepted list. +func checkStatus(resp *http.Response, accepted ...int) error { + for _, code := range accepted { + if resp.StatusCode == code { + return nil + } + } + body, _ := io.ReadAll(resp.Body) + // Try to extract error message from JSON. + var errResp struct { + Error string `json:"error"` + } + if json.Unmarshal(body, &errResp) == nil && errResp.Error != "" { + return fmt.Errorf("HTTP %d: %s", resp.StatusCode, errResp.Error) + } + return fmt.Errorf("HTTP %d: %s", resp.StatusCode, string(body)) +} diff --git a/weed/storage/blockvol/blockapi/client_test.go b/weed/storage/blockvol/blockapi/client_test.go new file mode 100644 index 0000000000..970da58c1e --- /dev/null +++ b/weed/storage/blockvol/blockapi/client_test.go @@ -0,0 +1,103 @@ +package blockapi + +import ( + "context" + "encoding/json" + "net/http" + "net/http/httptest" + "testing" +) + +func TestClientCreateVolume(t *testing.T) { + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.Method != "POST" || r.URL.Path != "/block/volume" { + t.Errorf("unexpected request: %s %s", r.Method, r.URL.Path) + w.WriteHeader(http.StatusNotFound) + return + } + var req CreateVolumeRequest + if err := json.NewDecoder(r.Body).Decode(&req); err != nil { + t.Errorf("decode request: %v", err) + w.WriteHeader(http.StatusBadRequest) + return + } + if req.Name != "test-vol" { + t.Errorf("expected name test-vol, got %s", req.Name) + } + w.Header().Set("Content-Type", "application/json") + json.NewEncoder(w).Encode(VolumeInfo{ + Name: req.Name, + VolumeServer: "vs1:9333", + SizeBytes: req.SizeBytes, + Epoch: 1, + Role: "primary", + Status: "active", + }) + })) + defer ts.Close() + + client := NewClient(ts.URL) + info, err := client.CreateVolume(context.Background(), CreateVolumeRequest{ + Name: "test-vol", + SizeBytes: 1 << 30, + }) + if err != nil { + t.Fatal(err) + } + if info.Name != "test-vol" { + t.Errorf("expected name test-vol, got %s", info.Name) + } + if info.VolumeServer != "vs1:9333" { + t.Errorf("expected vs1:9333, got %s", info.VolumeServer) + } +} + +func TestClientListVolumes(t *testing.T) { + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.Method != "GET" || r.URL.Path != "/block/volumes" { + w.WriteHeader(http.StatusNotFound) + return + } + w.Header().Set("Content-Type", "application/json") + json.NewEncoder(w).Encode([]VolumeInfo{ + {Name: "alpha", VolumeServer: "vs1:9333"}, + {Name: "beta", VolumeServer: "vs2:9333"}, + }) + })) + defer ts.Close() + + client := NewClient(ts.URL) + vols, err := client.ListVolumes(context.Background()) + if err != nil { + t.Fatal(err) + } + if len(vols) != 2 { + t.Fatalf("expected 2 volumes, got %d", len(vols)) + } + if vols[0].Name != "alpha" || vols[1].Name != "beta" { + t.Errorf("unexpected volumes: %+v", vols) + } +} + +func TestClientMultiMasterFallback(t *testing.T) { + // First server immediately rejects connections (closed listener). + dead := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {})) + deadURL := dead.URL + dead.Close() // close it so connections are refused + + // Second server responds normally. + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + json.NewEncoder(w).Encode([]VolumeInfo{}) + })) + defer ts.Close() + + client := NewClient(deadURL + "," + ts.URL) + vols, err := client.ListVolumes(context.Background()) + if err != nil { + t.Fatal(err) + } + if vols == nil { + t.Error("expected non-nil result") + } +} diff --git a/weed/storage/blockvol/blockapi/types.go b/weed/storage/blockvol/blockapi/types.go new file mode 100644 index 0000000000..605bf2dd6c --- /dev/null +++ b/weed/storage/blockvol/blockapi/types.go @@ -0,0 +1,81 @@ +package blockapi + +import ( + "strings" + + "github.com/seaweedfs/seaweedfs/weed/storage/blockvol" +) + +// CreateVolumeRequest is the request body for POST /block/volume. +type CreateVolumeRequest struct { + Name string `json:"name"` + SizeBytes uint64 `json:"size_bytes"` + ReplicaPlacement string `json:"replica_placement"` // SeaweedFS placement string: "000", "001", "010", "100" + DiskType string `json:"disk_type"` // e.g. "ssd", "hdd" +} + +// VolumeInfo describes a block volume. +type VolumeInfo struct { + Name string `json:"name"` + VolumeServer string `json:"volume_server"` + SizeBytes uint64 `json:"size_bytes"` + ReplicaPlacement string `json:"replica_placement,omitempty"` + Epoch uint64 `json:"epoch"` + Role string `json:"role"` + Status string `json:"status"` + ISCSIAddr string `json:"iscsi_addr"` + IQN string `json:"iqn"` + ReplicaServer string `json:"replica_server,omitempty"` + ReplicaISCSIAddr string `json:"replica_iscsi_addr,omitempty"` + ReplicaIQN string `json:"replica_iqn,omitempty"` + ReplicaDataAddr string `json:"replica_data_addr,omitempty"` + ReplicaCtrlAddr string `json:"replica_ctrl_addr,omitempty"` + // CP8-2: Multi-replica fields. + ReplicaFactor int `json:"replica_factor"` + Replicas []ReplicaDetail `json:"replicas,omitempty"` + HealthScore float64 `json:"health_score"` + ReplicaDegraded bool `json:"replica_degraded,omitempty"` +} + +// ReplicaDetail describes one replica in the API response. +type ReplicaDetail struct { + Server string `json:"server"` + ISCSIAddr string `json:"iscsi_addr,omitempty"` + IQN string `json:"iqn,omitempty"` + HealthScore float64 `json:"health_score"` + WALLag uint64 `json:"wal_lag,omitempty"` +} + +// AssignRequest is the request body for POST /block/assign. +type AssignRequest struct { + Name string `json:"name"` + Epoch uint64 `json:"epoch"` + Role string `json:"role"` // "primary" | "replica" + LeaseTTLMs uint64 `json:"lease_ttl_ms"` // lease TTL in milliseconds +} + +// ServerInfo describes a block-capable volume server. +type ServerInfo struct { + Address string `json:"address"` + VolumeCount int `json:"volume_count"` + BlockCapable bool `json:"block_capable"` +} + +// RoleFromString converts a role string to its uint32 wire value. +// Returns 0 (RoleNone) for unrecognized strings. +func RoleFromString(s string) uint32 { + switch strings.ToLower(s) { + case "primary": + return uint32(blockvol.RolePrimary) + case "replica": + return uint32(blockvol.RoleReplica) + case "stale": + return uint32(blockvol.RoleStale) + case "rebuilding": + return uint32(blockvol.RoleRebuilding) + case "draining": + return uint32(blockvol.RoleDraining) + default: + return uint32(blockvol.RoleNone) + } +} diff --git a/weed/storage/blockvol/blockvol.go b/weed/storage/blockvol/blockvol.go index 828c17e9ef..c712979b87 100644 --- a/weed/storage/blockvol/blockvol.go +++ b/weed/storage/blockvol/blockvol.go @@ -44,9 +44,9 @@ type BlockVol struct { opsOutstanding atomic.Int64 // in-flight Read/Write/Trim/SyncCache ops opsDrained chan struct{} - // Replication fields (Phase 4A CP2). - shipper *WALShipper - replRecv *ReplicaReceiver + // Replication fields (Phase 4A CP2, generalized to N replicas in CP8-2). + shipperGroup *ShipperGroup + replRecv *ReplicaReceiver // Fencing fields (Phase 4A). epoch atomic.Uint64 // current persisted epoch @@ -60,6 +60,13 @@ type BlockVol struct { assignMu sync.Mutex // serializes HandleAssignment calls drainTimeout time.Duration // default 10s, for demote drain + // Health score and scrub (CP8-2). + healthScore *HealthScore + scrubber *Scrubber + + // Observability (CP8-4). + Metrics *EngineMetrics + // Snapshot fields (Phase 5 CP5-2). snapMu sync.RWMutex snapshots map[uint32]*activeSnapshot @@ -116,14 +123,16 @@ func CreateBlockVol(path string, opts CreateOptions, cfgs ...BlockVolConfig) (*B dm := NewDirtyMap(cfg.DirtyMapShards) wal := NewWALWriter(fd, sb.WALOffset, sb.WALSize, 0, 0) v := &BlockVol{ - fd: fd, - path: path, - super: sb, - config: cfg, - wal: wal, - dirtyMap: dm, - opsDrained: make(chan struct{}, 1), - snapshots: make(map[uint32]*activeSnapshot), + fd: fd, + path: path, + super: sb, + config: cfg, + wal: wal, + dirtyMap: dm, + healthScore: NewHealthScore(), + Metrics: NewEngineMetrics(), + opsDrained: make(chan struct{}, 1), + snapshots: make(map[uint32]*activeSnapshot), } v.nextLSN.Store(1) v.healthy.Store(true) @@ -134,6 +143,7 @@ func CreateBlockVol(path string, opts CreateOptions, cfgs ...BlockVolConfig) (*B LowWatermark: cfg.GroupCommitLowWatermark, OnDegraded: func() { v.healthy.Store(false) }, PostSyncCheck: v.writeGate, + Metrics: v.Metrics, }) go v.groupCommit.Run() v.flusher = NewFlusher(FlusherConfig{ @@ -142,6 +152,7 @@ func CreateBlockVol(path string, opts CreateOptions, cfgs ...BlockVolConfig) (*B WAL: wal, DirtyMap: dm, Interval: cfg.FlushInterval, + Metrics: v.Metrics, }) go v.flusher.Run() return v, nil @@ -189,13 +200,15 @@ func OpenBlockVol(path string, cfgs ...BlockVolConfig) (*BlockVol, error) { wal := NewWALWriter(fd, sb.WALOffset, sb.WALSize, sb.WALHead, sb.WALTail) v := &BlockVol{ - fd: fd, - path: path, - super: sb, - config: cfg, - wal: wal, - dirtyMap: dirtyMap, - opsDrained: make(chan struct{}, 1), + fd: fd, + path: path, + super: sb, + config: cfg, + wal: wal, + dirtyMap: dirtyMap, + healthScore: NewHealthScore(), + Metrics: NewEngineMetrics(), + opsDrained: make(chan struct{}, 1), } v.nextLSN.Store(nextLSN) v.epoch.Store(sb.Epoch) @@ -207,6 +220,7 @@ func OpenBlockVol(path string, cfgs ...BlockVolConfig) (*BlockVol, error) { LowWatermark: cfg.GroupCommitLowWatermark, OnDegraded: func() { v.healthy.Store(false) }, PostSyncCheck: v.writeGate, + Metrics: v.Metrics, }) go v.groupCommit.Run() v.flusher = NewFlusher(FlusherConfig{ @@ -215,6 +229,7 @@ func OpenBlockVol(path string, cfgs ...BlockVolConfig) (*BlockVol, error) { WAL: wal, DirtyMap: dirtyMap, Interval: cfg.FlushInterval, + Metrics: v.Metrics, }) go v.flusher.Run() @@ -292,9 +307,14 @@ func (v *BlockVol) appendWithRetry(entry *WALEntry) (uint64, error) { return 0, ErrVolumeClosed } - // Ship to replica if configured (fire-and-forget). - if v.shipper != nil { - v.shipper.Ship(entry) + // Ship to replicas if configured (fire-and-forget). + if v.shipperGroup != nil { + v.shipperGroup.ShipAll(entry) + } + + // Notify scrubber of written LBAs for CRC invalidation. + if v.scrubber != nil && entry.Type == EntryTypeWrite { + v.scrubber.NotifyWrite(entry.LBA, entry.Length/v.super.BlockSize) } return walOff, nil @@ -567,17 +587,33 @@ func (v *BlockVol) SyncCache() error { return v.groupCommit.Submit() } -// SetReplicaAddr configures the replica endpoint and creates a WAL shipper -// with distributed group commit. Must be called before any writes. +// ReplicaAddr holds the data and control addresses for one replica. +type ReplicaAddr struct { + DataAddr string + CtrlAddr string +} + +// SetReplicaAddr configures a single replica endpoint. Backward-compatible wrapper +// around SetReplicaAddrs for RF=2 callers. func (v *BlockVol) SetReplicaAddr(dataAddr, ctrlAddr string) { - v.shipper = NewWALShipper(dataAddr, ctrlAddr, func() uint64 { - return v.epoch.Load() - }) + v.SetReplicaAddrs([]ReplicaAddr{{DataAddr: dataAddr, CtrlAddr: ctrlAddr}}) +} + +// SetReplicaAddrs configures N replica endpoints and creates a ShipperGroup +// with distributed group commit. Creates fresh shippers (old group is GC'd). +func (v *BlockVol) SetReplicaAddrs(addrs []ReplicaAddr) { + shippers := make([]*WALShipper, len(addrs)) + for i, a := range addrs { + shippers[i] = NewWALShipper(a.DataAddr, a.CtrlAddr, func() uint64 { + return v.epoch.Load() + }, v.Metrics) + } + v.shipperGroup = NewShipperGroup(shippers) // Replace the group committer's sync function with a distributed version. v.groupCommit.Stop() v.groupCommit = NewGroupCommitter(GroupCommitterConfig{ - SyncFunc: MakeDistributedSync(v.fd.Sync, v.shipper, v), + SyncFunc: MakeDistributedSync(v.fd.Sync, v.shipperGroup, v), MaxDelay: v.config.GroupCommitMaxDelay, MaxBatch: v.config.GroupCommitMaxBatch, LowWatermark: v.config.GroupCommitLowWatermark, @@ -622,21 +658,21 @@ func (v *BlockVol) StopRebuildServer() { } } -// degradeReplica marks the shipper as degraded and logs a warning. +// degradeReplica logs a warning about a replica barrier failure. +// Individual shippers are already degraded by Barrier itself. func (v *BlockVol) degradeReplica(err error) { - if v.shipper != nil { - v.shipper.degraded.Store(true) - } log.Printf("blockvol: replica degraded: %v", err) } // BlockVolumeStatus contains block volume state for heartbeat reporting. type BlockVolumeStatus struct { - Epoch uint64 - WALHeadLSN uint64 - Role Role - CheckpointLSN uint64 - HasLease bool + Epoch uint64 + WALHeadLSN uint64 + Role Role + CheckpointLSN uint64 + HasLease bool + HealthScore float64 + ReplicaDegraded bool } // Status returns the current block volume status for heartbeat reporting. @@ -645,13 +681,79 @@ func (v *BlockVol) Status() BlockVolumeStatus { if v.flusher != nil { cpLSN = v.flusher.CheckpointLSN() } - return BlockVolumeStatus{ - Epoch: v.epoch.Load(), - WALHeadLSN: v.nextLSN.Load() - 1, - Role: v.Role(), - CheckpointLSN: cpLSN, - HasLease: v.lease.IsValid(), + var hs float64 = 1.0 + if v.healthScore != nil { + hs = v.healthScore.Score() } + var degraded bool + if v.shipperGroup != nil { + degraded = v.shipperGroup.AnyDegraded() + } + return BlockVolumeStatus{ + Epoch: v.epoch.Load(), + WALHeadLSN: v.nextLSN.Load() - 1, + Role: v.Role(), + CheckpointLSN: cpLSN, + HasLease: v.lease.IsValid(), + HealthScore: hs, + ReplicaDegraded: degraded, + } +} + +// CheckpointLSN returns the last LSN flushed to the extent region. +func (v *BlockVol) CheckpointLSN() uint64 { + if v.flusher != nil { + return v.flusher.CheckpointLSN() + } + return 0 +} + +// HealthScore returns the current health score (0.0-1.0). +func (v *BlockVol) HealthScore() float64 { + if v.healthScore == nil { + return 1.0 + } + return v.healthScore.Score() +} + +// HealthStats returns detailed health statistics. +func (v *BlockVol) HealthStats() HealthStats { + if v.healthScore == nil { + return HealthStats{Score: 1.0} + } + return v.healthScore.Stats() +} + +// StartScrub starts background scrubbing with the given interval. +func (v *BlockVol) StartScrub(interval time.Duration) { + if v.scrubber != nil { + return + } + v.scrubber = NewScrubber(v, interval) + v.scrubber.Start() +} + +// StopScrub stops the background scrubber. +func (v *BlockVol) StopScrub() { + if v.scrubber != nil { + v.scrubber.Stop() + v.scrubber = nil + } +} + +// TriggerScrub triggers an immediate scrub pass. +func (v *BlockVol) TriggerScrub() { + if v.scrubber != nil { + v.scrubber.TriggerNow() + } +} + +// ScrubStats returns the current scrub statistics. +func (v *BlockVol) ScrubStats() ScrubStats { + if v.scrubber == nil { + return ScrubStats{} + } + return v.scrubber.Stats() } // CreateSnapshot creates a point-in-time snapshot. The snapshot captures the @@ -945,13 +1047,16 @@ func (v *BlockVol) persistSuperblock() error { } // Close shuts down the block volume and closes the file. -// Shutdown order: shipper -> replica receiver -> rebuild server -> drain ops -> group committer -> flusher -> final flush -> close fd. +// Shutdown order: shippers -> replica receiver -> rebuild server -> drain ops -> group committer -> flusher -> final flush -> close fd. func (v *BlockVol) Close() error { v.closed.Store(true) - // Stop shipper first: no more Ship calls. - if v.shipper != nil { - v.shipper.Stop() + // Stop scrubber first (no more extent reads). + v.StopScrub() + + // Stop shippers: no more Ship calls. + if v.shipperGroup != nil { + v.shipperGroup.StopAll() } // Stop replica receiver: no more barrier waits. if v.replRecv != nil { diff --git a/weed/storage/blockvol/blockvol_test.go b/weed/storage/blockvol/blockvol_test.go index f05fef41ea..adfae06b45 100644 --- a/weed/storage/blockvol/blockvol_test.go +++ b/weed/storage/blockvol/blockvol_test.go @@ -2982,7 +2982,7 @@ func testDistCommitBothPass(t *testing.T) { shipper := NewWALShipper(dataAddr, ctrlAddr, func() uint64 { return 0 }) defer shipper.Stop() - distSync := MakeDistributedSync(walSync, shipper, v) + distSync := MakeDistributedSync(walSync, NewShipperGroup([]*WALShipper{shipper}), v) if err := distSync(); err != nil { t.Fatalf("distSync: %v", err) } @@ -3005,7 +3005,7 @@ func testDistCommitLocalFail(t *testing.T) { shipper := NewWALShipper(dataAddr, ctrlAddr, func() uint64 { return 0 }) defer shipper.Stop() - distSync := MakeDistributedSync(walSync, shipper, v) + distSync := MakeDistributedSync(walSync, NewShipperGroup([]*WALShipper{shipper}), v) err := distSync() if err == nil { t.Fatal("expected error from local sync failure") @@ -3025,10 +3025,10 @@ func testDistCommitRemoteFailDegrades(t *testing.T) { shipper := NewWALShipper(dataAddr, ctrlAddr, func() uint64 { return 0 }) defer shipper.Stop() - // Set shipper on vol so degradeReplica can mark it degraded. - v.shipper = shipper + // Set shipper group on vol so degradeReplica can see it. + v.shipperGroup = NewShipperGroup([]*WALShipper{shipper}) - distSync := MakeDistributedSync(walSync, shipper, v) + distSync := MakeDistributedSync(walSync, v.shipperGroup, v) // Should NOT return error (local succeeded, remote degraded). if err := distSync(); err != nil { t.Fatalf("distSync should not fail on remote error: %v", err) @@ -3054,7 +3054,7 @@ func testDistCommitNoReplica(t *testing.T) { v := createTestVol(t) defer v.Close() - // nil shipper --local-only mode. + // nil group --local-only mode. distSync := MakeDistributedSync(walSync, nil, v) if err := distSync(); err != nil { t.Fatalf("distSync: %v", err) @@ -3133,9 +3133,9 @@ func testBlockvolNoReplicaCompat(t *testing.T) { t.Error("data mismatch in no-replica mode") } - // Verify shipper is nil. - if v.shipper != nil { - t.Error("shipper should be nil without SetReplicaAddr") + // Verify shipperGroup is nil. + if v.shipperGroup != nil { + t.Error("shipperGroup should be nil without SetReplicaAddr") } } @@ -3541,15 +3541,16 @@ func testDemoteStopsShipper(t *testing.T) { v := setupPrimary(t) defer v.Close() - // Create a shipper (won't connect but that's fine for this test). - v.shipper = NewWALShipper("127.0.0.1:0", "127.0.0.1:0", func() uint64 { + // Create a shipper group (won't connect but that's fine for this test). + shipper := NewWALShipper("127.0.0.1:0", "127.0.0.1:0", func() uint64 { return v.epoch.Load() }) + v.shipperGroup = NewShipperGroup([]*WALShipper{shipper}) if err := v.HandleAssignment(2, RoleStale, 0); err != nil { t.Fatalf("demote: %v", err) } - if !v.shipper.stopped.Load() { + if !shipper.stopped.Load() { t.Error("shipper should be stopped after demotion") } } diff --git a/weed/storage/blockvol/csi/controller.go b/weed/storage/blockvol/csi/controller.go index ba483c7b44..c9d33bce39 100644 --- a/weed/storage/blockvol/csi/controller.go +++ b/weed/storage/blockvol/csi/controller.go @@ -3,10 +3,14 @@ package csi import ( "context" "errors" + "fmt" + "strings" + "time" "github.com/container-storage-interface/spec/lib/go/csi" "google.golang.org/grpc/codes" "google.golang.org/grpc/status" + "google.golang.org/protobuf/types/known/timestamppb" ) const ( @@ -32,7 +36,7 @@ func (s *controllerServer) CreateVolume(_ context.Context, req *csi.CreateVolume if req.CapacityRange != nil && req.CapacityRange.RequiredBytes > 0 { sizeBytes = req.CapacityRange.RequiredBytes } else if req.CapacityRange != nil && req.CapacityRange.LimitBytes > 0 { - // No RequiredBytes set — use LimitBytes as the target size. + // No RequiredBytes set -- use LimitBytes as the target size. sizeBytes = req.CapacityRange.LimitBytes } if req.CapacityRange != nil && req.CapacityRange.LimitBytes > 0 { @@ -126,27 +130,201 @@ func (s *controllerServer) ControllerUnpublishVolume(_ context.Context, req *csi return &csi.ControllerUnpublishVolumeResponse{}, nil } -func (s *controllerServer) ControllerGetCapabilities(_ context.Context, _ *csi.ControllerGetCapabilitiesRequest) (*csi.ControllerGetCapabilitiesResponse, error) { - return &csi.ControllerGetCapabilitiesResponse{ - Capabilities: []*csi.ControllerServiceCapability{ - { - Type: &csi.ControllerServiceCapability_Rpc{ - Rpc: &csi.ControllerServiceCapability_RPC{ - Type: csi.ControllerServiceCapability_RPC_CREATE_DELETE_VOLUME, - }, - }, - }, - { - Type: &csi.ControllerServiceCapability_Rpc{ - Rpc: &csi.ControllerServiceCapability_RPC{ - Type: csi.ControllerServiceCapability_RPC_PUBLISH_UNPUBLISH_VOLUME, - }, - }, - }, +func (s *controllerServer) CreateSnapshot(_ context.Context, req *csi.CreateSnapshotRequest) (*csi.CreateSnapshotResponse, error) { + if req.SourceVolumeId == "" { + return nil, status.Error(codes.InvalidArgument, "source volume ID is required") + } + if req.Name == "" { + return nil, status.Error(codes.InvalidArgument, "snapshot name is required") + } + + snapID := snapshotNameToID(req.Name) + info, err := s.backend.CreateSnapshot(context.Background(), req.SourceVolumeId, snapID) + if err != nil { + errMsg := err.Error() + // CSI idempotency: if the snapshot already exists on the same volume, + // return it as success. Only return AlreadyExists for true collisions + // (different source volume using the same FNV hash). + if strings.Contains(errMsg, "snapshot exists") || strings.Contains(errMsg, "already exists") { + snaps, lerr := s.backend.ListSnapshots(context.Background(), req.SourceVolumeId) + if lerr == nil { + for _, snap := range snaps { + if snap.SnapshotID == snapID { + // Same name retried on same volume -- idempotent success. + csiSnapID := FormatSnapshotID(req.SourceVolumeId, snapID) + return &csi.CreateSnapshotResponse{ + Snapshot: &csi.Snapshot{ + SnapshotId: csiSnapID, + SourceVolumeId: req.SourceVolumeId, + CreationTime: timestamppb.New(time.Unix(snap.CreatedAt, 0)), + ReadyToUse: true, + SizeBytes: int64(snap.SizeBytes), + }, + }, nil + } + } + } + // Snapshot exists but not on this volume -- true FNV collision. + return nil, status.Errorf(codes.AlreadyExists, + "snapshot ID collision: FNV hash %d already used by a different snapshot", snapID) + } + if strings.Contains(errMsg, "not found") { + return nil, status.Errorf(codes.NotFound, "volume %q not found", req.SourceVolumeId) + } + return nil, status.Errorf(codes.Internal, "create snapshot: %v", err) + } + + csiSnapID := FormatSnapshotID(req.SourceVolumeId, snapID) + return &csi.CreateSnapshotResponse{ + Snapshot: &csi.Snapshot{ + SnapshotId: csiSnapID, + SourceVolumeId: req.SourceVolumeId, + CreationTime: timestamppb.New(time.Unix(info.CreatedAt, 0)), + ReadyToUse: true, + SizeBytes: int64(info.SizeBytes), }, }, nil } +func (s *controllerServer) DeleteSnapshot(_ context.Context, req *csi.DeleteSnapshotRequest) (*csi.DeleteSnapshotResponse, error) { + if req.SnapshotId == "" { + return nil, status.Error(codes.InvalidArgument, "snapshot ID is required") + } + + volumeID, snapID, err := ParseSnapshotID(req.SnapshotId) + if err != nil { + // Can't parse → treat as already deleted (idempotent). + return &csi.DeleteSnapshotResponse{}, nil + } + + if berr := s.backend.DeleteSnapshot(context.Background(), volumeID, snapID); berr != nil { + // Volume not found → snapshot doesn't exist → success (idempotent). + if strings.Contains(berr.Error(), "not found") { + return &csi.DeleteSnapshotResponse{}, nil + } + return nil, status.Errorf(codes.Internal, "delete snapshot: %v", berr) + } + + return &csi.DeleteSnapshotResponse{}, nil +} + +func (s *controllerServer) ListSnapshots(_ context.Context, req *csi.ListSnapshotsRequest) (*csi.ListSnapshotsResponse, error) { + // If filtering by snapshot_id, parse it to get the volume. + if req.SnapshotId != "" { + volumeID, snapID, err := ParseSnapshotID(req.SnapshotId) + if err != nil { + return &csi.ListSnapshotsResponse{}, nil + } + + snaps, lerr := s.backend.ListSnapshots(context.Background(), volumeID) + if lerr != nil { + if strings.Contains(lerr.Error(), "not found") { + return &csi.ListSnapshotsResponse{}, nil + } + return nil, status.Errorf(codes.Internal, "list snapshots: %v", lerr) + } + + for _, snap := range snaps { + if snap.SnapshotID == snapID { + return &csi.ListSnapshotsResponse{ + Entries: []*csi.ListSnapshotsResponse_Entry{{ + Snapshot: &csi.Snapshot{ + SnapshotId: req.SnapshotId, + SourceVolumeId: volumeID, + SizeBytes: int64(snap.SizeBytes), + ReadyToUse: true, + }, + }}, + }, nil + } + } + return &csi.ListSnapshotsResponse{}, nil + } + + // If filtering by source_volume_id, list snapshots for that volume. + if req.SourceVolumeId != "" { + snaps, err := s.backend.ListSnapshots(context.Background(), req.SourceVolumeId) + if err != nil { + if strings.Contains(err.Error(), "not found") { + return &csi.ListSnapshotsResponse{}, nil + } + return nil, status.Errorf(codes.Internal, "list snapshots: %v", err) + } + + resp := &csi.ListSnapshotsResponse{} + for _, snap := range snaps { + resp.Entries = append(resp.Entries, &csi.ListSnapshotsResponse_Entry{ + Snapshot: &csi.Snapshot{ + SnapshotId: FormatSnapshotID(req.SourceVolumeId, snap.SnapshotID), + SourceVolumeId: req.SourceVolumeId, + SizeBytes: int64(snap.SizeBytes), + ReadyToUse: true, + }, + }) + } + return resp, nil + } + + // No filter: not supported for distributed mode (would need to enumerate all volumes). + return &csi.ListSnapshotsResponse{}, nil +} + +func (s *controllerServer) ControllerExpandVolume(_ context.Context, req *csi.ControllerExpandVolumeRequest) (*csi.ControllerExpandVolumeResponse, error) { + if req.VolumeId == "" { + return nil, status.Error(codes.InvalidArgument, "volume ID is required") + } + if req.CapacityRange == nil || req.CapacityRange.RequiredBytes <= 0 { + return nil, status.Error(codes.InvalidArgument, "capacity_range.required_bytes must be > 0") + } + + newSize := uint64(req.CapacityRange.RequiredBytes) + // Round up to block size. + if newSize%blockSize != 0 { + newSize = (newSize/blockSize + 1) * blockSize + } + + capacity, err := s.backend.ExpandVolume(context.Background(), req.VolumeId, newSize) + if err != nil { + errMsg := err.Error() + if strings.Contains(errMsg, "shrink") { + return nil, status.Errorf(codes.InvalidArgument, "cannot shrink volume") + } + if strings.Contains(errMsg, "snapshots") { + return nil, status.Errorf(codes.FailedPrecondition, + "cannot resize with active snapshots: delete snapshots first") + } + if strings.Contains(errMsg, "not found") { + return nil, status.Errorf(codes.NotFound, "volume %q not found", req.VolumeId) + } + return nil, status.Errorf(codes.Internal, "expand volume: %v", err) + } + + return &csi.ControllerExpandVolumeResponse{ + CapacityBytes: int64(capacity), + NodeExpansionRequired: true, + }, nil +} + +func (s *controllerServer) ControllerGetCapabilities(_ context.Context, _ *csi.ControllerGetCapabilitiesRequest) (*csi.ControllerGetCapabilitiesResponse, error) { + caps := []csi.ControllerServiceCapability_RPC_Type{ + csi.ControllerServiceCapability_RPC_CREATE_DELETE_VOLUME, + csi.ControllerServiceCapability_RPC_PUBLISH_UNPUBLISH_VOLUME, + csi.ControllerServiceCapability_RPC_CREATE_DELETE_SNAPSHOT, + csi.ControllerServiceCapability_RPC_LIST_SNAPSHOTS, + csi.ControllerServiceCapability_RPC_EXPAND_VOLUME, + } + + var result []*csi.ControllerServiceCapability + for _, c := range caps { + result = append(result, &csi.ControllerServiceCapability{ + Type: &csi.ControllerServiceCapability_Rpc{ + Rpc: &csi.ControllerServiceCapability_RPC{Type: c}, + }, + }) + } + return &csi.ControllerGetCapabilitiesResponse{Capabilities: result}, nil +} + func (s *controllerServer) ValidateVolumeCapabilities(_ context.Context, req *csi.ValidateVolumeCapabilitiesRequest) (*csi.ValidateVolumeCapabilitiesResponse, error) { if req.VolumeId == "" { return nil, status.Error(codes.InvalidArgument, "volume ID is required") @@ -164,3 +342,6 @@ func (s *controllerServer) ValidateVolumeCapabilities(_ context.Context, req *cs }, }, nil } + +// errSnapshotsPreventResize is used by the mock backend for testing. +var errSnapshotsPreventResize = fmt.Errorf("blockvol: cannot resize with active snapshots") diff --git a/weed/storage/blockvol/csi/controller_test.go b/weed/storage/blockvol/csi/controller_test.go index b390f6550a..0ce5cf22a7 100644 --- a/weed/storage/blockvol/csi/controller_test.go +++ b/weed/storage/blockvol/csi/controller_test.go @@ -220,7 +220,7 @@ func TestControllerUnpublish_Success(t *testing.T) { } } -func TestController_Capabilities_IncludesPublish(t *testing.T) { +func TestController_Capabilities(t *testing.T) { mgr := newTestManager(t) backend := NewLocalVolumeBackend(mgr) cs := &controllerServer{backend: backend} @@ -230,24 +230,225 @@ func TestController_Capabilities_IncludesPublish(t *testing.T) { t.Fatalf("ControllerGetCapabilities: %v", err) } - hasCreate := false - hasPublish := false + want := map[csi.ControllerServiceCapability_RPC_Type]bool{ + csi.ControllerServiceCapability_RPC_CREATE_DELETE_VOLUME: true, + csi.ControllerServiceCapability_RPC_PUBLISH_UNPUBLISH_VOLUME: true, + csi.ControllerServiceCapability_RPC_CREATE_DELETE_SNAPSHOT: true, + csi.ControllerServiceCapability_RPC_LIST_SNAPSHOTS: true, + csi.ControllerServiceCapability_RPC_EXPAND_VOLUME: true, + } + + got := make(map[csi.ControllerServiceCapability_RPC_Type]bool) for _, cap := range resp.Capabilities { rpc := cap.GetRpc() - if rpc == nil { - continue - } - switch rpc.Type { - case csi.ControllerServiceCapability_RPC_CREATE_DELETE_VOLUME: - hasCreate = true - case csi.ControllerServiceCapability_RPC_PUBLISH_UNPUBLISH_VOLUME: - hasPublish = true + if rpc != nil { + got[rpc.Type] = true } } - if !hasCreate { - t.Fatal("expected CREATE_DELETE_VOLUME capability") + + for capType := range want { + if !got[capType] { + t.Fatalf("missing capability: %v", capType) + } } - if !hasPublish { - t.Fatal("expected PUBLISH_UNPUBLISH_VOLUME capability") +} + +func TestController_CreateSnapshot(t *testing.T) { + mgr := newTestManager(t) + backend := NewLocalVolumeBackend(mgr) + cs := &controllerServer{backend: backend} + + // Create volume first. + cs.CreateVolume(context.Background(), &csi.CreateVolumeRequest{ + Name: "snap-vol", + VolumeCapabilities: testVolCaps(), + CapacityRange: &csi.CapacityRange{RequiredBytes: 4 * 1024 * 1024}, + }) + + resp, err := cs.CreateSnapshot(context.Background(), &csi.CreateSnapshotRequest{ + Name: "my-snapshot", + SourceVolumeId: "snap-vol", + }) + if err != nil { + t.Fatalf("CreateSnapshot: %v", err) + } + if resp.Snapshot.SnapshotId == "" { + t.Fatal("snapshot ID should not be empty") + } + if resp.Snapshot.SourceVolumeId != "snap-vol" { + t.Fatalf("SourceVolumeId: got %q, want snap-vol", resp.Snapshot.SourceVolumeId) + } + if !resp.Snapshot.ReadyToUse { + t.Fatal("snapshot should be ready") + } +} + +func TestController_CreateSnapshotIdempotent(t *testing.T) { + mgr := newTestManager(t) + backend := NewLocalVolumeBackend(mgr) + cs := &controllerServer{backend: backend} + + cs.CreateVolume(context.Background(), &csi.CreateVolumeRequest{ + Name: "snap-vol", + VolumeCapabilities: testVolCaps(), + CapacityRange: &csi.CapacityRange{RequiredBytes: 4 * 1024 * 1024}, + }) + + // First create. + resp1, err := cs.CreateSnapshot(context.Background(), &csi.CreateSnapshotRequest{ + Name: "my-snapshot", + SourceVolumeId: "snap-vol", + }) + if err != nil { + t.Fatalf("first CreateSnapshot: %v", err) + } + + // Same name on same volume -- CSI idempotency: should return existing snapshot. + resp2, err := cs.CreateSnapshot(context.Background(), &csi.CreateSnapshotRequest{ + Name: "my-snapshot", + SourceVolumeId: "snap-vol", + }) + if err != nil { + t.Fatalf("idempotent CreateSnapshot: %v", err) + } + if resp2.Snapshot.SnapshotId != resp1.Snapshot.SnapshotId { + t.Fatalf("snapshot IDs differ: %q vs %q", resp2.Snapshot.SnapshotId, resp1.Snapshot.SnapshotId) + } + if resp2.Snapshot.SourceVolumeId != "snap-vol" { + t.Fatalf("source volume: got %q", resp2.Snapshot.SourceVolumeId) + } +} + +func TestController_DeleteSnapshot(t *testing.T) { + mgr := newTestManager(t) + backend := NewLocalVolumeBackend(mgr) + cs := &controllerServer{backend: backend} + + cs.CreateVolume(context.Background(), &csi.CreateVolumeRequest{ + Name: "snap-vol", + VolumeCapabilities: testVolCaps(), + CapacityRange: &csi.CapacityRange{RequiredBytes: 4 * 1024 * 1024}, + }) + + resp, _ := cs.CreateSnapshot(context.Background(), &csi.CreateSnapshotRequest{ + Name: "del-snap", + SourceVolumeId: "snap-vol", + }) + + _, err := cs.DeleteSnapshot(context.Background(), &csi.DeleteSnapshotRequest{ + SnapshotId: resp.Snapshot.SnapshotId, + }) + if err != nil { + t.Fatalf("DeleteSnapshot: %v", err) + } +} + +func TestController_DeleteSnapshotNotFound(t *testing.T) { + mgr := newTestManager(t) + backend := NewLocalVolumeBackend(mgr) + cs := &controllerServer{backend: backend} + + // Delete non-existent snapshot should succeed (idempotent). + _, err := cs.DeleteSnapshot(context.Background(), &csi.DeleteSnapshotRequest{ + SnapshotId: "snap-nonexistent-vol-42", + }) + if err != nil { + t.Fatalf("DeleteSnapshot non-existent: %v", err) + } +} + +func TestController_ListSnapshotsByVolume(t *testing.T) { + mgr := newTestManager(t) + backend := NewLocalVolumeBackend(mgr) + cs := &controllerServer{backend: backend} + + cs.CreateVolume(context.Background(), &csi.CreateVolumeRequest{ + Name: "list-vol", + VolumeCapabilities: testVolCaps(), + CapacityRange: &csi.CapacityRange{RequiredBytes: 4 * 1024 * 1024}, + }) + + cs.CreateSnapshot(context.Background(), &csi.CreateSnapshotRequest{ + Name: "snap-a", SourceVolumeId: "list-vol", + }) + cs.CreateSnapshot(context.Background(), &csi.CreateSnapshotRequest{ + Name: "snap-b", SourceVolumeId: "list-vol", + }) + + resp, err := cs.ListSnapshots(context.Background(), &csi.ListSnapshotsRequest{ + SourceVolumeId: "list-vol", + }) + if err != nil { + t.Fatalf("ListSnapshots: %v", err) + } + if len(resp.Entries) != 2 { + t.Fatalf("expected 2 snapshots, got %d", len(resp.Entries)) + } +} + +func TestController_ExpandVolume(t *testing.T) { + mgr := newTestManager(t) + backend := NewLocalVolumeBackend(mgr) + cs := &controllerServer{backend: backend} + + cs.CreateVolume(context.Background(), &csi.CreateVolumeRequest{ + Name: "expand-vol", + VolumeCapabilities: testVolCaps(), + CapacityRange: &csi.CapacityRange{RequiredBytes: 4 * 1024 * 1024}, + }) + + resp, err := cs.ControllerExpandVolume(context.Background(), &csi.ControllerExpandVolumeRequest{ + VolumeId: "expand-vol", + CapacityRange: &csi.CapacityRange{RequiredBytes: 8 * 1024 * 1024}, + }) + if err != nil { + t.Fatalf("ControllerExpandVolume: %v", err) + } + if resp.CapacityBytes != 8*1024*1024 { + t.Fatalf("CapacityBytes: got %d, want %d", resp.CapacityBytes, 8*1024*1024) + } + if !resp.NodeExpansionRequired { + t.Fatal("NodeExpansionRequired should be true") + } +} + +func TestController_ExpandNoShrink(t *testing.T) { + mgr := newTestManager(t) + backend := NewLocalVolumeBackend(mgr) + cs := &controllerServer{backend: backend} + + cs.CreateVolume(context.Background(), &csi.CreateVolumeRequest{ + Name: "noshrink-vol", + VolumeCapabilities: testVolCaps(), + CapacityRange: &csi.CapacityRange{RequiredBytes: 8 * 1024 * 1024}, + }) + + _, err := cs.ControllerExpandVolume(context.Background(), &csi.ControllerExpandVolumeRequest{ + VolumeId: "noshrink-vol", + CapacityRange: &csi.CapacityRange{RequiredBytes: 4 * 1024 * 1024}, + }) + if err == nil { + t.Fatal("expected error for shrink") + } + st, _ := status.FromError(err) + if st.Code() != codes.InvalidArgument { + t.Fatalf("expected InvalidArgument, got %v", st.Code()) + } +} + +func TestController_ExpandMissingVolumeID(t *testing.T) { + mgr := newTestManager(t) + backend := NewLocalVolumeBackend(mgr) + cs := &controllerServer{backend: backend} + + _, err := cs.ControllerExpandVolume(context.Background(), &csi.ControllerExpandVolumeRequest{ + CapacityRange: &csi.CapacityRange{RequiredBytes: 8 * 1024 * 1024}, + }) + if err == nil { + t.Fatal("expected error for missing volume ID") + } + st, _ := status.FromError(err) + if st.Code() != codes.InvalidArgument { + t.Fatalf("expected InvalidArgument, got %v", st.Code()) } } diff --git a/weed/storage/blockvol/csi/iscsi_util.go b/weed/storage/blockvol/csi/iscsi_util.go index a7d9f8b945..01f479f98e 100644 --- a/weed/storage/blockvol/csi/iscsi_util.go +++ b/weed/storage/blockvol/csi/iscsi_util.go @@ -16,6 +16,7 @@ type ISCSIUtil interface { Logout(ctx context.Context, iqn string) error GetDeviceByIQN(ctx context.Context, iqn string) (string, error) IsLoggedIn(ctx context.Context, iqn string) (bool, error) + RescanDevice(ctx context.Context, iqn string) error } // MountUtil provides filesystem mount operations. @@ -26,6 +27,7 @@ type MountUtil interface { Unmount(ctx context.Context, target string) error IsFormatted(ctx context.Context, device string) (bool, error) IsMounted(ctx context.Context, target string) (bool, error) + ResizeFS(ctx context.Context, devicePath, mountPath, fsType string) error } // realISCSIUtil uses iscsiadm CLI. @@ -115,6 +117,15 @@ func (r *realISCSIUtil) IsLoggedIn(ctx context.Context, iqn string) (bool, error return strings.Contains(string(out), iqn), nil } +func (r *realISCSIUtil) RescanDevice(ctx context.Context, iqn string) error { + cmd := exec.CommandContext(ctx, "iscsiadm", "-m", "node", "-T", iqn, "--rescan") + out, err := cmd.CombinedOutput() + if err != nil { + return fmt.Errorf("iscsiadm rescan: %s: %w", string(out), err) + } + return nil +} + // realMountUtil uses mount/umount/mkfs CLI. type realMountUtil struct{} @@ -200,11 +211,32 @@ func (r *realMountUtil) IsMounted(ctx context.Context, target string) (bool, err return false, nil } +func (r *realMountUtil) ResizeFS(ctx context.Context, devicePath, mountPath, fsType string) error { + switch fsType { + case "ext4", "ext3", "ext2": + cmd := exec.CommandContext(ctx, "resize2fs", devicePath) + out, err := cmd.CombinedOutput() + if err != nil { + return fmt.Errorf("resize2fs: %s: %w", string(out), err) + } + case "xfs": + cmd := exec.CommandContext(ctx, "xfs_growfs", mountPath) + out, err := cmd.CombinedOutput() + if err != nil { + return fmt.Errorf("xfs_growfs: %s: %w", string(out), err) + } + default: + return fmt.Errorf("unsupported filesystem type for resize: %s", fsType) + } + return nil +} + // mockISCSIUtil is a test double for ISCSIUtil. type mockISCSIUtil struct { discoveryErr error loginErr error logoutErr error + rescanErr error getDeviceResult string getDeviceErr error loggedIn map[string]bool @@ -247,11 +279,17 @@ func (m *mockISCSIUtil) IsLoggedIn(_ context.Context, iqn string) (bool, error) return m.loggedIn[iqn], nil } +func (m *mockISCSIUtil) RescanDevice(_ context.Context, iqn string) error { + m.calls = append(m.calls, "rescan:"+iqn) + return m.rescanErr +} + // mockMountUtil is a test double for MountUtil. type mockMountUtil struct { formatAndMountErr error mountErr error unmountErr error + resizeFSErr error isFormattedResult bool isMountedTargets map[string]bool calls []string @@ -304,3 +342,8 @@ func (m *mockMountUtil) IsFormatted(_ context.Context, device string) (bool, err func (m *mockMountUtil) IsMounted(_ context.Context, target string) (bool, error) { return m.isMountedTargets[target], nil } + +func (m *mockMountUtil) ResizeFS(_ context.Context, devicePath, mountPath, fsType string) error { + m.calls = append(m.calls, "resizefs:"+devicePath+":"+mountPath+":"+fsType) + return m.resizeFSErr +} diff --git a/weed/storage/blockvol/csi/node.go b/weed/storage/blockvol/csi/node.go index 787855ed33..c62a87cb2b 100644 --- a/weed/storage/blockvol/csi/node.go +++ b/weed/storage/blockvol/csi/node.go @@ -13,11 +13,13 @@ import ( "google.golang.org/grpc/status" ) -// stagedVolumeInfo tracks info needed for NodeUnstageVolume. +// stagedVolumeInfo tracks info needed for NodeUnstageVolume and NodeExpandVolume. type stagedVolumeInfo struct { - iqn string - iscsiAddr string - isLocal bool // true if volume is served by local VolumeManager + iqn string + iscsiAddr string + isLocal bool // true if volume is served by local VolumeManager + fsType string // filesystem type (ext4, xfs, etc.) + stagingPath string // staging mount path } type nodeServer struct { @@ -140,9 +142,11 @@ func (s *nodeServer) NodeStageVolume(ctx context.Context, req *csi.NodeStageVolu s.staged = make(map[string]*stagedVolumeInfo) } s.staged[volumeID] = &stagedVolumeInfo{ - iqn: iqn, - iscsiAddr: portal, - isLocal: isLocal, + iqn: iqn, + iscsiAddr: portal, + isLocal: isLocal, + fsType: fsType, + stagingPath: stagingPath, } s.stagedMu.Unlock() @@ -293,20 +297,82 @@ func (s *nodeServer) NodeUnpublishVolume(ctx context.Context, req *csi.NodeUnpub return &csi.NodeUnpublishVolumeResponse{}, nil } -func (s *nodeServer) NodeGetCapabilities(_ context.Context, _ *csi.NodeGetCapabilitiesRequest) (*csi.NodeGetCapabilitiesResponse, error) { - return &csi.NodeGetCapabilitiesResponse{ - Capabilities: []*csi.NodeServiceCapability{ - { - Type: &csi.NodeServiceCapability_Rpc{ - Rpc: &csi.NodeServiceCapability_RPC{ - Type: csi.NodeServiceCapability_RPC_STAGE_UNSTAGE_VOLUME, - }, - }, - }, - }, +func (s *nodeServer) NodeExpandVolume(ctx context.Context, req *csi.NodeExpandVolumeRequest) (*csi.NodeExpandVolumeResponse, error) { + if req.VolumeId == "" { + return nil, status.Error(codes.InvalidArgument, "volume ID is required") + } + + // Look up staged volume info. + s.stagedMu.Lock() + info, ok := s.staged[req.VolumeId] + s.stagedMu.Unlock() + if !ok { + return nil, status.Errorf(codes.FailedPrecondition, "volume %q not staged", req.VolumeId) + } + + // Check that iSCSI session is active. + loggedIn, err := s.iscsiUtil.IsLoggedIn(ctx, info.iqn) + if err != nil { + return nil, status.Errorf(codes.Internal, "check iSCSI session: %v", err) + } + if !loggedIn { + return nil, status.Errorf(codes.FailedPrecondition, "volume %q not staged: iSCSI session not active", req.VolumeId) + } + + // Rescan device to pick up new size. + if err := s.iscsiUtil.RescanDevice(ctx, info.iqn); err != nil { + return nil, status.Errorf(codes.Internal, "rescan iSCSI device: %v", err) + } + + // Find the device path. + device, err := s.iscsiUtil.GetDeviceByIQN(ctx, info.iqn) + if err != nil { + return nil, status.Errorf(codes.Internal, "find device: %v", err) + } + + // Determine mount path and fsType. + mountPath := info.stagingPath + if req.VolumePath != "" { + mountPath = req.VolumePath + } + fsType := info.fsType + if fsType == "" { + fsType = "ext4" + } + + // Resize the filesystem. + if err := s.mountUtil.ResizeFS(ctx, device, mountPath, fsType); err != nil { + return nil, status.Errorf(codes.Internal, "resize filesystem: %v", err) + } + + s.logger.Printf("NodeExpandVolume: %s expanded (device=%s, fs=%s)", req.VolumeId, device, fsType) + + capacity := int64(0) + if req.CapacityRange != nil { + capacity = req.CapacityRange.RequiredBytes + } + return &csi.NodeExpandVolumeResponse{ + CapacityBytes: capacity, }, nil } +func (s *nodeServer) NodeGetCapabilities(_ context.Context, _ *csi.NodeGetCapabilitiesRequest) (*csi.NodeGetCapabilitiesResponse, error) { + caps := []csi.NodeServiceCapability_RPC_Type{ + csi.NodeServiceCapability_RPC_STAGE_UNSTAGE_VOLUME, + csi.NodeServiceCapability_RPC_EXPAND_VOLUME, + } + + var result []*csi.NodeServiceCapability + for _, c := range caps { + result = append(result, &csi.NodeServiceCapability{ + Type: &csi.NodeServiceCapability_Rpc{ + Rpc: &csi.NodeServiceCapability_RPC{Type: c}, + }, + }) + } + return &csi.NodeGetCapabilitiesResponse{Capabilities: result}, nil +} + func (s *nodeServer) NodeGetInfo(_ context.Context, _ *csi.NodeGetInfoRequest) (*csi.NodeGetInfoResponse, error) { return &csi.NodeGetInfoResponse{ NodeId: s.nodeID, diff --git a/weed/storage/blockvol/csi/node_test.go b/weed/storage/blockvol/csi/node_test.go index a181616c12..021d1f0200 100644 --- a/weed/storage/blockvol/csi/node_test.go +++ b/weed/storage/blockvol/csi/node_test.go @@ -5,6 +5,7 @@ import ( "errors" "log" "os" + "strings" "testing" "github.com/container-storage-interface/spec/lib/go/csi" @@ -511,6 +512,239 @@ func TestNode_UnstageRetryKeepsStagedEntry(t *testing.T) { } } +// TestNode_ExpandVolume verifies happy path for NodeExpandVolume. +func TestNode_ExpandVolume(t *testing.T) { + ns, mi, mm := newTestNodeServer(t) + + stagingPath := t.TempDir() + + // Stage the volume first. + _, err := ns.NodeStageVolume(context.Background(), &csi.NodeStageVolumeRequest{ + VolumeId: "test-vol", + StagingTargetPath: stagingPath, + VolumeCapability: testVolCap(), + }) + if err != nil { + t.Fatalf("NodeStageVolume: %v", err) + } + + // Reset calls to track expand-specific calls. + mi.calls = nil + mm.calls = nil + + _, err = ns.NodeExpandVolume(context.Background(), &csi.NodeExpandVolumeRequest{ + VolumeId: "test-vol", + CapacityRange: &csi.CapacityRange{RequiredBytes: 8 * 1024 * 1024}, + }) + if err != nil { + t.Fatalf("NodeExpandVolume: %v", err) + } + + // Verify rescan was called. + foundRescan := false + for _, c := range mi.calls { + if strings.HasPrefix(c, "rescan:") { + foundRescan = true + } + } + if !foundRescan { + t.Fatalf("expected rescan call, got: %v", mi.calls) + } + + // Verify resize was called. + foundResize := false + for _, c := range mm.calls { + if strings.HasPrefix(c, "resizefs:") { + foundResize = true + } + } + if !foundResize { + t.Fatalf("expected resizefs call, got: %v", mm.calls) + } +} + +// TestNode_ExpandNotStaged verifies error when volume is not staged. +func TestNode_ExpandNotStaged(t *testing.T) { + ns, _, _ := newTestNodeServer(t) + + _, err := ns.NodeExpandVolume(context.Background(), &csi.NodeExpandVolumeRequest{ + VolumeId: "not-staged", + CapacityRange: &csi.CapacityRange{RequiredBytes: 8 * 1024 * 1024}, + }) + if err == nil { + t.Fatal("expected error for not-staged volume") + } + st, _ := status.FromError(err) + if st.Code() != codes.FailedPrecondition { + t.Fatalf("expected FailedPrecondition, got %v", st.Code()) + } +} + +// TestNode_ExpandMissingVolumeID verifies error for missing volume ID. +func TestNode_ExpandMissingVolumeID(t *testing.T) { + ns, _, _ := newTestNodeServer(t) + + _, err := ns.NodeExpandVolume(context.Background(), &csi.NodeExpandVolumeRequest{}) + if err == nil { + t.Fatal("expected error for missing volume ID") + } + st, _ := status.FromError(err) + if st.Code() != codes.InvalidArgument { + t.Fatalf("expected InvalidArgument, got %v", st.Code()) + } +} + +// TestNode_ExpandXFSUsesMountPath verifies xfs_growfs uses mount path. +func TestNode_ExpandXFSUsesMountPath(t *testing.T) { + mi := newMockISCSIUtil() + mi.getDeviceResult = "/dev/sdb" + mm := newMockMountUtil() + + ns := &nodeServer{ + mgr: nil, + nodeID: "test-node-1", + iqnPrefix: "iqn.2024.com.seaweedfs", + iscsiUtil: mi, + mountUtil: mm, + logger: log.New(os.Stderr, "[test-node] ", log.LstdFlags), + staged: map[string]*stagedVolumeInfo{ + "xfs-vol": { + iqn: "iqn.2024.com.seaweedfs:xfs-vol", + iscsiAddr: "10.0.0.5:3260", + isLocal: false, + fsType: "xfs", + stagingPath: "/staging/xfs-vol", + }, + }, + } + + // Mark as logged in so the check passes. + mi.loggedIn["iqn.2024.com.seaweedfs:xfs-vol"] = true + + _, err := ns.NodeExpandVolume(context.Background(), &csi.NodeExpandVolumeRequest{ + VolumeId: "xfs-vol", + CapacityRange: &csi.CapacityRange{RequiredBytes: 16 * 1024 * 1024}, + }) + if err != nil { + t.Fatalf("NodeExpandVolume: %v", err) + } + + // Verify resizefs was called with xfs type. + foundXFS := false + for _, c := range mm.calls { + if strings.Contains(c, "resizefs:") && strings.Contains(c, ":xfs") { + foundXFS = true + // Verify mount path is used (not device path). + if !strings.Contains(c, "/staging/xfs-vol") { + t.Fatalf("xfs_growfs should use mount path, got: %s", c) + } + } + } + if !foundXFS { + t.Fatalf("expected resizefs call with xfs, got: %v", mm.calls) + } +} + +// TestNode_ExpandRejectsNoISCSISession verifies error when iSCSI not logged in. +func TestNode_ExpandRejectsNoISCSISession(t *testing.T) { + mi := newMockISCSIUtil() + mm := newMockMountUtil() + + ns := &nodeServer{ + mgr: nil, + nodeID: "test-node-1", + iqnPrefix: "iqn.2024.com.seaweedfs", + iscsiUtil: mi, + mountUtil: mm, + logger: log.New(os.Stderr, "[test-node] ", log.LstdFlags), + staged: map[string]*stagedVolumeInfo{ + "no-session-vol": { + iqn: "iqn.2024.com.seaweedfs:no-session-vol", + iscsiAddr: "10.0.0.5:3260", + isLocal: false, + fsType: "ext4", + stagingPath: "/staging/no-session-vol", + }, + }, + } + // NOT logged in (loggedIn map is empty). + + _, err := ns.NodeExpandVolume(context.Background(), &csi.NodeExpandVolumeRequest{ + VolumeId: "no-session-vol", + }) + if err == nil { + t.Fatal("expected error when iSCSI session not active") + } + st, _ := status.FromError(err) + if st.Code() != codes.FailedPrecondition { + t.Fatalf("expected FailedPrecondition, got %v", st.Code()) + } +} + +// TestNode_NodeGetCapabilities verifies EXPAND_VOLUME capability is present. +func TestNode_NodeGetCapabilities(t *testing.T) { + ns, _, _ := newTestNodeServer(t) + + resp, err := ns.NodeGetCapabilities(context.Background(), &csi.NodeGetCapabilitiesRequest{}) + if err != nil { + t.Fatalf("NodeGetCapabilities: %v", err) + } + + hasStage := false + hasExpand := false + for _, cap := range resp.Capabilities { + rpc := cap.GetRpc() + if rpc == nil { + continue + } + switch rpc.Type { + case csi.NodeServiceCapability_RPC_STAGE_UNSTAGE_VOLUME: + hasStage = true + case csi.NodeServiceCapability_RPC_EXPAND_VOLUME: + hasExpand = true + } + } + if !hasStage { + t.Fatal("expected STAGE_UNSTAGE_VOLUME capability") + } + if !hasExpand { + t.Fatal("expected EXPAND_VOLUME capability") + } +} + +// TestNode_StageCapturesFSType verifies that fsType is captured in staged info. +func TestNode_StageCapturesFSType(t *testing.T) { + ns, _, _ := newTestNodeServer(t) + + stagingPath := t.TempDir() + + _, err := ns.NodeStageVolume(context.Background(), &csi.NodeStageVolumeRequest{ + VolumeId: "test-vol", + StagingTargetPath: stagingPath, + VolumeCapability: &csi.VolumeCapability{ + AccessType: &csi.VolumeCapability_Mount{ + Mount: &csi.VolumeCapability_MountVolume{FsType: "xfs"}, + }, + }, + }) + if err != nil { + t.Fatalf("NodeStageVolume: %v", err) + } + + ns.stagedMu.Lock() + info := ns.staged["test-vol"] + ns.stagedMu.Unlock() + if info == nil { + t.Fatal("expected test-vol in staged map") + } + if info.fsType != "xfs" { + t.Fatalf("fsType: got %q, want xfs", info.fsType) + } + if info.stagingPath != stagingPath { + t.Fatalf("stagingPath: got %q, want %q", info.stagingPath, stagingPath) + } +} + // TestNode_StageFallbackLocal verifies local fallback when no volume_context is provided. func TestNode_StageFallbackLocal(t *testing.T) { ns, mi, _ := newTestNodeServer(t) diff --git a/weed/storage/blockvol/csi/qa_cp83_adversarial_test.go b/weed/storage/blockvol/csi/qa_cp83_adversarial_test.go new file mode 100644 index 0000000000..1290ceb374 --- /dev/null +++ b/weed/storage/blockvol/csi/qa_cp83_adversarial_test.go @@ -0,0 +1,690 @@ +package csi + +import ( + "context" + "fmt" + "sync" + "sync/atomic" + "testing" + + "github.com/container-storage-interface/spec/lib/go/csi" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" +) + +// ============================================================================ +// QA Adversarial Tests for CP8-3: CSI Snapshots + Expansion +// Covers concurrency, edge cases, error injection, and lifecycle races. +// ============================================================================ + +// --- A. Concurrent Snapshot Operations --- + +// TestQA_CP83_ConcurrentCreateSameSnapshot: 10 goroutines create the same +// snapshot name on the same volume. Exactly one should create; the rest +// should get idempotent success (same snapID on same volume). +func TestQA_CP83_ConcurrentCreateSameSnapshot(t *testing.T) { + mgr := newTestManager(t) + backend := NewLocalVolumeBackend(mgr) + cs := &controllerServer{backend: backend} + + cs.CreateVolume(context.Background(), &csi.CreateVolumeRequest{ + Name: "conc-snap-vol", + VolumeCapabilities: testVolCaps(), + CapacityRange: &csi.CapacityRange{RequiredBytes: 4 * 1024 * 1024}, + }) + + const N = 10 + var wg sync.WaitGroup + errs := make([]error, N) + resps := make([]*csi.CreateSnapshotResponse, N) + + wg.Add(N) + for i := 0; i < N; i++ { + go func(idx int) { + defer wg.Done() + resp, err := cs.CreateSnapshot(context.Background(), &csi.CreateSnapshotRequest{ + Name: "same-snap-name", + SourceVolumeId: "conc-snap-vol", + }) + errs[idx] = err + resps[idx] = resp + }(i) + } + wg.Wait() + + // All should succeed (one creates, rest get idempotent). + var snapID string + for i := 0; i < N; i++ { + if errs[i] != nil { + t.Errorf("goroutine %d: unexpected error: %v", i, errs[i]) + continue + } + if snapID == "" { + snapID = resps[i].Snapshot.SnapshotId + } else if resps[i].Snapshot.SnapshotId != snapID { + t.Errorf("goroutine %d: snapshot ID mismatch: %s vs %s", i, resps[i].Snapshot.SnapshotId, snapID) + } + } +} + +// TestQA_CP83_ConcurrentCreateDifferentSnapshots: 5 goroutines each create +// a uniquely named snapshot on the same volume. All should succeed. +func TestQA_CP83_ConcurrentCreateDifferentSnapshots(t *testing.T) { + mgr := newTestManager(t) + backend := NewLocalVolumeBackend(mgr) + cs := &controllerServer{backend: backend} + + cs.CreateVolume(context.Background(), &csi.CreateVolumeRequest{ + Name: "multi-snap-vol", + VolumeCapabilities: testVolCaps(), + CapacityRange: &csi.CapacityRange{RequiredBytes: 4 * 1024 * 1024}, + }) + + const N = 5 + var wg sync.WaitGroup + errs := make([]error, N) + + wg.Add(N) + for i := 0; i < N; i++ { + go func(idx int) { + defer wg.Done() + _, errs[idx] = cs.CreateSnapshot(context.Background(), &csi.CreateSnapshotRequest{ + Name: fmt.Sprintf("snap-%d", idx), + SourceVolumeId: "multi-snap-vol", + }) + }(i) + } + wg.Wait() + + for i := 0; i < N; i++ { + if errs[i] != nil { + t.Errorf("goroutine %d: unexpected error: %v", i, errs[i]) + } + } + + // Verify all snapshots exist. + listResp, err := cs.ListSnapshots(context.Background(), &csi.ListSnapshotsRequest{ + SourceVolumeId: "multi-snap-vol", + }) + if err != nil { + t.Fatal(err) + } + if len(listResp.Entries) != N { + t.Fatalf("expected %d snapshots, got %d", N, len(listResp.Entries)) + } +} + +// TestQA_CP83_ConcurrentSnapshotAndExpand: one goroutine creates a snapshot, +// another tries to expand. The expand should fail (FailedPrecondition) if +// the snapshot was created first, or the snapshot should succeed. +func TestQA_CP83_ConcurrentSnapshotAndExpand(t *testing.T) { + mgr := newTestManager(t) + backend := NewLocalVolumeBackend(mgr) + cs := &controllerServer{backend: backend} + + cs.CreateVolume(context.Background(), &csi.CreateVolumeRequest{ + Name: "race-vol", + VolumeCapabilities: testVolCaps(), + CapacityRange: &csi.CapacityRange{RequiredBytes: 4 * 1024 * 1024}, + }) + + var wg sync.WaitGroup + var snapErr, expandErr error + + wg.Add(2) + go func() { + defer wg.Done() + _, snapErr = cs.CreateSnapshot(context.Background(), &csi.CreateSnapshotRequest{ + Name: "race-snap", + SourceVolumeId: "race-vol", + }) + }() + go func() { + defer wg.Done() + _, expandErr = cs.ControllerExpandVolume(context.Background(), &csi.ControllerExpandVolumeRequest{ + VolumeId: "race-vol", + CapacityRange: &csi.CapacityRange{RequiredBytes: 8 * 1024 * 1024}, + }) + }() + wg.Wait() + + // One of two outcomes: + // 1. Snapshot wins → expand gets FailedPrecondition + // 2. Expand wins → snapshot still succeeds (taken after expand) + if snapErr != nil { + t.Fatalf("snapshot should always succeed, got: %v", snapErr) + } + if expandErr != nil { + st, _ := status.FromError(expandErr) + if st.Code() != codes.FailedPrecondition { + t.Fatalf("expand error should be FailedPrecondition, got %v: %s", st.Code(), st.Message()) + } + } + // No panic = pass. +} + +// --- B. Expand Edge Cases --- + +// TestQA_CP83_ExpandToSameSize: expand to current size should succeed (no-op). +func TestQA_CP83_ExpandToSameSize(t *testing.T) { + mgr := newTestManager(t) + backend := NewLocalVolumeBackend(mgr) + cs := &controllerServer{backend: backend} + + cs.CreateVolume(context.Background(), &csi.CreateVolumeRequest{ + Name: "same-size-vol", + VolumeCapabilities: testVolCaps(), + CapacityRange: &csi.CapacityRange{RequiredBytes: 4 * 1024 * 1024}, + }) + + resp, err := cs.ControllerExpandVolume(context.Background(), &csi.ControllerExpandVolumeRequest{ + VolumeId: "same-size-vol", + CapacityRange: &csi.CapacityRange{RequiredBytes: 4 * 1024 * 1024}, + }) + // Engine may return success (no-op) or error (can't shrink to same). + // Either way, no panic. + if err != nil { + st, _ := status.FromError(err) + // InvalidArgument (shrink guard) is acceptable for same-size. + if st.Code() != codes.InvalidArgument { + t.Fatalf("unexpected error: %v", err) + } + } else if resp.CapacityBytes != 4*1024*1024 { + t.Fatalf("expected 4MB, got %d", resp.CapacityBytes) + } +} + +// TestQA_CP83_ExpandToSmallerSize: shrink should be rejected. +func TestQA_CP83_ExpandToSmallerSize(t *testing.T) { + mgr := newTestManager(t) + backend := NewLocalVolumeBackend(mgr) + cs := &controllerServer{backend: backend} + + cs.CreateVolume(context.Background(), &csi.CreateVolumeRequest{ + Name: "shrink-vol", + VolumeCapabilities: testVolCaps(), + CapacityRange: &csi.CapacityRange{RequiredBytes: 8 * 1024 * 1024}, + }) + + _, err := cs.ControllerExpandVolume(context.Background(), &csi.ControllerExpandVolumeRequest{ + VolumeId: "shrink-vol", + CapacityRange: &csi.CapacityRange{RequiredBytes: 4 * 1024 * 1024}, + }) + if err == nil { + t.Fatal("expected error for shrink") + } + st, _ := status.FromError(err) + if st.Code() != codes.InvalidArgument { + t.Fatalf("expected InvalidArgument for shrink, got %v", st.Code()) + } +} + +// TestQA_CP83_ExpandWithZeroBytes: expand with 0 bytes should be rejected. +func TestQA_CP83_ExpandWithZeroBytes(t *testing.T) { + mgr := newTestManager(t) + backend := NewLocalVolumeBackend(mgr) + cs := &controllerServer{backend: backend} + + cs.CreateVolume(context.Background(), &csi.CreateVolumeRequest{ + Name: "zero-expand-vol", + VolumeCapabilities: testVolCaps(), + CapacityRange: &csi.CapacityRange{RequiredBytes: 4 * 1024 * 1024}, + }) + + _, err := cs.ControllerExpandVolume(context.Background(), &csi.ControllerExpandVolumeRequest{ + VolumeId: "zero-expand-vol", + CapacityRange: &csi.CapacityRange{RequiredBytes: 0}, + }) + if err == nil { + t.Fatal("expected error for zero expand") + } + st, _ := status.FromError(err) + if st.Code() != codes.InvalidArgument { + t.Fatalf("expected InvalidArgument, got %v", st.Code()) + } +} + +// TestQA_CP83_ExpandNilCapacity: expand with nil CapacityRange should be rejected. +func TestQA_CP83_ExpandNilCapacity(t *testing.T) { + mgr := newTestManager(t) + backend := NewLocalVolumeBackend(mgr) + cs := &controllerServer{backend: backend} + + _, err := cs.ControllerExpandVolume(context.Background(), &csi.ControllerExpandVolumeRequest{ + VolumeId: "nil-cap-vol", + CapacityRange: nil, + }) + if err == nil { + t.Fatal("expected error for nil capacity range") + } + st, _ := status.FromError(err) + if st.Code() != codes.InvalidArgument { + t.Fatalf("expected InvalidArgument, got %v", st.Code()) + } +} + +// TestQA_CP83_ExpandNonAligned: expand to a non-block-aligned size should +// round up and succeed. +func TestQA_CP83_ExpandNonAligned(t *testing.T) { + mgr := newTestManager(t) + backend := NewLocalVolumeBackend(mgr) + cs := &controllerServer{backend: backend} + + cs.CreateVolume(context.Background(), &csi.CreateVolumeRequest{ + Name: "align-vol", + VolumeCapabilities: testVolCaps(), + CapacityRange: &csi.CapacityRange{RequiredBytes: 4 * 1024 * 1024}, + }) + + // Request non-aligned size (5MB + 1 byte). + requested := int64(5*1024*1024 + 1) + resp, err := cs.ControllerExpandVolume(context.Background(), &csi.ControllerExpandVolumeRequest{ + VolumeId: "align-vol", + CapacityRange: &csi.CapacityRange{RequiredBytes: requested}, + }) + if err != nil { + t.Fatalf("non-aligned expand should succeed: %v", err) + } + // Result should be rounded up to next 4096 boundary. + expected := int64((5*1024*1024+1)/4096+1) * 4096 + if resp.CapacityBytes != expected { + t.Fatalf("expected rounded %d, got %d", expected, resp.CapacityBytes) + } +} + +// TestQA_CP83_ExpandMissingVolume: expand on nonexistent volume. +func TestQA_CP83_ExpandMissingVolume(t *testing.T) { + mgr := newTestManager(t) + backend := NewLocalVolumeBackend(mgr) + cs := &controllerServer{backend: backend} + + _, err := cs.ControllerExpandVolume(context.Background(), &csi.ControllerExpandVolumeRequest{ + VolumeId: "no-such-vol", + CapacityRange: &csi.CapacityRange{RequiredBytes: 8 * 1024 * 1024}, + }) + if err == nil { + t.Fatal("expected error for missing volume") + } + st, _ := status.FromError(err) + if st.Code() != codes.NotFound { + t.Fatalf("expected NotFound, got %v: %s", st.Code(), st.Message()) + } +} + +// --- C. Snapshot Lifecycle --- + +// TestQA_CP83_DoubleSnapshot: create two snapshots, delete first, verify +// second still exists, delete second, verify empty. +func TestQA_CP83_DoubleSnapshot(t *testing.T) { + mgr := newTestManager(t) + backend := NewLocalVolumeBackend(mgr) + cs := &controllerServer{backend: backend} + ctx := context.Background() + + cs.CreateVolume(ctx, &csi.CreateVolumeRequest{ + Name: "double-snap-vol", + VolumeCapabilities: testVolCaps(), + CapacityRange: &csi.CapacityRange{RequiredBytes: 4 * 1024 * 1024}, + }) + + snap1, err := cs.CreateSnapshot(ctx, &csi.CreateSnapshotRequest{ + Name: "snap-alpha", SourceVolumeId: "double-snap-vol", + }) + if err != nil { + t.Fatal(err) + } + snap2, err := cs.CreateSnapshot(ctx, &csi.CreateSnapshotRequest{ + Name: "snap-beta", SourceVolumeId: "double-snap-vol", + }) + if err != nil { + t.Fatal(err) + } + + // Delete first. + cs.DeleteSnapshot(ctx, &csi.DeleteSnapshotRequest{SnapshotId: snap1.Snapshot.SnapshotId}) + + // List should show 1. + list, _ := cs.ListSnapshots(ctx, &csi.ListSnapshotsRequest{SourceVolumeId: "double-snap-vol"}) + if len(list.Entries) != 1 { + t.Fatalf("expected 1 snapshot, got %d", len(list.Entries)) + } + if list.Entries[0].Snapshot.SnapshotId != snap2.Snapshot.SnapshotId { + t.Fatalf("wrong snapshot survived") + } + + // Delete second. + cs.DeleteSnapshot(ctx, &csi.DeleteSnapshotRequest{SnapshotId: snap2.Snapshot.SnapshotId}) + + // List should be empty. + list, _ = cs.ListSnapshots(ctx, &csi.ListSnapshotsRequest{SourceVolumeId: "double-snap-vol"}) + if len(list.Entries) != 0 { + t.Fatalf("expected 0 snapshots, got %d", len(list.Entries)) + } +} + +// TestQA_CP83_SnapshotOnDeletedVolume: delete volume, then try snapshot. +func TestQA_CP83_SnapshotOnDeletedVolume(t *testing.T) { + mgr := newTestManager(t) + backend := NewLocalVolumeBackend(mgr) + cs := &controllerServer{backend: backend} + ctx := context.Background() + + cs.CreateVolume(ctx, &csi.CreateVolumeRequest{ + Name: "doomed-vol", + VolumeCapabilities: testVolCaps(), + CapacityRange: &csi.CapacityRange{RequiredBytes: 4 * 1024 * 1024}, + }) + cs.DeleteVolume(ctx, &csi.DeleteVolumeRequest{VolumeId: "doomed-vol"}) + + _, err := cs.CreateSnapshot(ctx, &csi.CreateSnapshotRequest{ + Name: "orphan", SourceVolumeId: "doomed-vol", + }) + if err == nil { + t.Fatal("expected error for snapshot on deleted volume") + } + st, _ := status.FromError(err) + if st.Code() != codes.NotFound { + t.Fatalf("expected NotFound, got %v", st.Code()) + } +} + +// TestQA_CP83_DeleteVolumeWithActiveSnapshots: volume with snapshots can be +// deleted (engine cleans up snap files). +func TestQA_CP83_DeleteVolumeWithActiveSnapshots(t *testing.T) { + mgr := newTestManager(t) + backend := NewLocalVolumeBackend(mgr) + cs := &controllerServer{backend: backend} + ctx := context.Background() + + cs.CreateVolume(ctx, &csi.CreateVolumeRequest{ + Name: "snap-then-delete-vol", + VolumeCapabilities: testVolCaps(), + CapacityRange: &csi.CapacityRange{RequiredBytes: 4 * 1024 * 1024}, + }) + cs.CreateSnapshot(ctx, &csi.CreateSnapshotRequest{ + Name: "pre-delete-snap", SourceVolumeId: "snap-then-delete-vol", + }) + + _, err := cs.DeleteVolume(ctx, &csi.DeleteVolumeRequest{VolumeId: "snap-then-delete-vol"}) + if err != nil { + t.Fatalf("delete volume with snapshots should succeed: %v", err) + } + + // Verify volume is gone. + _, err = cs.CreateSnapshot(ctx, &csi.CreateSnapshotRequest{ + Name: "after-delete-snap", SourceVolumeId: "snap-then-delete-vol", + }) + if err == nil { + t.Fatal("expected error for snapshot on deleted volume") + } +} + +// TestQA_CP83_FullCycleSnapExpandSnap: create → snap → delete snap → expand → +// snap at new size → verify snapshot reports new size. +func TestQA_CP83_FullCycleSnapExpandSnap(t *testing.T) { + mgr := newTestManager(t) + backend := NewLocalVolumeBackend(mgr) + cs := &controllerServer{backend: backend} + ctx := context.Background() + + cs.CreateVolume(ctx, &csi.CreateVolumeRequest{ + Name: "cycle-vol", + VolumeCapabilities: testVolCaps(), + CapacityRange: &csi.CapacityRange{RequiredBytes: 4 * 1024 * 1024}, + }) + + // Snapshot at 4MB. + snap1, _ := cs.CreateSnapshot(ctx, &csi.CreateSnapshotRequest{ + Name: "pre-expand-snap", SourceVolumeId: "cycle-vol", + }) + if snap1.Snapshot.SizeBytes != 4*1024*1024 { + t.Fatalf("snap1 size: got %d, want %d", snap1.Snapshot.SizeBytes, 4*1024*1024) + } + + // Delete snap so we can expand. + cs.DeleteSnapshot(ctx, &csi.DeleteSnapshotRequest{SnapshotId: snap1.Snapshot.SnapshotId}) + + // Expand to 8MB. + _, err := cs.ControllerExpandVolume(ctx, &csi.ControllerExpandVolumeRequest{ + VolumeId: "cycle-vol", + CapacityRange: &csi.CapacityRange{RequiredBytes: 8 * 1024 * 1024}, + }) + if err != nil { + t.Fatalf("expand: %v", err) + } + + // Snapshot at 8MB. + snap2, _ := cs.CreateSnapshot(ctx, &csi.CreateSnapshotRequest{ + Name: "post-expand-snap", SourceVolumeId: "cycle-vol", + }) + if snap2.Snapshot.SizeBytes != 8*1024*1024 { + t.Fatalf("snap2 size: got %d, want %d", snap2.Snapshot.SizeBytes, 8*1024*1024) + } +} + +// --- D. Error Injection --- + +// TestQA_CP83_ExpandBackendFailure: backend returns error on expand. +func TestQA_CP83_ExpandBackendFailure(t *testing.T) { + mock := &expandFailBackend{expandErr: fmt.Errorf("disk full")} + cs := &controllerServer{backend: mock} + + _, err := cs.ControllerExpandVolume(context.Background(), &csi.ControllerExpandVolumeRequest{ + VolumeId: "fail-vol", + CapacityRange: &csi.CapacityRange{RequiredBytes: 8 * 1024 * 1024}, + }) + if err == nil { + t.Fatal("expected error from backend") + } + st, _ := status.FromError(err) + if st.Code() != codes.Internal { + t.Fatalf("expected Internal, got %v", st.Code()) + } +} + +// TestQA_CP83_SnapshotBackendFailure: backend returns error on create snapshot. +func TestQA_CP83_SnapshotBackendFailure(t *testing.T) { + mock := &expandFailBackend{snapErr: fmt.Errorf("I/O error")} + cs := &controllerServer{backend: mock} + + _, err := cs.CreateSnapshot(context.Background(), &csi.CreateSnapshotRequest{ + Name: "fail-snap", + SourceVolumeId: "fail-vol", + }) + if err == nil { + t.Fatal("expected error from backend") + } + st, _ := status.FromError(err) + if st.Code() != codes.Internal { + t.Fatalf("expected Internal, got %v", st.Code()) + } +} + +// --- E. ListSnapshots Edge Cases --- + +// TestQA_CP83_ListSnapshotsBySnapshotID: filter by snapshot ID returns exactly 1. +func TestQA_CP83_ListSnapshotsBySnapshotID(t *testing.T) { + mgr := newTestManager(t) + backend := NewLocalVolumeBackend(mgr) + cs := &controllerServer{backend: backend} + ctx := context.Background() + + cs.CreateVolume(ctx, &csi.CreateVolumeRequest{ + Name: "list-by-id-vol", + VolumeCapabilities: testVolCaps(), + CapacityRange: &csi.CapacityRange{RequiredBytes: 4 * 1024 * 1024}, + }) + snapResp, _ := cs.CreateSnapshot(ctx, &csi.CreateSnapshotRequest{ + Name: "list-snap", SourceVolumeId: "list-by-id-vol", + }) + + list, err := cs.ListSnapshots(ctx, &csi.ListSnapshotsRequest{ + SnapshotId: snapResp.Snapshot.SnapshotId, + }) + if err != nil { + t.Fatal(err) + } + if len(list.Entries) != 1 { + t.Fatalf("expected 1, got %d", len(list.Entries)) + } + if list.Entries[0].Snapshot.SnapshotId != snapResp.Snapshot.SnapshotId { + t.Fatalf("wrong snapshot") + } +} + +// TestQA_CP83_ListSnapshotsBySnapshotID_NotFound: filter by non-existent +// snapshot ID returns empty. +func TestQA_CP83_ListSnapshotsBySnapshotID_NotFound(t *testing.T) { + mgr := newTestManager(t) + backend := NewLocalVolumeBackend(mgr) + cs := &controllerServer{backend: backend} + ctx := context.Background() + + cs.CreateVolume(ctx, &csi.CreateVolumeRequest{ + Name: "list-miss-vol", + VolumeCapabilities: testVolCaps(), + CapacityRange: &csi.CapacityRange{RequiredBytes: 4 * 1024 * 1024}, + }) + + list, err := cs.ListSnapshots(ctx, &csi.ListSnapshotsRequest{ + SnapshotId: FormatSnapshotID("list-miss-vol", 99999), + }) + if err != nil { + t.Fatal(err) + } + if len(list.Entries) != 0 { + t.Fatalf("expected 0, got %d", len(list.Entries)) + } +} + +// TestQA_CP83_ListSnapshotsNoFilter: no filter returns empty (documented behavior). +func TestQA_CP83_ListSnapshotsNoFilter(t *testing.T) { + mgr := newTestManager(t) + backend := NewLocalVolumeBackend(mgr) + cs := &controllerServer{backend: backend} + ctx := context.Background() + + cs.CreateVolume(ctx, &csi.CreateVolumeRequest{ + Name: "no-filter-vol", + VolumeCapabilities: testVolCaps(), + CapacityRange: &csi.CapacityRange{RequiredBytes: 4 * 1024 * 1024}, + }) + cs.CreateSnapshot(ctx, &csi.CreateSnapshotRequest{ + Name: "no-filter-snap", SourceVolumeId: "no-filter-vol", + }) + + list, err := cs.ListSnapshots(ctx, &csi.ListSnapshotsRequest{}) + if err != nil { + t.Fatal(err) + } + if len(list.Entries) != 0 { + t.Fatalf("expected 0 (no filter not supported), got %d", len(list.Entries)) + } +} + +// --- F. Concurrent Expand --- + +// TestQA_CP83_ConcurrentExpand: 5 goroutines expand the same volume to +// increasing sizes. No panics, final size should be the largest requested. +func TestQA_CP83_ConcurrentExpand(t *testing.T) { + mgr := newTestManager(t) + backend := NewLocalVolumeBackend(mgr) + cs := &controllerServer{backend: backend} + + cs.CreateVolume(context.Background(), &csi.CreateVolumeRequest{ + Name: "conc-expand-vol", + VolumeCapabilities: testVolCaps(), + CapacityRange: &csi.CapacityRange{RequiredBytes: 4 * 1024 * 1024}, + }) + + const N = 5 + var wg sync.WaitGroup + var maxCapacity atomic.Int64 + + wg.Add(N) + for i := 0; i < N; i++ { + go func(idx int) { + defer wg.Done() + newSize := int64((idx + 2) * 4 * 1024 * 1024) // 8MB, 12MB, 16MB, 20MB, 24MB + resp, err := cs.ControllerExpandVolume(context.Background(), &csi.ControllerExpandVolumeRequest{ + VolumeId: "conc-expand-vol", + CapacityRange: &csi.CapacityRange{RequiredBytes: newSize}, + }) + if err != nil { + // Some may fail with "shrink" if a larger expand completed first. + return + } + for { + old := maxCapacity.Load() + if resp.CapacityBytes <= old { + break + } + if maxCapacity.CompareAndSwap(old, resp.CapacityBytes) { + break + } + } + }(i) + } + wg.Wait() + + // Final capacity should be at least 24MB. + if maxCapacity.Load() < 24*1024*1024 { + t.Fatalf("expected max capacity >= 24MB, got %d", maxCapacity.Load()) + } +} + +// --- G. Snapshot ID Helpers --- + +// TestQA_CP83_FormatParseRoundTrip: verify format/parse round-trip for various names. +func TestQA_CP83_FormatParseRoundTrip(t *testing.T) { + cases := []struct { + vol string + snapID uint32 + }{ + {"simple", 42}, + {"pvc-12345", 99}, + {"vol-with-many-dashes", 1}, + {"vol", 0}, // ID 0 + {"vol", 0xFFFFFFFF}, // max uint32 + } + for _, tc := range cases { + formatted := FormatSnapshotID(tc.vol, tc.snapID) + gotVol, gotSnap, err := ParseSnapshotID(formatted) + if err != nil { + t.Errorf("parse(%q): %v", formatted, err) + continue + } + if gotVol != tc.vol || gotSnap != tc.snapID { + t.Errorf("round-trip failed: vol=%q snap=%d → %q → vol=%q snap=%d", + tc.vol, tc.snapID, formatted, gotVol, gotSnap) + } + } +} + +// --- Mock backends for error injection --- + +type expandFailBackend struct { + expandErr error + snapErr error +} + +func (m *expandFailBackend) CreateVolume(_ context.Context, name string, sizeBytes uint64) (*VolumeInfo, error) { + return &VolumeInfo{VolumeID: name, CapacityBytes: sizeBytes}, nil +} +func (m *expandFailBackend) DeleteVolume(_ context.Context, _ string) error { return nil } +func (m *expandFailBackend) LookupVolume(_ context.Context, name string) (*VolumeInfo, error) { + return &VolumeInfo{VolumeID: name}, nil +} +func (m *expandFailBackend) CreateSnapshot(_ context.Context, volumeID string, snapID uint32) (*SnapshotInfo, error) { + if m.snapErr != nil { + return nil, m.snapErr + } + return &SnapshotInfo{SnapshotID: snapID, VolumeID: volumeID}, nil +} +func (m *expandFailBackend) DeleteSnapshot(_ context.Context, _ string, _ uint32) error { return nil } +func (m *expandFailBackend) ListSnapshots(_ context.Context, _ string) ([]*SnapshotInfo, error) { + return nil, nil +} +func (m *expandFailBackend) ExpandVolume(_ context.Context, _ string, _ uint64) (uint64, error) { + return 0, m.expandErr +} diff --git a/weed/storage/blockvol/csi/qa_cp83_test.go b/weed/storage/blockvol/csi/qa_cp83_test.go new file mode 100644 index 0000000000..83626b0957 --- /dev/null +++ b/weed/storage/blockvol/csi/qa_cp83_test.go @@ -0,0 +1,446 @@ +package csi + +import ( + "context" + "errors" + "fmt" + "log" + "os" + "strings" + "testing" + + "github.com/container-storage-interface/spec/lib/go/csi" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" +) + +// TestQA_Snap_CreateOnNonExistentVolume verifies NotFound when creating snapshot +// on a volume that doesn't exist. +func TestQA_Snap_CreateOnNonExistentVolume(t *testing.T) { + mgr := newTestManager(t) + backend := NewLocalVolumeBackend(mgr) + cs := &controllerServer{backend: backend} + + _, err := cs.CreateSnapshot(context.Background(), &csi.CreateSnapshotRequest{ + Name: "orphan-snap", + SourceVolumeId: "nonexistent-volume", + }) + if err == nil { + t.Fatal("expected error for snapshot on nonexistent volume") + } + st, _ := status.FromError(err) + if st.Code() != codes.NotFound { + t.Fatalf("expected NotFound, got %v: %s", st.Code(), st.Message()) + } +} + +// TestQA_Snap_DeleteNonExistentSnapshot verifies idempotent success when deleting +// a snapshot that doesn't exist (CSI spec: delete should succeed). +func TestQA_Snap_DeleteNonExistentSnapshot(t *testing.T) { + mgr := newTestManager(t) + backend := NewLocalVolumeBackend(mgr) + cs := &controllerServer{backend: backend} + + // Create volume but no snapshot. + cs.CreateVolume(context.Background(), &csi.CreateVolumeRequest{ + Name: "del-snap-vol", + VolumeCapabilities: testVolCaps(), + CapacityRange: &csi.CapacityRange{RequiredBytes: 4 * 1024 * 1024}, + }) + + // Delete a well-formed snapshot ID for an existing volume, but snapshot doesn't exist. + snapID := FormatSnapshotID("del-snap-vol", 99999) + _, err := cs.DeleteSnapshot(context.Background(), &csi.DeleteSnapshotRequest{ + SnapshotId: snapID, + }) + if err != nil { + t.Fatalf("delete nonexistent snapshot should succeed (idempotent), got: %v", err) + } +} + +// TestQA_Snap_SnapshotIDParsing exercises ParseSnapshotID with weird, empty, +// and unicode strings. Delete should treat unparseable IDs as idempotent success. +func TestQA_Snap_SnapshotIDParsing(t *testing.T) { + mgr := newTestManager(t) + backend := NewLocalVolumeBackend(mgr) + cs := &controllerServer{backend: backend} + + cases := []struct { + name string + id string + }{ + {"empty", ""}, + {"no prefix", "garbage-123"}, + {"only prefix", "snap-"}, + {"no numeric suffix", "snap-vol-abc"}, + {"unicode volume", "snap-日本語ボリューム-42"}, + {"missing separator", "snap-noseparator"}, + {"double dash", "snap--42"}, + {"huge number", "snap-vol-99999999999999999999"}, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + if tc.id == "" { + // Empty ID → InvalidArgument from DeleteSnapshot. + _, err := cs.DeleteSnapshot(context.Background(), &csi.DeleteSnapshotRequest{ + SnapshotId: tc.id, + }) + if err == nil { + t.Fatal("expected error for empty snapshot ID") + } + st, _ := status.FromError(err) + if st.Code() != codes.InvalidArgument { + t.Fatalf("expected InvalidArgument for empty ID, got %v", st.Code()) + } + return + } + + // Non-empty but unparseable → idempotent success. + _, err := cs.DeleteSnapshot(context.Background(), &csi.DeleteSnapshotRequest{ + SnapshotId: tc.id, + }) + if err != nil { + t.Fatalf("delete unparseable snapshot ID should succeed (idempotent), got: %v", err) + } + }) + } +} + +// TestQA_Snap_ExpandWithActiveSnapshots verifies FailedPrecondition when expanding +// a volume that has active snapshots. +func TestQA_Snap_ExpandWithActiveSnapshots(t *testing.T) { + mgr := newTestManager(t) + backend := NewLocalVolumeBackend(mgr) + cs := &controllerServer{backend: backend} + + // Create volume. + cs.CreateVolume(context.Background(), &csi.CreateVolumeRequest{ + Name: "snap-expand-vol", + VolumeCapabilities: testVolCaps(), + CapacityRange: &csi.CapacityRange{RequiredBytes: 4 * 1024 * 1024}, + }) + + // Create snapshot. + _, err := cs.CreateSnapshot(context.Background(), &csi.CreateSnapshotRequest{ + Name: "blocking-snap", + SourceVolumeId: "snap-expand-vol", + }) + if err != nil { + t.Fatalf("CreateSnapshot: %v", err) + } + + // Now try to expand --engine should reject due to active snapshots. + _, err = cs.ControllerExpandVolume(context.Background(), &csi.ControllerExpandVolumeRequest{ + VolumeId: "snap-expand-vol", + CapacityRange: &csi.CapacityRange{RequiredBytes: 8 * 1024 * 1024}, + }) + if err == nil { + t.Fatal("expected error for expand with active snapshots") + } + st, _ := status.FromError(err) + if st.Code() != codes.FailedPrecondition { + t.Fatalf("expected FailedPrecondition, got %v: %s", st.Code(), st.Message()) + } + if !strings.Contains(st.Message(), "snapshots") { + t.Fatalf("error message should mention snapshots: %s", st.Message()) + } +} + +// TestQA_Snap_SnapshotCollisionHandled tests that two different names hashing to +// the same FNV-32a ID are handled with AlreadyExists. +func TestQA_Snap_SnapshotCollisionHandled(t *testing.T) { + // Use a mock backend that always returns "already exists" after the first create, + // simulating what happens when two names produce the same FNV-32a hash. + mock := &collisionMockBackend{ + createCount: 0, + } + cs := &controllerServer{backend: mock} + + // First snapshot succeeds. + _, err := cs.CreateSnapshot(context.Background(), &csi.CreateSnapshotRequest{ + Name: "first-name", + SourceVolumeId: "collision-vol", + }) + if err != nil { + t.Fatalf("first CreateSnapshot: %v", err) + } + + // Second snapshot: mock simulates FNV collision by returning "already exists". + _, err = cs.CreateSnapshot(context.Background(), &csi.CreateSnapshotRequest{ + Name: "second-name", + SourceVolumeId: "collision-vol", + }) + if err == nil { + t.Fatal("expected error for collision") + } + st, _ := status.FromError(err) + if st.Code() != codes.AlreadyExists { + t.Fatalf("expected AlreadyExists, got %v: %s", st.Code(), st.Message()) + } + if !strings.Contains(st.Message(), "collision") { + t.Fatalf("error message should mention collision: %s", st.Message()) + } +} + +// TestQA_Expand_RescanFailure verifies that NodeExpandVolume handles rescan failure +// by returning Internal error. +func TestQA_Expand_RescanFailure(t *testing.T) { + mi := newMockISCSIUtil() + mi.getDeviceResult = "/dev/sdb" + mi.rescanErr = errors.New("iscsiadm rescan: device not found") + mm := newMockMountUtil() + + ns := &nodeServer{ + mgr: nil, + nodeID: "test-node-1", + iqnPrefix: "iqn.2024.com.seaweedfs", + iscsiUtil: mi, + mountUtil: mm, + logger: log.New(os.Stderr, "[test-node] ", log.LstdFlags), + staged: map[string]*stagedVolumeInfo{ + "rescan-fail-vol": { + iqn: "iqn.2024.com.seaweedfs:rescan-fail-vol", + iscsiAddr: "10.0.0.5:3260", + isLocal: false, + fsType: "ext4", + stagingPath: "/staging/rescan-fail-vol", + }, + }, + } + + // Mark as logged in so session check passes. + mi.loggedIn["iqn.2024.com.seaweedfs:rescan-fail-vol"] = true + + _, err := ns.NodeExpandVolume(context.Background(), &csi.NodeExpandVolumeRequest{ + VolumeId: "rescan-fail-vol", + CapacityRange: &csi.CapacityRange{RequiredBytes: 16 * 1024 * 1024}, + }) + if err == nil { + t.Fatal("expected error from rescan failure") + } + st, _ := status.FromError(err) + if st.Code() != codes.Internal { + t.Fatalf("expected Internal, got %v: %s", st.Code(), st.Message()) + } +} + +// TestQA_FullLifecycleWithSnapshotExpand tests the full lifecycle: +// create volume → snapshot → delete snapshot → expand → verify. +func TestQA_FullLifecycleWithSnapshotExpand(t *testing.T) { + mgr := newTestManager(t) + backend := NewLocalVolumeBackend(mgr) + cs := &controllerServer{backend: backend} + ctx := context.Background() + + // 1. Create volume. + volResp, err := cs.CreateVolume(ctx, &csi.CreateVolumeRequest{ + Name: "lifecycle-vol", + VolumeCapabilities: testVolCaps(), + CapacityRange: &csi.CapacityRange{RequiredBytes: 4 * 1024 * 1024}, + }) + if err != nil { + t.Fatalf("CreateVolume: %v", err) + } + if volResp.Volume.CapacityBytes != 4*1024*1024 { + t.Fatalf("initial capacity: got %d", volResp.Volume.CapacityBytes) + } + + // 2. Create snapshot. + snapResp, err := cs.CreateSnapshot(ctx, &csi.CreateSnapshotRequest{ + Name: "lifecycle-snap", + SourceVolumeId: "lifecycle-vol", + }) + if err != nil { + t.Fatalf("CreateSnapshot: %v", err) + } + if !snapResp.Snapshot.ReadyToUse { + t.Fatal("snapshot should be ready") + } + snapID := snapResp.Snapshot.SnapshotId + + // 3. List snapshots --should find it. + listResp, err := cs.ListSnapshots(ctx, &csi.ListSnapshotsRequest{ + SourceVolumeId: "lifecycle-vol", + }) + if err != nil { + t.Fatalf("ListSnapshots: %v", err) + } + if len(listResp.Entries) != 1 { + t.Fatalf("expected 1 snapshot, got %d", len(listResp.Entries)) + } + + // 4. Expand should fail with active snapshot. + _, err = cs.ControllerExpandVolume(ctx, &csi.ControllerExpandVolumeRequest{ + VolumeId: "lifecycle-vol", + CapacityRange: &csi.CapacityRange{RequiredBytes: 8 * 1024 * 1024}, + }) + if err == nil { + t.Fatal("expected expand to fail with active snapshot") + } + st, _ := status.FromError(err) + if st.Code() != codes.FailedPrecondition { + t.Fatalf("expected FailedPrecondition, got %v", st.Code()) + } + + // 5. Delete snapshot. + _, err = cs.DeleteSnapshot(ctx, &csi.DeleteSnapshotRequest{ + SnapshotId: snapID, + }) + if err != nil { + t.Fatalf("DeleteSnapshot: %v", err) + } + + // 6. Now expand should succeed. + expandResp, err := cs.ControllerExpandVolume(ctx, &csi.ControllerExpandVolumeRequest{ + VolumeId: "lifecycle-vol", + CapacityRange: &csi.CapacityRange{RequiredBytes: 8 * 1024 * 1024}, + }) + if err != nil { + t.Fatalf("ControllerExpandVolume: %v", err) + } + if expandResp.CapacityBytes != 8*1024*1024 { + t.Fatalf("expanded capacity: got %d, want %d", expandResp.CapacityBytes, 8*1024*1024) + } + if !expandResp.NodeExpansionRequired { + t.Fatal("NodeExpansionRequired should be true") + } + + // 7. List snapshots --should be empty now. + listResp, err = cs.ListSnapshots(ctx, &csi.ListSnapshotsRequest{ + SourceVolumeId: "lifecycle-vol", + }) + if err != nil { + t.Fatalf("ListSnapshots after delete: %v", err) + } + if len(listResp.Entries) != 0 { + t.Fatalf("expected 0 snapshots after delete, got %d", len(listResp.Entries)) + } +} + +// TestQA_Snap_ListSnapshotsBackendError verifies that ListSnapshots propagates +// backend errors as Internal instead of silently returning empty list. +func TestQA_Snap_ListSnapshotsBackendError(t *testing.T) { + mock := &errorBackend{ + listErr: fmt.Errorf("connection refused"), + } + cs := &controllerServer{backend: mock} + + // By snapshot_id. + _, err := cs.ListSnapshots(context.Background(), &csi.ListSnapshotsRequest{ + SnapshotId: FormatSnapshotID("vol1", 42), + }) + if err == nil { + t.Fatal("expected error from backend failure") + } + st, _ := status.FromError(err) + if st.Code() != codes.Internal { + t.Fatalf("expected Internal, got %v: %s", st.Code(), st.Message()) + } + + // By source_volume_id. + _, err = cs.ListSnapshots(context.Background(), &csi.ListSnapshotsRequest{ + SourceVolumeId: "vol1", + }) + if err == nil { + t.Fatal("expected error from backend failure") + } + st, _ = status.FromError(err) + if st.Code() != codes.Internal { + t.Fatalf("expected Internal, got %v: %s", st.Code(), st.Message()) + } +} + +// TestQA_Snap_ListSnapshotsNotFoundVolume verifies that ListSnapshots returns +// empty list (not error) when volume is not found. +func TestQA_Snap_ListSnapshotsNotFoundVolume(t *testing.T) { + mgr := newTestManager(t) + backend := NewLocalVolumeBackend(mgr) + cs := &controllerServer{backend: backend} + + resp, err := cs.ListSnapshots(context.Background(), &csi.ListSnapshotsRequest{ + SourceVolumeId: "nonexistent-vol", + }) + if err != nil { + t.Fatalf("ListSnapshots on missing volume should succeed, got: %v", err) + } + if len(resp.Entries) != 0 { + t.Fatalf("expected 0 entries, got %d", len(resp.Entries)) + } +} + +// --- collisionMockBackend --- + +// collisionMockBackend is a VolumeBackend that simulates FNV-32a hash collision +// on CreateSnapshot. The first CreateSnapshot succeeds; all subsequent ones +// return "snapshot already exists" to simulate what happens when two different +// names produce the same FNV-32a hash. +type collisionMockBackend struct { + createCount int +} + +func (m *collisionMockBackend) CreateVolume(_ context.Context, name string, sizeBytes uint64) (*VolumeInfo, error) { + return &VolumeInfo{VolumeID: name, CapacityBytes: sizeBytes}, nil +} + +func (m *collisionMockBackend) DeleteVolume(_ context.Context, name string) error { + return nil +} + +func (m *collisionMockBackend) LookupVolume(_ context.Context, name string) (*VolumeInfo, error) { + return &VolumeInfo{VolumeID: name}, nil +} + +func (m *collisionMockBackend) CreateSnapshot(_ context.Context, volumeID string, snapID uint32) (*SnapshotInfo, error) { + m.createCount++ + if m.createCount > 1 { + return nil, fmt.Errorf("snapshot already exists") + } + return &SnapshotInfo{SnapshotID: snapID, VolumeID: volumeID, SizeBytes: 4 * 1024 * 1024}, nil +} + +func (m *collisionMockBackend) DeleteSnapshot(_ context.Context, volumeID string, snapID uint32) error { + return nil +} + +func (m *collisionMockBackend) ListSnapshots(_ context.Context, volumeID string) ([]*SnapshotInfo, error) { + return nil, nil +} + +func (m *collisionMockBackend) ExpandVolume(_ context.Context, volumeID string, newSizeBytes uint64) (uint64, error) { + return newSizeBytes, nil +} + +// --- errorBackend --- + +// errorBackend is a VolumeBackend that returns configurable errors for testing +// error propagation paths. +type errorBackend struct { + listErr error +} + +func (m *errorBackend) CreateVolume(_ context.Context, name string, sizeBytes uint64) (*VolumeInfo, error) { + return &VolumeInfo{VolumeID: name, CapacityBytes: sizeBytes}, nil +} + +func (m *errorBackend) DeleteVolume(_ context.Context, name string) error { return nil } + +func (m *errorBackend) LookupVolume(_ context.Context, name string) (*VolumeInfo, error) { + return &VolumeInfo{VolumeID: name}, nil +} + +func (m *errorBackend) CreateSnapshot(_ context.Context, volumeID string, snapID uint32) (*SnapshotInfo, error) { + return &SnapshotInfo{SnapshotID: snapID, VolumeID: volumeID}, nil +} + +func (m *errorBackend) DeleteSnapshot(_ context.Context, volumeID string, snapID uint32) error { + return nil +} + +func (m *errorBackend) ListSnapshots(_ context.Context, volumeID string) ([]*SnapshotInfo, error) { + return nil, m.listErr +} + +func (m *errorBackend) ExpandVolume(_ context.Context, volumeID string, newSizeBytes uint64) (uint64, error) { + return newSizeBytes, nil +} diff --git a/weed/storage/blockvol/csi/snapshot_ids.go b/weed/storage/blockvol/csi/snapshot_ids.go new file mode 100644 index 0000000000..5e24a785dc --- /dev/null +++ b/weed/storage/blockvol/csi/snapshot_ids.go @@ -0,0 +1,44 @@ +package csi + +import ( + "fmt" + "hash/fnv" + "strconv" + "strings" +) + +// FormatSnapshotID creates a CSI-standard snapshot ID: "snap-{volumeID}-{snapID}". +func FormatSnapshotID(volumeID string, snapID uint32) string { + return fmt.Sprintf("snap-%s-%d", volumeID, snapID) +} + +// ParseSnapshotID reverses FormatSnapshotID, returning volumeID and snapID. +// The volumeID may contain dashes, so we split on the last dash. +func ParseSnapshotID(csiSnapID string) (string, uint32, error) { + if !strings.HasPrefix(csiSnapID, "snap-") { + return "", 0, fmt.Errorf("invalid snapshot ID format: missing snap- prefix") + } + rest := csiSnapID[len("snap-"):] + + lastDash := strings.LastIndex(rest, "-") + if lastDash < 0 || lastDash == 0 { + return "", 0, fmt.Errorf("invalid snapshot ID format: no volume/snap separator") + } + + volumeID := rest[:lastDash] + numStr := rest[lastDash+1:] + + id, err := strconv.ParseUint(numStr, 10, 32) + if err != nil { + return "", 0, fmt.Errorf("invalid snapshot ID format: %w", err) + } + + return volumeID, uint32(id), nil +} + +// snapshotNameToID converts a snapshot name to a deterministic uint32 via FNV-32a hash. +func snapshotNameToID(name string) uint32 { + h := fnv.New32a() + h.Write([]byte(name)) + return h.Sum32() +} diff --git a/weed/storage/blockvol/csi/snapshot_ids_test.go b/weed/storage/blockvol/csi/snapshot_ids_test.go new file mode 100644 index 0000000000..e1db908d89 --- /dev/null +++ b/weed/storage/blockvol/csi/snapshot_ids_test.go @@ -0,0 +1,84 @@ +package csi + +import ( + "testing" +) + +func TestSnapshotID_FormatParseRoundTrip(t *testing.T) { + tests := []struct { + volumeID string + snapID uint32 + }{ + {"pvc-abc123", 42}, + {"vol-with-dashes-in-name", 999}, + {"simple", 0}, + {"a", 4294967295}, // max uint32 + } + for _, tt := range tests { + csiID := FormatSnapshotID(tt.volumeID, tt.snapID) + gotVol, gotSnap, err := ParseSnapshotID(csiID) + if err != nil { + t.Errorf("ParseSnapshotID(%q): %v", csiID, err) + continue + } + if gotVol != tt.volumeID { + t.Errorf("volumeID: got %q, want %q", gotVol, tt.volumeID) + } + if gotSnap != tt.snapID { + t.Errorf("snapID: got %d, want %d", gotSnap, tt.snapID) + } + } +} + +func TestSnapshotID_ParseErrors(t *testing.T) { + bad := []string{ + "", + "not-a-snap", + "snap-", // no volume or snap + "snap-vol", // no dash separator for snap ID + "snap-vol-abc", // non-numeric snap ID + } + for _, s := range bad { + _, _, err := ParseSnapshotID(s) + if err == nil { + t.Errorf("ParseSnapshotID(%q): expected error", s) + } + } +} + +func TestSnapshotNameToID_Deterministic(t *testing.T) { + name := "my-snapshot-name" + id1 := snapshotNameToID(name) + id2 := snapshotNameToID(name) + if id1 != id2 { + t.Fatalf("FNV hash not deterministic: %d vs %d", id1, id2) + } + if id1 == 0 { + t.Fatal("expected non-zero hash") + } +} + +func TestSnapshotNameToID_DifferentNames(t *testing.T) { + id1 := snapshotNameToID("snap-alpha") + id2 := snapshotNameToID("snap-beta") + if id1 == id2 { + t.Fatalf("different names produced same hash: %d", id1) + } +} + +func TestSnapshotID_EmptyVolumeID(t *testing.T) { + // snap--42 should parse: volumeID="" but we reject it + _, _, err := ParseSnapshotID("snap--42") + if err != nil { + // We accept this is an error since lastDash==0 means empty volumeID + return + } + // If it doesn't error, that's also fine as long as parse is correct +} + +func TestSnapshotID_FormatContainsComponents(t *testing.T) { + id := FormatSnapshotID("pvc-123", 7) + if id != "snap-pvc-123-7" { + t.Fatalf("got %q, want snap-pvc-123-7", id) + } +} diff --git a/weed/storage/blockvol/csi/volume_backend.go b/weed/storage/blockvol/csi/volume_backend.go index 7300dfc882..49d3c9f006 100644 --- a/weed/storage/blockvol/csi/volume_backend.go +++ b/weed/storage/blockvol/csi/volume_backend.go @@ -17,11 +17,24 @@ type VolumeInfo struct { CapacityBytes uint64 } +// SnapshotInfo holds snapshot metadata returned by the backend. +type SnapshotInfo struct { + SnapshotID uint32 + VolumeID string + CreatedAt int64 // Unix timestamp in seconds + SizeBytes uint64 +} + // VolumeBackend abstracts volume lifecycle for the CSI controller. type VolumeBackend interface { CreateVolume(ctx context.Context, name string, sizeBytes uint64) (*VolumeInfo, error) DeleteVolume(ctx context.Context, name string) error LookupVolume(ctx context.Context, name string) (*VolumeInfo, error) + + CreateSnapshot(ctx context.Context, volumeID string, snapID uint32) (*SnapshotInfo, error) + DeleteSnapshot(ctx context.Context, volumeID string, snapID uint32) error + ListSnapshots(ctx context.Context, volumeID string) ([]*SnapshotInfo, error) + ExpandVolume(ctx context.Context, volumeID string, newSizeBytes uint64) (uint64, error) } // LocalVolumeBackend wraps VolumeManager for standalone/local mode (CP6-1). @@ -66,6 +79,26 @@ func (b *LocalVolumeBackend) LookupVolume(ctx context.Context, name string) (*Vo }, nil } +func (b *LocalVolumeBackend) CreateSnapshot(ctx context.Context, volumeID string, snapID uint32) (*SnapshotInfo, error) { + info, err := b.mgr.CreateSnapshot(volumeID, snapID) + if err != nil { + return nil, err + } + return info, nil +} + +func (b *LocalVolumeBackend) DeleteSnapshot(ctx context.Context, volumeID string, snapID uint32) error { + return b.mgr.DeleteSnapshot(volumeID, snapID) +} + +func (b *LocalVolumeBackend) ListSnapshots(ctx context.Context, volumeID string) ([]*SnapshotInfo, error) { + return b.mgr.ListSnapshots(volumeID) +} + +func (b *LocalVolumeBackend) ExpandVolume(ctx context.Context, volumeID string, newSizeBytes uint64) (uint64, error) { + return b.mgr.ExpandVolume(volumeID, newSizeBytes) +} + // MasterVolumeClient calls master gRPC for volume operations. type MasterVolumeClient struct { masterAddr string @@ -129,3 +162,72 @@ func (c *MasterVolumeClient) LookupVolume(ctx context.Context, name string) (*Vo }) return info, err } + +func (c *MasterVolumeClient) CreateSnapshot(ctx context.Context, volumeID string, snapID uint32) (*SnapshotInfo, error) { + var info *SnapshotInfo + err := pb.WithMasterClient(false, pb.ServerAddress(c.masterAddr), c.dialOpt, false, func(client master_pb.SeaweedClient) error { + resp, err := client.CreateBlockSnapshot(ctx, &master_pb.CreateBlockSnapshotRequest{ + VolumeName: volumeID, + SnapshotId: snapID, + }) + if err != nil { + return err + } + info = &SnapshotInfo{ + SnapshotID: resp.SnapshotId, + VolumeID: volumeID, + CreatedAt: resp.CreatedAt, + SizeBytes: resp.SizeBytes, + } + return nil + }) + return info, err +} + +func (c *MasterVolumeClient) DeleteSnapshot(ctx context.Context, volumeID string, snapID uint32) error { + return pb.WithMasterClient(false, pb.ServerAddress(c.masterAddr), c.dialOpt, false, func(client master_pb.SeaweedClient) error { + _, err := client.DeleteBlockSnapshot(ctx, &master_pb.DeleteBlockSnapshotRequest{ + VolumeName: volumeID, + SnapshotId: snapID, + }) + return err + }) +} + +func (c *MasterVolumeClient) ListSnapshots(ctx context.Context, volumeID string) ([]*SnapshotInfo, error) { + var infos []*SnapshotInfo + err := pb.WithMasterClient(false, pb.ServerAddress(c.masterAddr), c.dialOpt, false, func(client master_pb.SeaweedClient) error { + resp, err := client.ListBlockSnapshots(ctx, &master_pb.ListBlockSnapshotsRequest{ + VolumeName: volumeID, + }) + if err != nil { + return err + } + for _, s := range resp.Snapshots { + infos = append(infos, &SnapshotInfo{ + SnapshotID: s.SnapshotId, + VolumeID: volumeID, + CreatedAt: s.CreatedAt, + SizeBytes: s.VolumeSizeBytes, + }) + } + return nil + }) + return infos, err +} + +func (c *MasterVolumeClient) ExpandVolume(ctx context.Context, volumeID string, newSizeBytes uint64) (uint64, error) { + var capacity uint64 + err := pb.WithMasterClient(false, pb.ServerAddress(c.masterAddr), c.dialOpt, false, func(client master_pb.SeaweedClient) error { + resp, err := client.ExpandBlockVolume(ctx, &master_pb.ExpandBlockVolumeRequest{ + Name: volumeID, + NewSizeBytes: newSizeBytes, + }) + if err != nil { + return err + } + capacity = resp.CapacityBytes + return nil + }) + return capacity, err +} diff --git a/weed/storage/blockvol/csi/volume_backend_test.go b/weed/storage/blockvol/csi/volume_backend_test.go index 0870b03ec9..8d53f35c60 100644 --- a/weed/storage/blockvol/csi/volume_backend_test.go +++ b/weed/storage/blockvol/csi/volume_backend_test.go @@ -90,3 +90,107 @@ func TestBackend_LocalLookupNotFound(t *testing.T) { t.Fatal("lookup missing should return error") } } + +func TestBackend_LocalCreateDeleteListSnapshots(t *testing.T) { + mgr := newTestManager(t) + backend := NewLocalVolumeBackend(mgr) + ctx := context.Background() + + // Create volume. + if _, err := backend.CreateVolume(ctx, "snap-vol", 4*1024*1024); err != nil { + t.Fatalf("create: %v", err) + } + + // Create snapshot. + snap, err := backend.CreateSnapshot(ctx, "snap-vol", 1) + if err != nil { + t.Fatalf("CreateSnapshot: %v", err) + } + if snap.SnapshotID != 1 { + t.Fatalf("SnapshotID: got %d, want 1", snap.SnapshotID) + } + if snap.CreatedAt == 0 { + t.Fatal("CreatedAt should not be zero") + } + + // List snapshots. + snaps, err := backend.ListSnapshots(ctx, "snap-vol") + if err != nil { + t.Fatalf("ListSnapshots: %v", err) + } + if len(snaps) != 1 { + t.Fatalf("expected 1 snapshot, got %d", len(snaps)) + } + + // Delete snapshot. + if err := backend.DeleteSnapshot(ctx, "snap-vol", 1); err != nil { + t.Fatalf("DeleteSnapshot: %v", err) + } + + // Idempotent delete. + if err := backend.DeleteSnapshot(ctx, "snap-vol", 1); err != nil { + t.Fatalf("idempotent delete: %v", err) + } + + // List should be empty now. + snaps, err = backend.ListSnapshots(ctx, "snap-vol") + if err != nil { + t.Fatalf("ListSnapshots after delete: %v", err) + } + if len(snaps) != 0 { + t.Fatalf("expected 0 snapshots after delete, got %d", len(snaps)) + } +} + +func TestBackend_LocalExpandVolume(t *testing.T) { + mgr := newTestManager(t) + backend := NewLocalVolumeBackend(mgr) + ctx := context.Background() + + if _, err := backend.CreateVolume(ctx, "expand-vol", 4*1024*1024); err != nil { + t.Fatalf("create: %v", err) + } + + newSize, err := backend.ExpandVolume(ctx, "expand-vol", 8*1024*1024) + if err != nil { + t.Fatalf("ExpandVolume: %v", err) + } + if newSize != 8*1024*1024 { + t.Fatalf("new size: got %d, want %d", newSize, 8*1024*1024) + } + + // Verify via lookup. + info, err := backend.LookupVolume(ctx, "expand-vol") + if err != nil { + t.Fatalf("lookup: %v", err) + } + if info.CapacityBytes != 8*1024*1024 { + t.Fatalf("capacity after expand: got %d, want %d", info.CapacityBytes, 8*1024*1024) + } +} + +func TestBackend_LocalExpandNoShrink(t *testing.T) { + mgr := newTestManager(t) + backend := NewLocalVolumeBackend(mgr) + ctx := context.Background() + + if _, err := backend.CreateVolume(ctx, "noshrink-vol", 8*1024*1024); err != nil { + t.Fatalf("create: %v", err) + } + + _, err := backend.ExpandVolume(ctx, "noshrink-vol", 4*1024*1024) + if err == nil { + t.Fatal("expected error when shrinking") + } +} + +func TestBackend_LocalSnapshotVolumeNotFound(t *testing.T) { + mgr := newTestManager(t) + backend := NewLocalVolumeBackend(mgr) + ctx := context.Background() + + _, err := backend.CreateSnapshot(ctx, "missing", 1) + if err == nil { + t.Fatal("expected error for missing volume") + } +} diff --git a/weed/storage/blockvol/csi/volume_manager.go b/weed/storage/blockvol/csi/volume_manager.go index 2725765ebe..e93b96f9a6 100644 --- a/weed/storage/blockvol/csi/volume_manager.go +++ b/weed/storage/blockvol/csi/volume_manager.go @@ -325,6 +325,93 @@ func (m *VolumeManager) ListenAddr() string { return "" } +// WithVolume runs fn while holding the manager lock with a reference to the volume. +func (m *VolumeManager) WithVolume(name string, fn func(*blockvol.BlockVol) error) error { + m.mu.RLock() + defer m.mu.RUnlock() + mv, ok := m.volumes[name] + if !ok { + return ErrVolumeNotFound + } + return fn(mv.vol) +} + +// CreateSnapshot creates a snapshot on the named volume. +func (m *VolumeManager) CreateSnapshot(name string, snapID uint32) (*SnapshotInfo, error) { + m.mu.RLock() + defer m.mu.RUnlock() + mv, ok := m.volumes[name] + if !ok { + return nil, ErrVolumeNotFound + } + if err := mv.vol.CreateSnapshot(snapID); err != nil { + return nil, err + } + // Find the snapshot we just created. + for _, s := range mv.vol.ListSnapshots() { + if s.ID == snapID { + return &SnapshotInfo{ + SnapshotID: snapID, + VolumeID: name, + CreatedAt: s.CreatedAt.Unix(), + SizeBytes: mv.sizeBytes, + }, nil + } + } + return nil, fmt.Errorf("snapshot %d created but not found", snapID) +} + +// DeleteSnapshot deletes a snapshot. Idempotent: returns nil if not found. +func (m *VolumeManager) DeleteSnapshot(name string, snapID uint32) error { + m.mu.RLock() + defer m.mu.RUnlock() + mv, ok := m.volumes[name] + if !ok { + return ErrVolumeNotFound + } + err := mv.vol.DeleteSnapshot(snapID) + if err != nil && err.Error() == "blockvol: snapshot not found" { + return nil + } + return err +} + +// ListSnapshots returns all snapshots for the named volume. +func (m *VolumeManager) ListSnapshots(name string) ([]*SnapshotInfo, error) { + m.mu.RLock() + defer m.mu.RUnlock() + mv, ok := m.volumes[name] + if !ok { + return nil, ErrVolumeNotFound + } + engineInfos := mv.vol.ListSnapshots() + result := make([]*SnapshotInfo, 0, len(engineInfos)) + for _, s := range engineInfos { + result = append(result, &SnapshotInfo{ + SnapshotID: s.ID, + VolumeID: name, + CreatedAt: s.CreatedAt.Unix(), + SizeBytes: mv.sizeBytes, + }) + } + return result, nil +} + +// ExpandVolume expands the named volume. Returns the new size. +func (m *VolumeManager) ExpandVolume(name string, newSizeBytes uint64) (uint64, error) { + m.mu.Lock() + defer m.mu.Unlock() + mv, ok := m.volumes[name] + if !ok { + return 0, ErrVolumeNotFound + } + if err := mv.vol.Expand(newSizeBytes); err != nil { + return 0, err + } + mv.sizeBytes = mv.vol.Info().VolumeSize + return mv.sizeBytes, nil +} + func (m *VolumeManager) volumePath(name string) string { return filepath.Join(m.dataDir, sanitizeFilename(name)+".blk") } diff --git a/weed/storage/blockvol/dist_group_commit.go b/weed/storage/blockvol/dist_group_commit.go index 9edf9d9c3e..43b9a13fb1 100644 --- a/weed/storage/blockvol/dist_group_commit.go +++ b/weed/storage/blockvol/dist_group_commit.go @@ -5,22 +5,24 @@ import ( ) // MakeDistributedSync creates a sync function that runs local WAL fsync and -// replica barrier in parallel. If no replica is configured or the replica is -// degraded, it falls back to local-only sync (Phase 3 behavior). +// replica barriers in parallel. Supports N replicas via ShipperGroup. // -// walSync: the local fsync function (typically fd.Sync) -// shipper: the WAL shipper to the replica (may be nil) -// vol: the BlockVol (used to read nextLSN and trigger degradation) -func MakeDistributedSync(walSync func() error, shipper *WALShipper, vol *BlockVol) func() error { +// If no replicas are configured (group is nil or empty) or all replicas are +// degraded, it falls back to local-only sync. +// +// Local fsync is the durability point. Replica barrier failures degrade +// individual shippers but never fail the write. +func MakeDistributedSync(walSync func() error, group *ShipperGroup, vol *BlockVol) func() error { return func() error { - if shipper == nil || shipper.IsDegraded() { + if group == nil || group.Len() == 0 || group.AllDegraded() { return walSync() } // The highest LSN that needs to be durable is nextLSN-1. lsnMax := vol.nextLSN.Load() - 1 - var localErr, remoteErr error + var localErr error + var barrierErrs []error var wg sync.WaitGroup wg.Add(2) go func() { @@ -29,17 +31,18 @@ func MakeDistributedSync(walSync func() error, shipper *WALShipper, vol *BlockVo }() go func() { defer wg.Done() - remoteErr = shipper.Barrier(lsnMax) + barrierErrs = group.BarrierAll(lsnMax) }() wg.Wait() if localErr != nil { return localErr } - if remoteErr != nil { - // Local succeeded, replica failed --degrade but don't fail the client. - vol.degradeReplica(remoteErr) - return nil + // Remote failures: degrade individual shippers (already done by Barrier). + for _, err := range barrierErrs { + if err != nil { + vol.degradeReplica(err) + } } return nil } diff --git a/weed/storage/blockvol/engine_metrics.go b/weed/storage/blockvol/engine_metrics.go new file mode 100644 index 0000000000..21731e4e4f --- /dev/null +++ b/weed/storage/blockvol/engine_metrics.go @@ -0,0 +1,119 @@ +package blockvol + +import ( + "sync/atomic" + "time" +) + +// EngineMetrics collects subsystem-level counters for Prometheus export. +// All fields are safe for concurrent access. +type EngineMetrics struct { + // Flusher + FlusherBytesTotal atomic.Uint64 + FlusherFlushesTotal atomic.Uint64 + FlusherErrorsTotal atomic.Uint64 + flusherLatencyNs atomicHistogram // flush cycle duration + + // Group Commit + GroupCommitBatchTotal atomic.Uint64 // alias for GroupCommitter.syncCount + groupCommitBatchSize atomicHistogram + groupCommitWaitNs atomicHistogram + + // WAL Shipper + WALShippedEntriesTotal atomic.Uint64 + WALBarrierRequestsTotal atomic.Uint64 + WALFailedBarriersTotal atomic.Uint64 + walBarrierLatencyNs atomicHistogram + + // Scrub + ScrubPassesTotal atomic.Uint64 + ScrubErrorsTotal atomic.Uint64 + scrubDurationNs atomicHistogram +} + +// NewEngineMetrics creates an EngineMetrics instance. +func NewEngineMetrics() *EngineMetrics { + return &EngineMetrics{} +} + +// RecordFlusherFlush records a completed flush cycle. +func (m *EngineMetrics) RecordFlusherFlush(bytesWritten uint64, dur time.Duration) { + m.FlusherBytesTotal.Add(bytesWritten) + m.FlusherFlushesTotal.Add(1) + m.flusherLatencyNs.record(dur.Nanoseconds()) +} + +// RecordFlusherError records a flush error. +func (m *EngineMetrics) RecordFlusherError() { + m.FlusherErrorsTotal.Add(1) +} + +// RecordGroupCommitBatch records a group commit batch. +func (m *EngineMetrics) RecordGroupCommitBatch(batchSize int, waitDur time.Duration) { + m.GroupCommitBatchTotal.Add(1) + m.groupCommitBatchSize.record(int64(batchSize)) + m.groupCommitWaitNs.record(waitDur.Nanoseconds()) +} + +// RecordWALShipped records a shipped WAL entry. +func (m *EngineMetrics) RecordWALShipped() { + m.WALShippedEntriesTotal.Add(1) +} + +// RecordWALBarrier records a barrier request with its result. +func (m *EngineMetrics) RecordWALBarrier(dur time.Duration, failed bool) { + m.WALBarrierRequestsTotal.Add(1) + m.walBarrierLatencyNs.record(dur.Nanoseconds()) + if failed { + m.WALFailedBarriersTotal.Add(1) + } +} + +// RecordScrubPass records a completed scrub pass. +func (m *EngineMetrics) RecordScrubPass(dur time.Duration, errors int64) { + m.ScrubPassesTotal.Add(1) + m.ScrubErrorsTotal.Add(uint64(errors)) + m.scrubDurationNs.record(dur.Nanoseconds()) +} + +// FlusherLatencySnapshot returns flusher latency stats (count, sum_ns). +func (m *EngineMetrics) FlusherLatencySnapshot() (count uint64, sumNs int64) { + return m.flusherLatencyNs.snapshot() +} + +// GroupCommitBatchSizeSnapshot returns batch size stats. +func (m *EngineMetrics) GroupCommitBatchSizeSnapshot() (count uint64, sum int64) { + return m.groupCommitBatchSize.snapshot() +} + +// GroupCommitWaitSnapshot returns group commit wait time stats. +func (m *EngineMetrics) GroupCommitWaitSnapshot() (count uint64, sumNs int64) { + return m.groupCommitWaitNs.snapshot() +} + +// WALBarrierLatencySnapshot returns barrier latency stats. +func (m *EngineMetrics) WALBarrierLatencySnapshot() (count uint64, sumNs int64) { + return m.walBarrierLatencyNs.snapshot() +} + +// ScrubDurationSnapshot returns scrub duration stats. +func (m *EngineMetrics) ScrubDurationSnapshot() (count uint64, sumNs int64) { + return m.scrubDurationNs.snapshot() +} + +// atomicHistogram is a minimal lock-free histogram that tracks count and sum. +// Prometheus histograms are registered in the iscsi-target metrics layer; +// this struct provides the raw observations for GaugeFunc export. +type atomicHistogram struct { + count atomic.Uint64 + sum atomic.Int64 +} + +func (h *atomicHistogram) record(val int64) { + h.count.Add(1) + h.sum.Add(val) +} + +func (h *atomicHistogram) snapshot() (uint64, int64) { + return h.count.Load(), h.sum.Load() +} diff --git a/weed/storage/blockvol/flusher.go b/weed/storage/blockvol/flusher.go index 805f83fb17..5af86ecd5d 100644 --- a/weed/storage/blockvol/flusher.go +++ b/weed/storage/blockvol/flusher.go @@ -37,6 +37,7 @@ type Flusher struct { logger *log.Logger lastErr bool // true if last FlushOnce returned error + metrics *EngineMetrics interval time.Duration notifyCh chan struct{} @@ -51,8 +52,9 @@ type FlusherConfig struct { Super *Superblock WAL *WALWriter DirtyMap *DirtyMap - Interval time.Duration // default 100ms - Logger *log.Logger // optional; defaults to log.Default() + Interval time.Duration // default 100ms + Logger *log.Logger // optional; defaults to log.Default() + Metrics *EngineMetrics // optional; if nil, no metrics recorded } // NewFlusher creates a flusher. Call Run() in a goroutine. @@ -73,6 +75,7 @@ func NewFlusher(cfg FlusherConfig) *Flusher { blockSize: cfg.Super.BlockSize, extentStart: cfg.Super.WALOffset + cfg.Super.WALSize, logger: cfg.Logger, + metrics: cfg.Metrics, checkpointLSN: cfg.Super.WALCheckpointLSN, checkpointTail: 0, interval: cfg.Interval, @@ -98,6 +101,9 @@ func (f *Flusher) Run() { f.logger.Printf("flusher error: %v", err) } f.lastErr = true + if f.metrics != nil { + f.metrics.RecordFlusherError() + } } else { f.lastErr = false } @@ -107,6 +113,9 @@ func (f *Flusher) Run() { f.logger.Printf("flusher error: %v", err) } f.lastErr = true + if f.metrics != nil { + f.metrics.RecordFlusherError() + } } else { f.lastErr = false } @@ -185,6 +194,7 @@ func (f *Flusher) FlushOnce() error { // flushOnceLocked is the inner FlushOnce. Caller must hold flushMu. func (f *Flusher) flushOnceLocked() error { + flushStart := time.Now() entries := f.dirtyMap.Snapshot() if len(entries) == 0 { return nil @@ -333,6 +343,12 @@ func (f *Flusher) flushOnceLocked() error { // Update superblock checkpoint. f.updateSuperblockCheckpoint(maxLSN, f.wal.Tail()) + // Record metrics. + if f.metrics != nil { + bytesWritten := uint64(len(entries)) * uint64(f.blockSize) + f.metrics.RecordFlusherFlush(bytesWritten, time.Since(flushStart)) + } + return nil } diff --git a/weed/storage/blockvol/group_commit.go b/weed/storage/blockvol/group_commit.go index 878d0abe8e..ae9614accb 100644 --- a/weed/storage/blockvol/group_commit.go +++ b/weed/storage/blockvol/group_commit.go @@ -22,6 +22,7 @@ type GroupCommitter struct { lowWatermark int // skip delay if fewer pending (0 = always wait) onDegraded func() // called when fsync fails postSyncCheck func() error // called after sync; error fails waiters + metrics *EngineMetrics mu sync.Mutex pending []chan error @@ -36,12 +37,13 @@ type GroupCommitter struct { // GroupCommitterConfig configures the group committer. type GroupCommitterConfig struct { - SyncFunc func() error // required: the fsync function - MaxDelay time.Duration // default 1ms - MaxBatch int // default 64 - LowWatermark int // skip delay if fewer pending (0 = always wait) - OnDegraded func() // optional: called on fsync error - PostSyncCheck func() error // optional: called after successful sync; error fails all waiters + SyncFunc func() error // required: the fsync function + MaxDelay time.Duration // default 1ms + MaxBatch int // default 64 + LowWatermark int // skip delay if fewer pending (0 = always wait) + OnDegraded func() // optional: called on fsync error + PostSyncCheck func() error // optional: called after successful sync; error fails all waiters + Metrics *EngineMetrics // optional; if nil, no metrics recorded } // NewGroupCommitter creates a new group committer. Call Run() to start it. @@ -62,6 +64,7 @@ func NewGroupCommitter(cfg GroupCommitterConfig) *GroupCommitter { lowWatermark: cfg.LowWatermark, onDegraded: cfg.OnDegraded, postSyncCheck: cfg.PostSyncCheck, + metrics: cfg.Metrics, notifyCh: make(chan struct{}, 1), stopCh: make(chan struct{}), done: make(chan struct{}), @@ -130,6 +133,7 @@ func (gc *GroupCommitter) Run() { // Perform fsync with panic recovery. A panic in syncFunc must // not leave waiters hung --notify them and drain stragglers. + syncStart := time.Now() err := gc.callSyncFunc() gc.syncCount.Add(1) if err == nil && gc.postSyncCheck != nil { @@ -139,6 +143,11 @@ func (gc *GroupCommitter) Run() { gc.onDegraded() } + // Record metrics. + if gc.metrics != nil { + gc.metrics.RecordGroupCommitBatch(len(batch), time.Since(syncStart)) + } + // Wake all waiters. for _, ch := range batch { ch <- err diff --git a/weed/storage/blockvol/health_score.go b/weed/storage/blockvol/health_score.go new file mode 100644 index 0000000000..dd2fe028b2 --- /dev/null +++ b/weed/storage/blockvol/health_score.go @@ -0,0 +1,77 @@ +package blockvol + +import ( + "sync/atomic" + "time" +) + +// HealthStats contains health score details for reporting. +type HealthStats struct { + Score float64 + ScrubErrors int64 + LastScrubError int64 // unix timestamp, 0 = never + LastScrubTime int64 // unix timestamp, 0 = never +} + +// HealthScore tracks the health of a block volume based on scrub results. +// WAL lag is NOT part of engine health — it is tracked at the registry level only. +type HealthScore struct { + scrubErrors atomic.Int64 + lastScrubError atomic.Int64 // unix seconds + lastScrubTime atomic.Int64 // unix seconds +} + +// NewHealthScore creates a new HealthScore with perfect health (1.0). +func NewHealthScore() *HealthScore { + return &HealthScore{} +} + +// RecordScrubError increments the scrub error counter and records timestamp. +func (h *HealthScore) RecordScrubError() { + h.scrubErrors.Add(1) + h.lastScrubError.Store(time.Now().Unix()) +} + +// RecordScrubComplete records when a scrub pass finishes. +func (h *HealthScore) RecordScrubComplete() { + h.lastScrubTime.Store(time.Now().Unix()) +} + +// Score returns the current health score in [0.0, 1.0]. +// Penalties: +// - Scrub errors > 0: penalty = min(0.5, errors × 0.1) +// - Last scrub > 48h ago: penalty = 0.1 +func (h *HealthScore) Score() float64 { + score := 1.0 + + // Scrub error penalty. + errs := h.scrubErrors.Load() + if errs > 0 { + penalty := float64(errs) * 0.1 + if penalty > 0.5 { + penalty = 0.5 + } + score -= penalty + } + + // Stale scrub penalty. + lastScrub := h.lastScrubTime.Load() + if lastScrub > 0 && time.Since(time.Unix(lastScrub, 0)) > 48*time.Hour { + score -= 0.1 + } + + if score < 0.0 { + score = 0.0 + } + return score +} + +// Stats returns detailed health statistics. +func (h *HealthScore) Stats() HealthStats { + return HealthStats{ + Score: h.Score(), + ScrubErrors: h.scrubErrors.Load(), + LastScrubError: h.lastScrubError.Load(), + LastScrubTime: h.lastScrubTime.Load(), + } +} diff --git a/weed/storage/blockvol/health_score_test.go b/weed/storage/blockvol/health_score_test.go new file mode 100644 index 0000000000..f9b142d090 --- /dev/null +++ b/weed/storage/blockvol/health_score_test.go @@ -0,0 +1,58 @@ +package blockvol + +import ( + "testing" + "time" +) + +func TestHealthScore_DefaultPerfect(t *testing.T) { + h := NewHealthScore() + if got := h.Score(); got != 1.0 { + t.Fatalf("default score: got %f, want 1.0", got) + } +} + +func TestHealthScore_ScrubErrorsLowerScore(t *testing.T) { + h := NewHealthScore() + h.RecordScrubError() + if got := h.Score(); got != 0.9 { + t.Fatalf("1 error: got %f, want 0.9", got) + } + h.RecordScrubError() + if got := h.Score(); got != 0.8 { + t.Fatalf("2 errors: got %f, want 0.8", got) + } +} + +func TestHealthScore_CappedPenalty(t *testing.T) { + h := NewHealthScore() + for i := 0; i < 10; i++ { + h.RecordScrubError() + } + // 10 errors × 0.1 = 1.0, but capped at 0.5 penalty → score 0.5 + if got := h.Score(); got != 0.5 { + t.Fatalf("10 errors: got %f, want 0.5", got) + } +} + +func TestHealthScore_StaleScrubPenalty(t *testing.T) { + h := NewHealthScore() + // Set last scrub to 49 hours ago. + h.lastScrubTime.Store(time.Now().Add(-49 * time.Hour).Unix()) + got := h.Score() + if got < 0.89 || got > 0.91 { + t.Fatalf("stale scrub: got %f, want ~0.9", got) + } +} + +func TestHealthScore_RecordScrubComplete(t *testing.T) { + h := NewHealthScore() + h.RecordScrubComplete() + stats := h.Stats() + if stats.LastScrubTime == 0 { + t.Fatal("LastScrubTime should be set after RecordScrubComplete") + } + if stats.Score != 1.0 { + t.Fatalf("score after complete: got %f, want 1.0", stats.Score) + } +} diff --git a/weed/storage/blockvol/iscsi/cmd/iscsi-target/metrics.go b/weed/storage/blockvol/iscsi/cmd/iscsi-target/metrics.go index 0e48fd16da..c930eacb65 100644 --- a/weed/storage/blockvol/iscsi/cmd/iscsi-target/metrics.go +++ b/weed/storage/blockvol/iscsi/cmd/iscsi-target/metrics.go @@ -101,6 +101,53 @@ func newMetricsAdapter(inner iscsi.BlockDevice, vol *blockvol.BlockVol, reg prom Name: "snapshot_count", Help: "Number of active snapshots", }, gs.snapshotCount)) + // --- Engine subsystem metrics (CP8-4) --- + em := vol.Metrics + + // Flusher + reg.MustRegister(prometheus.NewCounterFunc(prometheus.CounterOpts{ + Namespace: "seaweedfs", Subsystem: "blockvol", + Name: "flusher_bytes_total", Help: "Total bytes flushed to extent", + }, func() float64 { return float64(em.FlusherBytesTotal.Load()) })) + reg.MustRegister(prometheus.NewCounterFunc(prometheus.CounterOpts{ + Namespace: "seaweedfs", Subsystem: "blockvol", + Name: "flusher_flushes_total", Help: "Total flush cycles completed", + }, func() float64 { return float64(em.FlusherFlushesTotal.Load()) })) + reg.MustRegister(prometheus.NewCounterFunc(prometheus.CounterOpts{ + Namespace: "seaweedfs", Subsystem: "blockvol", + Name: "flusher_errors_total", Help: "Total flusher errors", + }, func() float64 { return float64(em.FlusherErrorsTotal.Load()) })) + reg.MustRegister(prometheus.NewGaugeFunc(prometheus.GaugeOpts{ + Namespace: "seaweedfs", Subsystem: "blockvol", + Name: "flusher_checkpoint_lsn", Help: "Last flushed checkpoint LSN", + }, gs.checkpointLSN)) + + // Group Commit + reg.MustRegister(prometheus.NewCounterFunc(prometheus.CounterOpts{ + Namespace: "seaweedfs", Subsystem: "blockvol", + Name: "group_commit_flushes_total", Help: "Total group commit fsyncs", + }, func() float64 { return float64(em.GroupCommitBatchTotal.Load()) })) + + // WAL Shipper + reg.MustRegister(prometheus.NewCounterFunc(prometheus.CounterOpts{ + Namespace: "seaweedfs", Subsystem: "blockvol", + Name: "wal_shipped_entries_total", Help: "Total WAL entries shipped to replicas", + }, func() float64 { return float64(em.WALShippedEntriesTotal.Load()) })) + reg.MustRegister(prometheus.NewCounterFunc(prometheus.CounterOpts{ + Namespace: "seaweedfs", Subsystem: "blockvol", + Name: "replica_failed_barriers_total", Help: "Total failed barrier requests", + }, func() float64 { return float64(em.WALFailedBarriersTotal.Load()) })) + + // Scrub + reg.MustRegister(prometheus.NewCounterFunc(prometheus.CounterOpts{ + Namespace: "seaweedfs", Subsystem: "blockvol", + Name: "scrub_passes_total", Help: "Total scrub passes completed", + }, func() float64 { return float64(em.ScrubPassesTotal.Load()) })) + reg.MustRegister(prometheus.NewCounterFunc(prometheus.CounterOpts{ + Namespace: "seaweedfs", Subsystem: "blockvol", + Name: "scrub_errors_total", Help: "Total scrub errors (CRC mismatches)", + }, func() float64 { return float64(em.ScrubErrorsTotal.Load()) })) + return &metricsAdapter{ inner: inner, writeOps: writeOps, @@ -192,3 +239,7 @@ func (gs *gaugeSource) role() float64 { func (gs *gaugeSource) snapshotCount() float64 { return float64(len(gs.vol.ListSnapshots())) } + +func (gs *gaugeSource) checkpointLSN() float64 { + return float64(gs.vol.CheckpointLSN()) +} diff --git a/weed/storage/blockvol/promotion.go b/weed/storage/blockvol/promotion.go index 1d6528032f..e3df40c932 100644 --- a/weed/storage/blockvol/promotion.go +++ b/weed/storage/blockvol/promotion.go @@ -115,9 +115,9 @@ func demote(vol *BlockVol, newEpoch uint64) error { } } - // Stop shipper if present. - if vol.shipper != nil { - vol.shipper.Stop() + // Stop shippers if present. + if vol.shipperGroup != nil { + vol.shipperGroup.StopAll() } // Transition Draining -> Stale. diff --git a/weed/storage/blockvol/qa_phase4a_cp2_test.go b/weed/storage/blockvol/qa_phase4a_cp2_test.go index 4f54bbd233..8160372096 100644 --- a/weed/storage/blockvol/qa_phase4a_cp2_test.go +++ b/weed/storage/blockvol/qa_phase4a_cp2_test.go @@ -922,12 +922,12 @@ func testQADSyncRemoteFailDegrades(t *testing.T) { replica.replRecv.Stop() // Force shipper to reconnect (old ctrl conn may still be open). - primary.shipper.ctrlMu.Lock() - if primary.shipper.ctrlConn != nil { - primary.shipper.ctrlConn.Close() - primary.shipper.ctrlConn = nil + primary.shipperGroup.Shipper(0).ctrlMu.Lock() + if primary.shipperGroup.Shipper(0).ctrlConn != nil { + primary.shipperGroup.Shipper(0).ctrlConn.Close() + primary.shipperGroup.Shipper(0).ctrlConn = nil } - primary.shipper.ctrlMu.Unlock() + primary.shipperGroup.Shipper(0).ctrlMu.Unlock() // SyncCache triggers distributed sync -> barrier fails -> degrade. err := primary.SyncCache() @@ -937,7 +937,7 @@ func testQADSyncRemoteFailDegrades(t *testing.T) { } // Shipper should be degraded. - if !primary.shipper.IsDegraded() { + if !primary.shipperGroup.Shipper(0).IsDegraded() { t.Error("shipper should be degraded after replica receiver stopped") } } @@ -953,7 +953,7 @@ func testQADSyncBothFail(t *testing.T) { syncFn := MakeDistributedSync( func() error { return localErr }, - shipper, + NewShipperGroup([]*WALShipper{shipper}), vol, ) @@ -992,7 +992,7 @@ func testQADSyncParallelExecution(t *testing.T) { return nil } - syncFn := MakeDistributedSync(slowLocalSync, primary.shipper, primary) + syncFn := MakeDistributedSync(slowLocalSync, primary.shipperGroup, primary) start := time.Now() err := syncFn() @@ -1051,7 +1051,7 @@ func testQAE2EReplicaDataMatchesPrimary(t *testing.T) { } // Verify receivedLSN on replica matches what primary shipped. - primaryLSN := primary.shipper.ShippedLSN() + primaryLSN := primary.shipperGroup.Shipper(0).ShippedLSN() replicaLSN := replica.replRecv.ReceivedLSN() if replicaLSN < primaryLSN { t.Errorf("replica receivedLSN = %d < primary shippedLSN = %d", replicaLSN, primaryLSN) diff --git a/weed/storage/blockvol/qa_phase4a_cp3_test.go b/weed/storage/blockvol/qa_phase4a_cp3_test.go index 2df826b964..ee5a66fbac 100644 --- a/weed/storage/blockvol/qa_phase4a_cp3_test.go +++ b/weed/storage/blockvol/qa_phase4a_cp3_test.go @@ -335,13 +335,14 @@ func testAssignDemoteStopsShipper(t *testing.T) { primary.HandleAssignment(2, RoleStale, 0) // After demote, shipper should be stopped -- Ship is a no-op. - if primary.shipper != nil { + if primary.shipperGroup != nil && primary.shipperGroup.Len() > 0 { + shipper := primary.shipperGroup.Shipper(0) entry := &WALEntry{LSN: 999, Epoch: 2, Type: EntryTypeWrite, LBA: 0, Length: 4096, Data: makeBlock('X')} - err := primary.shipper.Ship(entry) + err := shipper.Ship(entry) if err != nil { t.Errorf("Ship after stop: %v (should silently drop)", err) } - if primary.shipper.ShippedLSN() == 999 { + if shipper.ShippedLSN() == 999 { t.Error("shipper advanced LSN after demote -- should be stopped") } } diff --git a/weed/storage/blockvol/qa_phase4a_cp4b4_test.go b/weed/storage/blockvol/qa_phase4a_cp4b4_test.go index fdabc193ff..026b9ce061 100644 --- a/weed/storage/blockvol/qa_phase4a_cp4b4_test.go +++ b/weed/storage/blockvol/qa_phase4a_cp4b4_test.go @@ -146,15 +146,16 @@ func testQA4b4ShipDataOkCtrlDown(t *testing.T) { replica.replRecv.ctrlListener.Close() // Force primary shipper to reconnect ctrl. - primary.shipper.ctrlMu.Lock() - if primary.shipper.ctrlConn != nil { - primary.shipper.ctrlConn.Close() - primary.shipper.ctrlConn = nil + shipper := primary.shipperGroup.Shipper(0) + shipper.ctrlMu.Lock() + if shipper.ctrlConn != nil { + shipper.ctrlConn.Close() + shipper.ctrlConn = nil } - primary.shipper.ctrlMu.Unlock() + shipper.ctrlMu.Unlock() // Barrier must fail (ctrl down). - err := primary.shipper.Barrier(replicaLSN) + err := shipper.Barrier(replicaLSN) if err == nil { t.Error("Barrier should fail when ctrl listener is closed") } diff --git a/weed/storage/blockvol/scrub.go b/weed/storage/blockvol/scrub.go new file mode 100644 index 0000000000..b7a9d4b489 --- /dev/null +++ b/weed/storage/blockvol/scrub.go @@ -0,0 +1,266 @@ +package blockvol + +import ( + "hash/crc32" + "log" + "sync" + "sync/atomic" + "time" +) + +// ScrubStats contains scrub pass statistics. +type ScrubStats struct { + PassCount int64 + ErrorCount int64 + LastPassTime int64 // unix seconds + SegmentsTotal int64 + SegmentsClean int64 + SegmentsDirty int64 // skipped (recently written) + SegmentsErrors int64 +} + +// Scrubber performs periodic patrol-reads to detect silent data corruption. +// Uses segment-level CRC (1 MiB segments = 256 LBAs with 4K blocks) to reduce +// memory usage. Written LBAs are tracked via NotifyWrite to avoid false positives. +type Scrubber struct { + vol *BlockVol + interval time.Duration + throttle time.Duration // delay between segments + stopCh chan struct{} + triggerCh chan struct{} + stopped atomic.Bool + + // Segment CRC map: segment_index → CRC32 of concatenated block data. + mu sync.Mutex + segmentSize uint64 // LBAs per segment (default 256 = 1 MiB) + segmentCRCs map[uint64]uint32 // segment_index → CRC32 + stats ScrubStats + + // Write tracking: LBAs written since last scrub pass. + writtenMu sync.Mutex + writtenSegs map[uint64]struct{} // segment indices with writes since last pass +} + +// NewScrubber creates a scrubber for the given volume. metrics is optional. +func NewScrubber(vol *BlockVol, interval time.Duration) *Scrubber { + blockSize := uint64(vol.super.BlockSize) + segmentSize := uint64(256) // default: 256 blocks per segment (1 MiB with 4K blocks) + if blockSize == 0 { + blockSize = 4096 + } + _ = blockSize // used to document the 256 choice + + return &Scrubber{ + vol: vol, + interval: interval, + throttle: 100 * time.Microsecond, + stopCh: make(chan struct{}), + triggerCh: make(chan struct{}, 1), + segmentSize: segmentSize, + segmentCRCs: make(map[uint64]uint32), + writtenSegs: make(map[uint64]struct{}), + } +} + +// Start begins the background scrub loop. +func (s *Scrubber) Start() { + go s.loop() +} + +// Stop stops the scrub loop. +func (s *Scrubber) Stop() { + if s.stopped.Swap(true) { + return + } + close(s.stopCh) +} + +// TriggerNow requests an immediate scrub pass. +func (s *Scrubber) TriggerNow() { + select { + case s.triggerCh <- struct{}{}: + default: + } +} + +// Stats returns the current scrub statistics. +func (s *Scrubber) Stats() ScrubStats { + s.mu.Lock() + defer s.mu.Unlock() + return s.stats +} + +// NotifyWrite records that LBAs were written, invalidating their segment CRCs. +// Called from appendWithRetry after WAL append. +func (s *Scrubber) NotifyWrite(lba uint64, blockCount uint32) { + s.writtenMu.Lock() + defer s.writtenMu.Unlock() + for i := uint32(0); i < blockCount; i++ { + segIdx := (lba + uint64(i)) / s.segmentSize + s.writtenSegs[segIdx] = struct{}{} + } +} + +func (s *Scrubber) loop() { + ticker := time.NewTicker(s.interval) + defer ticker.Stop() + + for { + select { + case <-s.stopCh: + return + case <-ticker.C: + s.runPass() + case <-s.triggerCh: + s.runPass() + } + } +} + +func (s *Scrubber) runPass() { + if s.stopped.Load() { + return + } + passStart := time.Now() + + blockSize := uint64(s.vol.super.BlockSize) + if blockSize == 0 { + return + } + totalBlocks := s.vol.super.VolumeSize / blockSize + totalSegments := (totalBlocks + s.segmentSize - 1) / s.segmentSize + + var segClean, segDirty, segErrors int64 + + for segIdx := uint64(0); segIdx < totalSegments; segIdx++ { + if s.stopped.Load() { + return + } + + // Check written segments under lock — live check, not snapshot. + // This avoids false positives from writes that arrive during the pass. + s.writtenMu.Lock() + _, wasWritten := s.writtenSegs[segIdx] + if wasWritten { + delete(s.writtenSegs, segIdx) // consumed: clear this segment entry + } + s.writtenMu.Unlock() + + if wasWritten { + // Invalidate old CRC for this segment. + s.mu.Lock() + delete(s.segmentCRCs, segIdx) + s.mu.Unlock() + segDirty++ + continue + } + + // Skip segments with dirty map entries. + startLBA := segIdx * s.segmentSize + if s.segmentHasDirtyBlocks(startLBA) { + s.mu.Lock() + delete(s.segmentCRCs, segIdx) + s.mu.Unlock() + segDirty++ + continue + } + + // Read and checksum the segment. + segCRC, err := s.checksumSegment(segIdx, totalBlocks) + if err != nil { + log.Printf("scrub: error reading segment %d: %v", segIdx, err) + segErrors++ + s.vol.healthScore.RecordScrubError() + continue + } + + // Post-read double-check: if a write arrived during the read, skip comparison. + // This closes the race between the pre-read writtenSegs check and the actual I/O. + s.writtenMu.Lock() + _, writtenDuringRead := s.writtenSegs[segIdx] + if writtenDuringRead { + delete(s.writtenSegs, segIdx) + } + s.writtenMu.Unlock() + if writtenDuringRead { + s.mu.Lock() + delete(s.segmentCRCs, segIdx) + s.mu.Unlock() + segDirty++ + continue + } + + // Compare with previous CRC. + s.mu.Lock() + prevCRC, hasPrev := s.segmentCRCs[segIdx] + s.segmentCRCs[segIdx] = segCRC + s.mu.Unlock() + + if hasPrev && prevCRC != segCRC { + log.Printf("scrub: CORRUPTION detected in segment %d (LBAs %d-%d): CRC %08x != %08x", + segIdx, startLBA, startLBA+s.segmentSize-1, segCRC, prevCRC) + segErrors++ + s.vol.healthScore.RecordScrubError() + } else { + segClean++ + } + + // Throttle I/O. + if s.throttle > 0 { + time.Sleep(s.throttle) + } + } + + s.vol.healthScore.RecordScrubComplete() + + s.mu.Lock() + s.stats.PassCount++ + s.stats.ErrorCount += segErrors + s.stats.LastPassTime = time.Now().Unix() + s.stats.SegmentsTotal = int64(totalSegments) + s.stats.SegmentsClean = segClean + s.stats.SegmentsDirty = segDirty + s.stats.SegmentsErrors = segErrors + s.mu.Unlock() + + // Record metrics. + if s.vol.Metrics != nil { + s.vol.Metrics.RecordScrubPass(time.Since(passStart), segErrors) + } +} + +// segmentHasDirtyBlocks checks if any LBA in the segment has a dirty map entry. +func (s *Scrubber) segmentHasDirtyBlocks(startLBA uint64) bool { + for i := uint64(0); i < s.segmentSize; i++ { + lba := startLBA + i + if lba*uint64(s.vol.super.BlockSize) >= s.vol.super.VolumeSize { + break + } + if _, _, _, ok := s.vol.dirtyMap.Get(lba); ok { + return true + } + } + return false +} + +// checksumSegment reads all blocks in a segment from the extent and computes CRC32. +func (s *Scrubber) checksumSegment(segIdx, totalBlocks uint64) (uint32, error) { + startLBA := segIdx * s.segmentSize + blockSize := uint64(s.vol.super.BlockSize) + extentStart := s.vol.super.WALOffset + s.vol.super.WALSize + + h := crc32.NewIEEE() + for i := uint64(0); i < s.segmentSize; i++ { + lba := startLBA + i + if lba >= totalBlocks { + break + } + buf := make([]byte, blockSize) + off := int64(extentStart + lba*blockSize) + if _, err := s.vol.fd.ReadAt(buf, off); err != nil { + return 0, err + } + h.Write(buf) + } + return h.Sum32(), nil +} diff --git a/weed/storage/blockvol/scrub_test.go b/weed/storage/blockvol/scrub_test.go new file mode 100644 index 0000000000..cac6360775 --- /dev/null +++ b/weed/storage/blockvol/scrub_test.go @@ -0,0 +1,297 @@ +package blockvol + +import ( + "os" + "path/filepath" + "testing" + "time" +) + +func createScrubVol(t *testing.T) *BlockVol { + t.Helper() + dir := t.TempDir() + cfg := DefaultConfig() + cfg.FlushInterval = 5 * time.Millisecond + v, err := CreateBlockVol(filepath.Join(dir, "scrub.blk"), CreateOptions{ + VolumeSize: 64 * 1024, // 64 KiB = 16 blocks + BlockSize: 4096, + WALSize: 64 * 1024, + }, cfg) + if err != nil { + t.Fatalf("CreateBlockVol: %v", err) + } + return v +} + +func TestScrub_CleanVolume(t *testing.T) { + v := createScrubVol(t) + defer v.Close() + + scrubber := NewScrubber(v, 24*time.Hour) + scrubber.throttle = 0 + + // First pass: establishes baseline CRCs. + scrubber.runPass() + stats := scrubber.Stats() + if stats.PassCount != 1 { + t.Fatalf("PassCount: got %d, want 1", stats.PassCount) + } + if stats.ErrorCount != 0 { + t.Fatalf("ErrorCount: got %d, want 0", stats.ErrorCount) + } + + // Second pass: same data, no errors. + scrubber.runPass() + stats = scrubber.Stats() + if stats.PassCount != 2 { + t.Fatalf("PassCount: got %d, want 2", stats.PassCount) + } + if stats.ErrorCount != 0 { + t.Fatalf("ErrorCount: got %d, want 0", stats.ErrorCount) + } +} + +func TestScrub_DetectCorruption(t *testing.T) { + v := createScrubVol(t) + defer v.Close() + + // Write some data and flush it to extent. + data := make([]byte, 4096) + for i := range data { + data[i] = 0xAA + } + if err := v.WriteLBA(0, data); err != nil { + t.Fatalf("WriteLBA: %v", err) + } + if err := v.SyncCache(); err != nil { + t.Fatalf("SyncCache: %v", err) + } + // Wait for flusher to flush. + time.Sleep(50 * time.Millisecond) + + scrubber := NewScrubber(v, 24*time.Hour) + scrubber.throttle = 0 + + // First pass: establish baseline. + scrubber.runPass() + if scrubber.Stats().ErrorCount != 0 { + t.Fatalf("first pass errors: %d", scrubber.Stats().ErrorCount) + } + + // Corrupt the extent data directly. + extentStart := v.super.WALOffset + v.super.WALSize + corruptBuf := []byte{0xFF, 0xFF, 0xFF, 0xFF} + v.fd.WriteAt(corruptBuf, int64(extentStart)) + v.fd.Sync() + + // Second pass: should detect corruption. + scrubber.runPass() + stats := scrubber.Stats() + if stats.ErrorCount == 0 { + t.Fatal("expected corruption error but got none") + } +} + +func TestScrub_SkipDirtyBlocks(t *testing.T) { + v := createScrubVol(t) + defer v.Close() + + scrubber := NewScrubber(v, 24*time.Hour) + scrubber.throttle = 0 + + // First pass: baseline. + scrubber.runPass() + + // Write data (stays in dirty map since flusher interval is slow relative to us). + data := make([]byte, 4096) + for i := range data { + data[i] = 0xBB + } + if err := v.WriteLBA(0, data); err != nil { + t.Fatalf("WriteLBA: %v", err) + } + + // Second pass: segment 0 has dirty blocks, should be skipped. + scrubber.runPass() + stats := scrubber.Stats() + if stats.ErrorCount != 0 { + t.Fatalf("dirty block should be skipped, not flagged as error: errors=%d", stats.ErrorCount) + } +} + +func TestScrub_SkipRecentlyWritten(t *testing.T) { + v := createScrubVol(t) + defer v.Close() + + scrubber := NewScrubber(v, 24*time.Hour) + scrubber.throttle = 0 + + // First pass: baseline. + scrubber.runPass() + + // Write, sync, flush (data in extent, dirty map cleared). + data := make([]byte, 4096) + for i := range data { + data[i] = 0xCC + } + if err := v.WriteLBA(0, data); err != nil { + t.Fatalf("WriteLBA: %v", err) + } + v.SyncCache() + time.Sleep(50 * time.Millisecond) // let flusher clear dirty map + + // Manually notify scrubber of write (simulates hook in appendWithRetry). + scrubber.NotifyWrite(0, 1) + + // Second pass: segment 0 was recently written, skip CRC comparison. + scrubber.runPass() + stats := scrubber.Stats() + if stats.ErrorCount != 0 { + t.Fatalf("recently written segment should be skipped: errors=%d", stats.ErrorCount) + } +} + +func TestScrub_StatsUpdated(t *testing.T) { + v := createScrubVol(t) + defer v.Close() + + scrubber := NewScrubber(v, 24*time.Hour) + scrubber.throttle = 0 + scrubber.runPass() + + stats := scrubber.Stats() + if stats.LastPassTime == 0 { + t.Fatal("LastPassTime should be set") + } + if stats.SegmentsTotal == 0 { + t.Fatal("SegmentsTotal should be > 0") + } +} + +func TestScrub_TriggerNow(t *testing.T) { + v := createScrubVol(t) + defer v.Close() + + scrubber := NewScrubber(v, 1*time.Hour) // long interval + scrubber.throttle = 0 + scrubber.Start() + defer scrubber.Stop() + + scrubber.TriggerNow() + // Wait for the triggered pass to complete. + time.Sleep(100 * time.Millisecond) + + stats := scrubber.Stats() + if stats.PassCount < 1 { + t.Fatalf("TriggerNow did not cause a pass: PassCount=%d", stats.PassCount) + } +} + +func TestScrub_StopIdempotent(t *testing.T) { + v := createScrubVol(t) + defer v.Close() + + scrubber := NewScrubber(v, 24*time.Hour) + scrubber.Start() + scrubber.Stop() + scrubber.Stop() // second stop should be no-op +} + +func TestScrub_HealthScoreImpact(t *testing.T) { + dir := t.TempDir() + cfg := DefaultConfig() + cfg.FlushInterval = 5 * time.Millisecond + path := filepath.Join(dir, "health.blk") + v, err := CreateBlockVol(path, CreateOptions{ + VolumeSize: 64 * 1024, + BlockSize: 4096, + WALSize: 64 * 1024, + }, cfg) + if err != nil { + t.Fatalf("CreateBlockVol: %v", err) + } + defer v.Close() + + if v.HealthScore() != 1.0 { + t.Fatalf("initial health: got %f, want 1.0", v.HealthScore()) + } + + // Write + flush to extent. + data := make([]byte, 4096) + for i := range data { + data[i] = 0xDD + } + v.WriteLBA(0, data) + v.SyncCache() + time.Sleep(50 * time.Millisecond) + + scrubber := NewScrubber(v, 24*time.Hour) + scrubber.throttle = 0 + scrubber.runPass() + + // Corrupt extent. + extentStart := v.super.WALOffset + v.super.WALSize + corruptBuf := []byte{0xFF} + v.fd.WriteAt(corruptBuf, int64(extentStart)) + v.fd.Sync() + + scrubber.runPass() + if v.HealthScore() >= 1.0 { + t.Fatalf("health should drop after corruption: got %f", v.HealthScore()) + } + + // Cleanup temp files on Windows: remove delta files if any. + _ = os.RemoveAll(dir) +} + +// Fix #5: Write arriving during a scrub pass must not cause a false corruption detection. +func TestScrub_InPassWrite_NoFalsePositive(t *testing.T) { + v := createScrubVol(t) + defer v.Close() + + // Write initial data and flush to extent. + data := make([]byte, 4096) + for i := range data { + data[i] = 0xAA + } + if err := v.WriteLBA(0, data); err != nil { + t.Fatalf("WriteLBA: %v", err) + } + v.SyncCache() + time.Sleep(50 * time.Millisecond) // let flusher clear dirty map + + scrubber := NewScrubber(v, 24*time.Hour) + scrubber.throttle = 0 + + // First pass: establish baseline CRC for segment 0. + scrubber.runPass() + if scrubber.Stats().ErrorCount != 0 { + t.Fatalf("baseline pass should have no errors: %d", scrubber.Stats().ErrorCount) + } + + // Now write different data and flush — this changes the extent. + for i := range data { + data[i] = 0xBB + } + if err := v.WriteLBA(0, data); err != nil { + t.Fatalf("WriteLBA: %v", err) + } + v.SyncCache() + time.Sleep(50 * time.Millisecond) // flushed to extent, dirty map cleared + + // Simulate in-pass write: NotifyWrite arrives AFTER pass would have + // snapshotted writtenSegs (old bug), but BEFORE segment 0 is read. + // With the fix, writtenSegs is checked live during the pass. + scrubber.NotifyWrite(0, 1) + + // Second pass: segment 0 extent data changed, but NotifyWrite should + // prevent false corruption — segment should be skipped. + scrubber.runPass() + stats := scrubber.Stats() + if stats.ErrorCount != 0 { + t.Fatalf("in-pass write caused false corruption: errors=%d", stats.ErrorCount) + } + if stats.SegmentsDirty == 0 { + t.Fatal("segment 0 should be counted as dirty (recently written)") + } +} diff --git a/weed/storage/blockvol/shipper_group.go b/weed/storage/blockvol/shipper_group.go new file mode 100644 index 0000000000..385cf2d504 --- /dev/null +++ b/weed/storage/blockvol/shipper_group.go @@ -0,0 +1,114 @@ +package blockvol + +import ( + "sync" +) + +// ShipperGroup wraps multiple WALShippers for fan-out replication to N replicas. +// Len()==0 means standalone (no replicas), Len()==1 is RF=2, Len()==2 is RF=3. +type ShipperGroup struct { + mu sync.RWMutex + shippers []*WALShipper +} + +// NewShipperGroup creates a ShipperGroup from the given shippers. +func NewShipperGroup(shippers []*WALShipper) *ShipperGroup { + return &ShipperGroup{shippers: shippers} +} + +// ShipAll sends a WAL entry to every non-degraded shipper (fire-and-forget). +func (sg *ShipperGroup) ShipAll(entry *WALEntry) { + sg.mu.RLock() + defer sg.mu.RUnlock() + for _, s := range sg.shippers { + s.Ship(entry) + } +} + +// BarrierAll sends barriers to all shippers in parallel. +// Returns a per-shipper error slice (nil entry = success). +func (sg *ShipperGroup) BarrierAll(lsnMax uint64) []error { + sg.mu.RLock() + n := len(sg.shippers) + shippers := sg.shippers + sg.mu.RUnlock() + + errs := make([]error, n) + var wg sync.WaitGroup + wg.Add(n) + for i := 0; i < n; i++ { + go func(idx int) { + defer wg.Done() + errs[idx] = shippers[idx].Barrier(lsnMax) + }(i) + } + wg.Wait() + return errs +} + +// AllDegraded returns true only if every shipper is degraded. +// Returns false when there are no shippers (standalone). +func (sg *ShipperGroup) AllDegraded() bool { + sg.mu.RLock() + defer sg.mu.RUnlock() + if len(sg.shippers) == 0 { + return false + } + for _, s := range sg.shippers { + if !s.IsDegraded() { + return false + } + } + return true +} + +// AnyDegraded returns true if at least one shipper is degraded. +func (sg *ShipperGroup) AnyDegraded() bool { + sg.mu.RLock() + defer sg.mu.RUnlock() + for _, s := range sg.shippers { + if s.IsDegraded() { + return true + } + } + return false +} + +// DegradedCount returns the number of degraded shippers. +func (sg *ShipperGroup) DegradedCount() int { + sg.mu.RLock() + defer sg.mu.RUnlock() + count := 0 + for _, s := range sg.shippers { + if s.IsDegraded() { + count++ + } + } + return count +} + +// StopAll stops all shippers and closes their connections. +func (sg *ShipperGroup) StopAll() { + sg.mu.Lock() + defer sg.mu.Unlock() + for _, s := range sg.shippers { + s.Stop() + } +} + +// Len returns the number of shippers. 0=standalone, 1=RF2, 2=RF3. +func (sg *ShipperGroup) Len() int { + sg.mu.RLock() + defer sg.mu.RUnlock() + return len(sg.shippers) +} + +// Shipper returns the shipper at index i. For internal/test use. +func (sg *ShipperGroup) Shipper(i int) *WALShipper { + sg.mu.RLock() + defer sg.mu.RUnlock() + if i < 0 || i >= len(sg.shippers) { + return nil + } + return sg.shippers[i] +} diff --git a/weed/storage/blockvol/shipper_group_test.go b/weed/storage/blockvol/shipper_group_test.go new file mode 100644 index 0000000000..89d31efe20 --- /dev/null +++ b/weed/storage/blockvol/shipper_group_test.go @@ -0,0 +1,116 @@ +package blockvol + +import ( + "sync/atomic" + "testing" +) + +func TestShipperGroup_ShipAll_Single(t *testing.T) { + s := newTestShipper() + sg := NewShipperGroup([]*WALShipper{s}) + entry := &WALEntry{LSN: 1, Epoch: 1, Type: EntryTypeWrite} + sg.ShipAll(entry) + // Ship on a non-connected shipper degrades it but doesn't panic. + if sg.Len() != 1 { + t.Fatalf("Len: got %d, want 1", sg.Len()) + } +} + +func TestShipperGroup_ShipAll_Two(t *testing.T) { + s1 := newTestShipper() + s2 := newTestShipper() + sg := NewShipperGroup([]*WALShipper{s1, s2}) + entry := &WALEntry{LSN: 1, Epoch: 1, Type: EntryTypeWrite} + sg.ShipAll(entry) + if sg.Len() != 2 { + t.Fatalf("Len: got %d, want 2", sg.Len()) + } +} + +func TestShipperGroup_BarrierAll_AllSucceed(t *testing.T) { + // Create shippers that are stopped (Barrier returns ErrShipperStopped). + // We test the fan-out logic: all return errors, which is fine. + s1 := newTestShipper() + s2 := newTestShipper() + sg := NewShipperGroup([]*WALShipper{s1, s2}) + errs := sg.BarrierAll(10) + if len(errs) != 2 { + t.Fatalf("BarrierAll: got %d errors, want 2", len(errs)) + } +} + +func TestShipperGroup_BarrierAll_OneFail(t *testing.T) { + s1 := newTestShipper() + s1.degraded.Store(true) + s2 := newTestShipper() + sg := NewShipperGroup([]*WALShipper{s1, s2}) + errs := sg.BarrierAll(10) + if errs[0] == nil { + t.Fatalf("expected error for degraded shipper[0]") + } +} + +func TestShipperGroup_BarrierAll_AllFail(t *testing.T) { + s1 := newTestShipper() + s1.degraded.Store(true) + s2 := newTestShipper() + s2.degraded.Store(true) + sg := NewShipperGroup([]*WALShipper{s1, s2}) + errs := sg.BarrierAll(10) + for i, e := range errs { + if e == nil { + t.Fatalf("expected error for shipper[%d]", i) + } + } +} + +func TestShipperGroup_AllDegraded_Empty(t *testing.T) { + sg := NewShipperGroup(nil) + if sg.AllDegraded() { + t.Fatal("empty group should not be AllDegraded") + } +} + +func TestShipperGroup_AllDegraded_Mixed(t *testing.T) { + s1 := newTestShipper() + s1.degraded.Store(true) + s2 := newTestShipper() + sg := NewShipperGroup([]*WALShipper{s1, s2}) + if sg.AllDegraded() { + t.Fatal("mixed group should not be AllDegraded") + } + if !sg.AnyDegraded() { + t.Fatal("mixed group should be AnyDegraded") + } +} + +func TestShipperGroup_StopAll(t *testing.T) { + s1 := newTestShipper() + s2 := newTestShipper() + sg := NewShipperGroup([]*WALShipper{s1, s2}) + sg.StopAll() + if !s1.stopped.Load() || !s2.stopped.Load() { + t.Fatal("StopAll should stop all shippers") + } +} + +func TestShipperGroup_DegradedCount(t *testing.T) { + s1 := newTestShipper() + s2 := newTestShipper() + s1.degraded.Store(true) + sg := NewShipperGroup([]*WALShipper{s1, s2}) + if got := sg.DegradedCount(); got != 1 { + t.Fatalf("DegradedCount: got %d, want 1", got) + } +} + +// newTestShipper creates a WALShipper with a fixed epoch, not connected to anything. +func newTestShipper() *WALShipper { + var epoch atomic.Uint64 + epoch.Store(1) + return &WALShipper{ + dataAddr: "127.0.0.1:0", + controlAddr: "127.0.0.1:0", + epochFn: func() uint64 { return epoch.Load() }, + } +} diff --git a/weed/storage/blockvol/wal_shipper.go b/weed/storage/blockvol/wal_shipper.go index a4fa7bad74..d45a0c79fd 100644 --- a/weed/storage/blockvol/wal_shipper.go +++ b/weed/storage/blockvol/wal_shipper.go @@ -23,6 +23,7 @@ type WALShipper struct { dataAddr string controlAddr string epochFn func() uint64 + metrics *EngineMetrics mu sync.Mutex // protects dataConn dataConn net.Conn @@ -37,11 +38,17 @@ type WALShipper struct { // NewWALShipper creates a WAL shipper. Connections are established lazily on // first Ship/Barrier call. epochFn returns the current epoch for validation. -func NewWALShipper(dataAddr, controlAddr string, epochFn func() uint64) *WALShipper { +// metrics is optional; if nil, no metrics are recorded. +func NewWALShipper(dataAddr, controlAddr string, epochFn func() uint64, metrics ...*EngineMetrics) *WALShipper { + var m *EngineMetrics + if len(metrics) > 0 { + m = metrics[0] + } return &WALShipper{ dataAddr: dataAddr, controlAddr: controlAddr, epochFn: epochFn, + metrics: m, } } @@ -83,6 +90,9 @@ func (s *WALShipper) Ship(entry *WALEntry) error { s.dataConn.SetWriteDeadline(time.Time{}) // clear deadline s.shippedLSN.Store(entry.LSN) + if s.metrics != nil { + s.metrics.RecordWALShipped() + } return nil } @@ -97,6 +107,8 @@ func (s *WALShipper) Barrier(lsnMax uint64) error { return ErrReplicaDegraded } + barrierStart := time.Now() + req := EncodeBarrierRequest(BarrierRequest{ LSN: lsnMax, Epoch: s.epochFn(), @@ -107,6 +119,7 @@ func (s *WALShipper) Barrier(lsnMax uint64) error { if err := s.ensureCtrlConn(); err != nil { s.markDegraded() + s.recordBarrierMetric(barrierStart, true) return ErrReplicaDegraded } @@ -114,34 +127,52 @@ func (s *WALShipper) Barrier(lsnMax uint64) error { if err := WriteFrame(s.ctrlConn, MsgBarrierReq, req); err != nil { s.markDegraded() + s.recordBarrierMetric(barrierStart, true) return ErrReplicaDegraded } msgType, payload, err := ReadFrame(s.ctrlConn) if err != nil { s.markDegraded() + s.recordBarrierMetric(barrierStart, true) return ErrReplicaDegraded } if msgType != MsgBarrierResp || len(payload) < 1 { s.markDegraded() + s.recordBarrierMetric(barrierStart, true) return ErrReplicaDegraded } switch payload[0] { case BarrierOK: + s.recordBarrierMetric(barrierStart, false) return nil case BarrierEpochMismatch: + s.markDegraded() + s.recordBarrierMetric(barrierStart, true) return fmt.Errorf("wal_shipper: barrier epoch mismatch") case BarrierTimeout: + s.markDegraded() + s.recordBarrierMetric(barrierStart, true) return fmt.Errorf("wal_shipper: barrier timeout on replica") case BarrierFsyncFailed: + s.markDegraded() + s.recordBarrierMetric(barrierStart, true) return fmt.Errorf("wal_shipper: barrier fsync failed on replica") default: + s.markDegraded() + s.recordBarrierMetric(barrierStart, true) return fmt.Errorf("wal_shipper: unknown barrier status %d", payload[0]) } } +func (s *WALShipper) recordBarrierMetric(start time.Time, failed bool) { + if s.metrics != nil { + s.metrics.RecordWALBarrier(time.Since(start), failed) + } +} + // ShippedLSN returns the highest LSN successfully sent to the replica. func (s *WALShipper) ShippedLSN() uint64 { return s.shippedLSN.Load()