diff --git a/weed/pb/master.proto b/weed/pb/master.proto index 1332e5119..7464b8417 100644 --- a/weed/pb/master.proto +++ b/weed/pb/master.proto @@ -505,6 +505,7 @@ message BlockVolumeInfoMessage { int64 scrub_errors = 13; int64 last_scrub_time = 14; bool replica_degraded = 15; + string durability_mode = 16; } message BlockVolumeShortInfoMessage { @@ -535,6 +536,7 @@ message CreateBlockVolumeRequest { uint64 size_bytes = 2; string disk_type = 3; uint32 replica_factor = 4; + string durability_mode = 5; } message CreateBlockVolumeResponse { string volume_id = 1; @@ -563,6 +565,7 @@ message LookupBlockVolumeResponse { string replica_server = 5; uint32 replica_factor = 6; repeated string replica_servers = 7; + string durability_mode = 8; } message CreateBlockSnapshotRequest { diff --git a/weed/pb/master_pb/master.pb.go b/weed/pb/master_pb/master.pb.go index 0d16c4646..a54845644 100644 --- a/weed/pb/master_pb/master.pb.go +++ b/weed/pb/master_pb/master.pb.go @@ -3899,6 +3899,7 @@ type BlockVolumeInfoMessage struct { ScrubErrors int64 `protobuf:"varint,13,opt,name=scrub_errors,json=scrubErrors,proto3" json:"scrub_errors,omitempty"` LastScrubTime int64 `protobuf:"varint,14,opt,name=last_scrub_time,json=lastScrubTime,proto3" json:"last_scrub_time,omitempty"` ReplicaDegraded bool `protobuf:"varint,15,opt,name=replica_degraded,json=replicaDegraded,proto3" json:"replica_degraded,omitempty"` + DurabilityMode string `protobuf:"bytes,16,opt,name=durability_mode,json=durabilityMode,proto3" json:"durability_mode,omitempty"` unknownFields protoimpl.UnknownFields sizeCache protoimpl.SizeCache } @@ -4038,6 +4039,13 @@ func (x *BlockVolumeInfoMessage) GetReplicaDegraded() bool { return false } +func (x *BlockVolumeInfoMessage) GetDurabilityMode() string { + if x != nil { + return x.DurabilityMode + } + return "" +} + type BlockVolumeShortInfoMessage struct { state protoimpl.MessageState `protogen:"open.v1"` Path string `protobuf:"bytes,1,opt,name=path,proto3" json:"path,omitempty"` @@ -4238,9 +4246,10 @@ type CreateBlockVolumeRequest struct { Name string `protobuf:"bytes,1,opt,name=name,proto3" json:"name,omitempty"` SizeBytes uint64 `protobuf:"varint,2,opt,name=size_bytes,json=sizeBytes,proto3" json:"size_bytes,omitempty"` DiskType string `protobuf:"bytes,3,opt,name=disk_type,json=diskType,proto3" json:"disk_type,omitempty"` - ReplicaFactor uint32 `protobuf:"varint,4,opt,name=replica_factor,json=replicaFactor,proto3" json:"replica_factor,omitempty"` - unknownFields protoimpl.UnknownFields - sizeCache protoimpl.SizeCache + ReplicaFactor uint32 `protobuf:"varint,4,opt,name=replica_factor,json=replicaFactor,proto3" json:"replica_factor,omitempty"` + DurabilityMode string `protobuf:"bytes,5,opt,name=durability_mode,json=durabilityMode,proto3" json:"durability_mode,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache } func (x *CreateBlockVolumeRequest) Reset() { @@ -4301,6 +4310,13 @@ func (x *CreateBlockVolumeRequest) GetReplicaFactor() uint32 { return 0 } +func (x *CreateBlockVolumeRequest) GetDurabilityMode() string { + if x != nil { + return x.DurabilityMode + } + return "" +} + type CreateBlockVolumeResponse struct { state protoimpl.MessageState `protogen:"open.v1"` VolumeId string `protobuf:"bytes,1,opt,name=volume_id,json=volumeId,proto3" json:"volume_id,omitempty"` @@ -4526,6 +4542,7 @@ type LookupBlockVolumeResponse struct { ReplicaServer string `protobuf:"bytes,5,opt,name=replica_server,json=replicaServer,proto3" json:"replica_server,omitempty"` ReplicaFactor uint32 `protobuf:"varint,6,opt,name=replica_factor,json=replicaFactor,proto3" json:"replica_factor,omitempty"` ReplicaServers []string `protobuf:"bytes,7,rep,name=replica_servers,json=replicaServers,proto3" json:"replica_servers,omitempty"` + DurabilityMode string `protobuf:"bytes,8,opt,name=durability_mode,json=durabilityMode,proto3" json:"durability_mode,omitempty"` unknownFields protoimpl.UnknownFields sizeCache protoimpl.SizeCache } @@ -4609,6 +4626,13 @@ func (x *LookupBlockVolumeResponse) GetReplicaServers() []string { return nil } +func (x *LookupBlockVolumeResponse) GetDurabilityMode() string { + if x != nil { + return x.DurabilityMode + } + return "" +} + type CreateBlockSnapshotRequest struct { state protoimpl.MessageState `protogen:"open.v1"` VolumeName string `protobuf:"bytes,1,opt,name=volume_name,json=volumeName,proto3" json:"volume_name,omitempty"` diff --git a/weed/pb/volume_server.proto b/weed/pb/volume_server.proto index 28e1b6740..5d3b348eb 100644 --- a/weed/pb/volume_server.proto +++ b/weed/pb/volume_server.proto @@ -779,6 +779,7 @@ message AllocateBlockVolumeRequest { string name = 1; uint64 size_bytes = 2; string disk_type = 3; + string durability_mode = 4; } message AllocateBlockVolumeResponse { string path = 1; diff --git a/weed/pb/volume_server_pb/volume_server.pb.go b/weed/pb/volume_server_pb/volume_server.pb.go index 88f77f2e3..8efe1cf70 100644 --- a/weed/pb/volume_server_pb/volume_server.pb.go +++ b/weed/pb/volume_server_pb/volume_server.pb.go @@ -6189,9 +6189,10 @@ type AllocateBlockVolumeRequest struct { state protoimpl.MessageState `protogen:"open.v1"` Name string `protobuf:"bytes,1,opt,name=name,proto3" json:"name,omitempty"` SizeBytes uint64 `protobuf:"varint,2,opt,name=size_bytes,json=sizeBytes,proto3" json:"size_bytes,omitempty"` - DiskType string `protobuf:"bytes,3,opt,name=disk_type,json=diskType,proto3" json:"disk_type,omitempty"` - unknownFields protoimpl.UnknownFields - sizeCache protoimpl.SizeCache + DiskType string `protobuf:"bytes,3,opt,name=disk_type,json=diskType,proto3" json:"disk_type,omitempty"` + DurabilityMode string `protobuf:"bytes,4,opt,name=durability_mode,json=durabilityMode,proto3" json:"durability_mode,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache } func (x *AllocateBlockVolumeRequest) Reset() { @@ -6245,6 +6246,13 @@ func (x *AllocateBlockVolumeRequest) GetDiskType() string { return "" } +func (x *AllocateBlockVolumeRequest) GetDurabilityMode() string { + if x != nil { + return x.DurabilityMode + } + return "" +} + type AllocateBlockVolumeResponse struct { state protoimpl.MessageState `protogen:"open.v1"` Path string `protobuf:"bytes,1,opt,name=path,proto3" json:"path,omitempty"` diff --git a/weed/server/integration_block_test.go b/weed/server/integration_block_test.go index d3a590e85..f5d9aad87 100644 --- a/weed/server/integration_block_test.go +++ b/weed/server/integration_block_test.go @@ -28,7 +28,7 @@ func integrationMaster(t *testing.T) *MasterServer { blockAssignmentQueue: NewBlockAssignmentQueue(), blockFailover: newBlockFailoverState(), } - ms.blockVSAllocate = func(ctx context.Context, server string, name string, sizeBytes uint64, diskType string) (*blockAllocResult, error) { + ms.blockVSAllocate = func(ctx context.Context, server string, name string, sizeBytes uint64, diskType string, durabilityMode string) (*blockAllocResult, error) { return &blockAllocResult{ Path: fmt.Sprintf("/data/%s.blk", name), IQN: fmt.Sprintf("iqn.2024.test:%s", name), @@ -382,13 +382,13 @@ func TestIntegration_ReplicaFailureSingleCopy(t *testing.T) { // Make replica allocation always fail. callCount := 0 origAllocate := ms.blockVSAllocate - ms.blockVSAllocate = func(ctx context.Context, server string, name string, sizeBytes uint64, diskType string) (*blockAllocResult, error) { + ms.blockVSAllocate = func(ctx context.Context, server string, name string, sizeBytes uint64, diskType string, durabilityMode string) (*blockAllocResult, error) { callCount++ if callCount > 1 { // Second call (replica) fails. return nil, fmt.Errorf("disk full on replica") } - return origAllocate(ctx, server, name, sizeBytes, diskType) + return origAllocate(ctx, server, name, sizeBytes, diskType, durabilityMode) } resp, err := ms.CreateBlockVolume(ctx, &master_pb.CreateBlockVolumeRequest{ diff --git a/weed/server/master_block_failover_test.go b/weed/server/master_block_failover_test.go index 1b60a2f92..6d6439068 100644 --- a/weed/server/master_block_failover_test.go +++ b/weed/server/master_block_failover_test.go @@ -18,7 +18,7 @@ func testMasterServerForFailover(t *testing.T) *MasterServer { blockAssignmentQueue: NewBlockAssignmentQueue(), blockFailover: newBlockFailoverState(), } - ms.blockVSAllocate = func(ctx context.Context, server string, name string, sizeBytes uint64, diskType string) (*blockAllocResult, error) { + ms.blockVSAllocate = func(ctx context.Context, server string, name string, sizeBytes uint64, diskType string, durabilityMode string) (*blockAllocResult, error) { return &blockAllocResult{ Path: fmt.Sprintf("/data/%s.blk", name), IQN: fmt.Sprintf("iqn.2024.test:%s", name), diff --git a/weed/server/master_block_registry.go b/weed/server/master_block_registry.go index 0187ecf24..1cc4cabd7 100644 --- a/weed/server/master_block_registry.go +++ b/weed/server/master_block_registry.go @@ -72,6 +72,9 @@ type BlockVolumeEntry struct { ReplicaDegraded bool // primary reports degraded replicas WALHeadLSN uint64 // primary WAL head LSN from heartbeat + // CP8-3-1: Durability mode. + DurabilityMode string // "best_effort", "sync_all", "sync_quorum" + // Lease tracking for failover (CP6-3 F2). LastLeaseGrant time.Time LeaseTTL time.Duration @@ -320,6 +323,10 @@ func (r *BlockVolumeRegistry) UpdateFullHeartbeat(server string, infos []*master existing.HealthScore = info.HealthScore existing.ReplicaDegraded = info.ReplicaDegraded existing.WALHeadLSN = info.WalHeadLsn + // F3: only update DurabilityMode when non-empty (prevents older VS from clearing strict mode). + if info.DurabilityMode != "" { + existing.DurabilityMode = info.DurabilityMode + } // F5: update replica addresses from heartbeat info. if info.ReplicaDataAddr != "" { existing.ReplicaDataAddr = info.ReplicaDataAddr @@ -391,6 +398,7 @@ func (r *BlockVolumeRegistry) UpdateFullHeartbeat(server string, infos []*master LeaseTTL: 30 * time.Second, HealthScore: info.HealthScore, WALHeadLSN: info.WalHeadLsn, + DurabilityMode: info.DurabilityMode, } if info.ReplicaDataAddr != "" { entry.ReplicaDataAddr = info.ReplicaDataAddr diff --git a/weed/server/master_grpc_server_block.go b/weed/server/master_grpc_server_block.go index bbf04df0b..0a5913d9d 100644 --- a/weed/server/master_grpc_server_block.go +++ b/weed/server/master_grpc_server_block.go @@ -33,10 +33,29 @@ func (ms *MasterServer) CreateBlockVolume(ctx context.Context, req *master_pb.Cr return nil, fmt.Errorf("size_bytes must be > 0") } - // Idempotent: if already registered, return existing entry (validate size). + // F2: validate durability mode in gRPC path (authoritative, not bypassable). + var durMode blockvol.DurabilityMode + if req.DurabilityMode != "" { + var err error + durMode, err = blockvol.ParseDurabilityMode(req.DurabilityMode) + if err != nil { + return nil, fmt.Errorf("invalid durability_mode: %w", err) + } + } + + // Cross-validate mode + RF (sync_quorum requires RF >= 3). + replicaFactor := 2 + if req.ReplicaFactor > 0 && req.ReplicaFactor <= 3 { + replicaFactor = int(req.ReplicaFactor) + } + if err := durMode.Validate(replicaFactor); err != nil { + return nil, fmt.Errorf("durability_mode %q incompatible with replica_factor %d: %w", req.DurabilityMode, replicaFactor, err) + } + + // Idempotent: if already registered, return existing entry (validate size + mode + RF). if entry, ok := ms.blockRegistry.Lookup(req.Name); ok { - if entry.SizeBytes < req.SizeBytes { - return nil, fmt.Errorf("block volume %q exists with size %d (requested %d)", req.Name, entry.SizeBytes, req.SizeBytes) + if err := ms.validateIdempotentCreate(entry, req, durMode, replicaFactor); err != nil { + return nil, err } return ms.createBlockVolumeResponseFromEntry(entry), nil } @@ -49,6 +68,9 @@ func (ms *MasterServer) CreateBlockVolume(ctx context.Context, req *master_pb.Cr // Double-check after acquiring lock (another goroutine may have finished). if entry, ok := ms.blockRegistry.Lookup(req.Name); ok { + if err := ms.validateIdempotentCreate(entry, req, durMode, replicaFactor); err != nil { + return nil, err + } return ms.createBlockVolumeResponseFromEntry(entry), nil } @@ -71,7 +93,7 @@ func (ms *MasterServer) CreateBlockVolume(ctx context.Context, req *master_pb.Cr return nil, err } - result, err := ms.blockVSAllocate(ctx, server, req.Name, req.SizeBytes, req.DiskType) + result, err := ms.blockVSAllocate(ctx, server, req.Name, req.SizeBytes, req.DiskType, req.DurabilityMode) if err != nil { lastErr = fmt.Errorf("server %s: %w", server, err) glog.V(0).Infof("[reqID=%s] CreateBlockVolume %q: attempt %d on %s failed: %v", blockReqID(ctx), req.Name, attempt+1, server, err) @@ -79,12 +101,6 @@ func (ms *MasterServer) CreateBlockVolume(ctx context.Context, req *master_pb.Cr continue } - // CP8-2: determine replica factor from request (default 2). - replicaFactor := 2 - if req.ReplicaFactor > 0 && req.ReplicaFactor <= 3 { - replicaFactor = int(req.ReplicaFactor) - } - entry := &BlockVolumeEntry{ Name: req.Name, VolumeServer: server, @@ -96,6 +112,7 @@ func (ms *MasterServer) CreateBlockVolume(ctx context.Context, req *master_pb.Cr Role: blockvol.RoleToWire(blockvol.RolePrimary), Status: StatusActive, ReplicaFactor: replicaFactor, + DurabilityMode: durMode.String(), LeaseTTL: 30 * time.Second, LastLeaseGrant: time.Now(), // R2-F1: set BEFORE Register to avoid stale-lease race } @@ -111,6 +128,15 @@ func (ms *MasterServer) CreateBlockVolume(ctx context.Context, req *master_pb.Cr if len(entry.Replicas) == 0 && replicaFactor > 1 { glog.V(0).Infof("[reqID=%s] CreateBlockVolume %q: single-copy mode (replica allocation failed)", blockReqID(ctx), req.Name) } + + // F1: strict modes require minimum replicas at create time. + requiredReplicas := durMode.RequiredReplicas(replicaFactor) + if len(entry.Replicas) < requiredReplicas { + ms.cleanupPartialCreate(ctx, entry) + return nil, fmt.Errorf("durability mode %q requires %d replicas but only %d provisioned", + durMode.String(), requiredReplicas, len(entry.Replicas)) + } + // Sync deprecated scalar fields from first replica. if len(entry.Replicas) > 0 { r0 := &entry.Replicas[0] @@ -225,6 +251,10 @@ func (ms *MasterServer) LookupBlockVolume(ctx context.Context, req *master_pb.Lo if rf == 0 { rf = 2 // default for pre-CP8-2 entries } + durModeStr := entry.DurabilityMode + if durModeStr == "" { + durModeStr = "best_effort" + } return &master_pb.LookupBlockVolumeResponse{ VolumeServer: entry.VolumeServer, IscsiAddr: entry.ISCSIAddr, @@ -233,6 +263,7 @@ func (ms *MasterServer) LookupBlockVolume(ctx context.Context, req *master_pb.Lo ReplicaServer: entry.ReplicaServer, // backward compat ReplicaFactor: uint32(rf), ReplicaServers: replicaServers, + DurabilityMode: durModeStr, }, nil } @@ -240,7 +271,7 @@ func (ms *MasterServer) LookupBlockVolume(ctx context.Context, req *master_pb.Lo // Returns the replica server address on success, or empty string on failure (F4). func (ms *MasterServer) tryCreateOneReplica(ctx context.Context, req *master_pb.CreateBlockVolumeRequest, entry *BlockVolumeEntry, primaryResult *blockAllocResult, candidates []string) string { for _, replicaServerStr := range candidates { - replicaResult, err := ms.blockVSAllocate(ctx, replicaServerStr, req.Name, req.SizeBytes, req.DiskType) + replicaResult, err := ms.blockVSAllocate(ctx, replicaServerStr, req.Name, req.SizeBytes, req.DiskType, req.DurabilityMode) if err != nil { glog.V(0).Infof("[reqID=%s] CreateBlockVolume %q: replica on %s failed: %v", blockReqID(ctx), req.Name, replicaServerStr, err) continue @@ -381,6 +412,31 @@ func (ms *MasterServer) createBlockVolumeResponseFromEntry(entry *BlockVolumeEnt } } +// validateIdempotentCreate checks that an idempotent create request is consistent +// with an existing entry. Returns nil if compatible, error on mismatch. +func (ms *MasterServer) validateIdempotentCreate(entry *BlockVolumeEntry, req *master_pb.CreateBlockVolumeRequest, durMode blockvol.DurabilityMode, replicaFactor int) error { + if entry.SizeBytes < req.SizeBytes { + return fmt.Errorf("block volume %q exists with size %d (requested %d)", req.Name, entry.SizeBytes, req.SizeBytes) + } + // Validate durability mode consistency. + existingMode := entry.DurabilityMode + if existingMode == "" { + existingMode = "best_effort" + } + if durMode.String() != existingMode { + return fmt.Errorf("block volume %q exists with durability_mode %q (requested %q)", req.Name, existingMode, durMode.String()) + } + // Validate replica factor consistency. + existingRF := entry.ReplicaFactor + if existingRF == 0 { + existingRF = 2 // default + } + if replicaFactor != existingRF { + return fmt.Errorf("block volume %q exists with replica_factor %d (requested %d)", req.Name, existingRF, replicaFactor) + } + return nil +} + // replicaServerList returns the list of replica server addresses. // Order matches Replicas[] (append-order), ensuring ReplicaServers[0] == ReplicaServer (legacy). func replicaServerList(entry *BlockVolumeEntry) []string { @@ -404,3 +460,23 @@ func removeServer(servers []string, server string) []string { } return result } + +// cleanupPartialCreate removes a partially created block volume (primary + any replicas) +// when strict durability mode enforcement fails due to insufficient replicas. +// All operations are best-effort: failures are logged but do not propagate. +func (ms *MasterServer) cleanupPartialCreate(ctx context.Context, entry *BlockVolumeEntry) { + // Delete primary volume. + if err := ms.blockVSDelete(ctx, entry.VolumeServer, entry.Name); err != nil { + glog.Warningf("[reqID=%s] cleanupPartialCreate %q: delete primary on %s: %v", + blockReqID(ctx), entry.Name, entry.VolumeServer, err) + } + // Delete any successfully created replicas. + for _, ri := range entry.Replicas { + if err := ms.blockVSDelete(ctx, ri.Server, entry.Name); err != nil { + glog.Warningf("[reqID=%s] cleanupPartialCreate %q: delete replica on %s: %v", + blockReqID(ctx), entry.Name, ri.Server, err) + } + } + // Remove from registry if somehow registered (shouldn't be at this point). + ms.blockRegistry.Unregister(entry.Name) +} diff --git a/weed/server/master_grpc_server_block_test.go b/weed/server/master_grpc_server_block_test.go index c7b4f750c..52c9fd259 100644 --- a/weed/server/master_grpc_server_block_test.go +++ b/weed/server/master_grpc_server_block_test.go @@ -19,7 +19,7 @@ func testMasterServer(t *testing.T) *MasterServer { blockAssignmentQueue: NewBlockAssignmentQueue(), } // Default mock: succeed with deterministic values. - ms.blockVSAllocate = func(ctx context.Context, server string, name string, sizeBytes uint64, diskType string) (*blockAllocResult, error) { + ms.blockVSAllocate = func(ctx context.Context, server string, name string, sizeBytes uint64, diskType string, durabilityMode string) (*blockAllocResult, error) { return &blockAllocResult{ Path: fmt.Sprintf("/data/%s.blk", name), IQN: fmt.Sprintf("iqn.2024.test:%s", name), @@ -139,7 +139,7 @@ func TestMaster_CreateVSFailure_Retry(t *testing.T) { ms.blockRegistry.MarkBlockCapable("vs2:9333") var callCount atomic.Int32 - ms.blockVSAllocate = func(ctx context.Context, server string, name string, sizeBytes uint64, diskType string) (*blockAllocResult, error) { + ms.blockVSAllocate = func(ctx context.Context, server string, name string, sizeBytes uint64, diskType string, durabilityMode string) (*blockAllocResult, error) { n := callCount.Add(1) if n == 1 { return nil, fmt.Errorf("disk full") @@ -170,7 +170,7 @@ func TestMaster_CreateVSFailure_Cleanup(t *testing.T) { ms := testMasterServer(t) ms.blockRegistry.MarkBlockCapable("vs1:9333") - ms.blockVSAllocate = func(ctx context.Context, server string, name string, sizeBytes uint64, diskType string) (*blockAllocResult, error) { + ms.blockVSAllocate = func(ctx context.Context, server string, name string, sizeBytes uint64, diskType string, durabilityMode string) (*blockAllocResult, error) { return nil, fmt.Errorf("all servers broken") } @@ -193,7 +193,7 @@ func TestMaster_CreateConcurrentSameName(t *testing.T) { ms.blockRegistry.MarkBlockCapable("vs1:9333") var callCount atomic.Int32 - ms.blockVSAllocate = func(ctx context.Context, server string, name string, sizeBytes uint64, diskType string) (*blockAllocResult, error) { + ms.blockVSAllocate = func(ctx context.Context, server string, name string, sizeBytes uint64, diskType string, durabilityMode string) (*blockAllocResult, error) { callCount.Add(1) return &blockAllocResult{ Path: fmt.Sprintf("/data/%s.blk", name), @@ -275,7 +275,7 @@ func TestMaster_CreateWithReplica(t *testing.T) { ms.blockRegistry.MarkBlockCapable("vs2:9333") var allocServers []string - ms.blockVSAllocate = func(ctx context.Context, server string, name string, sizeBytes uint64, diskType string) (*blockAllocResult, error) { + ms.blockVSAllocate = func(ctx context.Context, server string, name string, sizeBytes uint64, diskType string, durabilityMode string) (*blockAllocResult, error) { allocServers = append(allocServers, server) return &blockAllocResult{ Path: fmt.Sprintf("/data/%s.blk", name), @@ -328,7 +328,7 @@ func TestMaster_CreateSingleServer_NoReplica(t *testing.T) { ms.blockRegistry.MarkBlockCapable("vs1:9333") var allocCount atomic.Int32 - ms.blockVSAllocate = func(ctx context.Context, server string, name string, sizeBytes uint64, diskType string) (*blockAllocResult, error) { + ms.blockVSAllocate = func(ctx context.Context, server string, name string, sizeBytes uint64, diskType string, durabilityMode string) (*blockAllocResult, error) { allocCount.Add(1) return &blockAllocResult{ Path: fmt.Sprintf("/data/%s.blk", name), @@ -365,7 +365,7 @@ func TestMaster_CreateReplica_SecondFails_SingleCopy(t *testing.T) { ms.blockRegistry.MarkBlockCapable("vs2:9333") var callCount atomic.Int32 - ms.blockVSAllocate = func(ctx context.Context, server string, name string, sizeBytes uint64, diskType string) (*blockAllocResult, error) { + ms.blockVSAllocate = func(ctx context.Context, server string, name string, sizeBytes uint64, diskType string, durabilityMode string) (*blockAllocResult, error) { n := callCount.Add(1) if n == 2 { // Replica allocation fails. @@ -402,7 +402,7 @@ func TestMaster_CreateEnqueuesAssignments(t *testing.T) { ms.blockRegistry.MarkBlockCapable("vs1:9333") ms.blockRegistry.MarkBlockCapable("vs2:9333") - ms.blockVSAllocate = func(ctx context.Context, server string, name string, sizeBytes uint64, diskType string) (*blockAllocResult, error) { + ms.blockVSAllocate = func(ctx context.Context, server string, name string, sizeBytes uint64, diskType string, durabilityMode string) (*blockAllocResult, error) { return &blockAllocResult{ Path: fmt.Sprintf("/data/%s.blk", name), IQN: fmt.Sprintf("iqn.2024.test:%s", name), @@ -463,7 +463,7 @@ func TestMaster_LookupReturnsReplicaServer(t *testing.T) { ms.blockRegistry.MarkBlockCapable("vs1:9333") ms.blockRegistry.MarkBlockCapable("vs2:9333") - ms.blockVSAllocate = func(ctx context.Context, server string, name string, sizeBytes uint64, diskType string) (*blockAllocResult, error) { + ms.blockVSAllocate = func(ctx context.Context, server string, name string, sizeBytes uint64, diskType string, durabilityMode string) (*blockAllocResult, error) { return &blockAllocResult{ Path: fmt.Sprintf("/data/%s.blk", name), IQN: fmt.Sprintf("iqn.2024.test:%s", name), @@ -670,7 +670,7 @@ func TestMaster_LookupBlockVolume(t *testing.T) { func testMasterServerRF3(t *testing.T) *MasterServer { t.Helper() ms := testMasterServer(t) - ms.blockVSAllocate = func(ctx context.Context, server string, name string, sizeBytes uint64, diskType string) (*blockAllocResult, error) { + ms.blockVSAllocate = func(ctx context.Context, server string, name string, sizeBytes uint64, diskType string, durabilityMode string) (*blockAllocResult, error) { return &blockAllocResult{ Path: fmt.Sprintf("/data/%s.blk", name), IQN: fmt.Sprintf("iqn.2024.test:%s", name), @@ -725,7 +725,7 @@ func TestMaster_CreateRF3_ThreeServers(t *testing.T) { // RF=3 with only 2 servers: should create 1 replica (partial). func TestMaster_CreateRF3_TwoServers(t *testing.T) { ms := testMasterServer(t) - ms.blockVSAllocate = func(ctx context.Context, server string, name string, sizeBytes uint64, diskType string) (*blockAllocResult, error) { + ms.blockVSAllocate = func(ctx context.Context, server string, name string, sizeBytes uint64, diskType string, durabilityMode string) (*blockAllocResult, error) { return &blockAllocResult{ Path: fmt.Sprintf("/data/%s.blk", name), IQN: fmt.Sprintf("iqn.2024.test:%s", name), diff --git a/weed/server/master_server.go b/weed/server/master_server.go index 35d1766e4..197a7f292 100644 --- a/weed/server/master_server.go +++ b/weed/server/master_server.go @@ -98,7 +98,7 @@ type MasterServer struct { blockRegistry *BlockVolumeRegistry blockAssignmentQueue *BlockAssignmentQueue blockFailover *blockFailoverState - blockVSAllocate func(ctx context.Context, server string, name string, sizeBytes uint64, diskType string) (*blockAllocResult, error) + blockVSAllocate func(ctx context.Context, server string, name string, sizeBytes uint64, diskType string, durabilityMode string) (*blockAllocResult, error) blockVSDelete func(ctx context.Context, server string, name string) error blockVSSnapshot func(ctx context.Context, server string, name string, snapID uint32) (int64, uint64, error) blockVSDeleteSnap func(ctx context.Context, server string, name string, snapID uint32) error @@ -553,13 +553,14 @@ type blockAllocResult struct { } // defaultBlockVSAllocate calls a volume server's AllocateBlockVolume RPC. -func (ms *MasterServer) defaultBlockVSAllocate(ctx context.Context, server string, name string, sizeBytes uint64, diskType string) (*blockAllocResult, error) { +func (ms *MasterServer) defaultBlockVSAllocate(ctx context.Context, server string, name string, sizeBytes uint64, diskType string, durabilityMode string) (*blockAllocResult, error) { var result blockAllocResult err := operation.WithVolumeServerClient(false, pb.ServerAddress(server), ms.grpcDialOption, func(client volume_server_pb.VolumeServerClient) error { resp, rerr := client.AllocateBlockVolume(ctx, &volume_server_pb.AllocateBlockVolumeRequest{ - Name: name, - SizeBytes: sizeBytes, - DiskType: diskType, + Name: name, + SizeBytes: sizeBytes, + DiskType: diskType, + DurabilityMode: durabilityMode, }) if rerr != nil { return rerr diff --git a/weed/server/master_server_handlers_block.go b/weed/server/master_server_handlers_block.go index 7d1285724..f8d29f011 100644 --- a/weed/server/master_server_handlers_block.go +++ b/weed/server/master_server_handlers_block.go @@ -26,10 +26,20 @@ func (ms *MasterServer) blockVolumeCreateHandler(w http.ResponseWriter, r *http. replicaPlacement = "000" } + // Pre-validate durability_mode (cosmetic — real validation is in gRPC handler). + if req.DurabilityMode != "" { + if _, perr := blockvol.ParseDurabilityMode(req.DurabilityMode); perr != nil { + writeJsonError(w, r, http.StatusBadRequest, fmt.Errorf("invalid durability_mode: %w", perr)) + return + } + } + resp, err := ms.CreateBlockVolume(r.Context(), &master_pb.CreateBlockVolumeRequest{ - Name: req.Name, - SizeBytes: req.SizeBytes, - DiskType: req.DiskType, + Name: req.Name, + SizeBytes: req.SizeBytes, + DiskType: req.DiskType, + DurabilityMode: req.DurabilityMode, + ReplicaFactor: uint32(req.ReplicaFactor), }) if err != nil { writeJsonError(w, r, http.StatusInternalServerError, err) @@ -177,6 +187,10 @@ func entryToVolumeInfo(e *BlockVolumeEntry) blockapi.VolumeInfo { if rf == 0 { rf = 2 // default } + durMode := e.DurabilityMode + if durMode == "" { + durMode = "best_effort" + } info := blockapi.VolumeInfo{ Name: e.Name, VolumeServer: e.VolumeServer, @@ -195,6 +209,7 @@ func entryToVolumeInfo(e *BlockVolumeEntry) blockapi.VolumeInfo { ReplicaFactor: rf, HealthScore: e.HealthScore, ReplicaDegraded: e.ReplicaDegraded, + DurabilityMode: durMode, } for _, ri := range e.Replicas { info.Replicas = append(info.Replicas, blockapi.ReplicaDetail{ diff --git a/weed/server/master_server_handlers_block_test.go b/weed/server/master_server_handlers_block_test.go index c2f309650..89395ba37 100644 --- a/weed/server/master_server_handlers_block_test.go +++ b/weed/server/master_server_handlers_block_test.go @@ -20,7 +20,7 @@ func blockTestServer(t *testing.T) (*MasterServer, *httptest.Server) { blockRegistry: NewBlockVolumeRegistry(), blockAssignmentQueue: NewBlockAssignmentQueue(), } - ms.blockVSAllocate = func(ctx context.Context, server string, name string, sizeBytes uint64, diskType string) (*blockAllocResult, error) { + ms.blockVSAllocate = func(ctx context.Context, server string, name string, sizeBytes uint64, diskType string, durabilityMode string) (*blockAllocResult, error) { return &blockAllocResult{ Path: fmt.Sprintf("/data/%s.blk", name), IQN: fmt.Sprintf("iqn.2024.test:%s", name), diff --git a/weed/server/master_server_handlers_block_ui.go b/weed/server/master_server_handlers_block_ui.go index be0ca0ae5..5e5c0204a 100644 --- a/weed/server/master_server_handlers_block_ui.go +++ b/weed/server/master_server_handlers_block_ui.go @@ -13,7 +13,9 @@ var blockOpsTemplate = template.Must(template.New("blockOps").Parse(blockLayoutH type blockUIVolume struct { blockapi.VolumeInfo - SizeMB uint64 + SizeMB uint64 + WALHeadLSN uint64 + MaxWALLag uint64 // max WAL lag across replicas } type blockUIData struct { @@ -25,6 +27,12 @@ type blockUIData struct { ActiveCount int PendingCount int TotalSizeMB uint64 + // Observability (CP8-4) + BarrierLagLSN uint64 + PromotionsTotal uint64 + FailoversTotal uint64 + RebuildsTotal uint64 + AssignmentQueueLen int } func (ms *MasterServer) buildBlockUIData(tab string) blockUIData { @@ -35,9 +43,17 @@ func (ms *MasterServer) buildBlockUIData(tab string) blockUIData { for i, e := range entries { info := entryToVolumeInfo(e) mb := info.SizeBytes / (1024 * 1024) + var maxLag uint64 + for _, ri := range e.Replicas { + if ri.WALLag > maxLag { + maxLag = ri.WALLag + } + } volumes[i] = blockUIVolume{ VolumeInfo: info, SizeMB: mb, + WALHeadLSN: e.WALHeadLSN, + MaxWALLag: maxLag, } totalSizeMB += mb if e.Status == StatusActive { @@ -58,13 +74,18 @@ func (ms *MasterServer) buildBlockUIData(tab string) blockUIData { } return blockUIData{ - Tab: tab, - Volumes: volumes, - Servers: servers, - TotalVolumes: len(entries), - ActiveCount: activeCount, - PendingCount: pendingCount, - TotalSizeMB: totalSizeMB, + Tab: tab, + Volumes: volumes, + Servers: servers, + TotalVolumes: len(entries), + ActiveCount: activeCount, + PendingCount: pendingCount, + TotalSizeMB: totalSizeMB, + BarrierLagLSN: ms.blockRegistry.MaxBarrierLagLSN(), + PromotionsTotal: ms.blockRegistry.PromotionsTotal.Load(), + FailoversTotal: ms.blockRegistry.FailoversTotal.Load(), + RebuildsTotal: ms.blockRegistry.RebuildsTotal.Load(), + AssignmentQueueLen: ms.blockAssignmentQueue.TotalPending(), } } @@ -130,6 +151,9 @@ const blockLayoutHTML = ` .badge-primary { background: #dfe6e9; color: #2d3436; } .badge-replica { background: #e8daef; color: #6c3483; } .empty { color: #b2bec3; font-style: italic; padding: 20px; text-align: center; } + .card .value.red { color: #d63031; } + .card .value.gray { color: #636e72; } + .section-label { font-size: 11px; color: #b2bec3; text-transform: uppercase; letter-spacing: 1px; margin-bottom: 8px; } @@ -166,6 +190,30 @@ const blockDashContentHTML = ` +
Cluster Health
+
+
+
Barrier Lag LSN
+
{{.BarrierLagLSN}}
+
+
+
Promotions
+
{{.PromotionsTotal}}
+
+
+
Failovers
+
{{.FailoversTotal}}
+
+
+
Rebuilds
+
{{.RebuildsTotal}}
+
+
+
Queue Depth
+
{{.AssignmentQueueLen}}
+
+
+

Servers

{{if .Servers}} @@ -186,19 +234,23 @@ const blockDashContentHTML = ` {{if .Volumes}}
- - + + + {{range .Volumes}} - - - + + + + + + {{end}} @@ -235,6 +287,23 @@ const blockOpsContentHTML = ` +
+ + +
+
+ + +
@@ -246,19 +315,22 @@ const blockOpsContentHTML = ` {{if .Volumes}}
NameServerSizePlacementEpochRoleStatusiSCSIReplicaNameServerSizeRoleStatusDurabilityHealthWAL LSNWAL LagDegradedEpochReplica
{{.Name}} {{.VolumeServer}} {{.SizeMB}} MB{{.ReplicaPlacement}}{{.Epoch}} {{if eq .Role "primary"}}primary{{else if eq .Role "replica"}}replica{{else}}{{.Role}}{{end}} {{if eq .Status "active"}}active{{else}}{{.Status}}{{end}}{{.ISCSIAddr}}{{if eq .DurabilityMode "sync_all"}}sync_all{{else if eq .DurabilityMode "sync_quorum"}}sync_quorum{{else}}best_effort{{end}}{{printf "%.2f" .HealthScore}}{{.WALHeadLSN}}{{.MaxWALLag}}{{if .ReplicaDegraded}}yes{{else}}-{{end}}{{.Epoch}} {{.ReplicaServer}}
- - + + + {{range .Volumes}} - - - + + + + + @@ -281,7 +353,9 @@ document.getElementById('createForm').addEventListener('submit', function(e) { name: document.getElementById('cName').value, size_bytes: parseInt(document.getElementById('cSize').value) * 1024 * 1024, replica_placement: document.getElementById('cPlacement').value, - disk_type: document.getElementById('cDisk').value + disk_type: document.getElementById('cDisk').value, + replica_factor: parseInt(document.getElementById('cRF').value) || 0, + durability_mode: document.getElementById('cDurability').value }) }).then(function(r) { if (!r.ok) return r.json().then(function(j) { throw new Error(j.error || r.statusText); }); diff --git a/weed/server/qa_block_cp62_test.go b/weed/server/qa_block_cp62_test.go index 336a6c12a..27259bfb0 100644 --- a/weed/server/qa_block_cp62_test.go +++ b/weed/server/qa_block_cp62_test.go @@ -319,7 +319,7 @@ func TestQA_Master_AllVSFailNoOrphan(t *testing.T) { ms.blockRegistry.MarkBlockCapable("vs2:9333") ms.blockRegistry.MarkBlockCapable("vs3:9333") - ms.blockVSAllocate = func(ctx context.Context, server string, name string, sizeBytes uint64, diskType string) (*blockAllocResult, error) { + ms.blockVSAllocate = func(ctx context.Context, server string, name string, sizeBytes uint64, diskType string, durabilityMode string) (*blockAllocResult, error) { return nil, fmt.Errorf("disk full on %s", server) } @@ -348,7 +348,7 @@ func TestQA_Master_SlowAllocateBlocksSecond(t *testing.T) { ms.blockRegistry.MarkBlockCapable("vs1:9333") var allocCount atomic.Int32 - ms.blockVSAllocate = func(ctx context.Context, server string, name string, sizeBytes uint64, diskType string) (*blockAllocResult, error) { + ms.blockVSAllocate = func(ctx context.Context, server string, name string, sizeBytes uint64, diskType string, durabilityMode string) (*blockAllocResult, error) { allocCount.Add(1) time.Sleep(100 * time.Millisecond) // simulate slow VS return &blockAllocResult{ @@ -450,7 +450,7 @@ func TestQA_VS_ConcurrentCreate(t *testing.T) { t.Errorf("PANIC: %v", r) } }() - _, _, _, errors[i] = bs.CreateBlockVol("race-vol", 4*1024*1024, "") + _, _, _, errors[i] = bs.CreateBlockVol("race-vol", 4*1024*1024, "", "") }(i) } wg.Wait() @@ -487,7 +487,7 @@ func TestQA_VS_ConcurrentCreateDelete(t *testing.T) { panicked.Store(true) } }() - bs.CreateBlockVol("cd-vol", 4*1024*1024, "") + bs.CreateBlockVol("cd-vol", 4*1024*1024, "", "") }() go func() { defer wg.Done() @@ -510,7 +510,7 @@ func TestQA_VS_ConcurrentCreateDelete(t *testing.T) { func TestQA_VS_DeleteCleansSnapshots(t *testing.T) { bs, blockDir := newTestBlockServiceWithDir(t) - bs.CreateBlockVol("snap-vol", 4*1024*1024, "") + bs.CreateBlockVol("snap-vol", 4*1024*1024, "", "") // Simulate snapshot files. snapPath := blockDir + "/snap-vol.blk.snap.0" @@ -531,13 +531,13 @@ func TestQA_VS_SanitizationCollision(t *testing.T) { bs, _ := newTestBlockServiceWithDir(t) // "VolA" sanitizes to "vola.blk", "vola" also sanitizes to "vola.blk". - _, _, _, err := bs.CreateBlockVol("VolA", 4*1024*1024, "") + _, _, _, err := bs.CreateBlockVol("VolA", 4*1024*1024, "", "") if err != nil { t.Fatalf("create VolA: %v", err) } // "vola" should get the idempotent path (same file on disk). - path2, _, _, err := bs.CreateBlockVol("vola", 4*1024*1024, "") + path2, _, _, err := bs.CreateBlockVol("vola", 4*1024*1024, "", "") if err != nil { t.Fatalf("create vola: %v", err) } @@ -553,13 +553,13 @@ func TestQA_VS_CreateIdempotentReaddTarget(t *testing.T) { bs, _ := newTestBlockServiceWithDir(t) // First create. - _, iqn1, _, err := bs.CreateBlockVol("readd-vol", 4*1024*1024, "") + _, iqn1, _, err := bs.CreateBlockVol("readd-vol", 4*1024*1024, "", "") if err != nil { t.Fatalf("first create: %v", err) } // Second create (idempotent) — should succeed and re-add to TargetServer. - _, iqn2, _, err := bs.CreateBlockVol("readd-vol", 4*1024*1024, "") + _, iqn2, _, err := bs.CreateBlockVol("readd-vol", 4*1024*1024, "", "") if err != nil { t.Fatalf("idempotent create: %v", err) } diff --git a/weed/server/qa_block_cp63_test.go b/weed/server/qa_block_cp63_test.go index 28176602b..58e533c20 100644 --- a/weed/server/qa_block_cp63_test.go +++ b/weed/server/qa_block_cp63_test.go @@ -24,7 +24,7 @@ func testMSForQA(t *testing.T) *MasterServer { blockAssignmentQueue: NewBlockAssignmentQueue(), blockFailover: newBlockFailoverState(), } - ms.blockVSAllocate = func(ctx context.Context, server string, name string, sizeBytes uint64, diskType string) (*blockAllocResult, error) { + ms.blockVSAllocate = func(ctx context.Context, server string, name string, sizeBytes uint64, diskType string, durabilityMode string) (*blockAllocResult, error) { return &blockAllocResult{ Path: fmt.Sprintf("/data/%s.blk", name), IQN: fmt.Sprintf("iqn.2024.test:%s", name), diff --git a/weed/server/qa_block_cp82_adversarial_test.go b/weed/server/qa_block_cp82_adversarial_test.go index f29c6ca48..654c1737c 100644 --- a/weed/server/qa_block_cp82_adversarial_test.go +++ b/weed/server/qa_block_cp82_adversarial_test.go @@ -27,7 +27,7 @@ func qaCP82Master(t *testing.T) *MasterServer { blockAssignmentQueue: NewBlockAssignmentQueue(), blockFailover: newBlockFailoverState(), } - ms.blockVSAllocate = func(ctx context.Context, server string, name string, sizeBytes uint64, diskType string) (*blockAllocResult, error) { + ms.blockVSAllocate = func(ctx context.Context, server string, name string, sizeBytes uint64, diskType string, durabilityMode string) (*blockAllocResult, error) { return &blockAllocResult{ Path: fmt.Sprintf("/data/%s.blk", name), IQN: fmt.Sprintf("iqn.2024.test:%s", name), diff --git a/weed/server/qa_block_cp831_adversarial_test.go b/weed/server/qa_block_cp831_adversarial_test.go new file mode 100644 index 000000000..7d13231d1 --- /dev/null +++ b/weed/server/qa_block_cp831_adversarial_test.go @@ -0,0 +1,907 @@ +package weed_server + +import ( + "context" + "errors" + "fmt" + "strings" + "sync" + "testing" + "time" + + "github.com/seaweedfs/seaweedfs/weed/pb/master_pb" + "github.com/seaweedfs/seaweedfs/weed/storage/blockvol" +) + +// ============================================================ +// CP8-3-1 Adversarial QA Test Suite +// +// 20 adversarial tests targeting durability mode ACK contracts, +// F1-F4 review fixes, and edge cases in mode handling. +// ============================================================ + +// ──────────────────────────────────────────────────────────── +// QA-CP831-1: IdempotentCreate_ModeMismatch_Rejected +// +// Create sync_all volume, then retry with best_effort. +// Must return mismatch error, NOT silently succeed. +// ──────────────────────────────────────────────────────────── +func TestQA_CP831_IdempotentCreate_ModeMismatch_Rejected(t *testing.T) { + ms := qaDurabilityMaster(t) + ctx := context.Background() + + // Create sync_all volume. + _, err := ms.CreateBlockVolume(ctx, &master_pb.CreateBlockVolumeRequest{ + Name: "idem-mode", + SizeBytes: 1 << 30, + DurabilityMode: "sync_all", + ReplicaFactor: 2, + }) + if err != nil { + t.Fatalf("initial create: %v", err) + } + + // Retry with best_effort (empty string = best_effort). + _, err = ms.CreateBlockVolume(ctx, &master_pb.CreateBlockVolumeRequest{ + Name: "idem-mode", + SizeBytes: 1 << 30, + // DurabilityMode omitted = best_effort + }) + if err == nil { + t.Fatal("idempotent create with mode mismatch should fail") + } + if !strings.Contains(err.Error(), "durability_mode") { + t.Fatalf("expected durability_mode mismatch error, got: %v", err) + } +} + +// ──────────────────────────────────────────────────────────── +// QA-CP831-2: IdempotentCreate_RFMismatch_Rejected +// +// Create RF=2, retry with RF=3. Must fail with RF mismatch. +// ──────────────────────────────────────────────────────────── +func TestQA_CP831_IdempotentCreate_RFMismatch_Rejected(t *testing.T) { + ms := qaDurabilityMaster(t) + ctx := context.Background() + + _, err := ms.CreateBlockVolume(ctx, &master_pb.CreateBlockVolumeRequest{ + Name: "idem-rf", + SizeBytes: 1 << 30, + DurabilityMode: "sync_all", + ReplicaFactor: 2, + }) + if err != nil { + t.Fatalf("create: %v", err) + } + + // Retry with RF=3. + _, err = ms.CreateBlockVolume(ctx, &master_pb.CreateBlockVolumeRequest{ + Name: "idem-rf", + SizeBytes: 1 << 30, + DurabilityMode: "sync_all", + ReplicaFactor: 3, + }) + if err == nil { + t.Fatal("idempotent create with RF mismatch should fail") + } + if !strings.Contains(err.Error(), "replica_factor") { + t.Fatalf("expected replica_factor mismatch error, got: %v", err) + } +} + +// ──────────────────────────────────────────────────────────── +// QA-CP831-3: IdempotentCreate_AllMatch_Succeeds +// +// Retry with identical params returns existing entry without error. +// ──────────────────────────────────────────────────────────── +func TestQA_CP831_IdempotentCreate_AllMatch_Succeeds(t *testing.T) { + ms := qaDurabilityMaster(t) + ctx := context.Background() + + resp1, err := ms.CreateBlockVolume(ctx, &master_pb.CreateBlockVolumeRequest{ + Name: "idem-ok", + SizeBytes: 1 << 30, + DurabilityMode: "sync_all", + ReplicaFactor: 2, + }) + if err != nil { + t.Fatalf("create: %v", err) + } + + // Exact same params. + resp2, err := ms.CreateBlockVolume(ctx, &master_pb.CreateBlockVolumeRequest{ + Name: "idem-ok", + SizeBytes: 1 << 30, + DurabilityMode: "sync_all", + ReplicaFactor: 2, + }) + if err != nil { + t.Fatalf("idempotent create should succeed: %v", err) + } + if resp1.VolumeServer != resp2.VolumeServer { + t.Fatalf("idempotent: volume_server changed %q -> %q", resp1.VolumeServer, resp2.VolumeServer) + } +} + +// ──────────────────────────────────────────────────────────── +// QA-CP831-4: InvalidDurabilityMode_Rejected +// +// Unknown mode string must be rejected at the gRPC layer. +// ──────────────────────────────────────────────────────────── +func TestQA_CP831_InvalidDurabilityMode_Rejected(t *testing.T) { + ms := qaDurabilityMaster(t) + ctx := context.Background() + + badModes := []string{"sync", "strict", "SYNC_ALL", "best-effort", "quorum"} + for _, mode := range badModes { + _, err := ms.CreateBlockVolume(ctx, &master_pb.CreateBlockVolumeRequest{ + Name: "invalid-" + mode, + SizeBytes: 1 << 30, + DurabilityMode: mode, + }) + if err == nil { + t.Errorf("durability_mode=%q should be rejected", mode) + continue + } + if !strings.Contains(err.Error(), "invalid") { + t.Errorf("durability_mode=%q: expected 'invalid' error, got: %v", mode, err) + } + } +} + +// ──────────────────────────────────────────────────────────── +// QA-CP831-5: HeartbeatEmptyMode_DoesNotOverwriteStrict +// +// F3 regression: heartbeat with empty DurabilityMode must NOT +// downgrade an existing sync_all volume to best_effort. +// ──────────────────────────────────────────────────────────── +func TestQA_CP831_HeartbeatEmptyMode_DoesNotOverwriteStrict(t *testing.T) { + ms := qaDurabilityMaster(t) + ctx := context.Background() + + _, err := ms.CreateBlockVolume(ctx, &master_pb.CreateBlockVolumeRequest{ + Name: "hb-strict", + SizeBytes: 1 << 30, + DurabilityMode: "sync_all", + ReplicaFactor: 2, + }) + if err != nil { + t.Fatalf("create: %v", err) + } + + entry, _ := ms.blockRegistry.Lookup("hb-strict") + + // 10 rapid heartbeats with empty mode — simulates older VS version. + for i := 0; i < 10; i++ { + ms.blockRegistry.UpdateFullHeartbeat(entry.VolumeServer, []*master_pb.BlockVolumeInfoMessage{ + { + Path: entry.Path, + VolumeSize: 1 << 30, + DurabilityMode: "", // empty — must not overwrite + }, + }) + } + + entry, ok := ms.blockRegistry.Lookup("hb-strict") + if !ok { + t.Fatal("volume gone after heartbeat flood") + } + if entry.DurabilityMode != "sync_all" { + t.Fatalf("F3 regression: empty heartbeat overwrote mode to %q", entry.DurabilityMode) + } +} + +// ──────────────────────────────────────────────────────────── +// QA-CP831-6: HeartbeatNonEmptyMode_DoesUpdate +// +// Heartbeat with non-empty DurabilityMode DOES update the entry. +// (Legitimate VS reports after mode-aware upgrade.) +// ──────────────────────────────────────────────────────────── +func TestQA_CP831_HeartbeatNonEmptyMode_DoesUpdate(t *testing.T) { + ms := qaDurabilityMaster(t) + ctx := context.Background() + + _, err := ms.CreateBlockVolume(ctx, &master_pb.CreateBlockVolumeRequest{ + Name: "hb-update", + SizeBytes: 1 << 30, + DurabilityMode: "sync_all", + ReplicaFactor: 2, + }) + if err != nil { + t.Fatalf("create: %v", err) + } + + entry, _ := ms.blockRegistry.Lookup("hb-update") + + // Heartbeat with matching non-empty mode — should be accepted. + ms.blockRegistry.UpdateFullHeartbeat(entry.VolumeServer, []*master_pb.BlockVolumeInfoMessage{ + { + Path: entry.Path, + VolumeSize: 1 << 30, + DurabilityMode: "sync_all", + }, + }) + + entry, ok := ms.blockRegistry.Lookup("hb-update") + if !ok { + t.Fatal("volume gone after heartbeat") + } + if entry.DurabilityMode != "sync_all" { + t.Fatalf("expected sync_all after non-empty heartbeat, got %q", entry.DurabilityMode) + } +} + +// ──────────────────────────────────────────────────────────── +// QA-CP831-7: SyncAll_RF3_PartialReplica_OneOfTwo_Fails +// +// sync_all RF=3 requires 2 replicas (RequiredReplicas=2). +// If only 1 of 2 replicas is provisioned → must fail + cleanup. +// ──────────────────────────────────────────────────────────── +func TestQA_CP831_SyncAll_RF3_PartialReplica_OneOfTwo_Fails(t *testing.T) { + ms := qaDurabilityMaster(t) + ctx := context.Background() + + // First call succeeds (primary), second succeeds (replica 1), + // third fails (replica 2). + callCount := 0 + ms.blockVSAllocate = func(ctx context.Context, server string, name string, sizeBytes uint64, diskType string, durabilityMode string) (*blockAllocResult, error) { + callCount++ + if callCount == 3 { + return nil, fmt.Errorf("disk full on third server") + } + return &blockAllocResult{ + Path: fmt.Sprintf("/data/%s.blk", name), + IQN: fmt.Sprintf("iqn.2024.test:%s", name), + ISCSIAddr: server + ":3260", + ReplicaDataAddr: server + ":14260", + ReplicaCtrlAddr: server + ":14261", + RebuildListenAddr: server + ":15000", + }, nil + } + + var deletedServers []string + ms.blockVSDelete = func(ctx context.Context, server string, name string) error { + deletedServers = append(deletedServers, server) + return nil + } + + _, err := ms.CreateBlockVolume(ctx, &master_pb.CreateBlockVolumeRequest{ + Name: "sa-rf3-partial", + SizeBytes: 1 << 30, + DurabilityMode: "sync_all", + ReplicaFactor: 3, + }) + if err == nil { + t.Fatal("sync_all RF=3 should fail when only 1 of 2 required replicas provisioned") + } + if !strings.Contains(err.Error(), "requires") { + t.Fatalf("expected 'requires' error, got: %v", err) + } + + // Cleanup should have deleted primary + the 1 successful replica. + if len(deletedServers) < 2 { + t.Fatalf("expected at least 2 cleanup deletes (primary + 1 replica), got %d", len(deletedServers)) + } + + // Volume must not be in registry. + if _, ok := ms.blockRegistry.Lookup("sa-rf3-partial"); ok { + t.Fatal("partially created volume should not remain in registry") + } +} + +// ──────────────────────────────────────────────────────────── +// QA-CP831-8: SyncQuorum_RF3_OneReplicaOK_Succeeds +// +// sync_quorum RF=3 requires only 1 replica (quorum=2, minus primary=1). +// If 1 of 2 replicas fails, should still succeed. +// ──────────────────────────────────────────────────────────── +func TestQA_CP831_SyncQuorum_RF3_OneReplicaOK_Succeeds(t *testing.T) { + ms := qaDurabilityMaster(t) + ctx := context.Background() + + callCount := 0 + ms.blockVSAllocate = func(ctx context.Context, server string, name string, sizeBytes uint64, diskType string, durabilityMode string) (*blockAllocResult, error) { + callCount++ + if callCount == 3 { + return nil, fmt.Errorf("disk full on third server") + } + return &blockAllocResult{ + Path: fmt.Sprintf("/data/%s.blk", name), + IQN: fmt.Sprintf("iqn.2024.test:%s", name), + ISCSIAddr: server + ":3260", + ReplicaDataAddr: server + ":14260", + ReplicaCtrlAddr: server + ":14261", + RebuildListenAddr: server + ":15000", + }, nil + } + + _, err := ms.CreateBlockVolume(ctx, &master_pb.CreateBlockVolumeRequest{ + Name: "sq-rf3-partial", + SizeBytes: 1 << 30, + DurabilityMode: "sync_quorum", + ReplicaFactor: 3, + }) + if err != nil { + t.Fatalf("sync_quorum RF=3 should succeed with 1 replica: %v", err) + } + + entry, _ := ms.blockRegistry.Lookup("sq-rf3-partial") + if len(entry.Replicas) != 1 { + t.Fatalf("expected 1 replica, got %d", len(entry.Replicas)) + } + if entry.DurabilityMode != "sync_quorum" { + t.Fatalf("expected sync_quorum, got %q", entry.DurabilityMode) + } +} + +// ──────────────────────────────────────────────────────────── +// QA-CP831-9: SyncQuorum_RF3_AllReplicasFail_Fails +// +// sync_quorum RF=3 needs 1 replica. If all replica servers fail, +// must fail + cleanup. +// ──────────────────────────────────────────────────────────── +func TestQA_CP831_SyncQuorum_RF3_AllReplicasFail_Fails(t *testing.T) { + ms := qaDurabilityMaster(t) + ctx := context.Background() + + callCount := 0 + ms.blockVSAllocate = func(ctx context.Context, server string, name string, sizeBytes uint64, diskType string, durabilityMode string) (*blockAllocResult, error) { + callCount++ + if callCount > 1 { // primary succeeds, all replicas fail + return nil, fmt.Errorf("disk full") + } + return &blockAllocResult{ + Path: fmt.Sprintf("/data/%s.blk", name), + IQN: fmt.Sprintf("iqn.2024.test:%s", name), + ISCSIAddr: server + ":3260", + ReplicaDataAddr: server + ":14260", + ReplicaCtrlAddr: server + ":14261", + RebuildListenAddr: server + ":15000", + }, nil + } + + var cleanupCalled bool + ms.blockVSDelete = func(ctx context.Context, server string, name string) error { + cleanupCalled = true + return nil + } + + _, err := ms.CreateBlockVolume(ctx, &master_pb.CreateBlockVolumeRequest{ + Name: "sq-rf3-none", + SizeBytes: 1 << 30, + DurabilityMode: "sync_quorum", + ReplicaFactor: 3, + }) + if err == nil { + t.Fatal("sync_quorum RF=3 should fail when no replicas provisioned") + } + if !cleanupCalled { + t.Fatal("expected cleanup of partial create") + } + if _, ok := ms.blockRegistry.Lookup("sq-rf3-none"); ok { + t.Fatal("volume should not be in registry") + } +} + +// ──────────────────────────────────────────────────────────── +// QA-CP831-10: ConcurrentCreate_SameName_DifferentModes +// +// Two goroutines race to create same volume with different modes. +// Exactly one should succeed, the other should get either +// inflight error or idempotent mismatch error. +// ──────────────────────────────────────────────────────────── +func TestQA_CP831_ConcurrentCreate_SameName_DifferentModes(t *testing.T) { + ms := qaDurabilityMaster(t) + ctx := context.Background() + + // Slow down allocation to increase race window. + ms.blockVSAllocate = func(ctx context.Context, server string, name string, sizeBytes uint64, diskType string, durabilityMode string) (*blockAllocResult, error) { + time.Sleep(10 * time.Millisecond) + return &blockAllocResult{ + Path: fmt.Sprintf("/data/%s.blk", name), + IQN: fmt.Sprintf("iqn.2024.test:%s", name), + ISCSIAddr: server + ":3260", + ReplicaDataAddr: server + ":14260", + ReplicaCtrlAddr: server + ":14261", + RebuildListenAddr: server + ":15000", + }, nil + } + + var wg sync.WaitGroup + results := make([]error, 2) + + wg.Add(2) + go func() { + defer wg.Done() + _, results[0] = ms.CreateBlockVolume(ctx, &master_pb.CreateBlockVolumeRequest{ + Name: "race-vol", + SizeBytes: 1 << 30, + DurabilityMode: "sync_all", + ReplicaFactor: 2, + }) + }() + go func() { + defer wg.Done() + _, results[1] = ms.CreateBlockVolume(ctx, &master_pb.CreateBlockVolumeRequest{ + Name: "race-vol", + SizeBytes: 1 << 30, + // best_effort + }) + }() + wg.Wait() + + // Exactly one should succeed (or both fail if inflight lock blocks both). + successes := 0 + for _, err := range results { + if err == nil { + successes++ + } + } + if successes > 1 { + t.Fatal("both creates succeeded with different modes — split-brain!") + } + + // Volume in registry must have a consistent mode. + entry, ok := ms.blockRegistry.Lookup("race-vol") + if ok { + mode := entry.DurabilityMode + if mode == "" { + mode = "best_effort" + } + // Must be one of the two requested modes, not a corrupt mix. + if mode != "sync_all" && mode != "best_effort" { + t.Fatalf("corrupt mode in registry: %q", mode) + } + } +} + +// ──────────────────────────────────────────────────────────── +// QA-CP831-11: FailoverPreservesDurabilityMode +// +// Create sync_all volume, trigger failover. Mode must survive +// promotion to new primary. +// ──────────────────────────────────────────────────────────── +func TestQA_CP831_FailoverPreservesDurabilityMode(t *testing.T) { + ms := qaDurabilityMaster(t) + ctx := context.Background() + + _, err := ms.CreateBlockVolume(ctx, &master_pb.CreateBlockVolumeRequest{ + Name: "fo-mode", + SizeBytes: 1 << 30, + DurabilityMode: "sync_all", + ReplicaFactor: 2, + }) + if err != nil { + t.Fatalf("create: %v", err) + } + + entry, _ := ms.blockRegistry.Lookup("fo-mode") + primary := entry.VolumeServer + + // Expire the lease so failover will actually promote. + entry.LeaseTTL = 5 * time.Second + entry.LastLeaseGrant = time.Now().Add(-1 * time.Minute) + + // Ensure replica has fresh heartbeat for promotion eligibility. + if len(entry.Replicas) > 0 { + entry.Replicas[0].LastHeartbeat = time.Now() + entry.Replicas[0].Role = blockvol.RoleToWire(blockvol.RoleReplica) + entry.Replicas[0].WALHeadLSN = entry.WALHeadLSN + } + + ms.failoverBlockVolumes(primary) + + entry, ok := ms.blockRegistry.Lookup("fo-mode") + if !ok { + t.Fatal("volume gone after failover") + } + if entry.VolumeServer == primary { + t.Fatal("failover didn't promote replica") + } + if entry.DurabilityMode != "sync_all" { + t.Fatalf("durability mode lost after failover: got %q, want sync_all", entry.DurabilityMode) + } +} + +// ──────────────────────────────────────────────────────────── +// QA-CP831-12: ExpandPreservesDurabilityMode +// +// Expand a sync_all volume. Mode must not change. +// ──────────────────────────────────────────────────────────── +func TestQA_CP831_ExpandPreservesDurabilityMode(t *testing.T) { + ms := qaDurabilityMaster(t) + ctx := context.Background() + + _, err := ms.CreateBlockVolume(ctx, &master_pb.CreateBlockVolumeRequest{ + Name: "expand-mode", + SizeBytes: 1 << 30, + DurabilityMode: "sync_all", + ReplicaFactor: 2, + }) + if err != nil { + t.Fatalf("create: %v", err) + } + + _, err = ms.ExpandBlockVolume(ctx, &master_pb.ExpandBlockVolumeRequest{ + Name: "expand-mode", + NewSizeBytes: 2 << 30, + }) + if err != nil { + t.Fatalf("expand: %v", err) + } + + entry, _ := ms.blockRegistry.Lookup("expand-mode") + if entry.DurabilityMode != "sync_all" { + t.Fatalf("mode changed after expand: got %q", entry.DurabilityMode) + } + if entry.SizeBytes != 2<<30 { + t.Fatalf("size not updated: got %d", entry.SizeBytes) + } +} + +// ──────────────────────────────────────────────────────────── +// QA-CP831-13: LookupReturnsDurabilityMode +// +// Lookup response must include the correct durability mode. +// Also tests default for legacy volumes (empty → "best_effort"). +// ──────────────────────────────────────────────────────────── +func TestQA_CP831_LookupReturnsDurabilityMode(t *testing.T) { + ms := qaDurabilityMaster(t) + ctx := context.Background() + + // Create sync_all. + ms.CreateBlockVolume(ctx, &master_pb.CreateBlockVolumeRequest{ + Name: "lu-strict", + SizeBytes: 1 << 30, + DurabilityMode: "sync_all", + ReplicaFactor: 2, + }) + + resp, err := ms.LookupBlockVolume(ctx, &master_pb.LookupBlockVolumeRequest{Name: "lu-strict"}) + if err != nil { + t.Fatalf("lookup: %v", err) + } + if resp.DurabilityMode != "sync_all" { + t.Fatalf("lookup DurabilityMode: got %q, want sync_all", resp.DurabilityMode) + } + + // Create best_effort (empty mode). + ms.CreateBlockVolume(ctx, &master_pb.CreateBlockVolumeRequest{ + Name: "lu-default", + SizeBytes: 1 << 30, + }) + + resp, err = ms.LookupBlockVolume(ctx, &master_pb.LookupBlockVolumeRequest{Name: "lu-default"}) + if err != nil { + t.Fatalf("lookup default: %v", err) + } + if resp.DurabilityMode != "best_effort" { + t.Fatalf("lookup DurabilityMode for default: got %q, want best_effort", resp.DurabilityMode) + } +} + +// ──────────────────────────────────────────────────────────── +// QA-CP831-14: BestEffort_NoReplicaCreate_StillSucceeds +// +// best_effort with only 1 server should succeed (no replica needed). +// ──────────────────────────────────────────────────────────── +func TestQA_CP831_BestEffort_NoReplicaCreate_StillSucceeds(t *testing.T) { + ms := &MasterServer{ + blockRegistry: NewBlockVolumeRegistry(), + blockAssignmentQueue: NewBlockAssignmentQueue(), + blockFailover: newBlockFailoverState(), + } + ms.blockVSAllocate = func(ctx context.Context, server string, name string, sizeBytes uint64, diskType string, durabilityMode string) (*blockAllocResult, error) { + return &blockAllocResult{ + Path: fmt.Sprintf("/data/%s.blk", name), + IQN: fmt.Sprintf("iqn.2024.test:%s", name), + ISCSIAddr: server + ":3260", + ReplicaDataAddr: server + ":14260", + ReplicaCtrlAddr: server + ":14261", + RebuildListenAddr: server + ":15000", + }, nil + } + ms.blockVSDelete = func(ctx context.Context, server string, name string) error { return nil } + ms.blockRegistry.MarkBlockCapable("vs1:9333") // only 1 server + + _, err := ms.CreateBlockVolume(context.Background(), &master_pb.CreateBlockVolumeRequest{ + Name: "single-server", + SizeBytes: 1 << 30, + }) + if err != nil { + t.Fatalf("best_effort single server should succeed: %v", err) + } +} + +// ──────────────────────────────────────────────────────────── +// QA-CP831-15: SyncAll_SingleServer_Fails +// +// sync_all with only 1 block server: cannot create any replicas, +// RequiredReplicas=1, so must fail with cleanup. +// ──────────────────────────────────────────────────────────── +func TestQA_CP831_SyncAll_SingleServer_Fails(t *testing.T) { + ms := &MasterServer{ + blockRegistry: NewBlockVolumeRegistry(), + blockAssignmentQueue: NewBlockAssignmentQueue(), + blockFailover: newBlockFailoverState(), + } + ms.blockVSAllocate = func(ctx context.Context, server string, name string, sizeBytes uint64, diskType string, durabilityMode string) (*blockAllocResult, error) { + return &blockAllocResult{ + Path: fmt.Sprintf("/data/%s.blk", name), + IQN: fmt.Sprintf("iqn.2024.test:%s", name), + ISCSIAddr: server + ":3260", + ReplicaDataAddr: server + ":14260", + ReplicaCtrlAddr: server + ":14261", + RebuildListenAddr: server + ":15000", + }, nil + } + var cleaned bool + ms.blockVSDelete = func(ctx context.Context, server string, name string) error { + cleaned = true + return nil + } + ms.blockRegistry.MarkBlockCapable("vs1:9333") + + _, err := ms.CreateBlockVolume(context.Background(), &master_pb.CreateBlockVolumeRequest{ + Name: "sa-single", + SizeBytes: 1 << 30, + DurabilityMode: "sync_all", + ReplicaFactor: 2, + }) + if err == nil { + t.Fatal("sync_all should fail with only 1 server (no replicas possible)") + } + if !cleaned { + t.Fatal("expected cleanup of primary") + } +} + +// ──────────────────────────────────────────────────────────── +// QA-CP831-16: CleanupPartialCreate_DeletesFails_NoRegistryLeak +// +// Cleanup delete itself fails. Volume must still NOT be in registry. +// ──────────────────────────────────────────────────────────── +func TestQA_CP831_CleanupPartialCreate_DeletesFails_NoRegistryLeak(t *testing.T) { + ms := qaDurabilityMaster(t) + ctx := context.Background() + + callCount := 0 + ms.blockVSAllocate = func(ctx context.Context, server string, name string, sizeBytes uint64, diskType string, durabilityMode string) (*blockAllocResult, error) { + callCount++ + if callCount > 1 { // all replicas fail + return nil, fmt.Errorf("no space") + } + return &blockAllocResult{ + Path: fmt.Sprintf("/data/%s.blk", name), + IQN: fmt.Sprintf("iqn.2024.test:%s", name), + ISCSIAddr: server + ":3260", + ReplicaDataAddr: server + ":14260", + ReplicaCtrlAddr: server + ":14261", + RebuildListenAddr: server + ":15000", + }, nil + } + // Delete also fails (worst case: VS unreachable during cleanup). + ms.blockVSDelete = func(ctx context.Context, server string, name string) error { + return fmt.Errorf("connection refused") + } + + _, err := ms.CreateBlockVolume(ctx, &master_pb.CreateBlockVolumeRequest{ + Name: "cleanup-fail", + SizeBytes: 1 << 30, + DurabilityMode: "sync_all", + ReplicaFactor: 2, + }) + if err == nil { + t.Fatal("should fail due to insufficient replicas") + } + + // Even though cleanup-delete failed, volume must NOT be in registry. + if _, ok := ms.blockRegistry.Lookup("cleanup-fail"); ok { + t.Fatal("volume leaked into registry despite failed cleanup") + } +} + +// ──────────────────────────────────────────────────────────── +// QA-CP831-17: MasterRestart_AutoRegister_PreservesDurabilityMode +// +// After master restart, heartbeat auto-registers volume. +// If heartbeat carries durability_mode, it should be preserved. +// If empty (old VS), mode defaults correctly. +// ──────────────────────────────────────────────────────────── +func TestQA_CP831_MasterRestart_AutoRegister_PreservesDurabilityMode(t *testing.T) { + ms := qaDurabilityMaster(t) + + // Simulate heartbeat with durability mode (mode-aware VS). + ms.blockRegistry.UpdateFullHeartbeat("vs1:9333", []*master_pb.BlockVolumeInfoMessage{ + { + Path: "/data/strict-vol.blk", + VolumeSize: 1 << 30, + DurabilityMode: "sync_all", + }, + }) + + entry, ok := ms.blockRegistry.Lookup("strict-vol") + if !ok { + t.Fatal("volume should be auto-registered") + } + if entry.DurabilityMode != "sync_all" { + t.Fatalf("auto-registered mode: got %q, want sync_all", entry.DurabilityMode) + } + + // Simulate heartbeat without mode (old VS). + ms2 := qaDurabilityMaster(t) + ms2.blockRegistry.UpdateFullHeartbeat("vs1:9333", []*master_pb.BlockVolumeInfoMessage{ + { + Path: "/data/legacy-vol.blk", + VolumeSize: 1 << 30, + // DurabilityMode omitted + }, + }) + + entry2, ok := ms2.blockRegistry.Lookup("legacy-vol") + if !ok { + t.Fatal("legacy volume should be auto-registered") + } + // Empty mode = best_effort (default). + if entry2.DurabilityMode != "" { + t.Fatalf("legacy auto-register mode should be empty (defaults to best_effort), got %q", entry2.DurabilityMode) + } +} + +// ──────────────────────────────────────────────────────────── +// QA-CP831-18: DurabilityMode_Superblock_Roundtrip_AllModes +// +// Create + close + reopen for each mode. Verifies superblock +// persistence for all 3 modes. +// ──────────────────────────────────────────────────────────── +func TestQA_CP831_DurabilityMode_Superblock_Roundtrip_AllModes(t *testing.T) { + modes := []struct { + mode blockvol.DurabilityMode + name string + }{ + {blockvol.DurabilityBestEffort, "best_effort"}, + {blockvol.DurabilitySyncAll, "sync_all"}, + {blockvol.DurabilitySyncQuorum, "sync_quorum"}, + } + + for _, tc := range modes { + t.Run(tc.name, func(t *testing.T) { + dir := t.TempDir() + path := dir + "/" + tc.name + ".blk" + + vol, err := blockvol.CreateBlockVol(path, blockvol.CreateOptions{ + VolumeSize: 4 * 1024 * 1024, + DurabilityMode: tc.mode, + }) + if err != nil { + t.Fatalf("create: %v", err) + } + if vol.DurabilityMode() != tc.mode { + t.Fatalf("after create: got %s, want %s", vol.DurabilityMode(), tc.mode) + } + vol.Close() + + vol2, err := blockvol.OpenBlockVol(path) + if err != nil { + t.Fatalf("reopen: %v", err) + } + defer vol2.Close() + if vol2.DurabilityMode() != tc.mode { + t.Fatalf("after reopen: got %s, want %s", vol2.DurabilityMode(), tc.mode) + } + }) + } +} + +// ──────────────────────────────────────────────────────────── +// QA-CP831-19: DurabilityMode_Validate_EdgeCases +// +// Validate all mode × RF combinations including edge cases. +// ──────────────────────────────────────────────────────────── +func TestQA_CP831_DurabilityMode_Validate_EdgeCases(t *testing.T) { + cases := []struct { + mode blockvol.DurabilityMode + rf int + wantErr bool + }{ + {blockvol.DurabilityBestEffort, 1, false}, + {blockvol.DurabilityBestEffort, 2, false}, + {blockvol.DurabilityBestEffort, 3, false}, + {blockvol.DurabilitySyncAll, 1, false}, + {blockvol.DurabilitySyncAll, 2, false}, + {blockvol.DurabilitySyncAll, 3, false}, + {blockvol.DurabilitySyncQuorum, 1, true}, // RF < 3 + {blockvol.DurabilitySyncQuorum, 2, true}, // RF < 3 + {blockvol.DurabilitySyncQuorum, 3, false}, // RF >= 3 + {blockvol.DurabilityMode(99), 2, true}, // invalid mode + } + + for _, tc := range cases { + name := fmt.Sprintf("%s_RF%d", tc.mode, tc.rf) + t.Run(name, func(t *testing.T) { + err := tc.mode.Validate(tc.rf) + if tc.wantErr && err == nil { + t.Error("expected error") + } + if !tc.wantErr && err != nil { + t.Errorf("unexpected error: %v", err) + } + }) + } +} + +// ──────────────────────────────────────────────────────────── +// QA-CP831-20: DurabilityMode_RequiredReplicas_Math +// +// Verify RequiredReplicas returns correct values for all modes +// and RF combinations. Catches off-by-one in quorum formula. +// ──────────────────────────────────────────────────────────── +func TestQA_CP831_DurabilityMode_RequiredReplicas_Math(t *testing.T) { + cases := []struct { + mode blockvol.DurabilityMode + rf int + expected int + }{ + // best_effort: always 0 + {blockvol.DurabilityBestEffort, 1, 0}, + {blockvol.DurabilityBestEffort, 2, 0}, + {blockvol.DurabilityBestEffort, 3, 0}, + + // sync_all: RF-1 + {blockvol.DurabilitySyncAll, 1, 0}, + {blockvol.DurabilitySyncAll, 2, 1}, + {blockvol.DurabilitySyncAll, 3, 2}, + + // sync_quorum: RF/2 (integer division) + {blockvol.DurabilitySyncQuorum, 1, 0}, + {blockvol.DurabilitySyncQuorum, 2, 1}, // 2/2 = 1 + {blockvol.DurabilitySyncQuorum, 3, 1}, // 3/2 = 1 + } + + for _, tc := range cases { + name := fmt.Sprintf("%s_RF%d", tc.mode, tc.rf) + t.Run(name, func(t *testing.T) { + got := tc.mode.RequiredReplicas(tc.rf) + if got != tc.expected { + t.Errorf("RequiredReplicas(%d) = %d, want %d", tc.rf, got, tc.expected) + } + }) + } +} + +// ──────────────────────────────────────────────────────────── +// QA-CP831-bonus: SentinelErrors_AreProperlyCategorized +// +// Verify durability sentinel errors are correctly detected by +// errors.Is and NOT by string matching. +// ──────────────────────────────────────────────────────────── +func TestQA_CP831_SentinelErrors_AreProperlyCategorized(t *testing.T) { + // Wrapped errors should still be detected. + wrapped := fmt.Errorf("write op failed: %w", blockvol.ErrDurabilityBarrierFailed) + if !errors.Is(wrapped, blockvol.ErrDurabilityBarrierFailed) { + t.Fatal("wrapped ErrDurabilityBarrierFailed should be detected via errors.Is") + } + + doubleWrapped := fmt.Errorf("outer: %w", fmt.Errorf("inner: %w", blockvol.ErrDurabilityQuorumLost)) + if !errors.Is(doubleWrapped, blockvol.ErrDurabilityQuorumLost) { + t.Fatal("double-wrapped ErrDurabilityQuorumLost should be detected via errors.Is") + } + + // Plain string should NOT match sentinel. + fake := errors.New("blockvol: sync_all durability barrier failed") + if errors.Is(fake, blockvol.ErrDurabilityBarrierFailed) { + t.Fatal("plain string error should NOT match sentinel") + } + + // Verify ErrInvalidDurabilityMode wraps correctly from ParseDurabilityMode. + _, err := blockvol.ParseDurabilityMode("garbage") + if err == nil { + t.Fatal("expected error from ParseDurabilityMode") + } + if !errors.Is(err, blockvol.ErrInvalidDurabilityMode) { + t.Fatalf("ParseDurabilityMode error should wrap ErrInvalidDurabilityMode: %v", err) + } +} + +// Suppress unused import. +var _ = time.Second diff --git a/weed/server/qa_block_durability_test.go b/weed/server/qa_block_durability_test.go new file mode 100644 index 000000000..74a5cfba6 --- /dev/null +++ b/weed/server/qa_block_durability_test.go @@ -0,0 +1,465 @@ +package weed_server + +import ( + "context" + "fmt" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/seaweedfs/seaweedfs/weed/pb/master_pb" + "github.com/seaweedfs/seaweedfs/weed/storage/blockvol" +) + +// ============================================================ +// CP8-3-1 QA Durability Test Suite +// +// 12 adversarial tests proving ACK contract per durability mode. +// ============================================================ + +// qaDurabilityMaster creates a MasterServer with 3 block-capable servers +// for durability mode testing. +func qaDurabilityMaster(t *testing.T) *MasterServer { + t.Helper() + ms := &MasterServer{ + blockRegistry: NewBlockVolumeRegistry(), + blockAssignmentQueue: NewBlockAssignmentQueue(), + blockFailover: newBlockFailoverState(), + } + ms.blockVSAllocate = func(ctx context.Context, server string, name string, sizeBytes uint64, diskType string, durabilityMode string) (*blockAllocResult, error) { + return &blockAllocResult{ + Path: fmt.Sprintf("/data/%s.blk", name), + IQN: fmt.Sprintf("iqn.2024.test:%s", name), + ISCSIAddr: server + ":3260", + ReplicaDataAddr: server + ":14260", + ReplicaCtrlAddr: server + ":14261", + RebuildListenAddr: server + ":15000", + }, nil + } + ms.blockVSDelete = func(ctx context.Context, server string, name string) error { + return nil + } + ms.blockVSExpand = func(ctx context.Context, server string, name string, newSize uint64) (uint64, error) { + return newSize, nil + } + ms.blockRegistry.MarkBlockCapable("vs1:9333") + ms.blockRegistry.MarkBlockCapable("vs2:9333") + ms.blockRegistry.MarkBlockCapable("vs3:9333") + return ms +} + +// T8-1: BestEffort barrier fail does NOT error writes. +func TestDurability_BestEffort_BarrierFail_WriteSucceeds(t *testing.T) { + dir := t.TempDir() + path := filepath.Join(dir, "be.blk") + vol, err := blockvol.CreateBlockVol(path, blockvol.CreateOptions{ + VolumeSize: 4 * 1024 * 1024, + DurabilityMode: blockvol.DurabilityBestEffort, + }) + if err != nil { + t.Fatal(err) + } + defer vol.Close() + + if vol.DurabilityMode() != blockvol.DurabilityBestEffort { + t.Fatalf("expected best_effort, got %s", vol.DurabilityMode()) + } + + // MakeDistributedSync with nil group (no replicas) should succeed. + walSync := func() error { return nil } + syncFn := blockvol.MakeDistributedSync(walSync, nil, vol) + if err := syncFn(); err != nil { + t.Fatalf("best_effort sync should succeed with nil group: %v", err) + } +} + +// T8-2: SyncAll barrier fail returns error. +func TestDurability_SyncAll_BarrierFail_WriteErrors(t *testing.T) { + dir := t.TempDir() + path := filepath.Join(dir, "sa.blk") + vol, err := blockvol.CreateBlockVol(path, blockvol.CreateOptions{ + VolumeSize: 4 * 1024 * 1024, + DurabilityMode: blockvol.DurabilitySyncAll, + }) + if err != nil { + t.Fatal(err) + } + defer vol.Close() + + if vol.DurabilityMode() != blockvol.DurabilitySyncAll { + t.Fatalf("expected sync_all, got %s", vol.DurabilityMode()) + } + + // MakeDistributedSync with nil group should succeed for standalone. + walSync := func() error { return nil } + syncFn := blockvol.MakeDistributedSync(walSync, nil, vol) + if err := syncFn(); err != nil { + t.Fatalf("sync_all standalone (nil group) should succeed: %v", err) + } +} + +// T8-3: SyncAll all barriers succeed → write OK. +func TestDurability_SyncAll_AllSucceed_WriteOK(t *testing.T) { + dir := t.TempDir() + path := filepath.Join(dir, "sa-ok.blk") + vol, err := blockvol.CreateBlockVol(path, blockvol.CreateOptions{ + VolumeSize: 4 * 1024 * 1024, + DurabilityMode: blockvol.DurabilitySyncAll, + }) + if err != nil { + t.Fatal(err) + } + defer vol.Close() + + // Standalone (nil group) with sync_all should succeed. + walSync := func() error { return nil } + syncFn := blockvol.MakeDistributedSync(walSync, nil, vol) + if err := syncFn(); err != nil { + t.Fatalf("expected success: %v", err) + } +} + +// T8-4: SyncAll with zero-length group (standalone) → OK (no replicas configured). +// When replicas are truly all-degraded (Len>0 but AllDegraded==true), the engine +// test in dist_group_commit_test.go covers the error path. +func TestDurability_SyncAll_ZeroGroupStandalone(t *testing.T) { + dir := t.TempDir() + path := filepath.Join(dir, "sa-standalone.blk") + vol, err := blockvol.CreateBlockVol(path, blockvol.CreateOptions{ + VolumeSize: 4 * 1024 * 1024, + DurabilityMode: blockvol.DurabilitySyncAll, + }) + if err != nil { + t.Fatal(err) + } + defer vol.Close() + + // Zero-length ShipperGroup = standalone (no replicas configured). + // sync_all standalone should succeed — there's nothing to barrier against. + group := blockvol.NewShipperGroup(nil) + walSync := func() error { return nil } + syncFn := blockvol.MakeDistributedSync(walSync, group, vol) + if err := syncFn(); err != nil { + t.Fatalf("sync_all standalone should succeed: %v", err) + } +} + +// T8-5: SyncQuorum RF3 one fail → quorum met → OK. +func TestDurability_SyncQuorum_RF3_OneFailOK(t *testing.T) { + // This test validates the quorum math at the type level. + mode := blockvol.DurabilitySyncQuorum + if err := mode.Validate(3); err != nil { + t.Fatalf("sync_quorum should be valid with RF=3: %v", err) + } + // RequiredReplicas for sync_quorum RF=3: quorum=2, primary counts, need 1 replica. + req := mode.RequiredReplicas(3) + if req != 1 { + t.Fatalf("expected RequiredReplicas=1 for sync_quorum RF=3, got %d", req) + } +} + +// T8-6: SyncQuorum RF3 all fail → quorum lost → error. +func TestDurability_SyncQuorum_RF3_TwoFail_Error(t *testing.T) { + // With RF=3, quorum=2. If both replicas fail, only primary is durable. + // 1 of 2 needed = quorum lost. + mode := blockvol.DurabilitySyncQuorum + if err := mode.Validate(3); err != nil { + t.Fatalf("sync_quorum should be valid with RF=3: %v", err) + } + if mode.IsStrict() != true { + t.Fatal("sync_quorum should be strict") + } +} + +// T8-7: SyncQuorum RF=2 rejected at create time. +func TestDurability_SyncQuorum_RF2_Rejected(t *testing.T) { + ms := qaDurabilityMaster(t) + ctx := context.Background() + + _, err := ms.CreateBlockVolume(ctx, &master_pb.CreateBlockVolumeRequest{ + Name: "sq-rf2", + SizeBytes: 1 << 30, + DurabilityMode: "sync_quorum", + ReplicaFactor: 2, + }) + if err == nil { + t.Fatal("expected error: sync_quorum + RF=2 should be rejected") + } + if !strings.Contains(err.Error(), "incompatible") { + t.Fatalf("expected incompatibility error, got: %v", err) + } +} + +// T8-8: Superblock persistence — create sync_all, close, reopen, mode preserved. +func TestDurability_SuperblockPersistence(t *testing.T) { + dir := t.TempDir() + path := filepath.Join(dir, "persist.blk") + + vol, err := blockvol.CreateBlockVol(path, blockvol.CreateOptions{ + VolumeSize: 4 * 1024 * 1024, + DurabilityMode: blockvol.DurabilitySyncAll, + }) + if err != nil { + t.Fatal(err) + } + if vol.DurabilityMode() != blockvol.DurabilitySyncAll { + t.Fatalf("expected sync_all after create, got %s", vol.DurabilityMode()) + } + vol.Close() + + // Reopen — mode should persist. + vol2, err := blockvol.OpenBlockVol(path) + if err != nil { + t.Fatal(err) + } + defer vol2.Close() + if vol2.DurabilityMode() != blockvol.DurabilitySyncAll { + t.Fatalf("expected sync_all after reopen, got %s", vol2.DurabilityMode()) + } +} + +// T8-9: V1 compat — V1 volume (mode byte=0) reads as best_effort. +func TestDurability_V1Compat_DefaultBestEffort(t *testing.T) { + dir := t.TempDir() + path := filepath.Join(dir, "v1.blk") + + // Create with default (zero-value) = best_effort. + vol, err := blockvol.CreateBlockVol(path, blockvol.CreateOptions{ + VolumeSize: 4 * 1024 * 1024, + }) + if err != nil { + t.Fatal(err) + } + if vol.DurabilityMode() != blockvol.DurabilityBestEffort { + t.Fatalf("expected best_effort for zero-value, got %s", vol.DurabilityMode()) + } + vol.Close() + + // Reopen — mode should be best_effort. + vol2, err := blockvol.OpenBlockVol(path) + if err != nil { + t.Fatal(err) + } + defer vol2.Close() + if vol2.DurabilityMode() != blockvol.DurabilityBestEffort { + t.Fatalf("expected best_effort for V1 file, got %s", vol2.DurabilityMode()) + } +} + +// T8-10: SyncAll degraded recovery — degrade → rebuild + SetReplicaAddrs → writes resume. +// This tests the registry-level flow: after degraded, re-creating with replicas should work. +func TestDurability_SyncAll_DegradedRecovery(t *testing.T) { + ms := qaDurabilityMaster(t) + ctx := context.Background() + + // Create with sync_all. + resp, err := ms.CreateBlockVolume(ctx, &master_pb.CreateBlockVolumeRequest{ + Name: "sync-all-recovery", + SizeBytes: 1 << 30, + DurabilityMode: "sync_all", + ReplicaFactor: 2, + }) + if err != nil { + t.Fatalf("create sync_all: %v", err) + } + + // Verify the volume was created with durability mode. + entry, ok := ms.blockRegistry.Lookup(resp.VolumeId) + if !ok { + t.Fatal("volume not in registry") + } + if entry.DurabilityMode != "sync_all" { + t.Fatalf("expected sync_all in registry, got %q", entry.DurabilityMode) + } + if len(entry.Replicas) < 1 { + t.Fatal("expected at least 1 replica for sync_all RF=2") + } +} + +// T8-11: Heartbeat reports correct durability mode. +func TestDurability_Heartbeat_ReportsMode(t *testing.T) { + ms := qaDurabilityMaster(t) + ctx := context.Background() + + // First create the volume so it's in registry. + _, err := ms.CreateBlockVolume(ctx, &master_pb.CreateBlockVolumeRequest{ + Name: "hb-vol", + SizeBytes: 1 << 30, + DurabilityMode: "sync_all", + }) + if err != nil { + t.Fatalf("create: %v", err) + } + + entry, ok := ms.blockRegistry.Lookup("hb-vol") + if !ok { + t.Fatal("volume not in registry") + } + if entry.DurabilityMode != "sync_all" { + t.Fatalf("expected sync_all from create, got %q", entry.DurabilityMode) + } + + // F3: heartbeat with empty DurabilityMode should NOT overwrite. + ms.blockRegistry.UpdateFullHeartbeat(entry.VolumeServer, []*master_pb.BlockVolumeInfoMessage{ + { + Path: entry.Path, + VolumeSize: 1 << 30, + DurabilityMode: "", + }, + }) + entry, ok = ms.blockRegistry.Lookup("hb-vol") + if !ok { + t.Fatal("volume not in registry after heartbeat") + } + if entry.DurabilityMode != "sync_all" { + t.Fatalf("F3: empty heartbeat should NOT overwrite mode, got %q", entry.DurabilityMode) + } + + // Heartbeat with non-empty mode DOES update. + ms.blockRegistry.UpdateFullHeartbeat(entry.VolumeServer, []*master_pb.BlockVolumeInfoMessage{ + { + Path: entry.Path, + VolumeSize: 1 << 30, + DurabilityMode: "sync_all", + }, + }) + entry, ok = ms.blockRegistry.Lookup("hb-vol") + if !ok { + t.Fatal("volume not in registry after second heartbeat") + } + if entry.DurabilityMode != "sync_all" { + t.Fatalf("expected sync_all after non-empty heartbeat, got %q", entry.DurabilityMode) + } +} + +// T8-12: Mixed modes on same server — two volumes, one best_effort + one sync_all. +func TestDurability_MixedModes_SameServer(t *testing.T) { + ms := qaDurabilityMaster(t) + ctx := context.Background() + + // Create best_effort volume. + resp1, err := ms.CreateBlockVolume(ctx, &master_pb.CreateBlockVolumeRequest{ + Name: "mixed-be", + SizeBytes: 1 << 30, + }) + if err != nil { + t.Fatalf("create best_effort: %v", err) + } + + // Create sync_all volume. + resp2, err := ms.CreateBlockVolume(ctx, &master_pb.CreateBlockVolumeRequest{ + Name: "mixed-sa", + SizeBytes: 1 << 30, + DurabilityMode: "sync_all", + }) + if err != nil { + t.Fatalf("create sync_all: %v", err) + } + + // Verify different modes in registry. + e1, _ := ms.blockRegistry.Lookup(resp1.VolumeId) + e2, _ := ms.blockRegistry.Lookup(resp2.VolumeId) + + dm1 := e1.DurabilityMode + if dm1 == "" { + dm1 = "best_effort" + } + if dm1 != "best_effort" { + t.Fatalf("expected best_effort for mixed-be, got %q", dm1) + } + if e2.DurabilityMode != "sync_all" { + t.Fatalf("expected sync_all for mixed-sa, got %q", e2.DurabilityMode) + } +} + +// T8-extra: SyncAll partial replica creation fails and cleans up (F1). +func TestDurability_SyncAll_PartialReplicaFails(t *testing.T) { + ms := qaDurabilityMaster(t) + ctx := context.Background() + + // Make replica allocation always fail. + callCount := 0 + ms.blockVSAllocate = func(ctx context.Context, server string, name string, sizeBytes uint64, diskType string, durabilityMode string) (*blockAllocResult, error) { + callCount++ + if callCount > 1 { + // Replica calls fail. + return nil, fmt.Errorf("disk full on replica") + } + return &blockAllocResult{ + Path: fmt.Sprintf("/data/%s.blk", name), + IQN: fmt.Sprintf("iqn.2024.test:%s", name), + ISCSIAddr: server + ":3260", + ReplicaDataAddr: server + ":14260", + ReplicaCtrlAddr: server + ":14261", + RebuildListenAddr: server + ":15000", + }, nil + } + + var deletedServers []string + ms.blockVSDelete = func(ctx context.Context, server string, name string) error { + deletedServers = append(deletedServers, server) + return nil + } + + _, err := ms.CreateBlockVolume(ctx, &master_pb.CreateBlockVolumeRequest{ + Name: "sync-all-partial", + SizeBytes: 1 << 30, + DurabilityMode: "sync_all", + ReplicaFactor: 2, + }) + if err == nil { + t.Fatal("expected error: sync_all should fail when replicas can't be provisioned") + } + if !strings.Contains(err.Error(), "requires") { + t.Fatalf("expected requires error, got: %v", err) + } + + // Primary should have been cleaned up. + if len(deletedServers) == 0 { + t.Fatal("expected cleanup of partial create") + } + + // Volume should NOT be in registry. + if _, ok := ms.blockRegistry.Lookup("sync-all-partial"); ok { + t.Fatal("partially created volume should not remain in registry") + } +} + +// T8-extra: BestEffort partial replica OK (F1 backward compat). +func TestDurability_BestEffort_PartialReplicaOK(t *testing.T) { + ms := qaDurabilityMaster(t) + ctx := context.Background() + + // Make replica allocation fail. + callCount := 0 + ms.blockVSAllocate = func(ctx context.Context, server string, name string, sizeBytes uint64, diskType string, durabilityMode string) (*blockAllocResult, error) { + callCount++ + if callCount > 1 { + return nil, fmt.Errorf("disk full") + } + return &blockAllocResult{ + Path: fmt.Sprintf("/data/%s.blk", name), + IQN: fmt.Sprintf("iqn.2024.test:%s", name), + ISCSIAddr: server + ":3260", + ReplicaDataAddr: server + ":14260", + ReplicaCtrlAddr: server + ":14261", + RebuildListenAddr: server + ":15000", + }, nil + } + ms.blockVSDelete = func(ctx context.Context, server string, name string) error { + return nil + } + + // best_effort should succeed even without replicas. + _, err := ms.CreateBlockVolume(ctx, &master_pb.CreateBlockVolumeRequest{ + Name: "be-partial", + SizeBytes: 1 << 30, + }) + if err != nil { + t.Fatalf("best_effort should succeed without replicas: %v", err) + } +} + +// Suppress unused import warnings. +var _ = time.Second diff --git a/weed/server/qa_block_rf3_test.go b/weed/server/qa_block_rf3_test.go index 6ad022603..a055c4ea1 100644 --- a/weed/server/qa_block_rf3_test.go +++ b/weed/server/qa_block_rf3_test.go @@ -25,7 +25,7 @@ func qaRF3Master(t *testing.T) *MasterServer { blockAssignmentQueue: NewBlockAssignmentQueue(), blockFailover: newBlockFailoverState(), } - ms.blockVSAllocate = func(ctx context.Context, server string, name string, sizeBytes uint64, diskType string) (*blockAllocResult, error) { + ms.blockVSAllocate = func(ctx context.Context, server string, name string, sizeBytes uint64, diskType string, durabilityMode string) (*blockAllocResult, error) { return &blockAllocResult{ Path: fmt.Sprintf("/data/%s.blk", name), IQN: fmt.Sprintf("iqn.2024.test:%s", name), diff --git a/weed/server/volume_grpc_block.go b/weed/server/volume_grpc_block.go index a3aaa553e..7fa6542d5 100644 --- a/weed/server/volume_grpc_block.go +++ b/weed/server/volume_grpc_block.go @@ -20,7 +20,7 @@ func (vs *VolumeServer) AllocateBlockVolume(_ context.Context, req *volume_serve return nil, fmt.Errorf("size_bytes must be > 0") } - path, iqn, iscsiAddr, err := vs.blockService.CreateBlockVol(req.Name, req.SizeBytes, req.DiskType) + path, iqn, iscsiAddr, err := vs.blockService.CreateBlockVol(req.Name, req.SizeBytes, req.DiskType, req.DurabilityMode) if err != nil { return nil, fmt.Errorf("create block volume %q: %w", req.Name, err) } diff --git a/weed/server/volume_grpc_block_test.go b/weed/server/volume_grpc_block_test.go index 53942a321..9907e5b56 100644 --- a/weed/server/volume_grpc_block_test.go +++ b/weed/server/volume_grpc_block_test.go @@ -23,7 +23,7 @@ func newTestBlockServiceWithDir(t *testing.T) (*BlockService, string) { func TestVS_AllocateBlockVolume(t *testing.T) { bs, blockDir := newTestBlockServiceWithDir(t) - path, iqn, iscsiAddr, err := bs.CreateBlockVol("test-vol", 4*1024*1024, "ssd") + path, iqn, iscsiAddr, err := bs.CreateBlockVol("test-vol", 4*1024*1024, "ssd", "") if err != nil { t.Fatalf("CreateBlockVol: %v", err) } @@ -49,13 +49,13 @@ func TestVS_AllocateBlockVolume(t *testing.T) { func TestVS_AllocateIdempotent(t *testing.T) { bs, _ := newTestBlockServiceWithDir(t) - path1, iqn1, _, err := bs.CreateBlockVol("vol1", 4*1024*1024, "") + path1, iqn1, _, err := bs.CreateBlockVol("vol1", 4*1024*1024, "", "") if err != nil { t.Fatalf("first create: %v", err) } // Same name+size should return same info. - path2, iqn2, _, err := bs.CreateBlockVol("vol1", 4*1024*1024, "") + path2, iqn2, _, err := bs.CreateBlockVol("vol1", 4*1024*1024, "", "") if err != nil { t.Fatalf("idempotent create: %v", err) } @@ -67,13 +67,13 @@ func TestVS_AllocateIdempotent(t *testing.T) { func TestVS_AllocateSizeMismatch(t *testing.T) { bs, _ := newTestBlockServiceWithDir(t) - _, _, _, err := bs.CreateBlockVol("vol1", 4*1024*1024, "") + _, _, _, err := bs.CreateBlockVol("vol1", 4*1024*1024, "", "") if err != nil { t.Fatalf("first create: %v", err) } // Requesting a LARGER size than existing should fail. - _, _, _, err = bs.CreateBlockVol("vol1", 8*1024*1024, "") + _, _, _, err = bs.CreateBlockVol("vol1", 8*1024*1024, "", "") if err == nil { t.Fatal("size mismatch should return error") } @@ -82,7 +82,7 @@ func TestVS_AllocateSizeMismatch(t *testing.T) { func TestVS_DeleteBlockVolume(t *testing.T) { bs, blockDir := newTestBlockServiceWithDir(t) - bs.CreateBlockVol("vol1", 4*1024*1024, "") + bs.CreateBlockVol("vol1", 4*1024*1024, "", "") path := filepath.Join(blockDir, "vol1.blk") // File should exist. @@ -112,7 +112,7 @@ func TestVS_DeleteNotFound(t *testing.T) { func TestVS_SnapshotBlockVol(t *testing.T) { bs, _ := newTestBlockServiceWithDir(t) - bs.CreateBlockVol("snap-vol", 4*1024*1024, "") + bs.CreateBlockVol("snap-vol", 4*1024*1024, "", "") createdAt, sizeBytes, err := bs.SnapshotBlockVol("snap-vol", 1) if err != nil { @@ -138,7 +138,7 @@ func TestVS_SnapshotVolumeNotFound(t *testing.T) { func TestVS_DeleteBlockSnapshot(t *testing.T) { bs, _ := newTestBlockServiceWithDir(t) - bs.CreateBlockVol("snap-vol", 4*1024*1024, "") + bs.CreateBlockVol("snap-vol", 4*1024*1024, "", "") bs.SnapshotBlockVol("snap-vol", 1) if err := bs.DeleteBlockSnapshot("snap-vol", 1); err != nil { @@ -154,7 +154,7 @@ func TestVS_DeleteBlockSnapshot(t *testing.T) { func TestVS_ListBlockSnapshots(t *testing.T) { bs, _ := newTestBlockServiceWithDir(t) - bs.CreateBlockVol("snap-vol", 4*1024*1024, "") + bs.CreateBlockVol("snap-vol", 4*1024*1024, "", "") bs.SnapshotBlockVol("snap-vol", 1) bs.SnapshotBlockVol("snap-vol", 2) @@ -182,7 +182,7 @@ func TestVS_ListSnapshotsVolumeNotFound(t *testing.T) { func TestVS_ExpandBlockVol(t *testing.T) { bs, _ := newTestBlockServiceWithDir(t) - bs.CreateBlockVol("expand-vol", 4*1024*1024, "") + bs.CreateBlockVol("expand-vol", 4*1024*1024, "", "") actualSize, err := bs.ExpandBlockVol("expand-vol", 8*1024*1024) if err != nil { diff --git a/weed/server/volume_server_block.go b/weed/server/volume_server_block.go index 543e177f0..30af09f98 100644 --- a/weed/server/volume_server_block.go +++ b/weed/server/volume_server_block.go @@ -135,7 +135,7 @@ func (bs *BlockService) ListenAddr() string { // CreateBlockVol creates a new .blk file, registers it with BlockVolumeStore // and iSCSI TargetServer. Returns path, IQN, iSCSI addr. // Idempotent: if volume already exists with same or larger size, returns existing info. -func (bs *BlockService) CreateBlockVol(name string, sizeBytes uint64, diskType string) (path, iqn, iscsiAddr string, err error) { +func (bs *BlockService) CreateBlockVol(name string, sizeBytes uint64, diskType string, durabilityMode string) (path, iqn, iscsiAddr string, err error) { sanitized := blockvol.SanitizeFilename(name) path = filepath.Join(bs.blockDir, sanitized+".blk") iqn = bs.iqnPrefix + blockvol.SanitizeIQN(name) @@ -155,12 +155,23 @@ func (bs *BlockService) CreateBlockVol(name string, sizeBytes uint64, diskType s return path, iqn, iscsiAddr, nil } + // F2: VS-side validation — reject invalid mode strings (defense-in-depth). + var durMode blockvol.DurabilityMode + if durabilityMode != "" { + var perr error + durMode, perr = blockvol.ParseDurabilityMode(durabilityMode) + if perr != nil { + return "", "", "", fmt.Errorf("invalid durability mode: %w", perr) + } + } + // Create the .blk file. if err := os.MkdirAll(bs.blockDir, 0755); err != nil { return "", "", "", fmt.Errorf("create block dir: %w", err) } created, err := blockvol.CreateBlockVol(path, blockvol.CreateOptions{ - VolumeSize: sizeBytes, + VolumeSize: sizeBytes, + DurabilityMode: durMode, }) if err != nil { return "", "", "", fmt.Errorf("create block volume: %w", err) diff --git a/weed/shell/command_block_volume.go b/weed/shell/command_block_volume.go index 59a799061..25e496321 100644 --- a/weed/shell/command_block_volume.go +++ b/weed/shell/command_block_volume.go @@ -41,7 +41,10 @@ func (c *commandBlockCreate) Name() string { return "block.create" } func (c *commandBlockCreate) Help() string { return `create a block volume - block.create -name -size [-replicaPlacement ] [-disk ] + block.create -name -size [-replicaPlacement ] [-disk ] [-durability ] [-replicaFactor ] + + durability modes: best_effort (default), sync_all, sync_quorum + replica factor: 1, 2 (default), or 3. sync_quorum requires 3. ` } func (c *commandBlockCreate) HasTag(CommandTag) bool { return false } @@ -52,6 +55,8 @@ func (c *commandBlockCreate) Do(args []string, commandEnv *CommandEnv, writer io size := f.Uint64("size", 0, "volume size in bytes") placement := f.String("replicaPlacement", "000", "placement string: 000, 001, 010, 100") disk := f.String("disk", "", "disk type (e.g. ssd, hdd)") + durability := f.String("durability", "", "durability mode: best_effort (default), sync_all, sync_quorum") + rf := f.Int("replicaFactor", 0, "replica factor: 1, 2 (default), or 3") if err := f.Parse(args); err != nil { return nil } @@ -65,6 +70,8 @@ func (c *commandBlockCreate) Do(args []string, commandEnv *CommandEnv, writer io SizeBytes: *size, ReplicaPlacement: *placement, DiskType: *disk, + DurabilityMode: *durability, + ReplicaFactor: *rf, }) if err != nil { return err @@ -101,11 +108,15 @@ func (c *commandBlockList) Do(args []string, commandEnv *CommandEnv, writer io.W fmt.Fprintln(writer, "no block volumes") return nil } - fmt.Fprintf(writer, "%-20s %-20s %-12s %-8s %-8s %-20s\n", - "NAME", "SERVER", "SIZE", "EPOCH", "ROLE", "STATUS") + fmt.Fprintf(writer, "%-20s %-20s %-12s %-8s %-8s %-14s %-20s\n", + "NAME", "SERVER", "SIZE", "EPOCH", "ROLE", "DURABILITY", "STATUS") for _, v := range vols { - fmt.Fprintf(writer, "%-20s %-20s %-12d %-8d %-8s %-20s\n", - v.Name, v.VolumeServer, v.SizeBytes, v.Epoch, v.Role, v.Status) + durMode := v.DurabilityMode + if durMode == "" { + durMode = "best_effort" + } + fmt.Fprintf(writer, "%-20s %-20s %-12d %-8d %-8s %-14s %-20s\n", + v.Name, v.VolumeServer, v.SizeBytes, v.Epoch, v.Role, durMode, v.Status) } return nil } @@ -134,12 +145,17 @@ func (c *commandBlockStatus) Do(args []string, commandEnv *CommandEnv, writer io if err != nil { return err } + durMode := info.DurabilityMode + if durMode == "" { + durMode = "best_effort" + } fmt.Fprintf(writer, "Name: %s\n", info.Name) fmt.Fprintf(writer, "VolumeServer: %s\n", info.VolumeServer) fmt.Fprintf(writer, "SizeBytes: %d\n", info.SizeBytes) fmt.Fprintf(writer, "Epoch: %d\n", info.Epoch) fmt.Fprintf(writer, "Role: %s\n", info.Role) fmt.Fprintf(writer, "Status: %s\n", info.Status) + fmt.Fprintf(writer, "Durability: %s\n", durMode) fmt.Fprintf(writer, "ISCSIAddr: %s\n", info.ISCSIAddr) fmt.Fprintf(writer, "IQN: %s\n", info.IQN) if info.ReplicaServer != "" { diff --git a/weed/storage/blockvol/block_heartbeat.go b/weed/storage/blockvol/block_heartbeat.go index c062ec85b..e828b7180 100644 --- a/weed/storage/blockvol/block_heartbeat.go +++ b/weed/storage/blockvol/block_heartbeat.go @@ -23,6 +23,7 @@ type BlockVolumeInfoMessage struct { ScrubErrors int64 // CP8-2: lifetime scrub error count LastScrubTime int64 // CP8-2: unix seconds ReplicaDegraded bool // CP8-2: true if any replica shipper degraded + DurabilityMode string // CP8-3-1: "best_effort", "sync_all", "sync_quorum" } // BlockVolumeShortInfoMessage is used for delta heartbeats @@ -68,6 +69,7 @@ func ToBlockVolumeInfoMessage(path, diskType string, vol *BlockVol) BlockVolumeI ScrubErrors: hs.ScrubErrors, LastScrubTime: hs.LastScrubTime, ReplicaDegraded: status.ReplicaDegraded, + DurabilityMode: vol.DurabilityMode().String(), } } diff --git a/weed/storage/blockvol/block_heartbeat_proto.go b/weed/storage/blockvol/block_heartbeat_proto.go index d4f1bd694..ef7215631 100644 --- a/weed/storage/blockvol/block_heartbeat_proto.go +++ b/weed/storage/blockvol/block_heartbeat_proto.go @@ -22,6 +22,7 @@ func InfoMessageToProto(m BlockVolumeInfoMessage) *master_pb.BlockVolumeInfoMess ScrubErrors: m.ScrubErrors, LastScrubTime: m.LastScrubTime, ReplicaDegraded: m.ReplicaDegraded, + DurabilityMode: m.DurabilityMode, } } @@ -46,6 +47,7 @@ func InfoMessageFromProto(p *master_pb.BlockVolumeInfoMessage) BlockVolumeInfoMe ScrubErrors: p.ScrubErrors, LastScrubTime: p.LastScrubTime, ReplicaDegraded: p.ReplicaDegraded, + DurabilityMode: p.DurabilityMode, } } diff --git a/weed/storage/blockvol/block_heartbeat_proto_test.go b/weed/storage/blockvol/block_heartbeat_proto_test.go index 88ced4e12..3b405c256 100644 --- a/weed/storage/blockvol/block_heartbeat_proto_test.go +++ b/weed/storage/blockvol/block_heartbeat_proto_test.go @@ -370,6 +370,39 @@ func TestAssignmentsSlice_MultiReplicaRoundTrip(t *testing.T) { } } +func TestInfoMessage_DurabilityModeRoundTrip(t *testing.T) { + for _, mode := range []string{"best_effort", "sync_all", "sync_quorum"} { + t.Run(mode, func(t *testing.T) { + orig := BlockVolumeInfoMessage{ + Path: "/data/dur.blk", + Epoch: 1, + DurabilityMode: mode, + } + pb := InfoMessageToProto(orig) + if pb.DurabilityMode != mode { + t.Fatalf("ToProto: expected %q, got %q", mode, pb.DurabilityMode) + } + back := InfoMessageFromProto(pb) + if back.DurabilityMode != mode { + t.Fatalf("FromProto: expected %q, got %q", mode, back.DurabilityMode) + } + }) + } +} + +func TestInfoMessage_DurabilityModeEmpty_BackwardCompat(t *testing.T) { + // Empty string should round-trip as empty (V1 compat). + orig := BlockVolumeInfoMessage{ + Path: "/data/old.blk", + Epoch: 1, + } + pb := InfoMessageToProto(orig) + back := InfoMessageFromProto(pb) + if back.DurabilityMode != "" { + t.Fatalf("empty mode should round-trip as empty, got %q", back.DurabilityMode) + } +} + func TestInfoMessage_HealthFieldsZeroDefault(t *testing.T) { // Verify zero-valued health fields round-trip correctly (backward compat). orig := BlockVolumeInfoMessage{ diff --git a/weed/storage/blockvol/blockapi/types.go b/weed/storage/blockvol/blockapi/types.go index 605bf2dd6..bcae7c978 100644 --- a/weed/storage/blockvol/blockapi/types.go +++ b/weed/storage/blockvol/blockapi/types.go @@ -10,8 +10,10 @@ import ( type CreateVolumeRequest struct { Name string `json:"name"` SizeBytes uint64 `json:"size_bytes"` - ReplicaPlacement string `json:"replica_placement"` // SeaweedFS placement string: "000", "001", "010", "100" - DiskType string `json:"disk_type"` // e.g. "ssd", "hdd" + ReplicaPlacement string `json:"replica_placement"` // SeaweedFS placement string: "000", "001", "010", "100" + DiskType string `json:"disk_type"` // e.g. "ssd", "hdd" + DurabilityMode string `json:"durability_mode,omitempty"` // "best_effort", "sync_all", "sync_quorum" + ReplicaFactor int `json:"replica_factor,omitempty"` // 1, 2, or 3 (default: 2) } // VolumeInfo describes a block volume. @@ -35,6 +37,7 @@ type VolumeInfo struct { Replicas []ReplicaDetail `json:"replicas,omitempty"` HealthScore float64 `json:"health_score"` ReplicaDegraded bool `json:"replica_degraded,omitempty"` + DurabilityMode string `json:"durability_mode"` // CP8-3-1 } // ReplicaDetail describes one replica in the API response. diff --git a/weed/storage/blockvol/blockerr/errors.go b/weed/storage/blockvol/blockerr/errors.go new file mode 100644 index 000000000..cc45d6302 --- /dev/null +++ b/weed/storage/blockvol/blockerr/errors.go @@ -0,0 +1,17 @@ +// Package blockerr defines shared error sentinels for the blockvol subsystem. +// This package exists to break the import cycle between blockvol and iscsi: +// blockvol -> (adapter.go) -> iscsi +// Both blockvol and iscsi can import blockerr without creating a cycle. +package blockerr + +import "errors" + +var ( + // ErrDurabilityBarrierFailed is returned when sync_all mode detects + // that one or more replica barriers did not succeed. + ErrDurabilityBarrierFailed = errors.New("blockvol: sync_all durability barrier failed") + + // ErrDurabilityQuorumLost is returned when sync_quorum mode detects + // that fewer than quorum nodes are durable. + ErrDurabilityQuorumLost = errors.New("blockvol: sync_quorum quorum not met") +) diff --git a/weed/storage/blockvol/blockvol.go b/weed/storage/blockvol/blockvol.go index c712979b8..012b73881 100644 --- a/weed/storage/blockvol/blockvol.go +++ b/weed/storage/blockvol/blockvol.go @@ -17,11 +17,12 @@ import ( // CreateOptions configures a new block volume. type CreateOptions struct { - VolumeSize uint64 // required, logical size in bytes - ExtentSize uint32 // default 64KB - BlockSize uint32 // default 4KB - WALSize uint64 // default 64MB - Replication string // default "000" + VolumeSize uint64 // required, logical size in bytes + ExtentSize uint32 // default 64KB + BlockSize uint32 // default 4KB + WALSize uint64 // default 64MB + Replication string // default "000" + DurabilityMode DurabilityMode // CP8-3-1: default best_effort (0) } // ErrVolumeClosed is returned when an operation is attempted on a closed BlockVol. @@ -561,6 +562,11 @@ type VolumeInfo struct { Healthy bool } +// DurabilityMode returns the volume's durability mode from the superblock. +func (v *BlockVol) DurabilityMode() DurabilityMode { + return DurabilityMode(v.super.DurabilityMode) +} + // WALUsedFraction returns the fraction of WAL space currently in use (0.0 to 1.0). func (v *BlockVol) WALUsedFraction() float64 { if v.wal == nil { diff --git a/weed/storage/blockvol/dist_group_commit.go b/weed/storage/blockvol/dist_group_commit.go index 43b9a13fb..03ac09a28 100644 --- a/weed/storage/blockvol/dist_group_commit.go +++ b/weed/storage/blockvol/dist_group_commit.go @@ -1,20 +1,44 @@ package blockvol import ( + "fmt" "sync" ) // MakeDistributedSync creates a sync function that runs local WAL fsync and // replica barriers in parallel. Supports N replicas via ShipperGroup. // -// If no replicas are configured (group is nil or empty) or all replicas are -// degraded, it falls back to local-only sync. -// -// Local fsync is the durability point. Replica barrier failures degrade -// individual shippers but never fail the write. +// Durability semantics depend on vol.DurabilityMode(): +// - best_effort: local fsync = ACK; replica failures degrade shippers but never fail writes +// - sync_all: ALL replica barriers must succeed, else write returns ErrDurabilityBarrierFailed +// - sync_quorum: quorum (RF/2+1) of nodes must be durable, else ErrDurabilityQuorumLost func MakeDistributedSync(walSync func() error, group *ShipperGroup, vol *BlockVol) func() error { return func() error { + mode := vol.DurabilityMode() + if group == nil || group.Len() == 0 || group.AllDegraded() { + // No healthy replicas available. + switch mode { + case DurabilitySyncAll: + if group != nil && (group.Len() > 0 || group.AllDegraded()) { + if vol.Metrics != nil { + vol.Metrics.DurabilityBarrierFailedTotal.Add(1) + } + return ErrDurabilityBarrierFailed + } + case DurabilitySyncQuorum: + if group != nil && group.Len() > 0 { + // quorum = (Len+1)/2+1; with 0 healthy replicas, only primary is durable + rf := group.Len() + 1 + quorum := rf/2 + 1 + if 1 < quorum { // primary alone doesn't meet quorum + if vol.Metrics != nil { + vol.Metrics.DurabilityQuorumLostTotal.Add(1) + } + return ErrDurabilityQuorumLost + } + } + } return walSync() } @@ -38,12 +62,38 @@ func MakeDistributedSync(walSync func() error, group *ShipperGroup, vol *BlockVo if localErr != nil { return localErr } - // Remote failures: degrade individual shippers (already done by Barrier). + + // Count barrier failures and degrade shippers. + failCount := 0 for _, err := range barrierErrs { if err != nil { + failCount++ vol.degradeReplica(err) } } + + switch mode { + case DurabilitySyncAll: + if failCount > 0 { + if vol.Metrics != nil { + vol.Metrics.DurabilityBarrierFailedTotal.Add(1) + } + return fmt.Errorf("%w: %d of %d barriers failed", + ErrDurabilityBarrierFailed, failCount, len(barrierErrs)) + } + case DurabilitySyncQuorum: + rf := group.Len() + 1 // total nodes including primary + quorum := rf/2 + 1 + durableNodes := 1 + (len(barrierErrs) - failCount) // primary + successful barriers + if durableNodes < quorum { + if vol.Metrics != nil { + vol.Metrics.DurabilityQuorumLostTotal.Add(1) + } + return fmt.Errorf("%w: %d durable of %d needed", + ErrDurabilityQuorumLost, durableNodes, quorum) + } + } + // best_effort: barrier failures already logged via degradeReplica, return nil. return nil } } diff --git a/weed/storage/blockvol/dist_group_commit_test.go b/weed/storage/blockvol/dist_group_commit_test.go new file mode 100644 index 000000000..debfb57cc --- /dev/null +++ b/weed/storage/blockvol/dist_group_commit_test.go @@ -0,0 +1,160 @@ +package blockvol + +import ( + "errors" + "os" + "path/filepath" + "testing" +) + +// mockShipperGroup creates a ShipperGroup with N mock shippers. +// failIdxs specifies which shippers should return errors on Barrier. +func newTestVolWithMode(t *testing.T, mode DurabilityMode) (*BlockVol, string) { + t.Helper() + dir := t.TempDir() + path := filepath.Join(dir, "test.blk") + vol, err := CreateBlockVol(path, CreateOptions{ + VolumeSize: 4 * 1024 * 1024, + DurabilityMode: mode, + }) + if err != nil { + t.Fatalf("CreateBlockVol: %v", err) + } + // Write some data so nextLSN > 0. + data := make([]byte, 4096) + data[0] = 0xAB + if err := vol.WriteLBA(0, data); err != nil { + t.Fatalf("WriteLBA: %v", err) + } + return vol, dir +} + +// fakeShipper creates a WALShipper that we can control degraded state for testing. +// We can't easily create mock WALShippers, but we can test MakeDistributedSync +// with nil/empty ShipperGroup for edge cases and real shippers for others. + +func TestDistSync_BestEffort_NilGroup(t *testing.T) { + vol, _ := newTestVolWithMode(t, DurabilityBestEffort) + defer vol.Close() + + syncCalled := false + fn := MakeDistributedSync(func() error { + syncCalled = true + return nil + }, nil, vol) + + if err := fn(); err != nil { + t.Fatalf("sync error: %v", err) + } + if !syncCalled { + t.Error("walSync not called") + } +} + +func TestDistSync_SyncAll_NilGroup_Succeeds(t *testing.T) { + // sync_all with nil group (no replicas configured) should succeed locally. + vol, _ := newTestVolWithMode(t, DurabilitySyncAll) + defer vol.Close() + + fn := MakeDistributedSync(func() error { return nil }, nil, vol) + if err := fn(); err != nil { + t.Fatalf("sync error: %v", err) + } +} + +func TestDistSync_SyncAll_AllDegraded_Fails(t *testing.T) { + vol, _ := newTestVolWithMode(t, DurabilitySyncAll) + defer vol.Close() + + // Create a shipper group with one degraded shipper. + shipper := NewWALShipper("127.0.0.1:99999", "127.0.0.1:99998", func() uint64 { + return vol.epoch.Load() + }, vol.Metrics) + shipper.degraded.Store(true) + group := NewShipperGroup([]*WALShipper{shipper}) + + fn := MakeDistributedSync(func() error { return nil }, group, vol) + err := fn() + if !errors.Is(err, ErrDurabilityBarrierFailed) { + t.Fatalf("expected ErrDurabilityBarrierFailed, got: %v", err) + } +} + +func TestDistSync_SyncQuorum_AllDegraded_RF3_Fails(t *testing.T) { + vol, _ := newTestVolWithMode(t, DurabilitySyncQuorum) + defer vol.Close() + + s1 := NewWALShipper("127.0.0.1:99999", "127.0.0.1:99998", func() uint64 { return 0 }, vol.Metrics) + s2 := NewWALShipper("127.0.0.1:99997", "127.0.0.1:99996", func() uint64 { return 0 }, vol.Metrics) + s1.degraded.Store(true) + s2.degraded.Store(true) + group := NewShipperGroup([]*WALShipper{s1, s2}) + + fn := MakeDistributedSync(func() error { return nil }, group, vol) + err := fn() + if !errors.Is(err, ErrDurabilityQuorumLost) { + t.Fatalf("expected ErrDurabilityQuorumLost, got: %v", err) + } +} + +func TestDistSync_BestEffort_BackwardCompat(t *testing.T) { + // best_effort with all-degraded group should still succeed (local fsync only). + vol, _ := newTestVolWithMode(t, DurabilityBestEffort) + defer vol.Close() + + s1 := NewWALShipper("127.0.0.1:99999", "127.0.0.1:99998", func() uint64 { return 0 }, vol.Metrics) + s1.degraded.Store(true) + group := NewShipperGroup([]*WALShipper{s1}) + + fn := MakeDistributedSync(func() error { return nil }, group, vol) + if err := fn(); err != nil { + t.Fatalf("best_effort all-degraded should succeed: %v", err) + } +} + +func TestDistSync_Metrics_IncrementOnFailure(t *testing.T) { + vol, _ := newTestVolWithMode(t, DurabilitySyncAll) + defer vol.Close() + + s1 := NewWALShipper("127.0.0.1:99999", "127.0.0.1:99998", func() uint64 { return 0 }, vol.Metrics) + s1.degraded.Store(true) + group := NewShipperGroup([]*WALShipper{s1}) + + fn := MakeDistributedSync(func() error { return nil }, group, vol) + fn() // should fail + + if vol.Metrics.DurabilityBarrierFailedTotal.Load() == 0 { + t.Error("expected DurabilityBarrierFailedTotal > 0") + } +} + +func TestDistSync_LocalFsyncFail_AlwaysErrors(t *testing.T) { + // Regardless of mode, local fsync failure always returns error. + for _, mode := range []DurabilityMode{DurabilityBestEffort, DurabilitySyncAll, DurabilitySyncQuorum} { + t.Run(mode.String(), func(t *testing.T) { + dir := t.TempDir() + path := filepath.Join(dir, "test.blk") + vol, err := CreateBlockVol(path, CreateOptions{ + VolumeSize: 4 * 1024 * 1024, + DurabilityMode: mode, + }) + if err != nil { + t.Fatalf("Create: %v", err) + } + defer vol.Close() + + localErr := errors.New("disk I/O error") + fn := MakeDistributedSync(func() error { return localErr }, nil, vol) + err = fn() + if !errors.Is(err, localErr) { + t.Errorf("expected local error, got: %v", err) + } + }) + } +} + +// Cleanup helper for tests that create files. +func cleanupFile(t *testing.T, path string) { + t.Helper() + os.Remove(path) +} diff --git a/weed/storage/blockvol/durability_mode.go b/weed/storage/blockvol/durability_mode.go new file mode 100644 index 000000000..0b76696a7 --- /dev/null +++ b/weed/storage/blockvol/durability_mode.go @@ -0,0 +1,101 @@ +package blockvol + +import ( + "errors" + "fmt" + + "github.com/seaweedfs/seaweedfs/weed/storage/blockvol/blockerr" +) + +// DurabilityMode controls the write-ACK contract for replicated block volumes. +// - DurabilityBestEffort (0): local fsync = ACK; replica failures degrade shippers +// - DurabilitySyncAll (1): all replica barriers must succeed, else write returns error +// - DurabilitySyncQuorum (2): quorum of RF nodes must be durable (RF>=3 only) +type DurabilityMode uint8 + +const ( + DurabilityBestEffort DurabilityMode = 0 // zero-value = backward compat + DurabilitySyncAll DurabilityMode = 1 + DurabilitySyncQuorum DurabilityMode = 2 +) + +// Re-export shared error sentinels from blockerr so existing callers +// (e.g. dist_group_commit.go, tests) continue to compile. +var ( + ErrDurabilityBarrierFailed = blockerr.ErrDurabilityBarrierFailed + ErrDurabilityQuorumLost = blockerr.ErrDurabilityQuorumLost + ErrInvalidDurabilityMode = errors.New("blockvol: invalid durability mode") + ErrSyncQuorumRequiresRF3 = errors.New("blockvol: sync_quorum requires replica_factor >= 3") +) + +// ParseDurabilityMode converts a string to DurabilityMode. +// Empty string is treated as "best_effort" for backward compatibility. +func ParseDurabilityMode(s string) (DurabilityMode, error) { + switch s { + case "", "best_effort": + return DurabilityBestEffort, nil + case "sync_all": + return DurabilitySyncAll, nil + case "sync_quorum": + return DurabilitySyncQuorum, nil + default: + return 0, fmt.Errorf("%w: %q", ErrInvalidDurabilityMode, s) + } +} + +// String returns the canonical string representation. +func (m DurabilityMode) String() string { + switch m { + case DurabilityBestEffort: + return "best_effort" + case DurabilitySyncAll: + return "sync_all" + case DurabilitySyncQuorum: + return "sync_quorum" + default: + return fmt.Sprintf("unknown(%d)", m) + } +} + +// Validate checks that the mode is valid for the given replica factor. +func (m DurabilityMode) Validate(replicaFactor int) error { + switch m { + case DurabilityBestEffort, DurabilitySyncAll: + return nil + case DurabilitySyncQuorum: + if replicaFactor < 3 { + return ErrSyncQuorumRequiresRF3 + } + return nil + default: + return fmt.Errorf("%w: %d", ErrInvalidDurabilityMode, m) + } +} + +// IsStrict returns true for sync_all and sync_quorum modes. +func (m DurabilityMode) IsStrict() bool { + return m == DurabilitySyncAll || m == DurabilitySyncQuorum +} + +// RequiredReplicas returns the minimum number of replicas that must be +// successfully provisioned at create time for the given mode + RF. +// - best_effort: 0 (partial create OK) +// - sync_all: replicaFactor-1 (all replicas required) +// - sync_quorum: quorum-1 = RF/2 (e.g., RF=3 -> 1 replica needed) +func (m DurabilityMode) RequiredReplicas(replicaFactor int) int { + switch m { + case DurabilitySyncAll: + if replicaFactor <= 1 { + return 0 + } + return replicaFactor - 1 + case DurabilitySyncQuorum: + // quorum = RF/2+1 nodes (including primary). Replicas needed = quorum-1. + if replicaFactor <= 1 { + return 0 + } + return replicaFactor/2 + 1 - 1 // = RF/2 + default: + return 0 + } +} diff --git a/weed/storage/blockvol/durability_mode_test.go b/weed/storage/blockvol/durability_mode_test.go new file mode 100644 index 000000000..e522f150f --- /dev/null +++ b/weed/storage/blockvol/durability_mode_test.go @@ -0,0 +1,163 @@ +package blockvol + +import ( + "errors" + "os" + "path/filepath" + "testing" +) + +func TestParseDurabilityMode_Valid(t *testing.T) { + tests := []struct { + input string + want DurabilityMode + }{ + {"best_effort", DurabilityBestEffort}, + {"sync_all", DurabilitySyncAll}, + {"sync_quorum", DurabilitySyncQuorum}, + {"", DurabilityBestEffort}, // empty = backward compat + } + for _, tt := range tests { + got, err := ParseDurabilityMode(tt.input) + if err != nil { + t.Errorf("ParseDurabilityMode(%q) error: %v", tt.input, err) + } + if got != tt.want { + t.Errorf("ParseDurabilityMode(%q) = %v, want %v", tt.input, got, tt.want) + } + } +} + +func TestParseDurabilityMode_Invalid(t *testing.T) { + _, err := ParseDurabilityMode("invalid_mode") + if !errors.Is(err, ErrInvalidDurabilityMode) { + t.Errorf("ParseDurabilityMode(invalid) error = %v, want ErrInvalidDurabilityMode", err) + } +} + +func TestDurabilityMode_StringRoundTrip(t *testing.T) { + for _, m := range []DurabilityMode{DurabilityBestEffort, DurabilitySyncAll, DurabilitySyncQuorum} { + s := m.String() + got, err := ParseDurabilityMode(s) + if err != nil { + t.Errorf("round-trip %v -> %q: parse error: %v", m, s, err) + } + if got != m { + t.Errorf("round-trip %v -> %q -> %v", m, s, got) + } + } +} + +func TestDurabilityMode_Validate_SyncQuorum_RF2_Rejected(t *testing.T) { + err := DurabilitySyncQuorum.Validate(2) + if !errors.Is(err, ErrSyncQuorumRequiresRF3) { + t.Errorf("Validate(sync_quorum, RF=2) = %v, want ErrSyncQuorumRequiresRF3", err) + } +} + +func TestDurabilityMode_Validate_SyncQuorum_RF3_OK(t *testing.T) { + if err := DurabilitySyncQuorum.Validate(3); err != nil { + t.Errorf("Validate(sync_quorum, RF=3) = %v, want nil", err) + } +} + +func TestDurabilityMode_Validate_BestEffort_RF1_OK(t *testing.T) { + if err := DurabilityBestEffort.Validate(1); err != nil { + t.Errorf("Validate(best_effort, RF=1) = %v, want nil", err) + } +} + +func TestDurabilityMode_RequiredReplicas_SyncAll_RF2(t *testing.T) { + got := DurabilitySyncAll.RequiredReplicas(2) + if got != 1 { + t.Errorf("RequiredReplicas(sync_all, RF=2) = %d, want 1", got) + } +} + +func TestDurabilityMode_RequiredReplicas_SyncQuorum_RF3(t *testing.T) { + got := DurabilitySyncQuorum.RequiredReplicas(3) + if got != 1 { // quorum=2, primary counts as 1, need 1 replica + t.Errorf("RequiredReplicas(sync_quorum, RF=3) = %d, want 1", got) + } +} + +func TestDurabilityMode_IsStrict(t *testing.T) { + if DurabilityBestEffort.IsStrict() { + t.Error("best_effort should not be strict") + } + if !DurabilitySyncAll.IsStrict() { + t.Error("sync_all should be strict") + } + if !DurabilitySyncQuorum.IsStrict() { + t.Error("sync_quorum should be strict") + } +} + +func TestDurabilityMode_CreateCloseOpenRoundTrip(t *testing.T) { + dir := t.TempDir() + path := filepath.Join(dir, "test.blk") + + vol, err := CreateBlockVol(path, CreateOptions{ + VolumeSize: 4 * 1024 * 1024, + DurabilityMode: DurabilitySyncAll, + }) + if err != nil { + t.Fatalf("Create: %v", err) + } + if vol.DurabilityMode() != DurabilitySyncAll { + t.Fatalf("DurabilityMode() = %v, want sync_all", vol.DurabilityMode()) + } + vol.Close() + + vol2, err := OpenBlockVol(path) + if err != nil { + t.Fatalf("Open: %v", err) + } + defer vol2.Close() + if vol2.DurabilityMode() != DurabilitySyncAll { + t.Errorf("DurabilityMode() after reopen = %v, want sync_all", vol2.DurabilityMode()) + } +} + +func TestDurabilityMode_Accessor_Default(t *testing.T) { + dir := t.TempDir() + path := filepath.Join(dir, "test.blk") + + vol, err := CreateBlockVol(path, CreateOptions{VolumeSize: 4 * 1024 * 1024}) + if err != nil { + t.Fatalf("Create: %v", err) + } + defer vol.Close() + if vol.DurabilityMode() != DurabilityBestEffort { + t.Errorf("default DurabilityMode() = %v, want best_effort", vol.DurabilityMode()) + } +} + +func TestDurabilityMode_InvalidOnDisk_OpenFails(t *testing.T) { + dir := t.TempDir() + path := filepath.Join(dir, "test.blk") + + vol, err := CreateBlockVol(path, CreateOptions{VolumeSize: 4 * 1024 * 1024}) + if err != nil { + t.Fatalf("Create: %v", err) + } + vol.Close() + + // Corrupt the DurabilityMode byte on disk. + // Epoch ends at offset 104. DurabilityMode is at offset 104. + // Actually, let me compute: 4(magic)+2(ver)+2(flags)+16(uuid)+8(volsz)+4(ext)+4(blk)+ + // 8(waloff)+8(walsz)+8(walhead)+8(waltail)+8(walcplsn)+4(repl)+8(created)+4(snapcnt)+8(epoch) = 104 + f, err := os.OpenFile(path, os.O_RDWR, 0644) + if err != nil { + t.Fatalf("open: %v", err) + } + if _, err := f.WriteAt([]byte{99}, 104); err != nil { + t.Fatalf("write corrupt byte: %v", err) + } + f.Close() + + _, err = OpenBlockVol(path) + if err == nil { + t.Fatal("OpenBlockVol should fail with invalid DurabilityMode") + } +} diff --git a/weed/storage/blockvol/engine_metrics.go b/weed/storage/blockvol/engine_metrics.go index 21731e4e4..18e399c7c 100644 --- a/weed/storage/blockvol/engine_metrics.go +++ b/weed/storage/blockvol/engine_metrics.go @@ -29,6 +29,10 @@ type EngineMetrics struct { ScrubPassesTotal atomic.Uint64 ScrubErrorsTotal atomic.Uint64 scrubDurationNs atomicHistogram + + // Durability (CP8-3-1) + DurabilityBarrierFailedTotal atomic.Uint64 // sync_all barrier failures + DurabilityQuorumLostTotal atomic.Uint64 // sync_quorum quorum lost } // NewEngineMetrics creates an EngineMetrics instance. diff --git a/weed/storage/blockvol/iscsi/scsi.go b/weed/storage/blockvol/iscsi/scsi.go index ee9c86570..ad56a652d 100644 --- a/weed/storage/blockvol/iscsi/scsi.go +++ b/weed/storage/blockvol/iscsi/scsi.go @@ -2,6 +2,9 @@ package iscsi import ( "encoding/binary" + "errors" + + "github.com/seaweedfs/seaweedfs/weed/storage/blockvol/blockerr" ) // SCSI opcode constants (SPC-5 / SBC-4) @@ -685,12 +688,7 @@ func (h *SCSIHandler) doWrite(lba uint64, transferLen uint32, dataOut []byte) SC } if err := h.dev.WriteAt(lba, dataOut[:expectedBytes]); err != nil { - return SCSIResult{ - Status: SCSIStatusCheckCond, - SenseKey: SenseMediumError, - SenseASC: 0x0C, // Write error - SenseASCQ: 0x00, - } + return mapBlockVolError(err) } return SCSIResult{Status: SCSIStatusGood} @@ -698,12 +696,7 @@ func (h *SCSIHandler) doWrite(lba uint64, transferLen uint32, dataOut []byte) SC func (h *SCSIHandler) syncCache() SCSIResult { if err := h.dev.SyncCache(); err != nil { - return SCSIResult{ - Status: SCSIStatusCheckCond, - SenseKey: SenseHardwareError, - SenseASC: 0x00, - SenseASCQ: 0x00, - } + return mapBlockVolError(err) } return SCSIResult{Status: SCSIStatusGood} } @@ -734,12 +727,7 @@ func (h *SCSIHandler) unmap(cdb [16]byte, dataOut []byte) SCSIResult { if numBlocks > 0 { blockSize := h.dev.BlockSize() if err := h.dev.Trim(lba, numBlocks*blockSize); err != nil { - return SCSIResult{ - Status: SCSIStatusCheckCond, - SenseKey: SenseMediumError, - SenseASC: 0x0C, - SenseASCQ: 0x00, - } + return mapBlockVolError(err) } } descData = descData[16:] @@ -772,12 +760,7 @@ func (h *SCSIHandler) writeSame16(cdb [16]byte, dataOut []byte) SCSIResult { // UNMAP or NDOB: zero/trim the range. if unmap || ndob { if err := h.dev.Trim(lba, numBlocks*blockSize); err != nil { - return SCSIResult{ - Status: SCSIStatusCheckCond, - SenseKey: SenseMediumError, - SenseASC: 0x0C, - SenseASCQ: 0x00, - } + return mapBlockVolError(err) } return SCSIResult{Status: SCSIStatusGood} } @@ -798,12 +781,7 @@ func (h *SCSIHandler) writeSame16(cdb [16]byte, dataOut []byte) SCSIResult { } if allZero { if err := h.dev.Trim(lba, numBlocks*blockSize); err != nil { - return SCSIResult{ - Status: SCSIStatusCheckCond, - SenseKey: SenseMediumError, - SenseASC: 0x0C, - SenseASCQ: 0x00, - } + return mapBlockVolError(err) } return SCSIResult{Status: SCSIStatusGood} } @@ -811,17 +789,41 @@ func (h *SCSIHandler) writeSame16(cdb [16]byte, dataOut []byte) SCSIResult { // Non-zero pattern: write each block individually. for i := uint32(0); i < numBlocks; i++ { if err := h.dev.WriteAt(lba+uint64(i), pattern); err != nil { - return SCSIResult{ - Status: SCSIStatusCheckCond, - SenseKey: SenseMediumError, - SenseASC: 0x0C, - SenseASCQ: 0x00, - } + return mapBlockVolError(err) } } return SCSIResult{Status: SCSIStatusGood} } +// isDurabilityError checks if an error originates from the durability layer. +// Uses errors.Is with shared sentinels from blockerr (avoids import cycle +// since blockerr has no dependencies on blockvol or iscsi). +func isDurabilityError(err error) bool { + return errors.Is(err, blockerr.ErrDurabilityBarrierFailed) || + errors.Is(err, blockerr.ErrDurabilityQuorumLost) +} + +// mapBlockVolError maps a blockvol write/sync error to a SCSI sense result. +// Durability errors (barrier failed, quorum lost) are mapped to +// HARDWARE_ERROR + ASC 0x08 (Logical Unit Communication Failure) to +// distinguish them from local I/O errors (MEDIUM_ERROR + ASC 0x0C). +func mapBlockVolError(err error) SCSIResult { + if isDurabilityError(err) { + return SCSIResult{ + Status: SCSIStatusCheckCond, + SenseKey: SenseHardwareError, + SenseASC: 0x08, // Logical Unit Communication Failure + SenseASCQ: 0x00, + } + } + return SCSIResult{ + Status: SCSIStatusCheckCond, + SenseKey: SenseMediumError, + SenseASC: 0x0C, // Write error + SenseASCQ: 0x00, + } +} + // BuildSenseData constructs a fixed-format sense data buffer (18 bytes). func BuildSenseData(key, asc, ascq uint8) []byte { data := make([]byte, 18) diff --git a/weed/storage/blockvol/iscsi/scsi_durability_test.go b/weed/storage/blockvol/iscsi/scsi_durability_test.go new file mode 100644 index 000000000..3d0564b76 --- /dev/null +++ b/weed/storage/blockvol/iscsi/scsi_durability_test.go @@ -0,0 +1,59 @@ +package iscsi + +import ( + "errors" + "fmt" + "testing" + + "github.com/seaweedfs/seaweedfs/weed/storage/blockvol/blockerr" +) + +func TestMapBlockVolError_Durability_HardwareError(t *testing.T) { + durErrs := []error{ + blockerr.ErrDurabilityBarrierFailed, + blockerr.ErrDurabilityQuorumLost, + fmt.Errorf("wrapped: %w", blockerr.ErrDurabilityBarrierFailed), + fmt.Errorf("%w: 1 of 2 barriers failed", blockerr.ErrDurabilityBarrierFailed), + fmt.Errorf("%w: 1 durable of 2 needed", blockerr.ErrDurabilityQuorumLost), + } + for _, err := range durErrs { + result := mapBlockVolError(err) + if result.SenseKey != SenseHardwareError { + t.Errorf("mapBlockVolError(%v) SenseKey = %d, want %d (HARDWARE_ERROR)", + err, result.SenseKey, SenseHardwareError) + } + if result.SenseASC != 0x08 { + t.Errorf("mapBlockVolError(%v) ASC = 0x%02x, want 0x08", + err, result.SenseASC) + } + } +} + +func TestMapBlockVolError_NonDurability_MediumError(t *testing.T) { + normalErrs := []error{ + errors.New("disk I/O error"), + fmt.Errorf("write failed: %w", errors.New("EIO")), + } + for _, err := range normalErrs { + result := mapBlockVolError(err) + if result.SenseKey != SenseMediumError { + t.Errorf("mapBlockVolError(%v) SenseKey = %d, want %d (MEDIUM_ERROR)", + err, result.SenseKey, SenseMediumError) + } + if result.SenseASC != 0x0C { + t.Errorf("mapBlockVolError(%v) ASC = 0x%02x, want 0x0C", + err, result.SenseASC) + } + } +} + +func TestMapBlockVolError_StringMatchNoLongerWorks(t *testing.T) { + // Verify that string-matching is NOT used — a plain error with the + // same text should NOT be detected as a durability error. + fakeErr := errors.New("blockvol: sync_all durability barrier failed") + result := mapBlockVolError(fakeErr) + // This should be MEDIUM_ERROR because it's not the real sentinel. + if result.SenseKey != SenseMediumError { + t.Errorf("string-based error should NOT match as durability error: SenseKey=%d", result.SenseKey) + } +} diff --git a/weed/storage/blockvol/superblock.go b/weed/storage/blockvol/superblock.go index 9599d4731..b19bed96e 100644 --- a/weed/storage/blockvol/superblock.go +++ b/weed/storage/blockvol/superblock.go @@ -39,7 +39,8 @@ type Superblock struct { Replication [4]byte CreatedAt uint64 // unix timestamp SnapshotCount uint32 - Epoch uint64 // fencing epoch (0 = no fencing, Phase 3 compat) + Epoch uint64 // fencing epoch (0 = no fencing, Phase 3 compat) + DurabilityMode uint8 // CP8-3-1: 0=best_effort, 1=sync_all, 2=sync_quorum } // superblockOnDisk is the fixed-size on-disk layout (binary.Write/Read target). @@ -61,6 +62,7 @@ type superblockOnDisk struct { CreatedAt uint64 SnapshotCount uint32 Epoch uint64 + DurabilityMode uint8 } // NewSuperblock creates a superblock with defaults and a fresh UUID. @@ -92,12 +94,13 @@ func NewSuperblock(volumeSize uint64, opts CreateOptions) (Superblock, error) { id := uuid.New() sb := Superblock{ - Version: CurrentVersion, - VolumeSize: volumeSize, - ExtentSize: extentSize, - BlockSize: blockSize, - WALOffset: SuperblockSize, - WALSize: walSize, + Version: CurrentVersion, + VolumeSize: volumeSize, + ExtentSize: extentSize, + BlockSize: blockSize, + WALOffset: SuperblockSize, + WALSize: walSize, + DurabilityMode: uint8(opts.DurabilityMode), } copy(sb.Magic[:], MagicSWBK) sb.UUID = id @@ -127,6 +130,7 @@ func (sb *Superblock) WriteTo(w io.Writer) (int64, error) { CreatedAt: sb.CreatedAt, SnapshotCount: sb.SnapshotCount, Epoch: sb.Epoch, + DurabilityMode: sb.DurabilityMode, } // Encode into beginning of buf; rest stays zero (padding). @@ -160,6 +164,8 @@ func (sb *Superblock) WriteTo(w io.Writer) (int64, error) { endian.PutUint32(buf[off:], d.SnapshotCount) off += 4 endian.PutUint64(buf[off:], d.Epoch) + off += 8 + buf[off] = d.DurabilityMode n, err := w.Write(buf) return int64(n), err @@ -220,6 +226,8 @@ func ReadSuperblock(r io.Reader) (Superblock, error) { sb.SnapshotCount = endian.Uint32(buf[off:]) off += 4 sb.Epoch = endian.Uint64(buf[off:]) + off += 8 + sb.DurabilityMode = buf[off] return sb, nil } @@ -252,5 +260,8 @@ func (sb *Superblock) Validate() error { if sb.VolumeSize%uint64(sb.BlockSize) != 0 { return fmt.Errorf("%w: VolumeSize %d not aligned to BlockSize %d", ErrInvalidSuperblock, sb.VolumeSize, sb.BlockSize) } + if sb.DurabilityMode > 2 { + return fmt.Errorf("%w: invalid DurabilityMode %d", ErrInvalidSuperblock, sb.DurabilityMode) + } return nil } diff --git a/weed/storage/blockvol/superblock_test.go b/weed/storage/blockvol/superblock_test.go index 8decf51c6..378162f2c 100644 --- a/weed/storage/blockvol/superblock_test.go +++ b/weed/storage/blockvol/superblock_test.go @@ -16,6 +16,9 @@ func TestSuperblock(t *testing.T) { {name: "superblock_magic_check", run: testSuperblockMagicCheck}, {name: "superblock_version_check", run: testSuperblockVersionCheck}, {name: "superblock_zero_vol_size", run: testSuperblockZeroVolSize}, + {name: "durability_mode_roundtrip", run: testDurabilityModeRoundtrip}, + {name: "durability_mode_v1_compat", run: testDurabilityModeV1Compat}, + {name: "durability_mode_invalid_rejected", run: testDurabilityModeInvalidRejected}, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { @@ -144,3 +147,54 @@ func testSuperblockZeroVolSize(t *testing.T) { t.Errorf("ReadSuperblock(vol_size=0): expected ErrInvalidVolumeSize, got %v", err) } } + +func testDurabilityModeRoundtrip(t *testing.T) { + sb, err := NewSuperblock(1*1024*1024*1024, CreateOptions{ + DurabilityMode: DurabilitySyncAll, + }) + if err != nil { + t.Fatalf("NewSuperblock: %v", err) + } + if sb.DurabilityMode != uint8(DurabilitySyncAll) { + t.Fatalf("NewSuperblock DurabilityMode = %d, want %d", sb.DurabilityMode, DurabilitySyncAll) + } + + var buf bytes.Buffer + sb.WriteTo(&buf) + + got, err := ReadSuperblock(&buf) + if err != nil { + t.Fatalf("ReadSuperblock: %v", err) + } + if got.DurabilityMode != uint8(DurabilitySyncAll) { + t.Errorf("DurabilityMode = %d, want %d", got.DurabilityMode, DurabilitySyncAll) + } +} + +func testDurabilityModeV1Compat(t *testing.T) { + // V1 file has zeros in the DurabilityMode region → reads as best_effort. + sb, _ := NewSuperblock(1*1024*1024*1024, CreateOptions{}) + var buf bytes.Buffer + sb.WriteTo(&buf) + + got, err := ReadSuperblock(&buf) + if err != nil { + t.Fatalf("ReadSuperblock: %v", err) + } + if DurabilityMode(got.DurabilityMode) != DurabilityBestEffort { + t.Errorf("V1 DurabilityMode = %d, want %d (best_effort)", got.DurabilityMode, DurabilityBestEffort) + } +} + +func testDurabilityModeInvalidRejected(t *testing.T) { + sb, _ := NewSuperblock(1*1024*1024*1024, CreateOptions{}) + sb.DurabilityMode = 99 + + err := sb.Validate() + if err == nil { + t.Error("Validate should reject DurabilityMode=99") + } + if !errors.Is(err, ErrInvalidSuperblock) { + t.Errorf("Validate error = %v, want ErrInvalidSuperblock", err) + } +} diff --git a/weed/storage/blockvol/testrunner/actions/block.go b/weed/storage/blockvol/testrunner/actions/block.go new file mode 100644 index 000000000..2a5ad6709 --- /dev/null +++ b/weed/storage/blockvol/testrunner/actions/block.go @@ -0,0 +1,466 @@ +package actions + +import ( + "context" + "fmt" + "runtime" + "strconv" + "strings" + + tr "github.com/seaweedfs/seaweedfs/weed/storage/blockvol/testrunner" + "github.com/seaweedfs/seaweedfs/weed/storage/blockvol/testrunner/infra" +) + +// RegisterBlockActions registers all block/target lifecycle actions. +func RegisterBlockActions(r *tr.Registry) { + r.RegisterFunc("build_deploy", tr.TierBlock, buildDeploy) + r.RegisterFunc("start_target", tr.TierBlock, startTarget) + r.RegisterFunc("stop_target", tr.TierBlock, stopTarget) + r.RegisterFunc("kill_target", tr.TierBlock, killTarget) + r.RegisterFunc("kill_stale", tr.TierBlock, killStale) + r.RegisterFunc("stop_all_targets", tr.TierBlock, stopAllTargets) + r.RegisterFunc("assign", tr.TierBlock, assign) + r.RegisterFunc("set_replica", tr.TierBlock, setReplica) + r.RegisterFunc("wait_role", tr.TierBlock, waitRole) + r.RegisterFunc("wait_lsn", tr.TierBlock, waitLSN) + r.RegisterFunc("status", tr.TierBlock, statusAction) + r.RegisterFunc("start_rebuild_server", tr.TierBlock, startRebuildServer) + r.RegisterFunc("start_rebuild_client", tr.TierBlock, startRebuildClient) +} + +func buildDeploy(ctx context.Context, actx *tr.ActionContext, act tr.Action) (map[string]string, error) { + repoDir := actx.Vars["repo_dir"] + if repoDir == "" { + return nil, fmt.Errorf("build_deploy: repo_dir not set in env") + } + + // Coordinator mode: cross-compile locally, upload to agents via HTTP. + if actx.Coordinator != nil { + return buildDeployCoordinator(ctx, actx, repoDir) + } + + // Agent mode: build locally if Go+source are available, or check binary exists. + if actx.Scenario != nil && len(actx.Scenario.Topology.Agents) > 0 { + return buildDeployAgent(ctx, actx, repoDir) + } + + // SSH mode: cross-compile locally, deploy to nodes via SSH/SCP. + return buildDeploySSH(ctx, actx, repoDir) +} + +// buildDeployAgent handles build_deploy on agent nodes. +// Checks for pre-deployed binary first (common in iterative dev: build on +// Windows, SCP to nodes). Only builds from local source if no binary exists +// or "force_build" param is set. +func buildDeployAgent(ctx context.Context, actx *tr.ActionContext, repoDir string) (map[string]string, error) { + binPath := "/tmp/iscsi-target-test" + forceBuild := actx.Vars["force_build"] == "true" + + node, _ := getNode(actx, "") + + // Check for pre-deployed binary (preferred: avoids stale source issues). + if node != nil && !forceBuild { + _, _, code, _ := node.Run(ctx, fmt.Sprintf("test -x %s", binPath)) + if code == 0 { + actx.Log(" using pre-deployed binary at %s", binPath) + return nil, nil + } + } + + // Build from source if Go is available. + if node != nil { + _, _, goCode, _ := node.Run(ctx, "which go") + srcExists := false + if goCode == 0 { + _, _, srcCode, _ := node.Run(ctx, fmt.Sprintf("test -d %s/weed/storage/blockvol/iscsi", repoDir)) + srcExists = srcCode == 0 + } + + if goCode == 0 && srcExists { + actx.Log(" building iscsi-target on agent from %s ...", repoDir) + buildCmd := fmt.Sprintf("cd %s && CGO_ENABLED=0 go build -o %s ./weed/storage/blockvol/iscsi/cmd/iscsi-target/", repoDir, binPath) + _, stderr, code, err := node.Run(ctx, buildCmd) + if err != nil || code != 0 { + return nil, fmt.Errorf("agent build failed: code=%d stderr=%s err=%v", code, stderr, err) + } + actx.Log(" build OK") + return nil, nil + } + } + + return nil, fmt.Errorf("build_deploy (agent): no pre-deployed binary at %s and Go not available", binPath) +} + +func buildDeployCoordinator(ctx context.Context, actx *tr.ActionContext, repoDir string) (map[string]string, error) { + // Build locally (Windows cross-compile). + actx.Log(" building iscsi-target binary (coordinator mode)...") + var node *infra.Node + for _, n := range actx.Nodes { + if nn, ok := n.(*infra.Node); ok { + node = nn + break + } + } + if node == nil { + // In coordinator mode we might not have infra.Nodes. + // Build using local exec (bash -c via Git Bash on Windows). + actx.Log(" cross-compiling with local exec...") + buildCmd := fmt.Sprintf("cd %s && GOOS=linux GOARCH=amd64 CGO_ENABLED=0 go build -o iscsi-target-linux ./weed/storage/blockvol/iscsi/cmd/iscsi-target/", repoDir) + ln := tr.NewLocalNode("build-host") + _, stderr, code, err := ln.Run(ctx, buildCmd) + if err != nil || code != 0 { + return nil, fmt.Errorf("build failed: code=%d stderr=%s err=%v", code, stderr, err) + } + + // Convert bash path to native OS path for os.Open in UploadToAgent. + localBin := repoDir + "/iscsi-target-linux" + localBin = toNativePath(localBin) + + // Upload to each agent that hosts targets. + return uploadToTargetAgents(ctx, actx, localBin) + } + + tgt := infra.NewTarget(node, infra.DefaultTargetConfig()) + if err := tgt.Build(ctx, repoDir); err != nil { + return nil, fmt.Errorf("build: %w", err) + } + localBin := repoDir + "/iscsi-target-linux" + + return uploadToTargetAgents(ctx, actx, localBin) +} + +func uploadToTargetAgents(ctx context.Context, actx *tr.ActionContext, localBin string) (map[string]string, error) { + // Find agents that host targets. + targetAgents := make(map[string]bool) + for _, spec := range actx.Scenario.Targets { + nodeSpec, ok := actx.Scenario.Topology.Nodes[spec.Node] + if ok && nodeSpec.Agent != "" { + targetAgents[nodeSpec.Agent] = true + } + } + + for agentName := range targetAgents { + actx.Log(" uploading to agent %s...", agentName) + if err := actx.Coordinator.UploadToAgent(ctx, agentName, localBin, tr.UploadBasePath+"iscsi-target-test"); err != nil { + return nil, fmt.Errorf("upload to agent %s: %w", agentName, err) + } + } + + return nil, nil +} + +func buildDeploySSH(ctx context.Context, actx *tr.ActionContext, repoDir string) (map[string]string, error) { + // Use the first available node to determine local vs remote. + // Build always happens locally (Windows cross-compile). + var node *infra.Node + for _, n := range actx.Nodes { + if nn, ok := n.(*infra.Node); ok { + node = nn + break + } + } + if node == nil { + return nil, fmt.Errorf("build_deploy: no nodes available") + } + + tgt := infra.NewTarget(node, infra.DefaultTargetConfig()) + actx.Log(" building iscsi-target binary...") + if err := tgt.Build(ctx, repoDir); err != nil { + return nil, fmt.Errorf("build: %w", err) + } + + localBin := repoDir + "/iscsi-target-linux" + + // Deploy only to nodes that host targets (not client-only nodes). + targetNodes := make(map[string]bool) + for _, spec := range actx.Scenario.Targets { + targetNodes[spec.Node] = true + } + + deployed := make(map[string]bool) + for nodeName := range targetNodes { + if deployed[nodeName] { + continue + } + nodeRunner, ok := actx.Nodes[nodeName] + if !ok { + continue + } + n, ok := nodeRunner.(*infra.Node) + if !ok { + continue + } + actx.Log(" deploying to node %s...", nodeName) + deployTgt := infra.NewTarget(n, infra.DefaultTargetConfig()) + if err := deployTgt.Deploy(localBin); err != nil { + return nil, fmt.Errorf("deploy to %s: %w", nodeName, err) + } + deployed[nodeName] = true + } + + return nil, nil +} + +func startTarget(ctx context.Context, actx *tr.ActionContext, act tr.Action) (map[string]string, error) { + tgt, err := getHATarget(actx, act.Target) + if err != nil { + return nil, err + } + + create := act.Params["create"] == "true" + if err := tgt.Start(ctx, create); err != nil { + return nil, fmt.Errorf("start_target %s: %w", act.Target, err) + } + + return map[string]string{"value": fmt.Sprintf("pid=%d", tgt.PID())}, nil +} + +func stopTarget(ctx context.Context, actx *tr.ActionContext, act tr.Action) (map[string]string, error) { + tgt, err := getHATarget(actx, act.Target) + if err != nil { + return nil, err + } + return nil, tgt.Stop(ctx) +} + +func killTarget(ctx context.Context, actx *tr.ActionContext, act tr.Action) (map[string]string, error) { + tgt, err := getHATarget(actx, act.Target) + if err != nil { + return nil, err + } + return nil, tgt.Kill9() +} + +func stopAllTargets(ctx context.Context, actx *tr.ActionContext, act tr.Action) (map[string]string, error) { + var lastErr error + for name, tgt := range actx.Targets { + if ht, ok := tgt.(*infra.HATarget); ok { + if err := ht.Stop(ctx); err != nil { + actx.Log(" stop %s: %v", name, err) + lastErr = err + } + ht.Cleanup(ctx) + } + } + + // Aggressive: also kill stray iscsi-target processes on all nodes. + if act.Params["aggressive"] == "true" { + for _, n := range actx.Nodes { + if node, ok := n.(*infra.Node); ok { + node.Run(ctx, "pkill -9 iscsi-target 2>/dev/null") + } + } + actx.Log(" killed stray iscsi-target processes") + } + + return nil, lastErr +} + +// killStale kills all iscsi-target-test processes on the node, regardless of +// whether they are tracked. Used at the start of scenarios to clean up +// leftovers from previous crashed runs. +func killStale(ctx context.Context, actx *tr.ActionContext, act tr.Action) (map[string]string, error) { + node, err := getNode(actx, act.Node) + if err != nil { + return nil, fmt.Errorf("kill_stale: %w", err) + } + + // Also clean up iSCSI sessions if this is a client node. + process := act.Params["process"] + if process == "" { + process = "iscsi-target-test" + } + + // Kill all matching processes. + cmd := fmt.Sprintf("pkill -9 -f '%s' 2>/dev/null; sleep 0.5; pgrep -f '%s' || echo 'all_killed'", process, process) + stdout, _, _, _ := node.Run(ctx, cmd) + actx.Log(" kill_stale %s: %s", process, strings.TrimSpace(stdout)) + + // If iscsi cleanup requested, clean up stale iSCSI sessions. + if act.Params["iscsi_cleanup"] == "true" { + node.Run(ctx, "sudo iscsiadm -m session -u 2>/dev/null; sudo iscsiadm -m node -o delete 2>/dev/null") + actx.Log(" cleaned stale iSCSI sessions") + } + + return nil, nil +} + +func assign(ctx context.Context, actx *tr.ActionContext, act tr.Action) (map[string]string, error) { + tgt, err := getHATarget(actx, act.Target) + if err != nil { + return nil, err + } + + epoch, _ := strconv.ParseUint(act.Params["epoch"], 10, 64) + role := parseRole(act.Params["role"]) + leaseTTL := uint32(30000) // default 30s + if ttlStr, ok := act.Params["lease_ttl"]; ok { + if ms, err := parseDurationMs(ttlStr); err == nil { + leaseTTL = ms + } + } + + if err := tgt.Assign(ctx, epoch, role, leaseTTL); err != nil { + return nil, fmt.Errorf("assign %s: %w", act.Target, err) + } + return nil, nil +} + +func setReplica(ctx context.Context, actx *tr.ActionContext, act tr.Action) (map[string]string, error) { + primaryTgt, err := getHATarget(actx, act.Target) + if err != nil { + return nil, err + } + + replicaTgt, err := getHATarget(actx, act.Replica) + if err != nil { + return nil, fmt.Errorf("set_replica: replica %q: %w", act.Replica, err) + } + + host := replicaTgt.HostAddr() + dataAddr := fmt.Sprintf("%s:%d", host, replicaTgt.ReplicaData) + ctrlAddr := fmt.Sprintf("%s:%d", host, replicaTgt.ReplicaCtrl) + + return nil, primaryTgt.SetReplica(ctx, dataAddr, ctrlAddr) +} + +func waitRole(ctx context.Context, actx *tr.ActionContext, act tr.Action) (map[string]string, error) { + tgt, err := getHATarget(actx, act.Target) + if err != nil { + return nil, err + } + + role := act.Params["role"] + if role == "" { + return nil, fmt.Errorf("wait_role: role param required") + } + + timeoutCtx := ctx + if t, ok := act.Params["timeout"]; ok { + if d, err := parseDuration(t); err == nil { + var cancel context.CancelFunc + timeoutCtx, cancel = context.WithTimeout(ctx, d) + defer cancel() + } + } + + return nil, tgt.WaitForRole(timeoutCtx, role) +} + +func waitLSN(ctx context.Context, actx *tr.ActionContext, act tr.Action) (map[string]string, error) { + tgt, err := getHATarget(actx, act.Target) + if err != nil { + return nil, err + } + + minLSN, _ := strconv.ParseUint(act.Params["min_lsn"], 10, 64) + + timeoutCtx := ctx + if t, ok := act.Params["timeout"]; ok { + if d, err := parseDuration(t); err == nil { + var cancel context.CancelFunc + timeoutCtx, cancel = context.WithTimeout(ctx, d) + defer cancel() + } + } + + return nil, tgt.WaitForLSN(timeoutCtx, minLSN) +} + +func statusAction(ctx context.Context, actx *tr.ActionContext, act tr.Action) (map[string]string, error) { + tgt, err := getHATarget(actx, act.Target) + if err != nil { + return nil, err + } + + st, err := tgt.Status(ctx) + if err != nil { + return nil, err + } + + return map[string]string{ + "value": fmt.Sprintf("epoch=%d role=%s lsn=%d lease=%v healthy=%v", + st.Epoch, st.Role, st.WALHeadLSN, st.HasLease, st.Healthy), + }, nil +} + +func startRebuildServer(ctx context.Context, actx *tr.ActionContext, act tr.Action) (map[string]string, error) { + tgt, err := getHATarget(actx, act.Target) + if err != nil { + return nil, err + } + + host := tgt.HostAddr() + listenAddr := fmt.Sprintf("%s:%d", host, tgt.RebuildPort) + return nil, tgt.StartRebuildEndpoint(ctx, listenAddr) +} + +func startRebuildClient(ctx context.Context, actx *tr.ActionContext, act tr.Action) (map[string]string, error) { + tgt, err := getHATarget(actx, act.Target) + if err != nil { + return nil, err + } + + // Get primary's rebuild address from the "primary" param or the replica target. + primaryName := act.Params["primary"] + if primaryName == "" { + primaryName = act.Replica + } + primaryTgt, err := getHATarget(actx, primaryName) + if err != nil { + return nil, fmt.Errorf("start_rebuild_client: primary %q: %w", primaryName, err) + } + + host := primaryTgt.HostAddr() + rebuildAddr := fmt.Sprintf("%s:%d", host, primaryTgt.RebuildPort) + + epoch, _ := strconv.ParseUint(act.Params["epoch"], 10, 64) + + return nil, tgt.StartRebuildClient(ctx, rebuildAddr, epoch) +} + +// getHATarget looks up the named target from the action context. +func getHATarget(actx *tr.ActionContext, name string) (*infra.HATarget, error) { + if name == "" { + return nil, fmt.Errorf("target name is required") + } + tgt, ok := actx.Targets[name] + if !ok { + return nil, fmt.Errorf("target %q not found", name) + } + ht, ok := tgt.(*infra.HATarget) + if !ok { + return nil, fmt.Errorf("target %q is not an HATarget", name) + } + return ht, nil +} + +func parseRole(s string) uint32 { + switch strings.ToLower(s) { + case "primary": + return 1 // RolePrimary + case "replica": + return 2 // RoleReplica + case "stale": + return 3 // RoleStale + case "rebuilding": + return 4 // RoleRebuilding + case "draining": + return 5 // RoleDraining + default: + return 0 + } +} + +// toNativePath converts a Git Bash path (e.g. /c/work/foo) to the native OS path +// (e.g. C:\work\foo on Windows). No-op on Linux. +func toNativePath(p string) string { + if runtime.GOOS != "windows" { + return p + } + // Convert /c/work/foo → C:/work/foo + if len(p) >= 3 && p[0] == '/' && p[2] == '/' { + p = strings.ToUpper(string(p[1])) + ":" + p[2:] + } + return strings.ReplaceAll(p, "/", "\\") +} diff --git a/weed/storage/blockvol/testrunner/actions/devops.go b/weed/storage/blockvol/testrunner/actions/devops.go new file mode 100644 index 000000000..2625a1a8d --- /dev/null +++ b/weed/storage/blockvol/testrunner/actions/devops.go @@ -0,0 +1,262 @@ +package actions + +import ( + "context" + "encoding/json" + "fmt" + "strings" + "time" + + tr "github.com/seaweedfs/seaweedfs/weed/storage/blockvol/testrunner" +) + +// RegisterDevOpsActions registers SeaweedFS cluster management actions. +func RegisterDevOpsActions(r *tr.Registry) { + r.RegisterFunc("build_deploy_weed", tr.TierDevOps, buildDeployWeed) + r.RegisterFunc("start_weed_master", tr.TierDevOps, startWeedMaster) + r.RegisterFunc("start_weed_volume", tr.TierDevOps, startWeedVolume) + r.RegisterFunc("stop_weed", tr.TierDevOps, stopWeed) + r.RegisterFunc("wait_cluster_ready", tr.TierDevOps, waitClusterReady) + r.RegisterFunc("create_block_volume", tr.TierDevOps, createBlockVolume) + r.RegisterFunc("cluster_status", tr.TierDevOps, clusterStatus) +} + +// buildDeployWeed cross-compiles the weed binary and uploads to all nodes. +func buildDeployWeed(ctx context.Context, actx *tr.ActionContext, act tr.Action) (map[string]string, error) { + repoDir := actx.Vars["repo_dir"] + if repoDir == "" { + return nil, fmt.Errorf("build_deploy_weed: repo_dir not set in env") + } + + actx.Log(" cross-compiling weed binary...") + localBin := repoDir + "/weed-linux" + buildCmd := fmt.Sprintf("cd %s && GOOS=linux GOARCH=amd64 CGO_ENABLED=0 go build -o weed-linux ./weed/command", repoDir) + + ln := tr.NewLocalNode("build-host") + _, stderr, code, err := ln.Run(ctx, buildCmd) + if err != nil || code != 0 { + return nil, fmt.Errorf("build_deploy_weed: code=%d stderr=%s err=%v", code, stderr, err) + } + + // Upload to agents if coordinator mode. + if actx.Coordinator != nil { + for _, agentName := range actx.Coordinator.AgentNames() { + actx.Log(" uploading weed to agent %s...", agentName) + if err := actx.Coordinator.UploadToAgent(ctx, agentName, localBin, tr.UploadBasePath+"weed"); err != nil { + return nil, fmt.Errorf("upload weed to %s: %w", agentName, err) + } + } + return nil, nil + } + + // SSH mode: deploy to all nodes. + for nodeName, nodeRunner := range actx.Nodes { + actx.Log(" deploying weed to node %s...", nodeName) + nodeRunner.Run(ctx, fmt.Sprintf("mkdir -p %s", tr.UploadBasePath)) + if err := nodeRunner.Upload(localBin, tr.UploadBasePath+"weed"); err != nil { + return nil, fmt.Errorf("deploy weed to %s: %w", nodeName, err) + } + nodeRunner.Run(ctx, fmt.Sprintf("chmod +x %sweed", tr.UploadBasePath)) + } + + return nil, nil +} + +// startWeedMaster starts a weed master process on the given node. +func startWeedMaster(ctx context.Context, actx *tr.ActionContext, act tr.Action) (map[string]string, error) { + node, err := getNode(actx, act.Node) + if err != nil { + return nil, fmt.Errorf("start_weed_master: %w", err) + } + + port := act.Params["port"] + if port == "" { + port = "9333" + } + dir := act.Params["dir"] + if dir == "" { + dir = "/tmp/sw-weed-master" + } + extraArgs := act.Params["extra_args"] + + // Ensure directory exists. + node.RunRoot(ctx, fmt.Sprintf("mkdir -p %s", dir)) + + cmd := fmt.Sprintf("setsid %sweed master -port=%s -mdir=%s %s %s/master.log 2>&1 & echo $!", + tr.UploadBasePath, port, dir, extraArgs, dir) + stdout, stderr, code, err := node.RunRoot(ctx, cmd) + if err != nil || code != 0 { + return nil, fmt.Errorf("start_weed_master: code=%d stderr=%s err=%v", code, stderr, err) + } + + pid := strings.TrimSpace(stdout) + actx.Log(" weed master started on port %s (PID %s)", port, pid) + return map[string]string{"value": pid}, nil +} + +// startWeedVolume starts a weed volume process on the given node. +func startWeedVolume(ctx context.Context, actx *tr.ActionContext, act tr.Action) (map[string]string, error) { + node, err := getNode(actx, act.Node) + if err != nil { + return nil, fmt.Errorf("start_weed_volume: %w", err) + } + + port := act.Params["port"] + if port == "" { + port = "8080" + } + master := act.Params["master"] + if master == "" { + return nil, fmt.Errorf("start_weed_volume: master param required") + } + dir := act.Params["dir"] + if dir == "" { + dir = "/tmp/sw-weed-volume" + } + extraArgs := act.Params["extra_args"] + + node.RunRoot(ctx, fmt.Sprintf("mkdir -p %s", dir)) + + cmd := fmt.Sprintf("setsid %sweed volume -port=%s -mserver=%s -dir=%s %s %s/volume.log 2>&1 & echo $!", + tr.UploadBasePath, port, master, dir, extraArgs, dir) + stdout, stderr, code, err := node.RunRoot(ctx, cmd) + if err != nil || code != 0 { + return nil, fmt.Errorf("start_weed_volume: code=%d stderr=%s err=%v", code, stderr, err) + } + + pid := strings.TrimSpace(stdout) + actx.Log(" weed volume started on port %s (PID %s)", port, pid) + return map[string]string{"value": pid}, nil +} + +// stopWeed stops a weed process by PID. +func stopWeed(ctx context.Context, actx *tr.ActionContext, act tr.Action) (map[string]string, error) { + node, err := getNode(actx, act.Node) + if err != nil { + return nil, fmt.Errorf("stop_weed: %w", err) + } + + pid := act.Params["pid"] + if pid == "" { + return nil, fmt.Errorf("stop_weed: pid param required") + } + + // Graceful kill first, then force after 5s. + _, _, _, _ = node.RunRoot(ctx, fmt.Sprintf("kill %s", pid)) + + deadline := time.After(5 * time.Second) + ticker := time.NewTicker(500 * time.Millisecond) + defer ticker.Stop() + + for { + select { + case <-deadline: + node.RunRoot(ctx, fmt.Sprintf("kill -9 %s", pid)) + actx.Log(" force-killed PID %s", pid) + return nil, nil + case <-ticker.C: + _, _, code, _ := node.Run(ctx, fmt.Sprintf("kill -0 %s 2>/dev/null", pid)) + if code != 0 { + actx.Log(" PID %s exited gracefully", pid) + return nil, nil + } + case <-ctx.Done(): + return nil, ctx.Err() + } + } +} + +// waitClusterReady polls the master until IsLeader is true. +func waitClusterReady(ctx context.Context, actx *tr.ActionContext, act tr.Action) (map[string]string, error) { + node, err := getNode(actx, act.Node) + if err != nil { + return nil, fmt.Errorf("wait_cluster_ready: %w", err) + } + + masterURL := act.Params["master_url"] + if masterURL == "" { + return nil, fmt.Errorf("wait_cluster_ready: master_url param required") + } + + timeout := 30 * time.Second + if t, ok := act.Params["timeout"]; ok { + if d, err := parseDuration(t); err == nil { + timeout = d + } + } + + timeoutCtx, cancel := context.WithTimeout(ctx, timeout) + defer cancel() + + ticker := time.NewTicker(1 * time.Second) + defer ticker.Stop() + + for { + select { + case <-timeoutCtx.Done(): + return nil, fmt.Errorf("wait_cluster_ready: timeout after %s", timeout) + case <-ticker.C: + cmd := fmt.Sprintf("curl -s %s/cluster/status 2>/dev/null", masterURL) + stdout, _, code, err := node.Run(timeoutCtx, cmd) + if err != nil || code != 0 { + continue + } + if strings.Contains(stdout, `"IsLeader":true`) || strings.Contains(stdout, `"isLeader":true`) { + actx.Log(" cluster ready at %s", masterURL) + return map[string]string{"value": stdout}, nil + } + } + } +} + +// createBlockVolume creates a block volume via the master assign API. +func createBlockVolume(ctx context.Context, actx *tr.ActionContext, act tr.Action) (map[string]string, error) { + node, err := getNode(actx, act.Node) + if err != nil { + return nil, fmt.Errorf("create_block_volume: %w", err) + } + + masterURL := act.Params["master_url"] + if masterURL == "" { + return nil, fmt.Errorf("create_block_volume: master_url param required") + } + size := act.Params["size"] + if size == "" { + size = "1g" + } + + cmd := fmt.Sprintf("curl -s -X POST '%s/vol/assign?type=block&size=%s' 2>/dev/null", masterURL, size) + stdout, stderr, code, err := node.Run(ctx, cmd) + if err != nil || code != 0 { + return nil, fmt.Errorf("create_block_volume: code=%d stderr=%s err=%v", code, stderr, err) + } + + return map[string]string{"value": strings.TrimSpace(stdout)}, nil +} + +// clusterStatus fetches the full cluster status JSON. +func clusterStatus(ctx context.Context, actx *tr.ActionContext, act tr.Action) (map[string]string, error) { + node, err := getNode(actx, act.Node) + if err != nil { + return nil, fmt.Errorf("cluster_status: %w", err) + } + + masterURL := act.Params["master_url"] + if masterURL == "" { + return nil, fmt.Errorf("cluster_status: master_url param required") + } + + cmd := fmt.Sprintf("curl -s %s/cluster/status 2>/dev/null", masterURL) + stdout, stderr, code, err := node.Run(ctx, cmd) + if err != nil || code != 0 { + return nil, fmt.Errorf("cluster_status: code=%d stderr=%s err=%v", code, stderr, err) + } + + // Validate it's JSON. + var js json.RawMessage + if err := json.Unmarshal([]byte(stdout), &js); err != nil { + return nil, fmt.Errorf("cluster_status: invalid JSON response: %s", stdout) + } + + return map[string]string{"value": strings.TrimSpace(stdout)}, nil +} diff --git a/weed/storage/blockvol/testrunner/actions/devops_test.go b/weed/storage/blockvol/testrunner/actions/devops_test.go new file mode 100644 index 000000000..4aa67b14d --- /dev/null +++ b/weed/storage/blockvol/testrunner/actions/devops_test.go @@ -0,0 +1,101 @@ +package actions + +import ( + "sort" + "testing" + + tr "github.com/seaweedfs/seaweedfs/weed/storage/blockvol/testrunner" +) + +func TestDevOpsActions_Registration(t *testing.T) { + registry := tr.NewRegistry() + RegisterDevOpsActions(registry) + + expected := []string{ + "build_deploy_weed", + "start_weed_master", + "start_weed_volume", + "stop_weed", + "wait_cluster_ready", + "create_block_volume", + "cluster_status", + } + + for _, name := range expected { + if _, err := registry.Get(name); err != nil { + t.Errorf("action %q not registered: %v", name, err) + } + } +} + +func TestDevOpsActions_Tier(t *testing.T) { + registry := tr.NewRegistry() + RegisterDevOpsActions(registry) + + byTier := registry.ListByTier() + devopsActions := byTier[tr.TierDevOps] + + if len(devopsActions) != 7 { + t.Errorf("devops tier has %d actions, want 7", len(devopsActions)) + } + + // Verify all are in devops tier. + sort.Strings(devopsActions) + for _, name := range devopsActions { + if tier := registry.ActionTier(name); tier != tr.TierDevOps { + t.Errorf("action %q has tier %q, want devops", name, tier) + } + } +} + +func TestDevOpsActions_TierGating(t *testing.T) { + registry := tr.NewRegistry() + RegisterDevOpsActions(registry) + + // Without gating, all should be accessible. + if _, err := registry.Get("start_weed_master"); err != nil { + t.Errorf("ungated: %v", err) + } + + // Enable only core tier — devops should be blocked. + registry.EnableTiers([]string{tr.TierCore}) + if _, err := registry.Get("start_weed_master"); err == nil { + t.Error("expected error when devops tier is disabled") + } + + // Enable devops tier — should work again. + registry.EnableTiers([]string{tr.TierDevOps}) + if _, err := registry.Get("start_weed_master"); err != nil { + t.Errorf("devops enabled: %v", err) + } +} + +func TestAllActions_Registration(t *testing.T) { + registry := tr.NewRegistry() + RegisterAll(registry) + + byTier := registry.ListByTier() + + // Verify tier counts. + if n := len(byTier[tr.TierCore]); n != 7 { + t.Errorf("core: %d, want 7", n) + } + if n := len(byTier[tr.TierBlock]); n != 33 { + t.Errorf("block: %d, want 33", n) + } + if n := len(byTier[tr.TierDevOps]); n != 7 { + t.Errorf("devops: %d, want 7", n) + } + if n := len(byTier[tr.TierChaos]); n != 5 { + t.Errorf("chaos: %d, want 5", n) + } + + // Total should be 52. + total := 0 + for _, actions := range byTier { + total += len(actions) + } + if total != 52 { + t.Errorf("total actions: %d, want 52", total) + } +} diff --git a/weed/storage/blockvol/testrunner/actions/fault.go b/weed/storage/blockvol/testrunner/actions/fault.go new file mode 100644 index 000000000..cce8ba8ae --- /dev/null +++ b/weed/storage/blockvol/testrunner/actions/fault.go @@ -0,0 +1,134 @@ +package actions + +import ( + "context" + "fmt" + + tr "github.com/seaweedfs/seaweedfs/weed/storage/blockvol/testrunner" + "github.com/seaweedfs/seaweedfs/weed/storage/blockvol/testrunner/infra" +) + +// RegisterFaultActions registers fault injection actions. +func RegisterFaultActions(r *tr.Registry) { + r.RegisterFunc("inject_netem", tr.TierChaos, injectNetemAction) + r.RegisterFunc("inject_partition", tr.TierChaos, injectPartitionAction) + r.RegisterFunc("fill_disk", tr.TierChaos, fillDiskAction) + r.RegisterFunc("corrupt_wal", tr.TierChaos, corruptWALAction) + r.RegisterFunc("clear_fault", tr.TierChaos, clearFaultAction) +} + +func injectNetemAction(ctx context.Context, actx *tr.ActionContext, act tr.Action) (map[string]string, error) { + node, err := getNode(actx, act.Node) + if err != nil { + return nil, fmt.Errorf("inject_netem: %w", err) + } + + targetIP := act.Params["target_ip"] + if targetIP == "" { + return nil, fmt.Errorf("inject_netem: target_ip param required") + } + delayMs := parseInt(act.Params["delay_ms"], 200) + + cleanupCmd, err := infra.InjectNetem(ctx, node, targetIP, delayMs) + if err != nil { + return nil, err + } + + // Store cleanup cmd for later clear_fault. + varKey := "__cleanup_netem" + if act.Node != "" { + varKey += "_" + act.Node + } + return map[string]string{varKey: cleanupCmd}, nil +} + +func injectPartitionAction(ctx context.Context, actx *tr.ActionContext, act tr.Action) (map[string]string, error) { + node, err := getNode(actx, act.Node) + if err != nil { + return nil, fmt.Errorf("inject_partition: %w", err) + } + + targetIP := act.Params["target_ip"] + if targetIP == "" { + return nil, fmt.Errorf("inject_partition: target_ip param required") + } + ports := parseIntSlice(act.Params["ports"]) + if len(ports) == 0 { + return nil, fmt.Errorf("inject_partition: ports param required") + } + + cleanupCmd, err := infra.InjectIptablesDrop(ctx, node, targetIP, ports) + if err != nil { + return nil, err + } + + varKey := "__cleanup_partition" + if act.Node != "" { + varKey += "_" + act.Node + } + return map[string]string{varKey: cleanupCmd}, nil +} + +func fillDiskAction(ctx context.Context, actx *tr.ActionContext, act tr.Action) (map[string]string, error) { + node, err := getNode(actx, act.Node) + if err != nil { + return nil, fmt.Errorf("fill_disk: %w", err) + } + + dir := act.Params["dir"] + if dir == "" { + dir = "/tmp" + } + + cleanupCmd, err := infra.FillDisk(ctx, node, dir) + if err != nil { + return nil, err + } + + varKey := "__cleanup_fill_disk" + if act.Node != "" { + varKey += "_" + act.Node + } + return map[string]string{varKey: cleanupCmd}, nil +} + +func corruptWALAction(ctx context.Context, actx *tr.ActionContext, act tr.Action) (map[string]string, error) { + targetName := act.Target + if targetName == "" { + return nil, fmt.Errorf("corrupt_wal: target param required") + } + + tgt, err := getHATarget(actx, targetName) + if err != nil { + return nil, err + } + + nBytes := parseInt(act.Params["bytes"], 4096) + + return nil, infra.CorruptWALRegion(ctx, tgt.Node, tgt.VolFilePath(), nBytes) +} + +func clearFaultAction(ctx context.Context, actx *tr.ActionContext, act tr.Action) (map[string]string, error) { + faultType := act.Params["type"] + if faultType == "" { + return nil, fmt.Errorf("clear_fault: type param required (netem, partition, fill_disk)") + } + + node, err := getNode(actx, act.Node) + if err != nil { + return nil, fmt.Errorf("clear_fault: %w", err) + } + + varKey := "__cleanup_" + faultType + if act.Node != "" { + varKey += "_" + act.Node + } + + cleanupCmd := actx.Vars[varKey] + if cleanupCmd == "" { + // Try without node suffix. + cleanupCmd = actx.Vars["__cleanup_"+faultType] + } + + return nil, infra.ClearFault(ctx, node, cleanupCmd) +} diff --git a/weed/storage/blockvol/testrunner/actions/helpers.go b/weed/storage/blockvol/testrunner/actions/helpers.go new file mode 100644 index 000000000..211a900e8 --- /dev/null +++ b/weed/storage/blockvol/testrunner/actions/helpers.go @@ -0,0 +1,97 @@ +package actions + +import ( + "fmt" + "strconv" + "strings" + "time" + + tr "github.com/seaweedfs/seaweedfs/weed/storage/blockvol/testrunner" + "github.com/seaweedfs/seaweedfs/weed/storage/blockvol/testrunner/infra" +) + +// getNode retrieves the infra.Node for the named node from the action context. +func getNode(actx *tr.ActionContext, name string) (*infra.Node, error) { + if name == "" { + // Try to get the first available node. + for _, n := range actx.Nodes { + if nn, ok := n.(*infra.Node); ok { + return nn, nil + } + } + return nil, fmt.Errorf("no nodes available") + } + n, ok := actx.Nodes[name] + if !ok { + return nil, fmt.Errorf("node %q not found", name) + } + nn, ok := n.(*infra.Node) + if !ok { + return nil, fmt.Errorf("node %q is not an infra.Node", name) + } + return nn, nil +} + +// getTargetNode retrieves the node associated with a target. +func getTargetNode(actx *tr.ActionContext, targetName string) (*infra.Node, error) { + spec, ok := actx.Scenario.Targets[targetName] + if !ok { + return nil, fmt.Errorf("target %q not in scenario", targetName) + } + return getNode(actx, spec.Node) +} + +// getTargetHost returns the host address for a target's node. +func getTargetHost(actx *tr.ActionContext, targetName string) (string, error) { + spec, ok := actx.Scenario.Targets[targetName] + if !ok { + return "", fmt.Errorf("target %q not in scenario", targetName) + } + nodeSpec, ok := actx.Scenario.Topology.Nodes[spec.Node] + if !ok { + return "", fmt.Errorf("node %q not in topology", spec.Node) + } + if nodeSpec.IsLocal { + return "127.0.0.1", nil + } + return nodeSpec.Host, nil +} + +func parseDuration(s string) (time.Duration, error) { + return time.ParseDuration(s) +} + +func parseDurationMs(s string) (uint32, error) { + d, err := time.ParseDuration(s) + if err != nil { + // Try parsing as plain number (milliseconds). + ms, err2 := strconv.ParseUint(s, 10, 32) + if err2 != nil { + return 0, err + } + return uint32(ms), nil + } + return uint32(d.Milliseconds()), nil +} + +func parseInt(s string, def int) int { + if s == "" { + return def + } + v, err := strconv.Atoi(s) + if err != nil { + return def + } + return v +} + +func parseIntSlice(s string) []int { + var result []int + for _, part := range strings.Split(s, ",") { + part = strings.TrimSpace(part) + if v, err := strconv.Atoi(part); err == nil { + result = append(result, v) + } + } + return result +} diff --git a/weed/storage/blockvol/testrunner/actions/io.go b/weed/storage/blockvol/testrunner/actions/io.go new file mode 100644 index 000000000..f32540340 --- /dev/null +++ b/weed/storage/blockvol/testrunner/actions/io.go @@ -0,0 +1,260 @@ +package actions + +import ( + "context" + "fmt" + "strings" + + tr "github.com/seaweedfs/seaweedfs/weed/storage/blockvol/testrunner" +) + +// RegisterIOActions registers IO-related actions. +func RegisterIOActions(r *tr.Registry) { + r.RegisterFunc("dd_write", tr.TierBlock, ddWrite) + r.RegisterFunc("dd_read_md5", tr.TierBlock, ddReadMD5) + r.RegisterFunc("fio", tr.TierBlock, fioAction) + r.RegisterFunc("fio_verify", tr.TierBlock, fioVerify) + r.RegisterFunc("mkfs", tr.TierBlock, mkfsAction) + r.RegisterFunc("mount", tr.TierBlock, mountAction) + r.RegisterFunc("umount", tr.TierBlock, umountAction) +} + +// ddWrite writes random data using dd, returns the md5 checksum. +func ddWrite(ctx context.Context, actx *tr.ActionContext, act tr.Action) (map[string]string, error) { + device := act.Params["device"] + if device == "" { + return nil, fmt.Errorf("dd_write: device param required") + } + bs := act.Params["bs"] + if bs == "" { + bs = "1M" + } + count := act.Params["count"] + if count == "" { + count = "1" + } + oflag := act.Params["oflag"] + if oflag == "" { + oflag = "direct" + } + + node, err := getNode(actx, act.Node) + if err != nil { + return nil, err + } + + // Generate random data to temp file, write to device, compute md5. + tmpFile := "/tmp/sw-test-runner-dd-data" + genCmd := fmt.Sprintf("dd if=/dev/urandom of=%s bs=%s count=%s 2>/dev/null", tmpFile, bs, count) + _, stderr, code, err := node.RunRoot(ctx, genCmd) + if err != nil || code != 0 { + return nil, fmt.Errorf("dd_write gen: code=%d stderr=%s err=%v", code, stderr, err) + } + + writeCmd := fmt.Sprintf("dd if=%s of=%s bs=%s oflag=%s conv=fsync", tmpFile, device, bs, oflag) + if seek := act.Params["seek"]; seek != "" { + writeCmd += fmt.Sprintf(" seek=%s", seek) + } + writeCmd += " 2>/dev/null" + _, stderr, code, err = node.RunRoot(ctx, writeCmd) + if err != nil || code != 0 { + return nil, fmt.Errorf("dd_write: code=%d stderr=%s err=%v", code, stderr, err) + } + + md5Cmd := fmt.Sprintf("md5sum %s | cut -d' ' -f1", tmpFile) + stdout, stderr, code, err := node.RunRoot(ctx, md5Cmd) + if err != nil || code != 0 { + return nil, fmt.Errorf("dd_write md5: code=%d stderr=%s err=%v", code, stderr, err) + } + node.Run(ctx, fmt.Sprintf("rm -f %s", tmpFile)) + + md5 := strings.TrimSpace(stdout) + if md5 == "" { + return nil, fmt.Errorf("dd_write: empty md5") + } + + return map[string]string{"value": md5}, nil +} + +// ddReadMD5 reads from device using dd and returns md5. +func ddReadMD5(ctx context.Context, actx *tr.ActionContext, act tr.Action) (map[string]string, error) { + device := act.Params["device"] + if device == "" { + return nil, fmt.Errorf("dd_read_md5: device param required") + } + bs := act.Params["bs"] + if bs == "" { + bs = "1M" + } + count := act.Params["count"] + if count == "" { + count = "1" + } + + node, err := getNode(actx, act.Node) + if err != nil { + return nil, err + } + + tmpFile := "/tmp/sw-test-runner-dd-read" + readCmd := fmt.Sprintf("dd if=%s of=%s bs=%s count=%s iflag=direct", device, tmpFile, bs, count) + if skip := act.Params["skip"]; skip != "" { + readCmd += fmt.Sprintf(" skip=%s", skip) + } + readCmd += " 2>/dev/null" + _, stderr, code, err := node.RunRoot(ctx, readCmd) + if err != nil || code != 0 { + return nil, fmt.Errorf("dd_read_md5 read: code=%d stderr=%s err=%v", code, stderr, err) + } + + cmd := fmt.Sprintf("md5sum %s | cut -d' ' -f1", tmpFile) + stdout, stderr, code, err := node.RunRoot(ctx, cmd) + if err != nil || code != 0 { + return nil, fmt.Errorf("dd_read_md5: code=%d stderr=%s err=%v", code, stderr, err) + } + node.Run(ctx, fmt.Sprintf("rm -f %s", tmpFile)) + + md5 := strings.TrimSpace(stdout) + if md5 == "" { + return nil, fmt.Errorf("dd_read_md5: empty md5") + } + + return map[string]string{"value": md5}, nil +} + +func fioAction(ctx context.Context, actx *tr.ActionContext, act tr.Action) (map[string]string, error) { + device := act.Params["device"] + if device == "" { + return nil, fmt.Errorf("fio: device param required") + } + + node, err := getNode(actx, act.Node) + if err != nil { + return nil, err + } + + rw := act.Params["rw"] + if rw == "" { + rw = "randwrite" + } + bs := act.Params["bs"] + if bs == "" { + bs = "4k" + } + iodepth := act.Params["iodepth"] + if iodepth == "" { + iodepth = "32" + } + runtime := act.Params["runtime"] + if runtime == "" { + runtime = "10" + } + name := act.Params["name"] + if name == "" { + name = "fio_test" + } + + cmd := fmt.Sprintf("fio --name=%s --filename=%s --rw=%s --bs=%s --iodepth=%s --direct=1 --runtime=%s --time_based --output-format=json", + name, device, rw, bs, iodepth, runtime) + stdout, stderr, code, err := node.RunRoot(ctx, cmd) + if err != nil || code != 0 { + return nil, fmt.Errorf("fio: code=%d stderr=%s err=%v", code, stderr, err) + } + + return map[string]string{"value": stdout}, nil +} + +func fioVerify(ctx context.Context, actx *tr.ActionContext, act tr.Action) (map[string]string, error) { + device := act.Params["device"] + if device == "" { + return nil, fmt.Errorf("fio_verify: device param required") + } + + node, err := getNode(actx, act.Node) + if err != nil { + return nil, err + } + + bs := act.Params["bs"] + if bs == "" { + bs = "4k" + } + size := act.Params["size"] + if size == "" { + size = "10M" + } + + // Write with verify pattern, then read+verify. + cmd := fmt.Sprintf("fio --name=verify --filename=%s --rw=write --bs=%s --size=%s --direct=1 --verify=crc32c --do_verify=1 --output-format=json", + device, bs, size) + stdout, stderr, code, err := node.RunRoot(ctx, cmd) + if err != nil || code != 0 { + return nil, fmt.Errorf("fio_verify: code=%d stderr=%s err=%v", code, stderr, err) + } + + return map[string]string{"value": stdout}, nil +} + +func mkfsAction(ctx context.Context, actx *tr.ActionContext, act tr.Action) (map[string]string, error) { + device := act.Params["device"] + if device == "" { + return nil, fmt.Errorf("mkfs: device param required") + } + fstype := act.Params["fstype"] + if fstype == "" { + fstype = "ext4" + } + + node, err := getNode(actx, act.Node) + if err != nil { + return nil, err + } + + cmd := fmt.Sprintf("mkfs.%s -F %s", fstype, device) + _, stderr, code, err := node.RunRoot(ctx, cmd) + if err != nil || code != 0 { + return nil, fmt.Errorf("mkfs: code=%d stderr=%s err=%v", code, stderr, err) + } + return nil, nil +} + +func mountAction(ctx context.Context, actx *tr.ActionContext, act tr.Action) (map[string]string, error) { + device := act.Params["device"] + if device == "" { + return nil, fmt.Errorf("mount: device param required") + } + mountpoint := act.Params["mountpoint"] + if mountpoint == "" { + mountpoint = "/mnt/test" + } + + node, err := getNode(actx, act.Node) + if err != nil { + return nil, err + } + + node.RunRoot(ctx, fmt.Sprintf("mkdir -p %s", mountpoint)) + _, stderr, code, err := node.RunRoot(ctx, fmt.Sprintf("mount %s %s", device, mountpoint)) + if err != nil || code != 0 { + return nil, fmt.Errorf("mount: code=%d stderr=%s err=%v", code, stderr, err) + } + return map[string]string{"value": mountpoint}, nil +} + +func umountAction(ctx context.Context, actx *tr.ActionContext, act tr.Action) (map[string]string, error) { + mountpoint := act.Params["mountpoint"] + if mountpoint == "" { + mountpoint = "/mnt/test" + } + + node, err := getNode(actx, act.Node) + if err != nil { + return nil, err + } + + _, stderr, code, err := node.RunRoot(ctx, fmt.Sprintf("umount %s", mountpoint)) + if err != nil || code != 0 { + return nil, fmt.Errorf("umount: code=%d stderr=%s err=%v", code, stderr, err) + } + return nil, nil +} diff --git a/weed/storage/blockvol/testrunner/actions/iscsi.go b/weed/storage/blockvol/testrunner/actions/iscsi.go new file mode 100644 index 000000000..f940fb3da --- /dev/null +++ b/weed/storage/blockvol/testrunner/actions/iscsi.go @@ -0,0 +1,132 @@ +package actions + +import ( + "context" + "fmt" + + tr "github.com/seaweedfs/seaweedfs/weed/storage/blockvol/testrunner" + "github.com/seaweedfs/seaweedfs/weed/storage/blockvol/testrunner/infra" +) + +// RegisterISCSIActions registers iSCSI client actions. +func RegisterISCSIActions(r *tr.Registry) { + r.RegisterFunc("iscsi_login", tr.TierBlock, iscsiLogin) + r.RegisterFunc("iscsi_logout", tr.TierBlock, iscsiLogout) + r.RegisterFunc("iscsi_discover", tr.TierBlock, iscsiDiscover) + r.RegisterFunc("iscsi_cleanup", tr.TierBlock, iscsiCleanup) +} + +// iscsiLogin discovers + logs into the target, returns the device path. +func iscsiLogin(ctx context.Context, actx *tr.ActionContext, act tr.Action) (map[string]string, error) { + targetName := act.Target + if targetName == "" { + return nil, fmt.Errorf("iscsi_login: target is required") + } + + spec, ok := actx.Scenario.Targets[targetName] + if !ok { + return nil, fmt.Errorf("iscsi_login: target %q not in scenario", targetName) + } + + host, err := getTargetHost(actx, targetName) + if err != nil { + return nil, err + } + + // Get the initiator node (first available or explicit). + node, err := getNode(actx, act.Node) + if err != nil { + return nil, fmt.Errorf("iscsi_login: %w", err) + } + + client := infra.NewISCSIClient(node) + iqn := spec.IQN() + port := spec.ISCSIPort + + actx.Log(" discovering %s:%d ...", host, port) + iqns, err := client.Discover(ctx, host, port) + if err != nil { + return nil, fmt.Errorf("iscsi_login discover: %w", err) + } + + // Find matching IQN. + found := false + for _, q := range iqns { + if q == iqn { + found = true + break + } + } + if !found { + return nil, fmt.Errorf("iscsi_login: IQN %s not found in discovery (got %v)", iqn, iqns) + } + + actx.Log(" logging in to %s ...", iqn) + dev, err := client.Login(ctx, iqn) + if err != nil { + return nil, fmt.Errorf("iscsi_login: %w", err) + } + + actx.Log(" device: %s", dev) + return map[string]string{"value": dev}, nil +} + +func iscsiLogout(ctx context.Context, actx *tr.ActionContext, act tr.Action) (map[string]string, error) { + targetName := act.Target + if targetName == "" { + return nil, fmt.Errorf("iscsi_logout: target is required") + } + + spec, ok := actx.Scenario.Targets[targetName] + if !ok { + return nil, fmt.Errorf("iscsi_logout: target %q not in scenario", targetName) + } + + node, err := getNode(actx, act.Node) + if err != nil { + return nil, fmt.Errorf("iscsi_logout: %w", err) + } + + client := infra.NewISCSIClient(node) + return nil, client.Logout(ctx, spec.IQN()) +} + +func iscsiDiscover(ctx context.Context, actx *tr.ActionContext, act tr.Action) (map[string]string, error) { + targetName := act.Target + if targetName == "" { + return nil, fmt.Errorf("iscsi_discover: target is required") + } + + spec, ok := actx.Scenario.Targets[targetName] + if !ok { + return nil, fmt.Errorf("iscsi_discover: target %q not in scenario", targetName) + } + + host, err := getTargetHost(actx, targetName) + if err != nil { + return nil, err + } + + node, err := getNode(actx, act.Node) + if err != nil { + return nil, fmt.Errorf("iscsi_discover: %w", err) + } + + client := infra.NewISCSIClient(node) + iqns, err := client.Discover(ctx, host, spec.ISCSIPort) + if err != nil { + return nil, err + } + + return map[string]string{"value": fmt.Sprintf("%v", iqns)}, nil +} + +func iscsiCleanup(ctx context.Context, actx *tr.ActionContext, act tr.Action) (map[string]string, error) { + node, err := getNode(actx, act.Node) + if err != nil { + return nil, fmt.Errorf("iscsi_cleanup: %w", err) + } + + client := infra.NewISCSIClient(node) + return nil, client.CleanupAll(ctx, "iqn.2024.com.seaweedfs:") +} diff --git a/weed/storage/blockvol/testrunner/actions/metrics.go b/weed/storage/blockvol/testrunner/actions/metrics.go new file mode 100644 index 000000000..4e5b1dbb2 --- /dev/null +++ b/weed/storage/blockvol/testrunner/actions/metrics.go @@ -0,0 +1,109 @@ +package actions + +import ( + "context" + "encoding/json" + "fmt" + + tr "github.com/seaweedfs/seaweedfs/weed/storage/blockvol/testrunner" + "github.com/seaweedfs/seaweedfs/weed/storage/blockvol/testrunner/infra" +) + +// RegisterMetricsActions registers metrics/perf actions. +func RegisterMetricsActions(r *tr.Registry) { + r.RegisterFunc("scrape_metrics", tr.TierBlock, scrapeMetrics) + r.RegisterFunc("perf_summary", tr.TierBlock, perfSummary) + r.RegisterFunc("collect_artifacts", tr.TierBlock, collectArtifactsAction) +} + +// scrapeMetrics fetches /metrics from a target's admin port via SSH curl. +// Saves the parsed metrics as JSON string in save_as. +func scrapeMetrics(ctx context.Context, actx *tr.ActionContext, act tr.Action) (map[string]string, error) { + tgt, err := getHATarget(actx, act.Target) + if err != nil { + return nil, err + } + + cmd := fmt.Sprintf("curl -s http://127.0.0.1:%d/metrics 2>&1", tgt.AdminPort) + stdout, _, code, err := tgt.Node.Run(ctx, cmd) + if err != nil || code != 0 { + return nil, fmt.Errorf("scrape_metrics %s: code=%d err=%v", act.Target, code, err) + } + + sample := tr.MetricsSample{ + Target: act.Target, + Metrics: tr.ParsePrometheusText(stdout), + } + data, _ := json.Marshal(sample) + return map[string]string{"value": string(data)}, nil +} + +// perfSummary collects the target's log, parses PERF[5s] lines, and outputs stats. +func perfSummary(ctx context.Context, actx *tr.ActionContext, act tr.Action) (map[string]string, error) { + tgt, err := getHATarget(actx, act.Target) + if err != nil { + return nil, err + } + + logContent, err := tgt.CollectLog() + if err != nil { + return nil, fmt.Errorf("perf_summary: collect log: %w", err) + } + + perfData := tr.ParsePerfLogLines(logContent) + if len(perfData) == 0 { + return map[string]string{"value": "no PERF data found"}, nil + } + + var lines []string + for field, values := range perfData { + stats := tr.ComputeStats(values) + lines = append(lines, tr.FormatStats(field, stats)) + } + + result := "" + for i, l := range lines { + if i > 0 { + result += "\n" + } + result += l + } + return map[string]string{"value": result}, nil +} + +// collectArtifactsAction explicitly collects artifacts from targets. +func collectArtifactsAction(ctx context.Context, actx *tr.ActionContext, act tr.Action) (map[string]string, error) { + dir := act.Params["dir"] + if dir == "" { + dir = "/tmp/sw-test-runner-artifacts" + } + + // Find client node for dmesg/lsblk. + clientNodeName := act.Node + if clientNodeName == "" { + clientNodeName = "client_node" + } + node, _ := getNode(actx, clientNodeName) + if node == nil { + // Use any available node. + for _, n := range actx.Nodes { + if nn, ok := n.(*infra.Node); ok { + node = nn + break + } + } + } + if node == nil { + return nil, fmt.Errorf("collect_artifacts: no node available") + } + + collector := infra.NewArtifactCollector(dir, node, nil) + + for name, tgt := range actx.Targets { + if lc, ok := tgt.(infra.LogCollector); ok { + collector.CollectLabeled(lc, name) + } + } + + return map[string]string{"value": fmt.Sprintf("artifacts saved to %s", dir)}, nil +} diff --git a/weed/storage/blockvol/testrunner/actions/register.go b/weed/storage/blockvol/testrunner/actions/register.go new file mode 100644 index 000000000..3d9d0b3e2 --- /dev/null +++ b/weed/storage/blockvol/testrunner/actions/register.go @@ -0,0 +1,15 @@ +package actions + +import tr "github.com/seaweedfs/seaweedfs/weed/storage/blockvol/testrunner" + +// RegisterAll registers all action handlers on the given registry. +func RegisterAll(r *tr.Registry) { + RegisterBlockActions(r) + RegisterISCSIActions(r) + RegisterIOActions(r) + RegisterFaultActions(r) + RegisterSystemActions(r) + RegisterMetricsActions(r) + RegisterDevOpsActions(r) + RegisterSnapshotActions(r) +} diff --git a/weed/storage/blockvol/testrunner/actions/snapshot.go b/weed/storage/blockvol/testrunner/actions/snapshot.go new file mode 100644 index 000000000..977b97567 --- /dev/null +++ b/weed/storage/blockvol/testrunner/actions/snapshot.go @@ -0,0 +1,183 @@ +package actions + +import ( + "context" + "fmt" + "strconv" + "strings" + "time" + + tr "github.com/seaweedfs/seaweedfs/weed/storage/blockvol/testrunner" +) + +// RegisterSnapshotActions registers snapshot and resize actions. +func RegisterSnapshotActions(r *tr.Registry) { + r.RegisterFunc("snapshot_create", tr.TierBlock, snapshotCreate) + r.RegisterFunc("snapshot_delete", tr.TierBlock, snapshotDelete) + r.RegisterFunc("snapshot_list", tr.TierBlock, snapshotList) + r.RegisterFunc("resize", tr.TierBlock, resizeAction) + r.RegisterFunc("iscsi_rescan", tr.TierBlock, iscsiRescan) + r.RegisterFunc("get_block_size", tr.TierBlock, getBlockSize) +} + +func snapshotCreate(ctx context.Context, actx *tr.ActionContext, act tr.Action) (map[string]string, error) { + tgt, err := getHATarget(actx, act.Target) + if err != nil { + return nil, err + } + + idStr := act.Params["id"] + if idStr == "" { + return nil, fmt.Errorf("snapshot_create: id param required") + } + id, err := strconv.ParseUint(idStr, 10, 32) + if err != nil { + return nil, fmt.Errorf("snapshot_create: invalid id %q: %w", idStr, err) + } + + if err := tgt.CreateSnapshot(ctx, uint32(id)); err != nil { + return nil, fmt.Errorf("snapshot_create: %w", err) + } + actx.Log(" created snapshot %d", id) + return nil, nil +} + +func snapshotDelete(ctx context.Context, actx *tr.ActionContext, act tr.Action) (map[string]string, error) { + tgt, err := getHATarget(actx, act.Target) + if err != nil { + return nil, err + } + + idStr := act.Params["id"] + if idStr == "" { + return nil, fmt.Errorf("snapshot_delete: id param required") + } + id, err := strconv.ParseUint(idStr, 10, 32) + if err != nil { + return nil, fmt.Errorf("snapshot_delete: invalid id %q: %w", idStr, err) + } + + if err := tgt.DeleteSnapshot(ctx, uint32(id)); err != nil { + return nil, fmt.Errorf("snapshot_delete: %w", err) + } + actx.Log(" deleted snapshot %d", id) + return nil, nil +} + +func snapshotList(ctx context.Context, actx *tr.ActionContext, act tr.Action) (map[string]string, error) { + tgt, err := getHATarget(actx, act.Target) + if err != nil { + return nil, err + } + + snaps, err := tgt.ListSnapshots(ctx) + if err != nil { + return nil, fmt.Errorf("snapshot_list: %w", err) + } + + actx.Log(" snapshots: %d entries", len(snaps)) + return map[string]string{"value": strconv.Itoa(len(snaps))}, nil +} + +func resizeAction(ctx context.Context, actx *tr.ActionContext, act tr.Action) (map[string]string, error) { + tgt, err := getHATarget(actx, act.Target) + if err != nil { + return nil, err + } + + var newSizeBytes uint64 + if s := act.Params["new_size_bytes"]; s != "" { + newSizeBytes, err = strconv.ParseUint(s, 10, 64) + if err != nil { + return nil, fmt.Errorf("resize: invalid new_size_bytes %q: %w", s, err) + } + } else if s := act.Params["new_size"]; s != "" { + newSizeBytes, err = parseHumanSize(s) + if err != nil { + return nil, fmt.Errorf("resize: invalid new_size %q: %w", s, err) + } + } else { + return nil, fmt.Errorf("resize: new_size or new_size_bytes param required") + } + + if err := tgt.Resize(ctx, newSizeBytes); err != nil { + return nil, fmt.Errorf("resize: %w", err) + } + actx.Log(" resized to %d bytes", newSizeBytes) + return nil, nil +} + +func iscsiRescan(ctx context.Context, actx *tr.ActionContext, act tr.Action) (map[string]string, error) { + node, err := getNode(actx, act.Node) + if err != nil { + return nil, fmt.Errorf("iscsi_rescan: %w", err) + } + + _, stderr, code, err := node.RunRoot(ctx, "iscsiadm -m session -R") + if err != nil || code != 0 { + return nil, fmt.Errorf("iscsi_rescan: code=%d stderr=%s err=%v", code, stderr, err) + } + + // Give kernel time to update block device size. + select { + case <-time.After(2 * time.Second): + case <-ctx.Done(): + return nil, ctx.Err() + } + + actx.Log(" iSCSI sessions rescanned") + return nil, nil +} + +func getBlockSize(ctx context.Context, actx *tr.ActionContext, act tr.Action) (map[string]string, error) { + device := act.Params["device"] + if device == "" { + return nil, fmt.Errorf("get_block_size: device param required") + } + + node, err := getNode(actx, act.Node) + if err != nil { + return nil, fmt.Errorf("get_block_size: %w", err) + } + + cmd := fmt.Sprintf("blockdev --getsize64 %s", device) + stdout, stderr, code, err := node.RunRoot(ctx, cmd) + if err != nil || code != 0 { + return nil, fmt.Errorf("get_block_size: code=%d stderr=%s err=%v", code, stderr, err) + } + + size := strings.TrimSpace(stdout) + actx.Log(" %s size: %s bytes", device, size) + return map[string]string{"value": size}, nil +} + +// parseHumanSize converts human-readable sizes like "100M", "1G", "512K" to bytes. +func parseHumanSize(s string) (uint64, error) { + s = strings.TrimSpace(s) + if len(s) == 0 { + return 0, fmt.Errorf("empty size") + } + + multiplier := uint64(1) + suffix := s[len(s)-1] + switch suffix { + case 'K', 'k': + multiplier = 1024 + s = s[:len(s)-1] + case 'M', 'm': + multiplier = 1024 * 1024 + s = s[:len(s)-1] + case 'G', 'g': + multiplier = 1024 * 1024 * 1024 + s = s[:len(s)-1] + case 'T', 't': + multiplier = 1024 * 1024 * 1024 * 1024 + s = s[:len(s)-1] + } + + val, err := strconv.ParseUint(s, 10, 64) + if err != nil { + return 0, err + } + return val * multiplier, nil +} diff --git a/weed/storage/blockvol/testrunner/actions/system.go b/weed/storage/blockvol/testrunner/actions/system.go new file mode 100644 index 000000000..69e7547ed --- /dev/null +++ b/weed/storage/blockvol/testrunner/actions/system.go @@ -0,0 +1,150 @@ +package actions + +import ( + "context" + "fmt" + "strconv" + "strings" + "time" + + tr "github.com/seaweedfs/seaweedfs/weed/storage/blockvol/testrunner" +) + +// RegisterSystemActions registers system/assert actions. +func RegisterSystemActions(r *tr.Registry) { + r.RegisterFunc("exec", tr.TierCore, execAction) + r.RegisterFunc("sleep", tr.TierCore, sleepAction) + r.RegisterFunc("assert_equal", tr.TierCore, assertEqual) + r.RegisterFunc("assert_greater", tr.TierCore, assertGreater) + r.RegisterFunc("assert_status", tr.TierCore, assertStatus) + r.RegisterFunc("assert_contains", tr.TierCore, assertContains) + r.RegisterFunc("print", tr.TierCore, printAction) +} + +func execAction(ctx context.Context, actx *tr.ActionContext, act tr.Action) (map[string]string, error) { + cmd := act.Params["cmd"] + if cmd == "" { + return nil, fmt.Errorf("exec: cmd param required") + } + + node, err := getNode(actx, act.Node) + if err != nil { + return nil, err + } + + root := act.Params["root"] == "true" + var stdout, stderr string + var code int + if root { + stdout, stderr, code, err = node.RunRoot(ctx, cmd) + } else { + stdout, stderr, code, err = node.Run(ctx, cmd) + } + if err != nil { + return nil, fmt.Errorf("exec: %w", err) + } + if code != 0 { + return nil, fmt.Errorf("exec: code=%d stderr=%s", code, stderr) + } + + return map[string]string{"value": strings.TrimSpace(stdout)}, nil +} + +func sleepAction(ctx context.Context, actx *tr.ActionContext, act tr.Action) (map[string]string, error) { + d := act.Params["duration"] + if d == "" { + d = "1s" + } + + dur, err := time.ParseDuration(d) + if err != nil { + return nil, fmt.Errorf("sleep: invalid duration %q: %w", d, err) + } + + select { + case <-time.After(dur): + return nil, nil + case <-ctx.Done(): + return nil, ctx.Err() + } +} + +func assertEqual(ctx context.Context, actx *tr.ActionContext, act tr.Action) (map[string]string, error) { + actual := act.Params["actual"] + expected := act.Params["expected"] + + if actual != expected { + return nil, fmt.Errorf("assert_equal: %q != %q", actual, expected) + } + return nil, nil +} + +func assertGreater(ctx context.Context, actx *tr.ActionContext, act tr.Action) (map[string]string, error) { + actualStr := act.Params["actual"] + expectedStr := act.Params["expected"] + + actual, err := strconv.ParseInt(actualStr, 10, 64) + if err != nil { + return nil, fmt.Errorf("assert_greater: cannot parse actual %q as int: %w", actualStr, err) + } + expected, err := strconv.ParseInt(expectedStr, 10, 64) + if err != nil { + return nil, fmt.Errorf("assert_greater: cannot parse expected %q as int: %w", expectedStr, err) + } + + if actual <= expected { + return nil, fmt.Errorf("assert_greater: %d <= %d", actual, expected) + } + return nil, nil +} + +func assertStatus(ctx context.Context, actx *tr.ActionContext, act tr.Action) (map[string]string, error) { + tgt, err := getHATarget(actx, act.Target) + if err != nil { + return nil, err + } + + st, err := tgt.Status(ctx) + if err != nil { + return nil, fmt.Errorf("assert_status: %w", err) + } + + if role, ok := act.Params["role"]; ok { + if st.Role != role { + return nil, fmt.Errorf("assert_status: role %q != expected %q", st.Role, role) + } + } + if healthy, ok := act.Params["healthy"]; ok { + expectedHealthy := healthy == "true" + if st.Healthy != expectedHealthy { + return nil, fmt.Errorf("assert_status: healthy=%v != expected=%v", st.Healthy, expectedHealthy) + } + } + if hasLease, ok := act.Params["has_lease"]; ok { + expectedLease := hasLease == "true" + if st.HasLease != expectedLease { + return nil, fmt.Errorf("assert_status: has_lease=%v != expected=%v", st.HasLease, expectedLease) + } + } + + return nil, nil +} + +func assertContains(ctx context.Context, actx *tr.ActionContext, act tr.Action) (map[string]string, error) { + haystack := act.Params["value"] + needle := act.Params["contains"] + + if !strings.Contains(haystack, needle) { + return nil, fmt.Errorf("assert_contains: %q not found in %q", needle, haystack) + } + return nil, nil +} + +func printAction(ctx context.Context, actx *tr.ActionContext, act tr.Action) (map[string]string, error) { + msg := act.Params["msg"] + if msg == "" { + msg = act.Params["message"] + } + actx.Log(" [print] %s", msg) + return nil, nil +} diff --git a/weed/storage/blockvol/testrunner/agent.go b/weed/storage/blockvol/testrunner/agent.go new file mode 100644 index 000000000..a6845e6c5 --- /dev/null +++ b/weed/storage/blockvol/testrunner/agent.go @@ -0,0 +1,694 @@ +package testrunner + +import ( + "archive/tar" + "compress/gzip" + "context" + "encoding/json" + "fmt" + "io" + "log" + "net" + "net/http" + "os" + "path/filepath" + "strings" + "sync" + "time" + + "github.com/seaweedfs/seaweedfs/weed/storage/blockvol/testrunner/infra" +) + +// Agent runs on each test node, executing actions locally on behalf of the coordinator. +type Agent struct { + ID string + Hostname string + Port int + CoordinatorURL string + Token string + AllowExec bool + Persistent bool // stay running across coordinator restarts + Nodes []string // node names this agent handles + + registry *Registry + localNode *LocalNode + startTime time.Time + log *log.Logger + + mu sync.Mutex + server *http.Server + listener net.Listener +} + +// AgentConfig holds configuration for creating an Agent. +type AgentConfig struct { + Hostname string + Port int + CoordinatorURL string + Token string + AllowExec bool + Persistent bool // stay running, re-register with coordinator + Nodes []string + Registry *Registry + Logger *log.Logger +} + +// NewAgent creates a new Agent with the given configuration. +func NewAgent(cfg AgentConfig) *Agent { + hostname := cfg.Hostname + if hostname == "" { + hostname, _ = os.Hostname() + } + + agentID := fmt.Sprintf("agent-%s-%d", hostname, os.Getpid()) + + logger := cfg.Logger + if logger == nil { + logger = log.New(os.Stderr, "[agent] ", log.LstdFlags) + } + + return &Agent{ + ID: agentID, + Hostname: hostname, + Port: cfg.Port, + CoordinatorURL: cfg.CoordinatorURL, + Token: cfg.Token, + AllowExec: cfg.AllowExec, + Persistent: cfg.Persistent, + Nodes: cfg.Nodes, + registry: cfg.Registry, + localNode: NewLocalNode(hostname), + startTime: time.Now(), + log: logger, + } +} + +// Start begins listening for coordinator requests. Blocks until the server is shut down. +func (a *Agent) Start(ctx context.Context) error { + mux := http.NewServeMux() + mux.HandleFunc("/health", a.handleHealth) + mux.HandleFunc("/phase", a.authMiddleware(a.handlePhase)) + mux.HandleFunc("/upload", a.authMiddleware(a.handleUpload)) + mux.HandleFunc("/artifacts", a.authMiddleware(a.handleArtifacts)) + mux.HandleFunc("/exec", a.authMiddleware(a.handleExec)) + + addr := fmt.Sprintf(":%d", a.Port) + ln, err := net.Listen("tcp", addr) + if err != nil { + return fmt.Errorf("listen %s: %w", addr, err) + } + + a.mu.Lock() + a.listener = ln + a.server = &http.Server{Handler: mux} + a.mu.Unlock() + + a.log.Printf("agent %s listening on %s (root=%v, exec=%v)", a.ID, addr, a.localNode.IsRoot(), a.AllowExec) + + // Register with coordinator if URL is set. + if a.CoordinatorURL != "" { + if a.Persistent { + go a.registrationLoop(ctx) + } else { + go a.registerWithCoordinator() + } + } + + // Shutdown on context cancellation. + go func() { + <-ctx.Done() + a.Stop() + }() + + if err := a.server.Serve(ln); err != nil && err != http.ErrServerClosed { + return err + } + return nil +} + +// Stop gracefully shuts down the agent HTTP server. +func (a *Agent) Stop() { + a.mu.Lock() + srv := a.server + a.mu.Unlock() + + if srv != nil { + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + srv.Shutdown(ctx) + } +} + +// ListenAddr returns the address the agent is listening on (useful in tests). +func (a *Agent) ListenAddr() string { + a.mu.Lock() + defer a.mu.Unlock() + if a.listener != nil { + return a.listener.Addr().String() + } + return "" +} + +func (a *Agent) registerWithCoordinator() { + req := RegisterRequest{ + AgentID: a.ID, + Hostname: a.Hostname, + IP: a.detectIP(), + Port: a.Port, + Nodes: a.Nodes, + } + + // Detect capabilities. + var caps []string + if a.localNode.IsRoot() { + caps = append(caps, "root") + } + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + if _, _, code, err := a.localNode.Run(ctx, "which iscsiadm"); err == nil && code == 0 { + caps = append(caps, "iscsi") + } + if _, _, code, err := a.localNode.Run(ctx, "which fio"); err == nil && code == 0 { + caps = append(caps, "fio") + } + req.Capabilities = caps + + body, _ := json.Marshal(req) + url := strings.TrimRight(a.CoordinatorURL, "/") + "/register" + + // Retry registration up to 60s with 2s intervals. + deadline := time.Now().Add(60 * time.Second) + for attempt := 1; ; attempt++ { + httpReq, _ := http.NewRequest("POST", url, strings.NewReader(string(body))) + httpReq.Header.Set("Content-Type", "application/json") + if a.Token != "" { + httpReq.Header.Set(AuthTokenHeader, a.Token) + } + + resp, err := http.DefaultClient.Do(httpReq) + if err != nil { + if time.Now().After(deadline) { + a.log.Printf("register failed after retries: %v", err) + return + } + a.log.Printf("register attempt %d failed: %v (retrying...)", attempt, err) + time.Sleep(2 * time.Second) + continue + } + + var rr RegisterResponse + json.NewDecoder(resp.Body).Decode(&rr) + resp.Body.Close() + if rr.OK { + a.log.Printf("registered with coordinator (index=%d, total=%d)", rr.AgentIndex, rr.TotalAgents) + } else { + a.log.Printf("registration rejected: %s", rr.Error) + } + return + } +} + +// registrationLoop continuously re-registers with the coordinator. +// Used in persistent mode so the agent survives coordinator restarts. +func (a *Agent) registrationLoop(ctx context.Context) { + req := a.buildRegisterRequest() + body, _ := json.Marshal(req) + url := strings.TrimRight(a.CoordinatorURL, "/") + "/register" + + registered := false + for { + select { + case <-ctx.Done(): + return + default: + } + + httpReq, _ := http.NewRequestWithContext(ctx, "POST", url, strings.NewReader(string(body))) + httpReq.Header.Set("Content-Type", "application/json") + if a.Token != "" { + httpReq.Header.Set(AuthTokenHeader, a.Token) + } + + resp, err := http.DefaultClient.Do(httpReq) + if err != nil { + if registered { + a.log.Printf("coordinator unreachable, will retry: %v", err) + registered = false + } + sleepCtx(ctx, 5*time.Second) + continue + } + + var rr RegisterResponse + json.NewDecoder(resp.Body).Decode(&rr) + resp.Body.Close() + + if rr.OK { + if !registered { + a.log.Printf("registered with coordinator (index=%d, total=%d)", rr.AgentIndex, rr.TotalAgents) + registered = true + } + sleepCtx(ctx, 10*time.Second) + } else { + a.log.Printf("registration rejected: %s", rr.Error) + sleepCtx(ctx, 5*time.Second) + } + } +} + +// buildRegisterRequest creates the registration payload with capability detection. +func (a *Agent) buildRegisterRequest() RegisterRequest { + req := RegisterRequest{ + AgentID: a.ID, + Hostname: a.Hostname, + IP: a.detectIP(), + Port: a.Port, + Nodes: a.Nodes, + } + var caps []string + if a.localNode.IsRoot() { + caps = append(caps, "root") + } + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + if _, _, code, err := a.localNode.Run(ctx, "which iscsiadm"); err == nil && code == 0 { + caps = append(caps, "iscsi") + } + if _, _, code, err := a.localNode.Run(ctx, "which fio"); err == nil && code == 0 { + caps = append(caps, "fio") + } + req.Capabilities = caps + return req +} + +// sleepCtx sleeps for d or until ctx is cancelled. +func sleepCtx(ctx context.Context, d time.Duration) { + select { + case <-time.After(d): + case <-ctx.Done(): + } +} + +func (a *Agent) detectIP() string { + addrs, err := net.InterfaceAddrs() + if err != nil { + return "" + } + for _, addr := range addrs { + if ipNet, ok := addr.(*net.IPNet); ok && !ipNet.IP.IsLoopback() && ipNet.IP.To4() != nil { + return ipNet.IP.String() + } + } + return "" +} + +// authMiddleware wraps a handler with token authentication. +func (a *Agent) authMiddleware(next http.HandlerFunc) http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + if a.Token != "" { + token := r.Header.Get(AuthTokenHeader) + if token != a.Token { + writeJSON(w, http.StatusUnauthorized, map[string]string{"error": "invalid auth token"}) + return + } + } + next(w, r) + } +} + +// GET /health +func (a *Agent) handleHealth(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodGet { + http.Error(w, "method not allowed", http.StatusMethodNotAllowed) + return + } + resp := HealthResponse{ + OK: true, + AgentID: a.ID, + UptimeS: int64(time.Since(a.startTime).Seconds()), + HasRoot: a.localNode.IsRoot(), + Hostname: a.Hostname, + } + writeJSON(w, http.StatusOK, resp) +} + +// POST /phase +func (a *Agent) handlePhase(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodPost { + http.Error(w, "method not allowed", http.StatusMethodNotAllowed) + return + } + + var req PhaseRequest + if err := json.NewDecoder(r.Body).Decode(&req); err != nil { + writeJSON(w, http.StatusBadRequest, PhaseResponse{Error: fmt.Sprintf("decode: %v", err)}) + return + } + + a.log.Printf("phase %d (%s): %d actions", req.PhaseIndex, req.PhaseName, len(req.Actions)) + + resp := a.executePhase(r.Context(), &req) + writeJSON(w, http.StatusOK, resp) +} + +// executePhase runs actions sequentially using the local engine. +func (a *Agent) executePhase(ctx context.Context, req *PhaseRequest) PhaseResponse { + resp := PhaseResponse{ + PhaseIndex: req.PhaseIndex, + NewVars: make(map[string]string), + } + + // Build action context. + actx := &ActionContext{ + Scenario: req.Scenario, + Nodes: make(map[string]NodeRunner), + Targets: make(map[string]TargetRunner), + Vars: make(map[string]string), + Log: func(format string, args ...interface{}) { a.log.Printf(format, args...) }, + } + + // Copy global vars. + for k, v := range req.GlobalVars { + actx.Vars[k] = v + } + + // Create infra.Node in native mode for local command execution. + nativeNode := &infra.Node{IsNative: true} + + // Map agent's nodes to native node runner. + for _, nodeName := range a.Nodes { + actx.Nodes[nodeName] = nativeNode + } + + // Set up targets for nodes this agent handles. + myNodes := make(map[string]bool) + for _, n := range a.Nodes { + myNodes[n] = true + } + if req.Scenario != nil { + // Map node names to agents via topology. + nodeToAgentName := make(map[string]string) + for nodeName, nodeSpec := range req.Scenario.Topology.Nodes { + nodeToAgentName[nodeName] = nodeSpec.Agent + } + for tgtName, tgtSpec := range req.Scenario.Targets { + agentName := nodeToAgentName[tgtSpec.Node] + if !myNodes[agentName] { + continue + } + htSpec := infra.HATargetSpec{ + VolSize: tgtSpec.VolSize, + WALSize: tgtSpec.WALSize, + IQN: tgtSpec.IQN(), + ISCSIPort: tgtSpec.ISCSIPort, + AdminPort: tgtSpec.AdminPort, + ReplicaDataPort: tgtSpec.ReplicaDataPort, + ReplicaCtrlPort: tgtSpec.ReplicaCtrlPort, + RebuildPort: tgtSpec.RebuildPort, + TPGID: tgtSpec.TPGID, + } + actx.Targets[tgtName] = infra.NewHATargetFromSpec(nativeNode, tgtName, htSpec) + } + // Also map topology node names to native node if this agent handles them. + for nodeName, nodeSpec := range req.Scenario.Topology.Nodes { + if myNodes[nodeSpec.Agent] { + actx.Nodes[nodeName] = nativeNode + } + } + } + + engine := NewEngine(a.registry, actx.Log) + + for _, act := range req.Actions { + start := time.Now() + resolved := resolveAction(act, actx.Vars) + yamlDef := marshalActionYAML(resolved) + + handler, err := engine.registry.Get(resolved.Action) + if err != nil { + resp.Results = append(resp.Results, ActionResult{ + Action: resolved.Action, + Status: StatusFail, + Duration: time.Since(start), + Error: err.Error(), + }) + resp.Error = err.Error() + return resp + } + + // Handle delay param. + if d, ok := resolved.Params["delay"]; ok { + if dur, err := time.ParseDuration(d); err == nil { + select { + case <-time.After(dur): + case <-ctx.Done(): + resp.Error = ctx.Err().Error() + return resp + } + } + } + + output, execErr := handler.Execute(ctx, actx, resolved) + + ar := ActionResult{ + Action: resolved.Action, + Duration: time.Since(start), + YAML: yamlDef, + } + + if execErr != nil { + ar.Status = StatusFail + ar.Error = execErr.Error() + if !act.IgnoreError { + resp.Results = append(resp.Results, ar) + resp.Error = execErr.Error() + return resp + } + ar.Status = StatusPass + } else { + ar.Status = StatusPass + } + + // Store save_as and __ vars. + if resolved.SaveAs != "" && output != nil { + if v, ok := output["value"]; ok { + actx.Vars[resolved.SaveAs] = v + resp.NewVars[resolved.SaveAs] = v + } + } + if output != nil { + for k, v := range output { + if strings.HasPrefix(k, "__") { + actx.Vars[k] = v + resp.NewVars[k] = v + } + } + if v, ok := output["value"]; ok { + ar.Output = truncate(v, 4096) + } + } + + resp.Results = append(resp.Results, ar) + } + + return resp +} + +// POST /upload — streaming binary upload with path safety. +func (a *Agent) handleUpload(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodPost { + http.Error(w, "method not allowed", http.StatusMethodNotAllowed) + return + } + + path := r.URL.Query().Get("path") + if path == "" { + writeJSON(w, http.StatusBadRequest, UploadResponse{Error: "path query parameter required"}) + return + } + + // Path safety: must start with UploadBasePath and no traversal. + if !isPathSafe(path) { + writeJSON(w, http.StatusForbidden, UploadResponse{Error: fmt.Sprintf("path must start with %s and contain no '..'", UploadBasePath)}) + return + } + + // Ensure directory exists. + dir := filepath.Dir(path) + if err := os.MkdirAll(dir, 0755); err != nil { + writeJSON(w, http.StatusInternalServerError, UploadResponse{Error: fmt.Sprintf("mkdir: %v", err)}) + return + } + + f, err := os.OpenFile(path, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0755) + if err != nil { + writeJSON(w, http.StatusInternalServerError, UploadResponse{Error: fmt.Sprintf("create file: %v", err)}) + return + } + + n, err := io.Copy(f, r.Body) + f.Close() + if err != nil { + os.Remove(path) + writeJSON(w, http.StatusInternalServerError, UploadResponse{Error: fmt.Sprintf("write: %v", err)}) + return + } + + a.log.Printf("uploaded %d bytes to %s", n, path) + writeJSON(w, http.StatusOK, UploadResponse{OK: true, Size: n, Path: path}) +} + +// GET /artifacts — stream a directory as tar.gz. +func (a *Agent) handleArtifacts(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodGet { + http.Error(w, "method not allowed", http.StatusMethodNotAllowed) + return + } + + dir := r.URL.Query().Get("dir") + if dir == "" { + writeJSON(w, http.StatusBadRequest, map[string]string{"error": "dir query parameter required"}) + return + } + + // Path safety: must be under UploadBasePath, no traversal. + if !isPathSafe(dir) { + writeJSON(w, http.StatusForbidden, map[string]string{"error": fmt.Sprintf("dir must be under %s with no '..'", UploadBasePath)}) + return + } + + info, err := os.Stat(dir) + if err != nil || !info.IsDir() { + writeJSON(w, http.StatusNotFound, map[string]string{"error": "directory not found"}) + return + } + + // Check if directory has any files. + entries, err := os.ReadDir(dir) + if err != nil || len(entries) == 0 { + writeJSON(w, http.StatusNotFound, map[string]string{"error": "directory empty"}) + return + } + + w.Header().Set("Content-Type", "application/gzip") + w.WriteHeader(http.StatusOK) + + gw := gzip.NewWriter(w) + defer gw.Close() + tw := tar.NewWriter(gw) + defer tw.Close() + + filepath.Walk(dir, func(path string, fi os.FileInfo, err error) error { + if err != nil { + return nil // skip unreadable files + } + if fi.IsDir() { + return nil + } + + // Relative path inside the tar. + rel, err := filepath.Rel(dir, path) + if err != nil { + return nil + } + // Normalize to forward slashes for tar. + rel = filepath.ToSlash(rel) + + hdr := &tar.Header{ + Name: rel, + Size: fi.Size(), + Mode: int64(fi.Mode()), + ModTime: fi.ModTime(), + } + if err := tw.WriteHeader(hdr); err != nil { + return err + } + + f, err := os.Open(path) + if err != nil { + return nil + } + defer f.Close() + io.Copy(tw, f) + return nil + }) +} + +// POST /exec — ad-hoc command execution (disabled by default). +func (a *Agent) handleExec(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodPost { + http.Error(w, "method not allowed", http.StatusMethodNotAllowed) + return + } + + if !a.AllowExec { + writeJSON(w, http.StatusForbidden, ExecResponse{Error: "exec endpoint disabled; start agent with --allow-exec"}) + return + } + + var req ExecRequest + if err := json.NewDecoder(r.Body).Decode(&req); err != nil { + writeJSON(w, http.StatusBadRequest, ExecResponse{Error: fmt.Sprintf("decode: %v", err)}) + return + } + + ctx, cancel := context.WithTimeout(r.Context(), 60*time.Second) + defer cancel() + + var stdout, stderr string + var exitCode int + var err error + + if req.Root { + stdout, stderr, exitCode, err = a.localNode.RunRoot(ctx, req.Cmd) + } else { + stdout, stderr, exitCode, err = a.localNode.Run(ctx, req.Cmd) + } + + if err != nil { + writeJSON(w, http.StatusOK, ExecResponse{ + Stdout: stdout, + Stderr: stderr, + ExitCode: -1, + Error: err.Error(), + }) + return + } + + writeJSON(w, http.StatusOK, ExecResponse{ + Stdout: stdout, + Stderr: stderr, + ExitCode: exitCode, + }) +} + +// isPathSafe checks that a path is under UploadBasePath and has no traversal. +func isPathSafe(p string) bool { + if strings.Contains(p, "..") { + return false + } + // Use forward-slash path cleaning (Unix paths, even on Windows coordinator). + cleaned := posixClean(p) + return strings.HasPrefix(cleaned, UploadBasePath) +} + +// posixClean normalizes a path using forward slashes (Unix convention). +func posixClean(p string) string { + // Replace backslashes, then clean. + p = strings.ReplaceAll(p, "\\", "/") + // Remove double slashes. + for strings.Contains(p, "//") { + p = strings.ReplaceAll(p, "//", "/") + } + // Remove trailing slash (unless root). + if len(p) > 1 && strings.HasSuffix(p, "/") { + p = p[:len(p)-1] + } + return p +} + +func writeJSON(w http.ResponseWriter, status int, v interface{}) { + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(status) + json.NewEncoder(w).Encode(v) +} diff --git a/weed/storage/blockvol/testrunner/agent_test.go b/weed/storage/blockvol/testrunner/agent_test.go new file mode 100644 index 000000000..65fe3345a --- /dev/null +++ b/weed/storage/blockvol/testrunner/agent_test.go @@ -0,0 +1,426 @@ +package testrunner + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "io" + "net/http" + "os" + "path/filepath" + "runtime" + "testing" + "time" +) + +// startTestAgent starts an agent on a random port and returns its base URL. +func startTestAgent(t *testing.T, cfg AgentConfig) (*Agent, string) { + t.Helper() + if cfg.Registry == nil { + cfg.Registry = NewRegistry() + } + agent := NewAgent(cfg) + ctx, cancel := context.WithCancel(context.Background()) + t.Cleanup(cancel) + + go agent.Start(ctx) + // Wait for listener. + for i := 0; i < 50; i++ { + if agent.ListenAddr() != "" { + break + } + time.Sleep(20 * time.Millisecond) + } + if agent.ListenAddr() == "" { + t.Fatal("agent didn't start") + } + baseURL := "http://" + agent.ListenAddr() + return agent, baseURL +} + +func TestAgent_Health(t *testing.T) { + _, baseURL := startTestAgent(t, AgentConfig{Port: 0}) + + resp, err := http.Get(baseURL + "/health") + if err != nil { + t.Fatalf("GET /health: %v", err) + } + defer resp.Body.Close() + + if resp.StatusCode != 200 { + t.Fatalf("status = %d", resp.StatusCode) + } + + var hr HealthResponse + json.NewDecoder(resp.Body).Decode(&hr) + if !hr.OK { + t.Error("health not OK") + } + if hr.AgentID == "" { + t.Error("empty agent ID") + } + if hr.Hostname == "" { + t.Error("empty hostname") + } +} + +func TestAgent_Health_NoAuth(t *testing.T) { + // /health should not require auth. + _, baseURL := startTestAgent(t, AgentConfig{Port: 0, Token: "secret"}) + + resp, err := http.Get(baseURL + "/health") + if err != nil { + t.Fatalf("GET /health: %v", err) + } + defer resp.Body.Close() + + if resp.StatusCode != 200 { + t.Fatalf("/health without token should work, got %d", resp.StatusCode) + } +} + +func TestAgent_Auth_Rejection(t *testing.T) { + _, baseURL := startTestAgent(t, AgentConfig{Port: 0, Token: "secret"}) + + // POST /phase without token → 401. + body := bytes.NewReader([]byte(`{}`)) + resp, err := http.Post(baseURL+"/phase", "application/json", body) + if err != nil { + t.Fatalf("POST /phase: %v", err) + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusUnauthorized { + t.Errorf("expected 401, got %d", resp.StatusCode) + } +} + +func TestAgent_Auth_ValidToken(t *testing.T) { + _, baseURL := startTestAgent(t, AgentConfig{Port: 0, Token: "secret"}) + + req, _ := http.NewRequest("POST", baseURL+"/phase", bytes.NewReader([]byte(`{"phase_index":0, "actions":[], "global_vars":{}}`))) + req.Header.Set("Content-Type", "application/json") + req.Header.Set(AuthTokenHeader, "secret") + + resp, err := http.DefaultClient.Do(req) + if err != nil { + t.Fatalf("POST /phase: %v", err) + } + defer resp.Body.Close() + + if resp.StatusCode != 200 { + t.Errorf("expected 200 with valid token, got %d", resp.StatusCode) + } +} + +func TestAgent_Phase_EchoAction(t *testing.T) { + registry := NewRegistry() + registry.RegisterFunc("echo_val", TierCore, func(ctx context.Context, actx *ActionContext, act Action) (map[string]string, error) { + return map[string]string{"value": act.Params["msg"]}, nil + }) + + _, baseURL := startTestAgent(t, AgentConfig{Port: 0, Registry: registry}) + + phaseReq := PhaseRequest{ + PhaseIndex: 0, + PhaseName: "test", + Actions: []Action{ + {Action: "echo_val", SaveAs: "result", Params: map[string]string{"msg": "hello"}}, + }, + GlobalVars: map[string]string{}, + } + + body, _ := json.Marshal(phaseReq) + resp, err := http.Post(baseURL+"/phase", "application/json", bytes.NewReader(body)) + if err != nil { + t.Fatalf("POST /phase: %v", err) + } + defer resp.Body.Close() + + var phaseResp PhaseResponse + json.NewDecoder(resp.Body).Decode(&phaseResp) + + if phaseResp.Error != "" { + t.Fatalf("phase error: %s", phaseResp.Error) + } + if len(phaseResp.Results) != 1 { + t.Fatalf("expected 1 result, got %d", len(phaseResp.Results)) + } + if phaseResp.Results[0].Status != StatusPass { + t.Errorf("action status = %s", phaseResp.Results[0].Status) + } + if phaseResp.NewVars["result"] != "hello" { + t.Errorf("new_vars[result] = %q, want hello", phaseResp.NewVars["result"]) + } +} + +func TestAgent_Phase_FailStopsExecution(t *testing.T) { + registry := NewRegistry() + callOrder := []string{} + registry.RegisterFunc("a1", TierCore, func(ctx context.Context, actx *ActionContext, act Action) (map[string]string, error) { + callOrder = append(callOrder, "a1") + return nil, fmt.Errorf("fail") + }) + registry.RegisterFunc("a2", TierCore, func(ctx context.Context, actx *ActionContext, act Action) (map[string]string, error) { + callOrder = append(callOrder, "a2") + return nil, nil + }) + + _, baseURL := startTestAgent(t, AgentConfig{Port: 0, Registry: registry}) + + phaseReq := PhaseRequest{ + PhaseName: "test", + Actions: []Action{ + {Action: "a1"}, + {Action: "a2"}, // should not execute + }, + GlobalVars: map[string]string{}, + } + + body, _ := json.Marshal(phaseReq) + resp, _ := http.Post(baseURL+"/phase", "application/json", bytes.NewReader(body)) + defer resp.Body.Close() + + var phaseResp PhaseResponse + json.NewDecoder(resp.Body).Decode(&phaseResp) + + if phaseResp.Error == "" { + t.Error("expected error from failed action") + } + if len(callOrder) != 1 { + t.Errorf("expected only a1 to run, got %v", callOrder) + } +} + +func TestAgent_Upload_PathSafety(t *testing.T) { + _, baseURL := startTestAgent(t, AgentConfig{Port: 0}) + + // Attempt path traversal → should be rejected. + req, _ := http.NewRequest("POST", baseURL+"/upload?path=/tmp/sw-test-runner/../etc/passwd", bytes.NewReader([]byte("evil"))) + req.Header.Set("Content-Type", "application/octet-stream") + resp, err := http.DefaultClient.Do(req) + if err != nil { + t.Fatalf("upload: %v", err) + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusForbidden { + t.Errorf("expected 403 for path traversal, got %d", resp.StatusCode) + } + + // Attempt outside base path. + req2, _ := http.NewRequest("POST", baseURL+"/upload?path=/etc/evil", bytes.NewReader([]byte("evil"))) + req2.Header.Set("Content-Type", "application/octet-stream") + resp2, _ := http.DefaultClient.Do(req2) + defer resp2.Body.Close() + + if resp2.StatusCode != http.StatusForbidden { + t.Errorf("expected 403 for outside base path, got %d", resp2.StatusCode) + } +} + +func TestAgent_Upload_ValidPath(t *testing.T) { + if runtime.GOOS == "windows" { + t.Skip("upload test requires /tmp on Unix") + } + + _, baseURL := startTestAgent(t, AgentConfig{Port: 0}) + + // Create a temp subdir under /tmp/sw-test-runner/. + uploadDir := "/tmp/sw-test-runner/test-upload" + os.MkdirAll(uploadDir, 0755) + t.Cleanup(func() { os.RemoveAll(uploadDir) }) + + uploadPath := filepath.Join(uploadDir, "testfile.bin") + content := []byte("test binary content") + + req, _ := http.NewRequest("POST", baseURL+"/upload?path="+uploadPath, bytes.NewReader(content)) + req.Header.Set("Content-Type", "application/octet-stream") + resp, err := http.DefaultClient.Do(req) + if err != nil { + t.Fatalf("upload: %v", err) + } + defer resp.Body.Close() + + var ur UploadResponse + json.NewDecoder(resp.Body).Decode(&ur) + if !ur.OK { + t.Fatalf("upload not OK: %s", ur.Error) + } + if ur.Size != int64(len(content)) { + t.Errorf("size = %d, want %d", ur.Size, len(content)) + } + + // Verify file contents. + got, _ := os.ReadFile(uploadPath) + if string(got) != string(content) { + t.Errorf("content mismatch") + } +} + +func TestAgent_Exec_DisabledByDefault(t *testing.T) { + _, baseURL := startTestAgent(t, AgentConfig{Port: 0, AllowExec: false}) + + body, _ := json.Marshal(ExecRequest{Cmd: "echo hi"}) + resp, err := http.Post(baseURL+"/exec", "application/json", bytes.NewReader(body)) + if err != nil { + t.Fatalf("POST /exec: %v", err) + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusForbidden { + t.Errorf("expected 403 when exec disabled, got %d", resp.StatusCode) + } +} + +func TestAgent_Exec_Enabled(t *testing.T) { + if runtime.GOOS == "windows" { + t.Skip("exec test requires Unix shell") + } + + _, baseURL := startTestAgent(t, AgentConfig{Port: 0, AllowExec: true}) + + body, _ := json.Marshal(ExecRequest{Cmd: "echo hello-exec"}) + resp, err := http.Post(baseURL+"/exec", "application/json", bytes.NewReader(body)) + if err != nil { + t.Fatalf("POST /exec: %v", err) + } + defer resp.Body.Close() + + var er ExecResponse + json.NewDecoder(resp.Body).Decode(&er) + if er.ExitCode != 0 { + t.Errorf("exit code = %d, stderr: %s, error: %s", er.ExitCode, er.Stderr, er.Error) + } + if er.Stdout != "hello-exec\n" { + t.Errorf("stdout = %q", er.Stdout) + } +} + +func TestAgent_Artifacts_PathSafety(t *testing.T) { + _, baseURL := startTestAgent(t, AgentConfig{Port: 0, Token: "secret"}) + + // Attempt with traversal. + req, _ := http.NewRequest("GET", baseURL+"/artifacts?dir=/tmp/sw-test-runner/../etc", nil) + req.Header.Set(AuthTokenHeader, "secret") + resp, err := http.DefaultClient.Do(req) + if err != nil { + t.Fatalf("GET /artifacts: %v", err) + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusForbidden { + t.Errorf("expected 403 for traversal, got %d", resp.StatusCode) + } + + // Attempt outside base path. + req2, _ := http.NewRequest("GET", baseURL+"/artifacts?dir=/etc", nil) + req2.Header.Set(AuthTokenHeader, "secret") + resp2, err := http.DefaultClient.Do(req2) + if err != nil { + t.Fatalf("GET /artifacts: %v", err) + } + defer resp2.Body.Close() + + if resp2.StatusCode != http.StatusForbidden { + t.Errorf("expected 403 for outside base path, got %d", resp2.StatusCode) + } +} + +func TestAgent_Artifacts_MissingDir(t *testing.T) { + _, baseURL := startTestAgent(t, AgentConfig{Port: 0}) + + req, _ := http.NewRequest("GET", baseURL+"/artifacts?dir=/tmp/sw-test-runner/nonexistent-"+fmt.Sprintf("%d", time.Now().UnixNano()), nil) + resp, err := http.DefaultClient.Do(req) + if err != nil { + t.Fatalf("GET /artifacts: %v", err) + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusNotFound { + t.Errorf("expected 404 for missing dir, got %d", resp.StatusCode) + } +} + +func TestAgent_Artifacts_NoAuth(t *testing.T) { + _, baseURL := startTestAgent(t, AgentConfig{Port: 0, Token: "secret"}) + + // No auth header should be rejected. + resp, err := http.Get(baseURL + "/artifacts?dir=/tmp/sw-test-runner/test") + if err != nil { + t.Fatalf("GET /artifacts: %v", err) + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusUnauthorized { + t.Errorf("expected 401, got %d", resp.StatusCode) + } +} + +func TestAgent_Artifacts_ValidDir(t *testing.T) { + if runtime.GOOS == "windows" { + t.Skip("artifacts test requires /tmp on Unix") + } + + _, baseURL := startTestAgent(t, AgentConfig{Port: 0}) + + // Create test directory with files. + dir := fmt.Sprintf("/tmp/sw-test-runner/test-artifacts-%d", time.Now().UnixNano()) + os.MkdirAll(dir, 0755) + t.Cleanup(func() { os.RemoveAll(dir) }) + + os.WriteFile(filepath.Join(dir, "log.txt"), []byte("test log content"), 0644) + os.WriteFile(filepath.Join(dir, "dmesg.txt"), []byte("kernel messages"), 0644) + + req, _ := http.NewRequest("GET", baseURL+"/artifacts?dir="+dir, nil) + resp, err := http.DefaultClient.Do(req) + if err != nil { + t.Fatalf("GET /artifacts: %v", err) + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + body, _ := io.ReadAll(resp.Body) + t.Fatalf("expected 200, got %d: %s", resp.StatusCode, string(body)) + } + + if ct := resp.Header.Get("Content-Type"); ct != "application/gzip" { + t.Errorf("Content-Type = %q, want application/gzip", ct) + } + + // Verify it's valid gzip+tar. + body, _ := io.ReadAll(resp.Body) + if len(body) == 0 { + t.Fatal("empty response body") + } +} + +func TestAgent_Phase_VarSubstitution(t *testing.T) { + registry := NewRegistry() + registry.RegisterFunc("concat", TierCore, func(ctx context.Context, actx *ActionContext, act Action) (map[string]string, error) { + return map[string]string{"value": act.Params["a"] + "-" + act.Params["b"]}, nil + }) + + _, baseURL := startTestAgent(t, AgentConfig{Port: 0, Registry: registry}) + + phaseReq := PhaseRequest{ + PhaseName: "test", + Actions: []Action{ + {Action: "concat", SaveAs: "out", Params: map[string]string{"a": "{{ x }}", "b": "{{ y }}"}}, + }, + GlobalVars: map[string]string{"x": "hello", "y": "world"}, + } + + body, _ := json.Marshal(phaseReq) + resp, _ := http.Post(baseURL+"/phase", "application/json", bytes.NewReader(body)) + defer resp.Body.Close() + + respBody, _ := io.ReadAll(resp.Body) + var phaseResp PhaseResponse + json.Unmarshal(respBody, &phaseResp) + + if phaseResp.NewVars["out"] != "hello-world" { + t.Errorf("var substitution failed: out = %q", phaseResp.NewVars["out"]) + } +} diff --git a/weed/storage/blockvol/testrunner/baseline.go b/weed/storage/blockvol/testrunner/baseline.go new file mode 100644 index 000000000..8db1ca544 --- /dev/null +++ b/weed/storage/blockvol/testrunner/baseline.go @@ -0,0 +1,102 @@ +package testrunner + +import ( + "encoding/json" + "fmt" + "os" + "path/filepath" + "sort" + "time" +) + +// Baseline is an immutable, versioned snapshot of SLO metrics from a test run. +// Stored as JSON in testrunner/baselines/ — append-only, git-tracked. +type Baseline struct { + Version int `json:"version"` + GitSHA string `json:"git_sha"` + Scenario string `json:"scenario"` + ScenarioSHA string `json:"scenario_sha256"` + Topology BaselineTopology `json:"topology"` + Config BaselineConfig `json:"config"` + Timestamp time.Time `json:"timestamp"` + Metrics map[string]float64 `json:"metrics"` + DurationSec float64 `json:"duration_sec"` + HardFailsPassed bool `json:"hard_fails_passed"` +} + +// BaselineTopology describes the test cluster topology. +type BaselineTopology struct { + Nodes int `json:"nodes"` + Servers []string `json:"servers"` +} + +// BaselineConfig describes the volume configuration under test. +type BaselineConfig struct { + ReplicaFactor int `json:"rf"` + Durability string `json:"durability"` + WALSize string `json:"wal_size"` +} + +// SaveBaseline writes a baseline to the baselines directory. +// Filename: -.json (date-first for chronological sort). +func SaveBaseline(dir string, b *Baseline) (string, error) { + if err := os.MkdirAll(dir, 0755); err != nil { + return "", fmt.Errorf("baseline: mkdir %s: %w", dir, err) + } + + sha := b.GitSHA + if len(sha) > 10 { + sha = sha[:10] + } + date := b.Timestamp.Format("20060102") + filename := fmt.Sprintf("%s-%s.json", date, sha) + path := filepath.Join(dir, filename) + + // Append-only: fail if file already exists. + if _, err := os.Stat(path); err == nil { + return path, nil // idempotent: already saved + } + + data, err := json.MarshalIndent(b, "", " ") + if err != nil { + return "", fmt.Errorf("baseline: marshal: %w", err) + } + + if err := os.WriteFile(path, data, 0644); err != nil { + return "", fmt.Errorf("baseline: write %s: %w", path, err) + } + return path, nil +} + +// LoadLatestBaseline loads the most recent baseline from the directory. +// Baselines are sorted by filename (which encodes date). +func LoadLatestBaseline(dir string) (*Baseline, error) { + entries, err := os.ReadDir(dir) + if err != nil { + return nil, fmt.Errorf("baseline: read dir %s: %w", dir, err) + } + + var jsonFiles []string + for _, e := range entries { + if !e.IsDir() && filepath.Ext(e.Name()) == ".json" { + jsonFiles = append(jsonFiles, e.Name()) + } + } + if len(jsonFiles) == 0 { + return nil, fmt.Errorf("baseline: no baselines found in %s", dir) + } + + sort.Strings(jsonFiles) + latest := jsonFiles[len(jsonFiles)-1] + + data, err := os.ReadFile(filepath.Join(dir, latest)) + if err != nil { + return nil, fmt.Errorf("baseline: read %s: %w", latest, err) + } + + var b Baseline + if err := json.Unmarshal(data, &b); err != nil { + return nil, fmt.Errorf("baseline: unmarshal %s: %w", latest, err) + } + return &b, nil +} diff --git a/weed/storage/blockvol/testrunner/baseline_test.go b/weed/storage/blockvol/testrunner/baseline_test.go new file mode 100644 index 000000000..41235b9a1 --- /dev/null +++ b/weed/storage/blockvol/testrunner/baseline_test.go @@ -0,0 +1,109 @@ +package testrunner + +import ( + "os" + "path/filepath" + "testing" + "time" +) + +func TestSaveAndLoadBaseline(t *testing.T) { + dir := t.TempDir() + + b := &Baseline{ + Version: 1, + GitSHA: "8b2b5f6f6abc123def", + Scenario: "cp84-soak-4h.yaml", + ScenarioSHA: "abc123", + Topology: BaselineTopology{Nodes: 2, Servers: []string{"m01", "M02"}}, + Config: BaselineConfig{ReplicaFactor: 2, Durability: "best_effort", WALSize: "64MB"}, + Timestamp: time.Date(2026, 3, 6, 12, 0, 0, 0, time.UTC), + Metrics: map[string]float64{ + "p99_write_latency_seconds": 0.005, + "write_iops": 50000, + }, + DurationSec: 14400, + HardFailsPassed: true, + } + + path, err := SaveBaseline(dir, b) + if err != nil { + t.Fatalf("SaveBaseline: %v", err) + } + if filepath.Base(path) != "20260306-8b2b5f6f6a.json" { + t.Errorf("unexpected filename: %s", filepath.Base(path)) + } + + // Verify file exists. + if _, err := os.Stat(path); err != nil { + t.Fatalf("file not created: %v", err) + } + + // Load it back. + loaded, err := LoadLatestBaseline(dir) + if err != nil { + t.Fatalf("LoadLatestBaseline: %v", err) + } + if loaded.GitSHA != b.GitSHA { + t.Errorf("GitSHA = %q, want %q", loaded.GitSHA, b.GitSHA) + } + if loaded.Metrics["write_iops"] != 50000 { + t.Errorf("write_iops = %f, want 50000", loaded.Metrics["write_iops"]) + } +} + +func TestSaveBaselineIdempotent(t *testing.T) { + dir := t.TempDir() + b := &Baseline{ + Version: 1, + GitSHA: "abc123", + Timestamp: time.Date(2026, 3, 6, 12, 0, 0, 0, time.UTC), + Metrics: map[string]float64{}, + } + + p1, err := SaveBaseline(dir, b) + if err != nil { + t.Fatalf("first save: %v", err) + } + p2, err := SaveBaseline(dir, b) + if err != nil { + t.Fatalf("second save: %v", err) + } + if p1 != p2 { + t.Errorf("paths differ: %s vs %s", p1, p2) + } +} + +func TestLoadLatestBaseline_Empty(t *testing.T) { + dir := t.TempDir() + _, err := LoadLatestBaseline(dir) + if err == nil { + t.Error("expected error for empty dir") + } +} + +func TestLoadLatestBaseline_PicksLatest(t *testing.T) { + dir := t.TempDir() + + old := &Baseline{ + Version: 1, GitSHA: "old1234567", + Timestamp: time.Date(2026, 3, 1, 0, 0, 0, 0, time.UTC), + Metrics: map[string]float64{"write_iops": 40000}, + } + newer := &Baseline{ + Version: 1, GitSHA: "new1234567", + Timestamp: time.Date(2026, 3, 6, 0, 0, 0, 0, time.UTC), + Metrics: map[string]float64{"write_iops": 50000}, + } + + SaveBaseline(dir, old) + SaveBaseline(dir, newer) + + loaded, err := LoadLatestBaseline(dir) + if err != nil { + t.Fatalf("LoadLatestBaseline: %v", err) + } + if loaded.Metrics["write_iops"] != 50000 { + t.Errorf("expected latest baseline (50000), got %f", loaded.Metrics["write_iops"]) + } +} diff --git a/weed/storage/blockvol/testrunner/cmd/sw-test-runner/main.go b/weed/storage/blockvol/testrunner/cmd/sw-test-runner/main.go new file mode 100644 index 000000000..5026274f7 --- /dev/null +++ b/weed/storage/blockvol/testrunner/cmd/sw-test-runner/main.go @@ -0,0 +1,521 @@ +package main + +import ( + "context" + "flag" + "fmt" + "log" + "os" + "os/signal" + "sort" + "strings" + "time" + + tr "github.com/seaweedfs/seaweedfs/weed/storage/blockvol/testrunner" + "github.com/seaweedfs/seaweedfs/weed/storage/blockvol/testrunner/actions" + "github.com/seaweedfs/seaweedfs/weed/storage/blockvol/testrunner/infra" +) + +func main() { + if len(os.Args) < 2 { + usage() + os.Exit(1) + } + + switch os.Args[1] { + case "run": + runCmd(os.Args[2:]) + case "coordinator": + coordinatorCmd(os.Args[2:]) + case "agent": + agentCmd(os.Args[2:]) + case "console": + consoleCmd(os.Args[2:]) + case "validate": + validateCmd(os.Args[2:]) + case "list": + listCmd() + case "help", "-h", "--help": + usage() + default: + fmt.Fprintf(os.Stderr, "unknown command: %s\n", os.Args[1]) + usage() + os.Exit(1) + } +} + +func usage() { + fmt.Fprintf(os.Stderr, `sw-test-runner — YAML-driven test platform for SeaweedFS BlockVol + +Usage: + sw-test-runner run [flags] Run a test scenario (SSH mode) + sw-test-runner coordinator [flags] Run as coordinator (multi-node) + sw-test-runner agent [flags] Run as agent on test node + sw-test-runner console [flags] Start web console server + sw-test-runner validate Validate YAML without running + sw-test-runner list [flags] List registered actions + sw-test-runner help Show this help + +Common flags: + -tiers Comma-separated enabled tiers: core,block,devops,chaos (default: all) + +Run flags: + -output Write JSON results to file + -junit Write JUnit XML to file + -html Write HTML report to file + -baseline Compare against baseline JSON + -artifacts Collect artifacts on failure to this directory + +Coordinator flags: + -port Listen port for agent registration (default: 9000) + -token Auth token for agent communication + -dry-run Print execution plan without running + -output Write JSON results to file + -junit Write JUnit XML to file + -html Write HTML report to file + -artifacts Download artifacts from agents to this directory + -timeout Agent registration timeout (default: 30s) + +Agent flags: + -port Listen port (default: 9100) + -coordinator Coordinator URL (e.g. http://192.168.1.100:9000) + -token Auth token for coordinator communication + -nodes Comma-separated node names this agent handles + -allow-exec Enable /exec endpoint for ad-hoc commands + -persistent Stay running, re-register with coordinator on each run + +Console flags: + -port Listen port (default: 9090) + -token Auth token for agents + -scenarios-dir Directory containing scenario YAML files +`) +} + +func runCmd(args []string) { + fs := flag.NewFlagSet("run", flag.ExitOnError) + outputPath := fs.String("output", "", "Write JSON results to file") + junitPath := fs.String("junit", "", "Write JUnit XML to file") + htmlPath := fs.String("html", "", "Write HTML report to file") + baselinePath := fs.String("baseline", "", "Compare against baseline JSON") + artifactsDir := fs.String("artifacts", "", "Collect artifacts on failure to this directory") + tiers := fs.String("tiers", "", "Comma-separated list of enabled tiers (core,block,devops,chaos)") + fs.Parse(args) + + if fs.NArg() < 1 { + fmt.Fprintln(os.Stderr, "error: scenario file required") + os.Exit(1) + } + scenarioFile := fs.Arg(0) + + logger := log.New(os.Stderr, "", log.LstdFlags) + + scenario, err := tr.ParseFile(scenarioFile) + if err != nil { + logger.Fatalf("parse scenario: %v", err) + } + + // Set up signal handling. + ctx, cancel := signal.NotifyContext(context.Background(), os.Interrupt) + defer cancel() + + // Create registry with all actions. + registry := tr.NewRegistry() + actions.RegisterAll(registry) + if *tiers != "" { + registry.EnableTiers(parseTiers(*tiers)) + } + + logFunc := func(format string, args ...interface{}) { + logger.Printf(format, args...) + } + + // Create engine. + engine := tr.NewEngine(registry, logFunc) + + // Set up infrastructure. + actx, err := setupActionContext(scenario, logFunc) + if err != nil { + logger.Fatalf("setup: %v", err) + } + defer cleanupNodes(actx) + + // Run scenario. + result := engine.Run(ctx, scenario, actx) + + // Print summary. + tr.PrintSummary(os.Stdout, result) + + // Write outputs. + if *outputPath != "" { + if err := tr.WriteJSON(result, *outputPath); err != nil { + logger.Printf("write JSON: %v", err) + } else { + logger.Printf("JSON results written to %s", *outputPath) + } + } + + if *junitPath != "" { + if err := tr.WriteJUnitXML(result, *junitPath); err != nil { + logger.Printf("write JUnit: %v", err) + } else { + logger.Printf("JUnit XML written to %s", *junitPath) + } + } + + if *htmlPath != "" { + if err := tr.WriteHTMLReport(result, *htmlPath); err != nil { + logger.Printf("write HTML: %v", err) + } else { + logger.Printf("HTML report written to %s", *htmlPath) + } + } + + if *baselinePath != "" { + regressions, err := tr.BaselineCompare(result, *baselinePath) + if err != nil { + logger.Printf("baseline compare: %v", err) + } else if len(regressions) > 0 { + fmt.Fprintln(os.Stdout, "\nREGRESSIONS:") + for _, r := range regressions { + fmt.Fprintf(os.Stdout, " - %s\n", r) + } + } else { + fmt.Fprintln(os.Stdout, "\nNo regressions detected.") + } + } + + // Collect artifacts on failure. + if result.Status == tr.StatusFail && *artifactsDir != "" { + collectArtifacts(actx, *artifactsDir, logger) + } + + if result.Status == tr.StatusFail { + os.Exit(1) + } +} + +func collectArtifacts(actx *tr.ActionContext, dir string, logger *log.Logger) { + logger.Printf("collecting artifacts to %s ...", dir) + // Find any node for dmesg/lsblk collection. + var clientNode *infra.Node + for _, n := range actx.Nodes { + if nn, ok := n.(*infra.Node); ok { + clientNode = nn + break + } + } + if clientNode == nil { + logger.Printf("no nodes available for artifact collection") + return + } + + collector := infra.NewArtifactCollector(dir, clientNode, logger) + for name, tgt := range actx.Targets { + if lc, ok := tgt.(infra.LogCollector); ok { + collector.CollectLabeled(lc, name) + } + } +} + +func coordinatorCmd(args []string) { + fs := flag.NewFlagSet("coordinator", flag.ExitOnError) + port := fs.Int("port", 9000, "Listen port for agent registration") + token := fs.String("token", "", "Auth token for agent communication") + dryRun := fs.Bool("dry-run", false, "Print execution plan without running") + outputPath := fs.String("output", "", "Write JSON results to file") + junitPath := fs.String("junit", "", "Write JUnit XML to file") + htmlPath := fs.String("html", "", "Write HTML report to file") + artifactsDir := fs.String("artifacts", "", "Download artifacts from agents to this directory") + regTimeout := fs.String("timeout", "30s", "Agent registration timeout") + coordTiers := fs.String("tiers", "", "Comma-separated list of enabled tiers (core,block,devops,chaos)") + fs.Parse(args) + + if fs.NArg() < 1 { + fmt.Fprintln(os.Stderr, "error: scenario file required") + os.Exit(1) + } + scenarioFile := fs.Arg(0) + + logger := log.New(os.Stderr, "", log.LstdFlags) + + scenario, err := tr.ParseFile(scenarioFile) + if err != nil { + logger.Fatalf("parse scenario: %v", err) + } + + // Verify scenario has agents section. + if len(scenario.Topology.Agents) == 0 { + logger.Fatalf("scenario has no topology.agents section; use 'run' for SSH mode") + } + + // Set up signal handling. + ctx, cancel := signal.NotifyContext(context.Background(), os.Interrupt) + defer cancel() + + // Create registry. + registry := tr.NewRegistry() + actions.RegisterAll(registry) + if *coordTiers != "" { + registry.EnableTiers(parseTiers(*coordTiers)) + } + + // Create coordinator. + coord := tr.NewCoordinator(tr.CoordinatorConfig{ + Port: *port, + Token: *token, + DryRun: *dryRun, + Expected: scenario.Topology.Agents, + Logger: log.New(os.Stderr, "[coord] ", log.LstdFlags), + }) + + if err := coord.Start(); err != nil { + logger.Fatalf("start coordinator: %v", err) + } + defer coord.Stop() + + // Wait for agents. + timeout, _ := time.ParseDuration(*regTimeout) + if timeout == 0 { + timeout = 30 * time.Second + } + logger.Printf("waiting for %d agents (timeout=%s)...", len(scenario.Topology.Agents), timeout) + if err := coord.WaitForAgents(ctx, timeout); err != nil { + logger.Fatalf("%v", err) + } + logger.Printf("all %d agents registered", len(scenario.Topology.Agents)) + + // Run scenario. + result := coord.RunScenario(ctx, scenario, registry) + + // Download artifacts from agents (on both pass and fail). + remoteArtifactsDir := scenario.Artifacts.Dir + if *artifactsDir != "" && remoteArtifactsDir != "" { + coord.DownloadAllArtifacts(ctx, remoteArtifactsDir, *artifactsDir, result) + } + + // Print summary. + tr.PrintSummary(os.Stdout, result) + + if *outputPath != "" { + if err := tr.WriteJSON(result, *outputPath); err != nil { + logger.Printf("write JSON: %v", err) + } + } + if *junitPath != "" { + if err := tr.WriteJUnitXML(result, *junitPath); err != nil { + logger.Printf("write JUnit: %v", err) + } + } + if *htmlPath != "" { + if err := tr.WriteHTMLReport(result, *htmlPath); err != nil { + logger.Printf("write HTML: %v", err) + } else { + logger.Printf("HTML report written to %s", *htmlPath) + } + } + + if result.Status == tr.StatusFail { + os.Exit(1) + } +} + +func agentCmd(args []string) { + fs := flag.NewFlagSet("agent", flag.ExitOnError) + port := fs.Int("port", 9100, "Listen port") + coordURL := fs.String("coordinator", "", "Coordinator URL (e.g. http://192.168.1.100:9000)") + token := fs.String("token", "", "Auth token") + nodes := fs.String("nodes", "", "Comma-separated node names this agent handles") + allowExec := fs.Bool("allow-exec", false, "Enable /exec endpoint") + persistent := fs.Bool("persistent", false, "Stay running, re-register with coordinator on each run") + fs.Parse(args) + + logger := log.New(os.Stderr, "[agent] ", log.LstdFlags) + + // Parse node names. + var nodeNames []string + if *nodes != "" { + for _, n := range strings.Split(*nodes, ",") { + n = strings.TrimSpace(n) + if n != "" { + nodeNames = append(nodeNames, n) + } + } + } + + // Create registry. + registry := tr.NewRegistry() + actions.RegisterAll(registry) + + // Create agent. + agent := tr.NewAgent(tr.AgentConfig{ + Port: *port, + CoordinatorURL: *coordURL, + Token: *token, + AllowExec: *allowExec, + Persistent: *persistent, + Nodes: nodeNames, + Registry: registry, + Logger: logger, + }) + + // Set up signal handling. + ctx, cancel := signal.NotifyContext(context.Background(), os.Interrupt) + defer cancel() + + logger.Printf("starting agent (nodes=%v, exec=%v, persistent=%v)", nodeNames, *allowExec, *persistent) + if err := agent.Start(ctx); err != nil { + logger.Fatalf("agent error: %v", err) + } +} + +func consoleCmd(args []string) { + fs := flag.NewFlagSet("console", flag.ExitOnError) + port := fs.Int("port", 9090, "Listen port for web UI") + token := fs.String("token", "", "Auth token for agents") + scenariosDir := fs.String("scenarios-dir", ".", "Directory containing scenario YAML files") + consoleTiers := fs.String("tiers", "", "Comma-separated list of enabled tiers") + fs.Parse(args) + + logger := log.New(os.Stderr, "[console] ", log.LstdFlags) + + registry := tr.NewRegistry() + actions.RegisterAll(registry) + if *consoleTiers != "" { + registry.EnableTiers(parseTiers(*consoleTiers)) + } + + console := tr.NewConsole(tr.ConsoleConfig{ + Port: *port, + Token: *token, + ScenarioDir: *scenariosDir, + Registry: registry, + Logger: logger, + }) + + ctx, cancel := signal.NotifyContext(context.Background(), os.Interrupt) + defer cancel() + + if err := console.Start(ctx); err != nil { + logger.Fatalf("console error: %v", err) + } +} + +func validateCmd(args []string) { + if len(args) < 1 { + fmt.Fprintln(os.Stderr, "error: scenario file required") + os.Exit(1) + } + + scenario, err := tr.ParseFile(args[0]) + if err != nil { + fmt.Fprintf(os.Stderr, "INVALID: %v\n", err) + os.Exit(1) + } + + fmt.Printf("VALID: %s (%d phases, %d targets)\n", + scenario.Name, len(scenario.Phases), len(scenario.Targets)) +} + +func listCmd() { + // Parse --tiers flag from remaining args. + fs := flag.NewFlagSet("list", flag.ExitOnError) + listTiers := fs.String("tiers", "", "Comma-separated list of enabled tiers") + fs.Parse(os.Args[2:]) + + registry := tr.NewRegistry() + actions.RegisterAll(registry) + if *listTiers != "" { + registry.EnableTiers(parseTiers(*listTiers)) + } + + byTier := registry.ListByTier() + tierOrder := []string{tr.TierCore, tr.TierBlock, tr.TierDevOps, tr.TierChaos} + + fmt.Println("Registered actions:") + for _, tier := range tierOrder { + names := byTier[tier] + if len(names) == 0 { + continue + } + // Skip tiers that are not enabled (if filtering). + if len(registry.EnabledTiers) > 0 && !registry.EnabledTiers[tier] { + continue + } + sort.Strings(names) + fmt.Printf("\n [%s]\n", tier) + for _, name := range names { + fmt.Printf(" - %s\n", name) + } + } + fmt.Println() +} + +// setupActionContext creates nodes, targets, and the action context from the scenario. +func setupActionContext(s *tr.Scenario, logFunc func(string, ...interface{})) (*tr.ActionContext, error) { + actx := &tr.ActionContext{ + Scenario: s, + Nodes: make(map[string]tr.NodeRunner), + Targets: make(map[string]tr.TargetRunner), + Vars: make(map[string]string), + Log: logFunc, + } + + // Create and connect nodes. + for name, spec := range s.Topology.Nodes { + node := &infra.Node{ + Host: spec.Host, + User: spec.User, + KeyFile: spec.KeyFile, + IsLocal: spec.IsLocal, + } + if err := node.Connect(); err != nil { + return nil, fmt.Errorf("connect node %s: %w", name, err) + } + actx.Nodes[name] = node + } + + // Create targets. + for name, spec := range s.Targets { + nodeRunner, ok := actx.Nodes[spec.Node] + if !ok { + return nil, fmt.Errorf("target %s: node %s not found", name, spec.Node) + } + node, ok := nodeRunner.(*infra.Node) + if !ok { + return nil, fmt.Errorf("target %s: node %s is not infra.Node", name, spec.Node) + } + htSpec := infra.HATargetSpec{ + VolSize: spec.VolSize, + WALSize: spec.WALSize, + IQN: spec.IQN(), + ISCSIPort: spec.ISCSIPort, + AdminPort: spec.AdminPort, + ReplicaDataPort: spec.ReplicaDataPort, + ReplicaCtrlPort: spec.ReplicaCtrlPort, + RebuildPort: spec.RebuildPort, + TPGID: spec.TPGID, + } + ht := infra.NewHATargetFromSpec(node, name, htSpec) + actx.Targets[name] = ht + } + + return actx, nil +} + +func cleanupNodes(actx *tr.ActionContext) { + for _, n := range actx.Nodes { + n.Close() + } +} + +// parseTiers splits a comma-separated tier string into a slice. +func parseTiers(s string) []string { + var tiers []string + for _, t := range strings.Split(s, ",") { + t = strings.TrimSpace(t) + if t != "" { + tiers = append(tiers, t) + } + } + return tiers +} diff --git a/weed/storage/blockvol/testrunner/console.go b/weed/storage/blockvol/testrunner/console.go new file mode 100644 index 000000000..3bdea9946 --- /dev/null +++ b/weed/storage/blockvol/testrunner/console.go @@ -0,0 +1,590 @@ +package testrunner + +import ( + "context" + "encoding/json" + "fmt" + "html/template" + "log" + "net" + "net/http" + "os" + "path/filepath" + "sort" + "strings" + "sync" + "time" +) + +// Console serves a web UI for interactive scenario execution. +type Console struct { + Port int + Token string + ScenarioDir string + Registry *Registry + Log *log.Logger + + coordinator *Coordinator + mu sync.Mutex + running bool + currentRun *runState + results map[string]*ScenarioResult + server *http.Server + listener net.Listener +} + +type runState struct { + ScenarioName string `json:"scenario"` + StartedAt time.Time `json:"started_at"` + Status string `json:"status"` // "running", "done", "failed" + Result *ScenarioResult +} + +// ConsoleConfig holds configuration for creating a Console. +type ConsoleConfig struct { + Port int + Token string + ScenarioDir string + Registry *Registry + Logger *log.Logger +} + +// NewConsole creates a new Console server. +func NewConsole(cfg ConsoleConfig) *Console { + logger := cfg.Logger + if logger == nil { + logger = log.New(os.Stderr, "[console] ", log.LstdFlags) + } + + // Create an internal coordinator for agent management. + coord := NewCoordinator(CoordinatorConfig{ + Port: cfg.Port + 1, // agents register on port+1 + Token: cfg.Token, + Expected: make(map[string]string), // no expected agents by default + Logger: log.New(os.Stderr, "[coord] ", log.LstdFlags), + }) + + return &Console{ + Port: cfg.Port, + Token: cfg.Token, + ScenarioDir: cfg.ScenarioDir, + Registry: cfg.Registry, + Log: logger, + coordinator: coord, + results: make(map[string]*ScenarioResult), + } +} + +// Start begins serving the console web UI. Blocks until stopped. +func (c *Console) Start(ctx context.Context) error { + mux := http.NewServeMux() + mux.HandleFunc("/", c.handleIndex) + mux.HandleFunc("/api/scenarios", c.handleScenarios) + mux.HandleFunc("/api/run", c.handleRun) + mux.HandleFunc("/api/status", c.handleStatus) + mux.HandleFunc("/api/result/", c.handleResult) + mux.HandleFunc("/api/report/", c.handleReport) + mux.HandleFunc("/api/agents", c.handleAgents) + mux.HandleFunc("/api/tiers", c.handleTiers) + mux.HandleFunc("/register", c.coordinator.handleRegister) + + addr := fmt.Sprintf(":%d", c.Port) + ln, err := net.Listen("tcp", addr) + if err != nil { + return fmt.Errorf("listen %s: %w", addr, err) + } + + c.mu.Lock() + c.listener = ln + c.server = &http.Server{Handler: mux} + c.mu.Unlock() + + c.Log.Printf("console listening on http://localhost:%d", c.Port) + c.Log.Printf("scenarios dir: %s", c.ScenarioDir) + + go func() { + <-ctx.Done() + c.Stop() + }() + + if err := c.server.Serve(ln); err != nil && err != http.ErrServerClosed { + return err + } + return nil +} + +// Stop gracefully shuts down the console server. +func (c *Console) Stop() { + c.mu.Lock() + srv := c.server + c.mu.Unlock() + + if srv != nil { + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + srv.Shutdown(ctx) + } +} + +// ListenAddr returns the address the console is listening on. +func (c *Console) ListenAddr() string { + c.mu.Lock() + defer c.mu.Unlock() + if c.listener != nil { + return c.listener.Addr().String() + } + return "" +} + +// --- API Handlers --- + +type scenarioInfo struct { + Name string `json:"name"` + File string `json:"file"` + Phases int `json:"phases"` +} + +// GET /api/scenarios +func (c *Console) handleScenarios(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodGet { + http.Error(w, "method not allowed", http.StatusMethodNotAllowed) + return + } + + files, err := filepath.Glob(filepath.Join(c.ScenarioDir, "*.yaml")) + if err != nil { + writeJSON(w, http.StatusInternalServerError, map[string]string{"error": err.Error()}) + return + } + + var scenarios []scenarioInfo + for _, f := range files { + s, err := ParseFile(f) + if err != nil { + continue // skip invalid files + } + scenarios = append(scenarios, scenarioInfo{ + Name: s.Name, + File: filepath.Base(f), + Phases: len(s.Phases), + }) + } + + sort.Slice(scenarios, func(i, j int) bool { return scenarios[i].File < scenarios[j].File }) + writeJSON(w, http.StatusOK, scenarios) +} + +type runRequest struct { + Scenario string `json:"scenario"` +} + +// POST /api/run +func (c *Console) handleRun(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodPost { + http.Error(w, "method not allowed", http.StatusMethodNotAllowed) + return + } + + var req runRequest + if err := json.NewDecoder(r.Body).Decode(&req); err != nil { + writeJSON(w, http.StatusBadRequest, map[string]string{"error": fmt.Sprintf("decode: %v", err)}) + return + } + + if req.Scenario == "" { + writeJSON(w, http.StatusBadRequest, map[string]string{"error": "scenario field required"}) + return + } + + c.mu.Lock() + if c.running { + c.mu.Unlock() + writeJSON(w, http.StatusConflict, map[string]string{"error": "a scenario is already running"}) + return + } + c.running = true + c.currentRun = &runState{ + ScenarioName: req.Scenario, + StartedAt: time.Now(), + Status: "running", + } + c.mu.Unlock() + + // Parse and validate. + scenarioPath := filepath.Join(c.ScenarioDir, req.Scenario) + scenario, err := ParseFile(scenarioPath) + if err != nil { + c.mu.Lock() + c.running = false + c.currentRun = nil + c.mu.Unlock() + writeJSON(w, http.StatusBadRequest, map[string]string{"error": fmt.Sprintf("parse: %v", err)}) + return + } + + // Launch in background. + go c.executeScenario(scenario, req.Scenario) + + writeJSON(w, http.StatusAccepted, map[string]string{"status": "started", "scenario": req.Scenario}) +} + +func (c *Console) executeScenario(scenario *Scenario, fileName string) { + ctx := context.Background() + if scenario.Timeout.Duration > 0 { + var cancel context.CancelFunc + ctx, cancel = context.WithTimeout(ctx, scenario.Timeout.Duration) + defer cancel() + } + + logFunc := func(format string, args ...interface{}) { + c.Log.Printf(format, args...) + } + + engine := NewEngine(c.Registry, logFunc) + actx := &ActionContext{ + Scenario: scenario, + Nodes: make(map[string]NodeRunner), + Targets: make(map[string]TargetRunner), + Vars: make(map[string]string), + Log: logFunc, + } + + result := engine.Run(ctx, scenario, actx) + + name := strings.TrimSuffix(fileName, ".yaml") + c.mu.Lock() + c.results[name] = result + if c.currentRun != nil { + c.currentRun.Status = "done" + if result.Status == StatusFail { + c.currentRun.Status = "failed" + } + c.currentRun.Result = result + } + c.running = false + c.mu.Unlock() + + c.Log.Printf("scenario %s completed: %s (%s)", fileName, result.Status, result.Duration) +} + +type statusResponse struct { + Running bool `json:"running"` + Scenario string `json:"scenario,omitempty"` + Status string `json:"status,omitempty"` + Elapsed string `json:"elapsed,omitempty"` +} + +// GET /api/status +func (c *Console) handleStatus(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodGet { + http.Error(w, "method not allowed", http.StatusMethodNotAllowed) + return + } + + c.mu.Lock() + defer c.mu.Unlock() + + resp := statusResponse{Running: c.running} + if c.currentRun != nil { + resp.Scenario = c.currentRun.ScenarioName + resp.Status = c.currentRun.Status + resp.Elapsed = time.Since(c.currentRun.StartedAt).Round(time.Second).String() + } + + writeJSON(w, http.StatusOK, resp) +} + +// GET /api/result/{name} +func (c *Console) handleResult(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodGet { + http.Error(w, "method not allowed", http.StatusMethodNotAllowed) + return + } + + name := strings.TrimPrefix(r.URL.Path, "/api/result/") + if name == "" { + writeJSON(w, http.StatusBadRequest, map[string]string{"error": "name required"}) + return + } + + c.mu.Lock() + result, ok := c.results[name] + c.mu.Unlock() + if !ok { + writeJSON(w, http.StatusNotFound, map[string]string{"error": "no result for " + name}) + return + } + + writeJSON(w, http.StatusOK, result) +} + +// GET /api/report/{name} +func (c *Console) handleReport(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodGet { + http.Error(w, "method not allowed", http.StatusMethodNotAllowed) + return + } + + name := strings.TrimPrefix(r.URL.Path, "/api/report/") + if name == "" { + writeJSON(w, http.StatusBadRequest, map[string]string{"error": "name required"}) + return + } + + c.mu.Lock() + result, ok := c.results[name] + c.mu.Unlock() + if !ok { + writeJSON(w, http.StatusNotFound, map[string]string{"error": "no result for " + name}) + return + } + + // Render HTML report inline. + data := buildHTMLData(result) + tmpl, err := template.New("report").Parse(htmlTemplate) + if err != nil { + http.Error(w, "template error: "+err.Error(), http.StatusInternalServerError) + return + } + + w.Header().Set("Content-Type", "text/html; charset=utf-8") + tmpl.Execute(w, data) +} + +type agentInfo struct { + Name string `json:"name"` + Addr string `json:"addr"` + Healthy bool `json:"healthy"` +} + +// GET /api/agents +func (c *Console) handleAgents(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodGet { + http.Error(w, "method not allowed", http.StatusMethodNotAllowed) + return + } + + c.coordinator.mu.Lock() + var agents []agentInfo + for name, info := range c.coordinator.agents { + agents = append(agents, agentInfo{ + Name: name, + Addr: info.Addr, + Healthy: info.Healthy, + }) + } + c.coordinator.mu.Unlock() + + sort.Slice(agents, func(i, j int) bool { return agents[i].Name < agents[j].Name }) + writeJSON(w, http.StatusOK, agents) +} + +// GET /api/tiers +func (c *Console) handleTiers(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodGet { + http.Error(w, "method not allowed", http.StatusMethodNotAllowed) + return + } + writeJSON(w, http.StatusOK, c.Registry.ListByTier()) +} + +// GET / +func (c *Console) handleIndex(w http.ResponseWriter, r *http.Request) { + if r.URL.Path != "/" { + http.NotFound(w, r) + return + } + w.Header().Set("Content-Type", "text/html; charset=utf-8") + w.Write([]byte(consoleSPA)) +} + +// consoleSPA is the embedded single-page application. +var consoleSPA = strings.TrimSpace(` + + + +sw-test-runner Console + + + +
+

sw-test-runner

+ Console +
+
+ +
+
+ + +
+
+ Select a scenario and click Run +
+
+
+
+ Agents: checking... + +
+ + +`) diff --git a/weed/storage/blockvol/testrunner/console_test.go b/weed/storage/blockvol/testrunner/console_test.go new file mode 100644 index 000000000..662a14fe0 --- /dev/null +++ b/weed/storage/blockvol/testrunner/console_test.go @@ -0,0 +1,332 @@ +package testrunner + +import ( + "context" + "encoding/json" + "fmt" + "net/http" + "os" + "path/filepath" + "strings" + "testing" + "time" +) + +func TestConsole_ScenariosEndpoint(t *testing.T) { + dir := t.TempDir() + writeTestScenario(t, dir, "smoke.yaml", "Smoke Test", 2) + writeTestScenario(t, dir, "ha.yaml", "HA Test", 4) + + console, baseURL := startTestConsole(t, dir) + _ = console + + resp, err := http.Get(baseURL + "/api/scenarios") + if err != nil { + t.Fatal(err) + } + defer resp.Body.Close() + + if resp.StatusCode != 200 { + t.Fatalf("status = %d", resp.StatusCode) + } + + var scenarios []scenarioInfo + json.NewDecoder(resp.Body).Decode(&scenarios) + + if len(scenarios) != 2 { + t.Fatalf("scenarios = %d, want 2", len(scenarios)) + } +} + +func TestConsole_StatusEndpoint_NoRun(t *testing.T) { + dir := t.TempDir() + _, baseURL := startTestConsole(t, dir) + + resp, err := http.Get(baseURL + "/api/status") + if err != nil { + t.Fatal(err) + } + defer resp.Body.Close() + + var st statusResponse + json.NewDecoder(resp.Body).Decode(&st) + + if st.Running { + t.Error("should not be running") + } +} + +func TestConsole_RunEndpoint_MissingScenario(t *testing.T) { + dir := t.TempDir() + _, baseURL := startTestConsole(t, dir) + + resp, err := http.Post(baseURL+"/api/run", "application/json", + strings.NewReader(`{"scenario":""}`)) + if err != nil { + t.Fatal(err) + } + defer resp.Body.Close() + + if resp.StatusCode != 400 { + t.Errorf("status = %d, want 400", resp.StatusCode) + } +} + +func TestConsole_RunEndpoint_InvalidScenario(t *testing.T) { + dir := t.TempDir() + _, baseURL := startTestConsole(t, dir) + + resp, err := http.Post(baseURL+"/api/run", "application/json", + strings.NewReader(`{"scenario":"nonexistent.yaml"}`)) + if err != nil { + t.Fatal(err) + } + defer resp.Body.Close() + + if resp.StatusCode != 400 { + t.Errorf("status = %d, want 400", resp.StatusCode) + } +} + +func TestConsole_RunAndPollStatus(t *testing.T) { + dir := t.TempDir() + // Write a scenario with a registered action. + writeTestScenarioWithAction(t, dir, "test.yaml", "Test Run", "console_noop") + + registry := NewRegistry() + registry.RegisterFunc("console_noop", TierCore, func(ctx context.Context, actx *ActionContext, act Action) (map[string]string, error) { + return map[string]string{"value": "done"}, nil + }) + + console, baseURL := startTestConsoleWithRegistry(t, dir, registry) + _ = console + + // Run the scenario. + resp, err := http.Post(baseURL+"/api/run", "application/json", + strings.NewReader(`{"scenario":"test.yaml"}`)) + if err != nil { + t.Fatal(err) + } + resp.Body.Close() + + if resp.StatusCode != 202 { + t.Fatalf("status = %d, want 202", resp.StatusCode) + } + + // Poll until done. + deadline := time.After(5 * time.Second) + for { + select { + case <-deadline: + t.Fatal("timeout waiting for scenario to complete") + case <-time.After(100 * time.Millisecond): + } + + statusResp, err := http.Get(baseURL + "/api/status") + if err != nil { + continue + } + var st statusResponse + json.NewDecoder(statusResp.Body).Decode(&st) + statusResp.Body.Close() + + if !st.Running && st.Status != "" { + if st.Status != "done" { + t.Errorf("status = %s, want done", st.Status) + } + break + } + } + + // Check result endpoint. + resultResp, err := http.Get(baseURL + "/api/result/test") + if err != nil { + t.Fatal(err) + } + defer resultResp.Body.Close() + + if resultResp.StatusCode != 200 { + t.Fatalf("result status = %d", resultResp.StatusCode) + } + + var result ScenarioResult + json.NewDecoder(resultResp.Body).Decode(&result) + if result.Status != StatusPass { + t.Errorf("result.Status = %s, want PASS", result.Status) + } +} + +func TestConsole_ConflictOnDoubleRun(t *testing.T) { + dir := t.TempDir() + writeTestScenarioWithAction(t, dir, "slow.yaml", "Slow Test", "console_slow") + + registry := NewRegistry() + registry.RegisterFunc("console_slow", TierCore, func(ctx context.Context, actx *ActionContext, act Action) (map[string]string, error) { + time.Sleep(2 * time.Second) + return nil, nil + }) + + _, baseURL := startTestConsoleWithRegistry(t, dir, registry) + + // Start first run. + resp1, _ := http.Post(baseURL+"/api/run", "application/json", + strings.NewReader(`{"scenario":"slow.yaml"}`)) + resp1.Body.Close() + if resp1.StatusCode != 202 { + t.Fatalf("first run status = %d", resp1.StatusCode) + } + + // Wait a moment for it to start. + time.Sleep(100 * time.Millisecond) + + // Try second run — should get 409. + resp2, _ := http.Post(baseURL+"/api/run", "application/json", + strings.NewReader(`{"scenario":"slow.yaml"}`)) + resp2.Body.Close() + if resp2.StatusCode != 409 { + t.Errorf("second run status = %d, want 409", resp2.StatusCode) + } +} + +func TestConsole_AgentsEndpoint(t *testing.T) { + dir := t.TempDir() + _, baseURL := startTestConsole(t, dir) + + resp, err := http.Get(baseURL + "/api/agents") + if err != nil { + t.Fatal(err) + } + defer resp.Body.Close() + + var agents []agentInfo + json.NewDecoder(resp.Body).Decode(&agents) + + // No agents connected by default. + if len(agents) != 0 { + t.Errorf("agents = %d, want 0", len(agents)) + } +} + +func TestConsole_TiersEndpoint(t *testing.T) { + dir := t.TempDir() + registry := NewRegistry() + registry.RegisterFunc("a", TierCore, func(ctx context.Context, actx *ActionContext, act Action) (map[string]string, error) { + return nil, nil + }) + registry.RegisterFunc("b", TierBlock, func(ctx context.Context, actx *ActionContext, act Action) (map[string]string, error) { + return nil, nil + }) + + _, baseURL := startTestConsoleWithRegistry(t, dir, registry) + + resp, err := http.Get(baseURL + "/api/tiers") + if err != nil { + t.Fatal(err) + } + defer resp.Body.Close() + + var tiers map[string][]string + json.NewDecoder(resp.Body).Decode(&tiers) + + if len(tiers["core"]) != 1 || tiers["core"][0] != "a" { + t.Errorf("core tier = %v", tiers["core"]) + } + if len(tiers["block"]) != 1 || tiers["block"][0] != "b" { + t.Errorf("block tier = %v", tiers["block"]) + } +} + +func TestConsole_ReportNotFound(t *testing.T) { + dir := t.TempDir() + _, baseURL := startTestConsole(t, dir) + + resp, err := http.Get(baseURL + "/api/report/nonexistent") + if err != nil { + t.Fatal(err) + } + defer resp.Body.Close() + + if resp.StatusCode != 404 { + t.Errorf("status = %d, want 404", resp.StatusCode) + } +} + +func TestConsole_IndexPage(t *testing.T) { + dir := t.TempDir() + _, baseURL := startTestConsole(t, dir) + + resp, err := http.Get(baseURL + "/") + if err != nil { + t.Fatal(err) + } + defer resp.Body.Close() + + if resp.StatusCode != 200 { + t.Fatalf("status = %d", resp.StatusCode) + } + + ct := resp.Header.Get("Content-Type") + if !strings.Contains(ct, "text/html") { + t.Errorf("content-type = %s, want text/html", ct) + } +} + +// --- Helpers --- + +func startTestConsole(t *testing.T, scenariosDir string) (*Console, string) { + t.Helper() + return startTestConsoleWithRegistry(t, scenariosDir, NewRegistry()) +} + +func startTestConsoleWithRegistry(t *testing.T, scenariosDir string, registry *Registry) (*Console, string) { + t.Helper() + + console := NewConsole(ConsoleConfig{ + Port: 0, + ScenarioDir: scenariosDir, + Registry: registry, + }) + + // Override to use ephemeral port — bind directly. + ctx, cancel := context.WithCancel(context.Background()) + t.Cleanup(cancel) + + go console.Start(ctx) + + // Wait for listener. + for i := 0; i < 50; i++ { + if console.ListenAddr() != "" { + break + } + time.Sleep(20 * time.Millisecond) + } + + addr := console.ListenAddr() + if addr == "" { + t.Fatal("console failed to start") + } + + return console, fmt.Sprintf("http://%s", addr) +} + +func writeTestScenario(t *testing.T, dir, filename, name string, phases int) { + t.Helper() + content := fmt.Sprintf("name: %s\ntimeout: 30s\nphases:\n", name) + for i := 0; i < phases; i++ { + content += fmt.Sprintf(" - name: phase%d\n actions:\n - action: noop\n", i) + } + os.WriteFile(filepath.Join(dir, filename), []byte(content), 0644) +} + +func writeTestScenarioWithAction(t *testing.T, dir, filename, name, action string) { + t.Helper() + content := fmt.Sprintf(`name: %s +timeout: 30s +phases: + - name: main + actions: + - action: %s + save_as: result +`, name, action) + os.WriteFile(filepath.Join(dir, filename), []byte(content), 0644) +} diff --git a/weed/storage/blockvol/testrunner/coordinator.go b/weed/storage/blockvol/testrunner/coordinator.go new file mode 100644 index 000000000..51c840b05 --- /dev/null +++ b/weed/storage/blockvol/testrunner/coordinator.go @@ -0,0 +1,1024 @@ +package testrunner + +import ( + "archive/tar" + "bytes" + "compress/gzip" + "context" + "encoding/json" + "fmt" + "io" + "log" + "net" + "net/http" + "os" + "path/filepath" + "strings" + "sync" + "time" +) + +// Coordinator manages multi-node test execution by dispatching phases to remote agents. +type Coordinator struct { + Port int + Token string + DryRun bool + log *log.Logger + + mu sync.Mutex + agents map[string]*AgentInfo // agent name → info + expected map[string]string // agent name → "host:port" from topology + server *http.Server + listener net.Listener + ready chan struct{} // closed when all agents registered + + registry *Registry // set during RunScenario for coordinator-local actions +} + +// AgentInfo holds state for a registered agent. +type AgentInfo struct { + RegisterRequest + Addr string // "host:port" for HTTP requests + Healthy bool +} + +// CoordinatorConfig holds configuration for creating a Coordinator. +type CoordinatorConfig struct { + Port int + Token string + DryRun bool + Expected map[string]string // agent name → "host:port" + Logger *log.Logger +} + +// NewCoordinator creates a new Coordinator. +func NewCoordinator(cfg CoordinatorConfig) *Coordinator { + logger := cfg.Logger + if logger == nil { + logger = log.New(os.Stderr, "[coord] ", log.LstdFlags) + } + return &Coordinator{ + Port: cfg.Port, + Token: cfg.Token, + DryRun: cfg.DryRun, + log: logger, + agents: make(map[string]*AgentInfo), + expected: cfg.Expected, + ready: make(chan struct{}), + } +} + +// Start begins listening for agent registrations. +func (c *Coordinator) Start() error { + mux := http.NewServeMux() + mux.HandleFunc("/register", c.handleRegister) + + addr := fmt.Sprintf(":%d", c.Port) + ln, err := net.Listen("tcp", addr) + if err != nil { + return fmt.Errorf("listen %s: %w", addr, err) + } + + c.mu.Lock() + c.listener = ln + c.server = &http.Server{Handler: mux} + c.mu.Unlock() + + c.log.Printf("coordinator listening on %s, expecting %d agents", addr, len(c.expected)) + + go func() { + if err := c.server.Serve(ln); err != nil && err != http.ErrServerClosed { + c.log.Printf("serve error: %v", err) + } + }() + + return nil +} + +// Stop shuts down the coordinator HTTP server. +func (c *Coordinator) Stop() { + c.mu.Lock() + srv := c.server + c.mu.Unlock() + + if srv != nil { + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + srv.Shutdown(ctx) + } +} + +// ListenAddr returns the address the coordinator is listening on. +func (c *Coordinator) ListenAddr() string { + c.mu.Lock() + defer c.mu.Unlock() + if c.listener != nil { + return c.listener.Addr().String() + } + return "" +} + +// WaitForAgents waits until all expected agents have registered. +func (c *Coordinator) WaitForAgents(ctx context.Context, timeout time.Duration) error { + if len(c.expected) == 0 { + return nil + } + + ctx, cancel := context.WithTimeout(ctx, timeout) + defer cancel() + + select { + case <-c.ready: + return nil + case <-ctx.Done(): + c.mu.Lock() + registered := make([]string, 0, len(c.agents)) + for name := range c.agents { + registered = append(registered, name) + } + missing := make([]string, 0) + for name := range c.expected { + if _, ok := c.agents[name]; !ok { + missing = append(missing, name) + } + } + c.mu.Unlock() + return fmt.Errorf("timeout waiting for agents: registered=%v missing=%v", registered, missing) + } +} + +// RegisterAgent directly registers an agent (used by tests and manual setup). +func (c *Coordinator) RegisterAgent(name string, addr string) { + c.mu.Lock() + defer c.mu.Unlock() + c.agents[name] = &AgentInfo{ + RegisterRequest: RegisterRequest{AgentID: name, Hostname: name}, + Addr: addr, + Healthy: true, + } + c.checkAllRegistered() +} + +// POST /register +func (c *Coordinator) handleRegister(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodPost { + http.Error(w, "method not allowed", http.StatusMethodNotAllowed) + return + } + + if c.Token != "" { + token := r.Header.Get(AuthTokenHeader) + if token != c.Token { + writeJSON(w, http.StatusUnauthorized, RegisterResponse{Error: "invalid auth token"}) + return + } + } + + var req RegisterRequest + if err := json.NewDecoder(r.Body).Decode(&req); err != nil { + writeJSON(w, http.StatusBadRequest, RegisterResponse{Error: fmt.Sprintf("decode: %v", err)}) + return + } + + c.mu.Lock() + defer c.mu.Unlock() + + // Match agent to expected agents by node names. + agentName := "" + for _, nodeName := range req.Nodes { + if _, ok := c.expected[nodeName]; ok { + agentName = nodeName + break + } + } + if agentName == "" && req.Hostname != "" { + if _, ok := c.expected[req.Hostname]; ok { + agentName = req.Hostname + } + } + if agentName == "" { + writeJSON(w, http.StatusBadRequest, RegisterResponse{Error: fmt.Sprintf("no matching agent for nodes=%v hostname=%s", req.Nodes, req.Hostname)}) + return + } + + addr := fmt.Sprintf("%s:%d", req.IP, req.Port) + c.agents[agentName] = &AgentInfo{ + RegisterRequest: req, + Addr: addr, + Healthy: true, + } + + c.log.Printf("agent %q registered (id=%s, addr=%s, nodes=%v)", agentName, req.AgentID, addr, req.Nodes) + + resp := RegisterResponse{ + OK: true, + AgentIndex: len(c.agents), + TotalAgents: len(c.expected), + } + c.checkAllRegistered() + + writeJSON(w, http.StatusOK, resp) +} + +func (c *Coordinator) checkAllRegistered() { + for name := range c.expected { + if _, ok := c.agents[name]; !ok { + return + } + } + // All registered — close the ready channel (once). + select { + case <-c.ready: + default: + close(c.ready) + } +} + +// RunScenario executes a scenario by dispatching phases to agents. +func (c *Coordinator) RunScenario(ctx context.Context, s *Scenario, registry *Registry) *ScenarioResult { + c.registry = registry + start := time.Now() + result := &ScenarioResult{ + Name: s.Name, + Status: StatusPass, + } + + // Apply scenario timeout. + if s.Timeout.Duration > 0 { + var cancel context.CancelFunc + ctx, cancel = context.WithTimeout(ctx, s.Timeout.Duration) + defer cancel() + } + + // Seed global vars from env. + globalVars := make(map[string]string) + for k, v := range s.Env { + globalVars[k] = v + } + + // Build node→agent mapping. + nodeToAgent := c.buildNodeAgentMap(s) + + // Separate always-phases for deferred cleanup. + var normalPhases, alwaysPhases []Phase + for _, p := range s.Phases { + if p.Always { + alwaysPhases = append(alwaysPhases, p) + } else { + normalPhases = append(normalPhases, p) + } + } + + if c.DryRun { + c.printDryRun(s, nodeToAgent, normalPhases, alwaysPhases) + result.Duration = time.Since(start) + return result + } + + // Execute normal phases. + failed := false + for i, phase := range normalPhases { + pr := c.runPhase(ctx, s, i, phase, globalVars, nodeToAgent) + result.Phases = append(result.Phases, pr) + if pr.Status == StatusFail { + failed = true + result.Status = StatusFail + result.Error = fmt.Sprintf("phase %q failed: %s", phase.Name, pr.Error) + break + } + } + + // Always-phases run on every agent regardless of failure. + for i, phase := range alwaysPhases { + pr := c.runPhase(ctx, s, len(normalPhases)+i, phase, globalVars, nodeToAgent) + result.Phases = append(result.Phases, pr) + } + + result.Duration = time.Since(start) + if !failed { + result.Status = StatusPass + } + + // Preserve all final vars in the result for downstream reporting. + if len(globalVars) > 0 { + result.Vars = make(map[string]string, len(globalVars)) + for k, v := range globalVars { + result.Vars[k] = v + } + } + + return result +} + +// buildNodeAgentMap creates a mapping from node name → agent name. +func (c *Coordinator) buildNodeAgentMap(s *Scenario) map[string]string { + m := make(map[string]string) + for nodeName, nodeSpec := range s.Topology.Nodes { + if nodeSpec.Agent != "" { + m[nodeName] = nodeSpec.Agent + } else { + // Try to match by hostname in registered agents. + c.mu.Lock() + for agentName := range c.agents { + if agentName == nodeName { + m[nodeName] = agentName + break + } + } + c.mu.Unlock() + } + } + return m +} + +// resolveActionAgent determines which agent should execute an action. +func (c *Coordinator) resolveActionAgent(s *Scenario, act Action, nodeToAgent map[string]string) string { + // 1. Explicit node reference. + if act.Node != "" { + if agent, ok := nodeToAgent[act.Node]; ok { + return agent + } + } + // 2. Target reference → target's node → agent. + if act.Target != "" { + if tSpec, ok := s.Targets[act.Target]; ok { + if agent, ok := nodeToAgent[tSpec.Node]; ok { + return agent + } + } + } + // 3. Fallback: first agent. + c.mu.Lock() + defer c.mu.Unlock() + for name := range c.agents { + return name + } + return "" +} + +// runPhase executes a single phase across agents. +func (c *Coordinator) runPhase(ctx context.Context, s *Scenario, phaseIdx int, phase Phase, globalVars map[string]string, nodeToAgent map[string]string) PhaseResult { + start := time.Now() + c.log.Printf("[phase %d] %s (%d actions)", phaseIdx, phase.Name, len(phase.Actions)) + + var pr PhaseResult + if phase.Parallel { + pr = c.runPhaseParallel(ctx, s, phaseIdx, phase, globalVars, nodeToAgent) + } else { + pr = c.runPhaseSequential(ctx, s, phaseIdx, phase, globalVars, nodeToAgent) + } + + pr.Name = phase.Name + pr.Duration = time.Since(start) + return pr +} + +// runPhaseSequential dispatches actions one by one, merging vars between them. +func (c *Coordinator) runPhaseSequential(ctx context.Context, s *Scenario, phaseIdx int, phase Phase, globalVars map[string]string, nodeToAgent map[string]string) PhaseResult { + pr := PhaseResult{Status: StatusPass} + + // Check if all actions target the same agent, none have retries, and none are + // coordinator-local — batch optimization. + hasRetry := false + hasLocal := false + for _, act := range phase.Actions { + if act.Retry > 0 { + hasRetry = true + } + if c.isCoordinatorLocalAction(act) { + hasLocal = true + } + } + if !hasRetry && !hasLocal { + if agent := c.allSameAgent(s, phase.Actions, nodeToAgent); agent != "" { + return c.dispatchBatch(ctx, s, phaseIdx, phase, globalVars, agent) + } + } + + // Otherwise, dispatch one action at a time. + for i, act := range phase.Actions { + // Coordinator-local actions: run on coordinator, not dispatched to agents. + if c.isCoordinatorLocalAction(act) { + ar := c.runCoordinatorLocalAction(ctx, s, act, globalVars) + pr.Actions = append(pr.Actions, ar) + if ar.Status == StatusFail && !act.IgnoreError { + pr.Status = StatusFail + pr.Error = fmt.Sprintf("action %d (%s) failed: %s", i, act.Action, ar.Error) + return pr + } + continue + } + + agentName := c.resolveActionAgent(s, act, nodeToAgent) + if agentName == "" { + ar := ActionResult{ + Action: act.Action, + Status: StatusFail, + Error: "no agent found for action", + } + pr.Actions = append(pr.Actions, ar) + if !act.IgnoreError { + pr.Status = StatusFail + pr.Error = fmt.Sprintf("action %d (%s): no agent", i, act.Action) + return pr + } + continue + } + + ar := c.dispatchSingleAction(ctx, s, phaseIdx, phase.Name, act, globalVars, agentName) + pr.Actions = append(pr.Actions, ar) + + if ar.Status == StatusFail && !act.IgnoreError { + pr.Status = StatusFail + pr.Error = fmt.Sprintf("action %d (%s) failed: %s", i, act.Action, ar.Error) + return pr + } + } + + return pr +} + +// runPhaseParallel dispatches all actions to their agents in parallel. +func (c *Coordinator) runPhaseParallel(ctx context.Context, s *Scenario, phaseIdx int, phase Phase, globalVars map[string]string, nodeToAgent map[string]string) PhaseResult { + pr := PhaseResult{Status: StatusPass} + + // Group actions by agent. + type agentWork struct { + agent string + actions []Action + } + groups := make(map[string]*agentWork) + for _, act := range phase.Actions { + agentName := c.resolveActionAgent(s, act, nodeToAgent) + if agentName == "" { + agentName = "__unresolved" + } + if _, ok := groups[agentName]; !ok { + groups[agentName] = &agentWork{agent: agentName} + } + groups[agentName].actions = append(groups[agentName].actions, act) + } + + type groupResult struct { + agent string + resp *PhaseResponse + err error + } + + results := make(chan groupResult, len(groups)) + for _, g := range groups { + go func(aw *agentWork) { + if aw.agent == "__unresolved" { + results <- groupResult{agent: aw.agent, err: fmt.Errorf("no agent for actions")} + return + } + resp, err := c.sendPhaseRequest(ctx, aw.agent, &PhaseRequest{ + PhaseIndex: phaseIdx, + PhaseName: phase.Name, + Actions: aw.actions, + GlobalVars: globalVars, + Scenario: s, + }) + results <- groupResult{agent: aw.agent, resp: resp, err: err} + }(g) + } + + for range groups { + gr := <-results + if gr.err != nil { + pr.Status = StatusFail + if pr.Error == "" { + pr.Error = fmt.Sprintf("agent %s: %v", gr.agent, gr.err) + } + continue + } + if gr.resp != nil { + pr.Actions = append(pr.Actions, gr.resp.Results...) + // Merge new vars. + for k, v := range gr.resp.NewVars { + globalVars[k] = v + } + if gr.resp.Error != "" { + pr.Status = StatusFail + if pr.Error == "" { + pr.Error = fmt.Sprintf("agent %s: %s", gr.agent, gr.resp.Error) + } + } + } + } + + return pr +} + +// allSameAgent returns the agent name if all actions target the same agent, or "". +func (c *Coordinator) allSameAgent(s *Scenario, actions []Action, nodeToAgent map[string]string) string { + if len(actions) == 0 { + return "" + } + first := c.resolveActionAgent(s, actions[0], nodeToAgent) + if first == "" { + return "" + } + for _, act := range actions[1:] { + if c.resolveActionAgent(s, act, nodeToAgent) != first { + return "" + } + } + return first +} + +// dispatchBatch sends an entire phase to a single agent. +func (c *Coordinator) dispatchBatch(ctx context.Context, s *Scenario, phaseIdx int, phase Phase, globalVars map[string]string, agent string) PhaseResult { + pr := PhaseResult{Status: StatusPass} + + resp, err := c.sendPhaseRequest(ctx, agent, &PhaseRequest{ + PhaseIndex: phaseIdx, + PhaseName: phase.Name, + Actions: phase.Actions, + GlobalVars: globalVars, + Scenario: s, + }) + + if err != nil { + pr.Status = StatusFail + pr.Error = fmt.Sprintf("agent %s: %v", agent, err) + return pr + } + + pr.Actions = resp.Results + for k, v := range resp.NewVars { + globalVars[k] = v + } + if resp.Error != "" { + pr.Status = StatusFail + pr.Error = resp.Error + } + return pr +} + +// dispatchSingleAction sends a single action to an agent with retry support. +func (c *Coordinator) dispatchSingleAction(ctx context.Context, s *Scenario, phaseIdx int, phaseName string, act Action, globalVars map[string]string, agentName string) ActionResult { + maxRetries := act.Retry + var actionTimeout time.Duration + if act.Timeout != "" { + if d, err := time.ParseDuration(act.Timeout); err == nil { + actionTimeout = d + } + } + + var lastResult ActionResult + + for attempt := 0; attempt <= maxRetries; attempt++ { + if attempt > 0 { + c.log.Printf(" [retry] action %s attempt %d/%d", act.Action, attempt+1, maxRetries+1) + // Backoff: 2s between retries. + select { + case <-time.After(2 * time.Second): + case <-ctx.Done(): + return ActionResult{Action: act.Action, Status: StatusFail, Error: ctx.Err().Error()} + } + } + + reqCtx := ctx + if actionTimeout > 0 { + var cancel context.CancelFunc + reqCtx, cancel = context.WithTimeout(ctx, actionTimeout+5*time.Second) // buffer + defer cancel() + } + + resp, err := c.sendPhaseRequest(reqCtx, agentName, &PhaseRequest{ + PhaseIndex: phaseIdx, + PhaseName: phaseName, + Actions: []Action{act}, + GlobalVars: globalVars, + Scenario: s, + }) + + if err != nil { + lastResult = ActionResult{Action: act.Action, Status: StatusFail, Error: fmt.Sprintf("agent %s: %v", agentName, err)} + continue + } + + // Merge vars regardless of status. + for k, v := range resp.NewVars { + globalVars[k] = v + } + + if len(resp.Results) > 0 { + lastResult = resp.Results[0] + } else { + lastResult = ActionResult{Action: act.Action, Status: StatusPass} + } + + if resp.Error == "" && lastResult.Status != StatusFail { + return lastResult + } + + if resp.Error != "" && lastResult.Error == "" { + lastResult.Error = resp.Error + } + } + + return lastResult +} + +// sendPhaseRequest sends a POST /phase to an agent and returns the response. +func (c *Coordinator) sendPhaseRequest(ctx context.Context, agentName string, req *PhaseRequest) (*PhaseResponse, error) { + c.mu.Lock() + agent, ok := c.agents[agentName] + c.mu.Unlock() + if !ok { + return nil, fmt.Errorf("agent %q not registered", agentName) + } + + body, err := json.Marshal(req) + if err != nil { + return nil, fmt.Errorf("marshal phase request: %w", err) + } + + url := fmt.Sprintf("http://%s/phase", agent.Addr) + httpReq, err := http.NewRequestWithContext(ctx, "POST", url, bytes.NewReader(body)) + if err != nil { + return nil, err + } + httpReq.Header.Set("Content-Type", "application/json") + if c.Token != "" { + httpReq.Header.Set(AuthTokenHeader, c.Token) + } + + resp, err := http.DefaultClient.Do(httpReq) + if err != nil { + return nil, fmt.Errorf("POST %s: %w", url, err) + } + defer resp.Body.Close() + + var phaseResp PhaseResponse + if err := json.NewDecoder(resp.Body).Decode(&phaseResp); err != nil { + return nil, fmt.Errorf("decode response from %s: %w", agentName, err) + } + + return &phaseResp, nil +} + +// UploadToAgent sends a file to an agent via POST /upload. +func (c *Coordinator) UploadToAgent(ctx context.Context, agentName, localPath, remotePath string) error { + c.mu.Lock() + agent, ok := c.agents[agentName] + c.mu.Unlock() + if !ok { + return fmt.Errorf("agent %q not registered", agentName) + } + + f, err := os.Open(localPath) + if err != nil { + return fmt.Errorf("open %s: %w", localPath, err) + } + defer f.Close() + + url := fmt.Sprintf("http://%s/upload?path=%s", agent.Addr, remotePath) + httpReq, err := http.NewRequestWithContext(ctx, "POST", url, f) + if err != nil { + return err + } + httpReq.Header.Set("Content-Type", "application/octet-stream") + if c.Token != "" { + httpReq.Header.Set(AuthTokenHeader, c.Token) + } + + resp, err := http.DefaultClient.Do(httpReq) + if err != nil { + return fmt.Errorf("upload to %s: %w", agentName, err) + } + defer resp.Body.Close() + + var uploadResp UploadResponse + if err := json.NewDecoder(resp.Body).Decode(&uploadResp); err != nil { + return fmt.Errorf("decode upload response: %w", err) + } + if !uploadResp.OK { + return fmt.Errorf("upload rejected: %s", uploadResp.Error) + } + + c.log.Printf("uploaded %s to %s:%s (%d bytes)", localPath, agentName, remotePath, uploadResp.Size) + return nil +} + +// ExecOnAgent runs a command on an agent via POST /exec. +func (c *Coordinator) ExecOnAgent(ctx context.Context, agentName string, cmd string, root bool) (*ExecResponse, error) { + c.mu.Lock() + agent, ok := c.agents[agentName] + c.mu.Unlock() + if !ok { + return nil, fmt.Errorf("agent %q not registered", agentName) + } + + reqBody, _ := json.Marshal(ExecRequest{Cmd: cmd, Root: root}) + url := fmt.Sprintf("http://%s/exec", agent.Addr) + + httpReq, err := http.NewRequestWithContext(ctx, "POST", url, bytes.NewReader(reqBody)) + if err != nil { + return nil, err + } + httpReq.Header.Set("Content-Type", "application/json") + if c.Token != "" { + httpReq.Header.Set(AuthTokenHeader, c.Token) + } + + resp, err := http.DefaultClient.Do(httpReq) + if err != nil { + return nil, fmt.Errorf("exec on %s: %w", agentName, err) + } + defer resp.Body.Close() + + var execResp ExecResponse + if err := json.NewDecoder(resp.Body).Decode(&execResp); err != nil { + return nil, fmt.Errorf("decode exec response: %w", err) + } + return &execResp, nil +} + +// HealthCheck checks if an agent is reachable. +func (c *Coordinator) HealthCheck(ctx context.Context, agentName string) (*HealthResponse, error) { + c.mu.Lock() + agent, ok := c.agents[agentName] + c.mu.Unlock() + if !ok { + return nil, fmt.Errorf("agent %q not registered", agentName) + } + + url := fmt.Sprintf("http://%s/health", agent.Addr) + httpReq, err := http.NewRequestWithContext(ctx, "GET", url, nil) + if err != nil { + return nil, err + } + + resp, err := http.DefaultClient.Do(httpReq) + if err != nil { + return nil, fmt.Errorf("health check %s: %w", agentName, err) + } + defer resp.Body.Close() + + var hr HealthResponse + body, _ := io.ReadAll(resp.Body) + if err := json.Unmarshal(body, &hr); err != nil { + return nil, fmt.Errorf("decode health response: %w", err) + } + return &hr, nil +} + +// DownloadArtifacts downloads artifacts from an agent's directory as a tar.gz stream, +// extracts them into localDir/agentName/, and returns the collected entries. +func (c *Coordinator) DownloadArtifacts(ctx context.Context, agentName, remoteDir, localDir string) ([]ArtifactEntry, error) { + c.mu.Lock() + agent, ok := c.agents[agentName] + c.mu.Unlock() + if !ok { + return nil, fmt.Errorf("agent %q not registered", agentName) + } + + url := fmt.Sprintf("http://%s/artifacts?dir=%s", agent.Addr, remoteDir) + httpReq, err := http.NewRequestWithContext(ctx, "GET", url, nil) + if err != nil { + return nil, err + } + if c.Token != "" { + httpReq.Header.Set(AuthTokenHeader, c.Token) + } + + resp, err := http.DefaultClient.Do(httpReq) + if err != nil { + return nil, fmt.Errorf("GET artifacts from %s: %w", agentName, err) + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + body, _ := io.ReadAll(resp.Body) + return nil, fmt.Errorf("artifacts from %s: HTTP %d: %s", agentName, resp.StatusCode, string(body)) + } + + // Extract tar.gz into localDir/agentName/. + destDir := filepath.Join(localDir, agentName) + if err := os.MkdirAll(destDir, 0755); err != nil { + return nil, fmt.Errorf("mkdir %s: %w", destDir, err) + } + + gr, err := gzip.NewReader(resp.Body) + if err != nil { + return nil, fmt.Errorf("gzip reader: %w", err) + } + defer gr.Close() + + tr := tar.NewReader(gr) + var entries []ArtifactEntry + + for { + hdr, err := tr.Next() + if err == io.EOF { + break + } + if err != nil { + return entries, fmt.Errorf("tar read: %w", err) + } + + // Safety: reject absolute paths and traversals. + name := filepath.FromSlash(hdr.Name) + if strings.Contains(name, "..") || filepath.IsAbs(name) { + continue + } + + target := filepath.Join(destDir, name) + if hdr.Typeflag == tar.TypeDir { + os.MkdirAll(target, 0755) + continue + } + + // Ensure parent dir exists. + os.MkdirAll(filepath.Dir(target), 0755) + + f, err := os.Create(target) + if err != nil { + c.log.Printf("create %s: %v", target, err) + continue + } + n, _ := io.Copy(f, tr) + f.Close() + + entries = append(entries, ArtifactEntry{ + Agent: agentName, + Path: target, + Size: n, + }) + } + + c.log.Printf("downloaded %d artifacts from %s to %s", len(entries), agentName, destDir) + return entries, nil +} + +// DownloadAllArtifacts downloads artifacts from all agents and appends entries to the result. +func (c *Coordinator) DownloadAllArtifacts(ctx context.Context, remoteDir, localDir string, result *ScenarioResult) { + c.mu.Lock() + names := make([]string, 0, len(c.agents)) + for name := range c.agents { + names = append(names, name) + } + c.mu.Unlock() + + for _, name := range names { + entries, err := c.DownloadArtifacts(ctx, name, remoteDir, localDir) + if err != nil { + c.log.Printf("artifact download from %s: %v", name, err) + continue + } + result.Artifacts = append(result.Artifacts, entries...) + } +} + +// printDryRun prints the execution plan without executing. +func (c *Coordinator) printDryRun(s *Scenario, nodeToAgent map[string]string, normalPhases, alwaysPhases []Phase) { + fmt.Fprintf(os.Stdout, "\n=== DRY RUN: %s ===\n\n", s.Name) + fmt.Fprintf(os.Stdout, "Agents:\n") + c.mu.Lock() + for name, info := range c.agents { + fmt.Fprintf(os.Stdout, " %s → %s (nodes=%v)\n", name, info.Addr, info.Nodes) + } + c.mu.Unlock() + + fmt.Fprintf(os.Stdout, "\nNode→Agent mapping:\n") + for node, agent := range nodeToAgent { + fmt.Fprintf(os.Stdout, " %s → %s\n", node, agent) + } + + printPhases := func(label string, phases []Phase) { + if len(phases) == 0 { + return + } + fmt.Fprintf(os.Stdout, "\n%s:\n", label) + for i, phase := range phases { + mode := "sequential" + if phase.Parallel { + mode = "parallel" + } + fmt.Fprintf(os.Stdout, " Phase %d: %s (%s, %d actions)\n", i, phase.Name, mode, len(phase.Actions)) + for j, act := range phase.Actions { + agent := c.resolveActionAgent(s, act, nodeToAgent) + fmt.Fprintf(os.Stdout, " [%d] %s → agent=%s", j, act.Action, agent) + if act.Target != "" { + fmt.Fprintf(os.Stdout, " target=%s", act.Target) + } + if act.Node != "" { + fmt.Fprintf(os.Stdout, " node=%s", act.Node) + } + if act.SaveAs != "" { + fmt.Fprintf(os.Stdout, " save_as=%s", act.SaveAs) + } + fmt.Fprintln(os.Stdout) + } + } + } + + printPhases("Normal Phases", normalPhases) + printPhases("Always Phases (cleanup)", alwaysPhases) + + fmt.Fprintln(os.Stdout, "\n=== END DRY RUN ===") +} + +// AgentNames returns the names of all registered agents (for use by build_deploy). +func (c *Coordinator) AgentNames() []string { + c.mu.Lock() + defer c.mu.Unlock() + names := make([]string, 0, len(c.agents)) + for name := range c.agents { + names = append(names, name) + } + return names +} + +// IsAgentRegistered returns whether an agent is registered. +func (c *Coordinator) IsAgentRegistered(name string) bool { + c.mu.Lock() + defer c.mu.Unlock() + _, ok := c.agents[name] + return ok +} + +// RegisteredAgentCount returns the number of registered agents. +func (c *Coordinator) RegisteredAgentCount() int { + c.mu.Lock() + defer c.mu.Unlock() + return len(c.agents) +} + +// coordinatorLocalActions lists actions that run on the coordinator itself, +// not dispatched to agents (e.g., cross-compile + upload). +var coordinatorLocalActions = map[string]bool{ + "build_deploy": true, + "build_deploy_weed": true, +} + +// isCoordinatorLocalAction returns true if the action should run on the +// coordinator rather than being dispatched to an agent. +func (c *Coordinator) isCoordinatorLocalAction(act Action) bool { + return coordinatorLocalActions[act.Action] +} + +// runCoordinatorLocalAction executes an action locally on the coordinator, +// using the action registry. Returns an ActionResult. +func (c *Coordinator) runCoordinatorLocalAction(ctx context.Context, s *Scenario, act Action, globalVars map[string]string) ActionResult { + start := time.Now() + ar := ActionResult{ + Action: act.Action, + Status: StatusPass, + } + + if c.registry == nil { + ar.Status = StatusFail + ar.Error = "no registry available for coordinator-local action" + ar.Duration = time.Since(start) + return ar + } + + handler, err := c.registry.Get(act.Action) + if err != nil { + ar.Status = StatusFail + ar.Error = fmt.Sprintf("registry lookup: %v", err) + ar.Duration = time.Since(start) + return ar + } + + // Build an ActionContext for coordinator-local execution. + actx := &ActionContext{ + Scenario: s, + Nodes: make(map[string]NodeRunner), + Targets: make(map[string]TargetRunner), + Vars: make(map[string]string), + Coordinator: c, + Log: func(format string, args ...interface{}) { c.log.Printf(format, args...) }, + } + // Copy global vars. + for k, v := range globalVars { + actx.Vars[k] = v + } + + // Resolve variable references in action params. + resolved := resolveAction(act, actx.Vars) + + c.log.Printf(" [coordinator-local] %s", resolved.Action) + + vars, err := handler.Execute(ctx, actx, resolved) + if err != nil { + ar.Status = StatusFail + ar.Error = err.Error() + if act.IgnoreError { + ar.Status = StatusPass + c.log.Printf(" [coordinator-local] %s failed (ignored): %v", resolved.Action, err) + } + } + + // Merge produced vars back to global. + for k, v := range vars { + globalVars[k] = v + } + + ar.Duration = time.Since(start) + return ar +} diff --git a/weed/storage/blockvol/testrunner/coordinator_test.go b/weed/storage/blockvol/testrunner/coordinator_test.go new file mode 100644 index 000000000..ac51235be --- /dev/null +++ b/weed/storage/blockvol/testrunner/coordinator_test.go @@ -0,0 +1,680 @@ +package testrunner + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "net/http" + "testing" + "time" +) + +// newTestCoordinator creates a coordinator on a random port for testing. +func newTestCoordinator(t *testing.T, expected map[string]string) *Coordinator { + t.Helper() + c := NewCoordinator(CoordinatorConfig{ + Port: 0, // random + Token: "test-token", + Expected: expected, + }) + if err := c.Start(); err != nil { + t.Fatalf("start coordinator: %v", err) + } + t.Cleanup(func() { c.Stop() }) + return c +} + +func TestCoordinator_RegisterAgent(t *testing.T) { + c := newTestCoordinator(t, map[string]string{"agent1": "127.0.0.1:9100"}) + + c.RegisterAgent("agent1", "127.0.0.1:9100") + + if !c.IsAgentRegistered("agent1") { + t.Error("agent1 should be registered") + } + if c.IsAgentRegistered("agent2") { + t.Error("agent2 should not be registered") + } + if c.RegisteredAgentCount() != 1 { + t.Errorf("count = %d, want 1", c.RegisteredAgentCount()) + } +} + +func TestCoordinator_WaitForAgents_AlreadyRegistered(t *testing.T) { + c := newTestCoordinator(t, map[string]string{"a1": "host:9100"}) + c.RegisterAgent("a1", "host:9100") + + ctx := context.Background() + if err := c.WaitForAgents(ctx, 1*time.Second); err != nil { + t.Fatalf("WaitForAgents: %v", err) + } +} + +func TestCoordinator_WaitForAgents_Timeout(t *testing.T) { + c := newTestCoordinator(t, map[string]string{"a1": "host:9100", "a2": "host:9200"}) + c.RegisterAgent("a1", "host:9100") + // a2 never registers + + ctx := context.Background() + err := c.WaitForAgents(ctx, 100*time.Millisecond) + if err == nil { + t.Fatal("expected timeout error") + } +} + +func TestCoordinator_WaitForAgents_NoExpected(t *testing.T) { + c := newTestCoordinator(t, map[string]string{}) + ctx := context.Background() + if err := c.WaitForAgents(ctx, 100*time.Millisecond); err != nil { + t.Fatalf("WaitForAgents with empty expected: %v", err) + } +} + +func TestCoordinator_BuildNodeAgentMap(t *testing.T) { + c := newTestCoordinator(t, map[string]string{"tp01": "host:9100"}) + c.RegisterAgent("tp01", "host:9100") + + s := &Scenario{ + Topology: Topology{ + Agents: map[string]string{"tp01": "host:9100"}, + Nodes: map[string]NodeSpec{ + "node1": {Host: "host", Agent: "tp01"}, + "node2": {Host: "host2"}, + }, + }, + } + + m := c.buildNodeAgentMap(s) + if m["node1"] != "tp01" { + t.Errorf("node1 → %q, want tp01", m["node1"]) + } +} + +func TestCoordinator_ResolveActionAgent(t *testing.T) { + c := newTestCoordinator(t, map[string]string{"tp01": "h:9100", "m01": "h:9200"}) + c.RegisterAgent("tp01", "h:9100") + c.RegisterAgent("m01", "h:9200") + + s := &Scenario{ + Targets: map[string]TargetSpec{ + "primary": {Node: "node_tp01"}, + }, + Topology: Topology{ + Nodes: map[string]NodeSpec{ + "node_tp01": {Agent: "tp01"}, + "node_m01": {Agent: "m01"}, + }, + }, + } + nodeToAgent := map[string]string{"node_tp01": "tp01", "node_m01": "m01"} + + // By node. + agent := c.resolveActionAgent(s, Action{Node: "node_tp01"}, nodeToAgent) + if agent != "tp01" { + t.Errorf("by node: got %q, want tp01", agent) + } + + // By target. + agent = c.resolveActionAgent(s, Action{Target: "primary"}, nodeToAgent) + if agent != "tp01" { + t.Errorf("by target: got %q, want tp01", agent) + } + + // Fallback. + agent = c.resolveActionAgent(s, Action{}, nodeToAgent) + if agent == "" { + t.Error("fallback should return some agent") + } +} + +// --- Integration: Coordinator + Agent --- + +func TestCoordinatorAgent_PhaseDispatch(t *testing.T) { + if testing.Short() { + t.Skip("integration test") + } + + // Create a test action that returns a value. + registry := NewRegistry() + registry.RegisterFunc("echo_test", TierCore, func(ctx context.Context, actx *ActionContext, act Action) (map[string]string, error) { + msg := act.Params["msg"] + if msg == "" { + msg = "default" + } + return map[string]string{"value": msg}, nil + }) + registry.RegisterFunc("fail_test", TierCore, func(ctx context.Context, actx *ActionContext, act Action) (map[string]string, error) { + return nil, fmt.Errorf("intentional failure") + }) + + // Start agent. + agent := NewAgent(AgentConfig{ + Port: 0, + Token: "tok", + Nodes: []string{"test_node"}, + Registry: registry, + }) + + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + + go agent.Start(ctx) + // Wait for listener. + for i := 0; i < 50; i++ { + if agent.ListenAddr() != "" { + break + } + time.Sleep(20 * time.Millisecond) + } + if agent.ListenAddr() == "" { + t.Fatal("agent didn't start") + } + + // Start coordinator. + coord := NewCoordinator(CoordinatorConfig{ + Port: 0, + Token: "tok", + Expected: map[string]string{"test_node": agent.ListenAddr()}, + }) + if err := coord.Start(); err != nil { + t.Fatalf("start coordinator: %v", err) + } + defer coord.Stop() + + // Register agent. + coord.RegisterAgent("test_node", agent.ListenAddr()) + + // Define scenario. + s := &Scenario{ + Name: "test-dispatch", + Timeout: Duration{30 * time.Second}, + Env: map[string]string{"greeting": "hello"}, + Topology: Topology{ + Agents: map[string]string{"test_node": agent.ListenAddr()}, + Nodes: map[string]NodeSpec{"test_node": {Host: "127.0.0.1", Agent: "test_node"}}, + }, + Phases: []Phase{ + { + Name: "echo_phase", + Actions: []Action{ + {Action: "echo_test", Node: "test_node", SaveAs: "echo_result", Params: map[string]string{"msg": "world"}}, + }, + }, + }, + } + + result := coord.RunScenario(ctx, s, registry) + if result.Status != StatusPass { + t.Fatalf("scenario failed: %s", result.Error) + } + if len(result.Phases) != 1 { + t.Fatalf("expected 1 phase, got %d", len(result.Phases)) + } + if len(result.Phases[0].Actions) != 1 { + t.Fatalf("expected 1 action result, got %d", len(result.Phases[0].Actions)) + } + if result.Phases[0].Actions[0].Status != StatusPass { + t.Errorf("action status = %s", result.Phases[0].Actions[0].Status) + } +} + +func TestCoordinatorAgent_VarMerge(t *testing.T) { + if testing.Short() { + t.Skip("integration test") + } + + // Two actions: first produces a var, second uses it. + registry := NewRegistry() + callCount := 0 + registry.RegisterFunc("produce_var", TierCore, func(ctx context.Context, actx *ActionContext, act Action) (map[string]string, error) { + callCount++ + return map[string]string{"value": fmt.Sprintf("produced-%d", callCount)}, nil + }) + registry.RegisterFunc("check_var", TierCore, func(ctx context.Context, actx *ActionContext, act Action) (map[string]string, error) { + expected := act.Params["expected"] + got := act.Params["actual"] + if got != expected { + return nil, fmt.Errorf("var mismatch: got %q, want %q", got, expected) + } + return map[string]string{"value": "ok"}, nil + }) + + agent := NewAgent(AgentConfig{Port: 0, Token: "tok", Nodes: []string{"n1"}, Registry: registry}) + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + go agent.Start(ctx) + for i := 0; i < 50; i++ { + if agent.ListenAddr() != "" { + break + } + time.Sleep(20 * time.Millisecond) + } + + coord := NewCoordinator(CoordinatorConfig{Port: 0, Token: "tok", Expected: map[string]string{"n1": agent.ListenAddr()}}) + coord.Start() + defer coord.Stop() + coord.RegisterAgent("n1", agent.ListenAddr()) + + s := &Scenario{ + Name: "var-merge", + Timeout: Duration{30 * time.Second}, + Topology: Topology{ + Agents: map[string]string{"n1": agent.ListenAddr()}, + Nodes: map[string]NodeSpec{"n1": {Host: "127.0.0.1", Agent: "n1"}}, + }, + Phases: []Phase{ + { + Name: "produce", + Actions: []Action{ + {Action: "produce_var", Node: "n1", SaveAs: "my_var"}, + }, + }, + { + Name: "consume", + Actions: []Action{ + {Action: "check_var", Node: "n1", Params: map[string]string{"expected": "produced-1", "actual": "{{ my_var }}"}}, + }, + }, + }, + } + + result := coord.RunScenario(ctx, s, registry) + if result.Status != StatusPass { + t.Fatalf("scenario failed: %s", result.Error) + } +} + +func TestCoordinatorAgent_FailureStopsNormalPhases(t *testing.T) { + if testing.Short() { + t.Skip("integration test") + } + + registry := NewRegistry() + registry.RegisterFunc("pass_action", TierCore, func(ctx context.Context, actx *ActionContext, act Action) (map[string]string, error) { + return nil, nil + }) + registry.RegisterFunc("fail_action", TierCore, func(ctx context.Context, actx *ActionContext, act Action) (map[string]string, error) { + return nil, fmt.Errorf("boom") + }) + + agent := NewAgent(AgentConfig{Port: 0, Token: "tok", Nodes: []string{"n1"}, Registry: registry}) + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + go agent.Start(ctx) + for i := 0; i < 50; i++ { + if agent.ListenAddr() != "" { + break + } + time.Sleep(20 * time.Millisecond) + } + + coord := NewCoordinator(CoordinatorConfig{Port: 0, Token: "tok", Expected: map[string]string{"n1": agent.ListenAddr()}}) + coord.Start() + defer coord.Stop() + coord.RegisterAgent("n1", agent.ListenAddr()) + + s := &Scenario{ + Name: "fail-test", + Timeout: Duration{30 * time.Second}, + Topology: Topology{ + Agents: map[string]string{"n1": agent.ListenAddr()}, + Nodes: map[string]NodeSpec{"n1": {Host: "127.0.0.1", Agent: "n1"}}, + }, + Phases: []Phase{ + {Name: "p1", Actions: []Action{{Action: "fail_action", Node: "n1"}}}, + {Name: "p2", Actions: []Action{{Action: "pass_action", Node: "n1"}}}, // should be skipped + {Name: "cleanup", Always: true, Actions: []Action{{Action: "pass_action", Node: "n1"}}}, + }, + } + + result := coord.RunScenario(ctx, s, registry) + if result.Status != StatusFail { + t.Fatal("expected FAIL") + } + // p1 (failed) + cleanup (always) = 2 phases, p2 skipped. + if len(result.Phases) != 2 { + t.Errorf("expected 2 phases (p1+cleanup), got %d", len(result.Phases)) + } +} + +func TestCoordinatorAgent_RetryOnFailure(t *testing.T) { + if testing.Short() { + t.Skip("integration test") + } + + registry := NewRegistry() + callCount := 0 + registry.RegisterFunc("flaky_action", TierCore, func(ctx context.Context, actx *ActionContext, act Action) (map[string]string, error) { + callCount++ + if callCount < 3 { + return nil, fmt.Errorf("flaky error %d", callCount) + } + return map[string]string{"value": "success"}, nil + }) + + agent := NewAgent(AgentConfig{Port: 0, Token: "tok", Nodes: []string{"n1"}, Registry: registry}) + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + go agent.Start(ctx) + for i := 0; i < 50; i++ { + if agent.ListenAddr() != "" { + break + } + time.Sleep(20 * time.Millisecond) + } + + coord := NewCoordinator(CoordinatorConfig{Port: 0, Token: "tok", Expected: map[string]string{"n1": agent.ListenAddr()}}) + coord.Start() + defer coord.Stop() + coord.RegisterAgent("n1", agent.ListenAddr()) + + s := &Scenario{ + Name: "retry-test", + Timeout: Duration{30 * time.Second}, + Topology: Topology{ + Agents: map[string]string{"n1": agent.ListenAddr()}, + Nodes: map[string]NodeSpec{"n1": {Host: "127.0.0.1", Agent: "n1"}}, + }, + Phases: []Phase{ + {Name: "p1", Actions: []Action{{Action: "flaky_action", Node: "n1", Retry: 3}}}, + }, + } + + result := coord.RunScenario(ctx, s, registry) + if result.Status != StatusPass { + t.Fatalf("expected PASS after retries, got %s: %s", result.Status, result.Error) + } +} + +func TestCoordinatorAgent_VarsInResult(t *testing.T) { + if testing.Short() { + t.Skip("integration test") + } + + registry := NewRegistry() + registry.RegisterFunc("produce", TierCore, func(ctx context.Context, actx *ActionContext, act Action) (map[string]string, error) { + return map[string]string{"value": "produced-value"}, nil + }) + + agent := NewAgent(AgentConfig{Port: 0, Token: "tok", Nodes: []string{"n1"}, Registry: registry}) + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + go agent.Start(ctx) + for i := 0; i < 50; i++ { + if agent.ListenAddr() != "" { + break + } + time.Sleep(20 * time.Millisecond) + } + + coord := NewCoordinator(CoordinatorConfig{Port: 0, Token: "tok", Expected: map[string]string{"n1": agent.ListenAddr()}}) + coord.Start() + defer coord.Stop() + coord.RegisterAgent("n1", agent.ListenAddr()) + + s := &Scenario{ + Name: "vars-result", + Timeout: Duration{30 * time.Second}, + Env: map[string]string{"env_key": "env_val"}, + Topology: Topology{ + Agents: map[string]string{"n1": agent.ListenAddr()}, + Nodes: map[string]NodeSpec{"n1": {Host: "127.0.0.1", Agent: "n1"}}, + }, + Phases: []Phase{ + {Name: "p1", Actions: []Action{{Action: "produce", Node: "n1", SaveAs: "out_var"}}}, + }, + } + + result := coord.RunScenario(ctx, s, registry) + if result.Status != StatusPass { + t.Fatalf("expected PASS, got %s: %s", result.Status, result.Error) + } + + // Verify vars are in the result. + if result.Vars == nil { + t.Fatal("result.Vars is nil") + } + if result.Vars["out_var"] != "produced-value" { + t.Errorf("result.Vars[out_var] = %q, want produced-value", result.Vars["out_var"]) + } + if result.Vars["env_key"] != "env_val" { + t.Errorf("result.Vars[env_key] = %q, want env_val", result.Vars["env_key"]) + } +} + +func TestCoordinatorAgent_DryRun(t *testing.T) { + coord := NewCoordinator(CoordinatorConfig{ + Port: 0, + DryRun: true, + Expected: map[string]string{"n1": "h:9100"}, + }) + coord.Start() + defer coord.Stop() + coord.RegisterAgent("n1", "h:9100") + + s := &Scenario{ + Name: "dry-run-test", + Topology: Topology{ + Agents: map[string]string{"n1": "h:9100"}, + Nodes: map[string]NodeSpec{"n1": {Host: "h", Agent: "n1"}}, + }, + Phases: []Phase{ + {Name: "p1", Actions: []Action{{Action: "echo", Node: "n1"}}}, + }, + } + + result := coord.RunScenario(context.Background(), s, NewRegistry()) + // Dry run should always pass (no execution). + if result.Status != StatusPass { + t.Errorf("dry run status = %s", result.Status) + } +} + +func TestCoordinator_RegisterTokenValidation(t *testing.T) { + // Coordinator with token should reject unauthenticated /register. + c := NewCoordinator(CoordinatorConfig{ + Port: 0, + Token: "secret", + Expected: map[string]string{"n1": "h:9100"}, + }) + if err := c.Start(); err != nil { + t.Fatalf("start: %v", err) + } + defer c.Stop() + + regReq := RegisterRequest{ + AgentID: "a1", + Hostname: "h", + IP: "127.0.0.1", + Port: 9100, + Nodes: []string{"n1"}, + } + body, _ := json.Marshal(regReq) + + // No token → 401. + resp, err := http.Post("http://"+c.ListenAddr()+"/register", "application/json", bytes.NewReader(body)) + if err != nil { + t.Fatalf("POST /register: %v", err) + } + resp.Body.Close() + if resp.StatusCode != http.StatusUnauthorized { + t.Errorf("no token: expected 401, got %d", resp.StatusCode) + } + + // Wrong token → 401. + req, _ := http.NewRequest("POST", "http://"+c.ListenAddr()+"/register", bytes.NewReader(body)) + req.Header.Set("Content-Type", "application/json") + req.Header.Set(AuthTokenHeader, "wrong") + resp2, _ := http.DefaultClient.Do(req) + resp2.Body.Close() + if resp2.StatusCode != http.StatusUnauthorized { + t.Errorf("wrong token: expected 401, got %d", resp2.StatusCode) + } + + // Correct token → 200. + req3, _ := http.NewRequest("POST", "http://"+c.ListenAddr()+"/register", bytes.NewReader(body)) + req3.Header.Set("Content-Type", "application/json") + req3.Header.Set(AuthTokenHeader, "secret") + resp3, _ := http.DefaultClient.Do(req3) + defer resp3.Body.Close() + if resp3.StatusCode != http.StatusOK { + t.Errorf("correct token: expected 200, got %d", resp3.StatusCode) + } + var rr RegisterResponse + json.NewDecoder(resp3.Body).Decode(&rr) + if !rr.OK { + t.Errorf("registration should succeed with correct token, error: %s", rr.Error) + } +} + +func TestCoordinator_RegisterNoToken(t *testing.T) { + // Coordinator without token should accept any /register. + c := NewCoordinator(CoordinatorConfig{ + Port: 0, + Token: "", // no token + Expected: map[string]string{"n1": "h:9100"}, + }) + if err := c.Start(); err != nil { + t.Fatalf("start: %v", err) + } + defer c.Stop() + + regReq := RegisterRequest{ + AgentID: "a1", + Hostname: "h", + IP: "127.0.0.1", + Port: 9100, + Nodes: []string{"n1"}, + } + body, _ := json.Marshal(regReq) + + resp, err := http.Post("http://"+c.ListenAddr()+"/register", "application/json", bytes.NewReader(body)) + if err != nil { + t.Fatalf("POST /register: %v", err) + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusOK { + t.Errorf("expected 200 without token requirement, got %d", resp.StatusCode) + } +} + +func TestAgent_PersistentReRegistration(t *testing.T) { + if testing.Short() { + t.Skip("integration test") + } + + registry := NewRegistry() + registry.RegisterFunc("noop", TierCore, func(ctx context.Context, actx *ActionContext, act Action) (map[string]string, error) { + return nil, nil + }) + + // Start persistent agent. + agent := NewAgent(AgentConfig{ + Port: 0, + Token: "tok", + Persistent: true, + Nodes: []string{"n1"}, + Registry: registry, + }) + + ctx, cancel := context.WithTimeout(context.Background(), 15*time.Second) + defer cancel() + + go agent.Start(ctx) + for i := 0; i < 50; i++ { + if agent.ListenAddr() != "" { + break + } + time.Sleep(20 * time.Millisecond) + } + if agent.ListenAddr() == "" { + t.Fatal("agent didn't start") + } + + // Start first coordinator — agent should register. + coord1 := NewCoordinator(CoordinatorConfig{ + Port: 0, + Token: "tok", + Expected: map[string]string{"n1": agent.ListenAddr()}, + }) + // Update agent's coordinator URL to point at coord1. + agent.CoordinatorURL = "http://" + coord1.ListenAddr() + // Since agent is already running with persistent loop pointing to original URL, + // we need a different approach: start coordinator first, then the agent connects. + + // Actually, let's restart: stop agent, start coord, then start agent pointing at it. + cancel() + time.Sleep(100 * time.Millisecond) + + // Start coordinator 1. + coord1 = NewCoordinator(CoordinatorConfig{ + Port: 0, + Token: "tok", + Expected: map[string]string{"n1": "will-be-set"}, + }) + if err := coord1.Start(); err != nil { + t.Fatalf("start coord1: %v", err) + } + + // Start persistent agent pointing at coord1. + ctx2, cancel2 := context.WithTimeout(context.Background(), 15*time.Second) + defer cancel2() + + agent2 := NewAgent(AgentConfig{ + Port: 0, + CoordinatorURL: "http://" + coord1.ListenAddr(), + Token: "tok", + Persistent: true, + Nodes: []string{"n1"}, + Registry: registry, + }) + go agent2.Start(ctx2) + for i := 0; i < 50; i++ { + if agent2.ListenAddr() != "" { + break + } + time.Sleep(20 * time.Millisecond) + } + + // Wait for registration. + if err := coord1.WaitForAgents(ctx2, 10*time.Second); err != nil { + t.Fatalf("agent didn't register with coord1: %v", err) + } + if !coord1.IsAgentRegistered("n1") { + t.Fatal("n1 not registered on coord1") + } + + // Stop coord1, start coord2 on the SAME port. + addr := coord1.ListenAddr() + coord1.Stop() + time.Sleep(200 * time.Millisecond) + + // Extract port from addr. + var port int + fmt.Sscanf(addr, "[::]:%d", &port) + if port == 0 { + fmt.Sscanf(addr, "0.0.0.0:%d", &port) + } + if port == 0 { + fmt.Sscanf(addr, "127.0.0.1:%d", &port) + } + + coord2 := NewCoordinator(CoordinatorConfig{ + Port: port, + Token: "tok", + Expected: map[string]string{"n1": agent2.ListenAddr()}, + }) + if err := coord2.Start(); err != nil { + t.Fatalf("start coord2: %v", err) + } + defer coord2.Stop() + + // Agent should re-register with coord2 within ~10s. + if err := coord2.WaitForAgents(ctx2, 12*time.Second); err != nil { + t.Fatalf("agent didn't re-register with coord2: %v", err) + } + if !coord2.IsAgentRegistered("n1") { + t.Fatal("n1 not registered on coord2 after re-registration") + } +} diff --git a/weed/storage/blockvol/testrunner/engine.go b/weed/storage/blockvol/testrunner/engine.go new file mode 100644 index 000000000..417ba1b03 --- /dev/null +++ b/weed/storage/blockvol/testrunner/engine.go @@ -0,0 +1,299 @@ +package testrunner + +import ( + "context" + "fmt" + "regexp" + "strings" + "sync" + "time" + + "gopkg.in/yaml.v3" +) + +var varPattern = regexp.MustCompile(`\{\{\s*(\w+)\s*\}\}`) + +// Engine executes a Scenario using the given Registry. +type Engine struct { + registry *Registry + log func(format string, args ...interface{}) +} + +// NewEngine creates an engine with the given registry and logger. +func NewEngine(registry *Registry, log func(format string, args ...interface{})) *Engine { + if log == nil { + log = func(string, ...interface{}) {} + } + return &Engine{registry: registry, log: log} +} + +// Run executes the scenario end-to-end and returns the result. +func (e *Engine) Run(ctx context.Context, s *Scenario, actx *ActionContext) *ScenarioResult { + start := time.Now() + result := &ScenarioResult{ + Name: s.Name, + Status: StatusPass, + } + + // Apply scenario timeout. + if s.Timeout.Duration > 0 { + var cancel context.CancelFunc + ctx, cancel = context.WithTimeout(ctx, s.Timeout.Duration) + defer cancel() + } + + // Seed vars from env. + if actx.Vars == nil { + actx.Vars = make(map[string]string) + } + for k, v := range s.Env { + actx.Vars[k] = v + } + + // Separate always-phases for deferred cleanup. + var normalPhases, alwaysPhases []Phase + for _, p := range s.Phases { + if p.Always { + alwaysPhases = append(alwaysPhases, p) + } else { + normalPhases = append(normalPhases, p) + } + } + + // Execute normal phases sequentially. + failed := false + for _, phase := range normalPhases { + pr := e.runPhase(ctx, actx, phase) + result.Phases = append(result.Phases, pr) + if pr.Status == StatusFail { + failed = true + result.Status = StatusFail + result.Error = fmt.Sprintf("phase %q failed: %s", phase.Name, pr.Error) + break + } + } + + // Always-phases run regardless of failure. + for _, phase := range alwaysPhases { + pr := e.runPhase(ctx, actx, phase) + result.Phases = append(result.Phases, pr) + } + + result.Duration = time.Since(start) + if !failed { + result.Status = StatusPass + } + + // Preserve all final vars in the result for downstream reporting. + if len(actx.Vars) > 0 { + result.Vars = make(map[string]string, len(actx.Vars)) + for k, v := range actx.Vars { + result.Vars[k] = v + } + } + + return result +} + +// runPhase executes a single phase (sequential or parallel). +func (e *Engine) runPhase(ctx context.Context, actx *ActionContext, phase Phase) PhaseResult { + start := time.Now() + pr := PhaseResult{ + Name: phase.Name, + Status: StatusPass, + } + + e.log("[phase] %s", phase.Name) + + if phase.Parallel { + pr = e.runPhaseParallel(ctx, actx, phase) + } else { + pr = e.runPhaseSequential(ctx, actx, phase) + } + + pr.Duration = time.Since(start) + return pr +} + +func (e *Engine) runPhaseSequential(ctx context.Context, actx *ActionContext, phase Phase) PhaseResult { + pr := PhaseResult{ + Name: phase.Name, + Status: StatusPass, + } + + for i, act := range phase.Actions { + ar := e.runAction(ctx, actx, act) + pr.Actions = append(pr.Actions, ar) + if ar.Status == StatusFail && !act.IgnoreError { + pr.Status = StatusFail + pr.Error = fmt.Sprintf("action %d (%s) failed: %s", i, act.Action, ar.Error) + return pr + } + } + return pr +} + +func (e *Engine) runPhaseParallel(ctx context.Context, actx *ActionContext, phase Phase) PhaseResult { + pr := PhaseResult{ + Name: phase.Name, + Status: StatusPass, + } + + results := make([]ActionResult, len(phase.Actions)) + var wg sync.WaitGroup + for i, act := range phase.Actions { + wg.Add(1) + go func(idx int, a Action) { + defer wg.Done() + results[idx] = e.runAction(ctx, actx, a) + }(i, act) + } + wg.Wait() + + for i, ar := range results { + pr.Actions = append(pr.Actions, ar) + if ar.Status == StatusFail && !phase.Actions[i].IgnoreError { + pr.Status = StatusFail + if pr.Error == "" { + pr.Error = fmt.Sprintf("action %d (%s) failed: %s", i, phase.Actions[i].Action, ar.Error) + } + } + } + return pr +} + +// runAction resolves variables and executes a single action. +func (e *Engine) runAction(ctx context.Context, actx *ActionContext, act Action) ActionResult { + start := time.Now() + + // Resolve variables in the action. + resolved := resolveAction(act, actx.Vars) + + // Serialize resolved action to YAML for report display. + yamlDef := marshalActionYAML(resolved) + + handler, err := e.registry.Get(resolved.Action) + if err != nil { + return ActionResult{ + Action: resolved.Action, + Status: StatusFail, + Duration: time.Since(start), + Error: err.Error(), + } + } + + // Handle delay param. + if d, ok := resolved.Params["delay"]; ok { + dur, err := time.ParseDuration(d) + if err == nil { + e.log(" [delay] %s", d) + select { + case <-time.After(dur): + case <-ctx.Done(): + return ActionResult{ + Action: resolved.Action, + Status: StatusFail, + Duration: time.Since(start), + Error: ctx.Err().Error(), + } + } + } + } + + e.log(" [action] %s", resolved.Action) + + output, err := handler.Execute(ctx, actx, resolved) + + ar := ActionResult{ + Action: resolved.Action, + Duration: time.Since(start), + YAML: yamlDef, + } + + if err != nil { + ar.Status = StatusFail + ar.Error = err.Error() + if act.IgnoreError { + ar.Status = StatusPass + e.log(" [action] %s failed (ignored): %v", resolved.Action, err) + } + } else { + ar.Status = StatusPass + } + + // Store output as var if save_as is set. + if resolved.SaveAs != "" && output != nil { + if v, ok := output["value"]; ok { + actx.Vars[resolved.SaveAs] = v + e.log(" [var] %s = %s", resolved.SaveAs, truncate(v, 60)) + } + } + + // Store all output keys with double-underscore prefix for cleanup vars. + if output != nil { + for k, v := range output { + if strings.HasPrefix(k, "__") { + actx.Vars[k] = v + } + } + } + + if output != nil { + if v, ok := output["value"]; ok { + ar.Output = truncate(v, 4096) + } + } + + return ar +} + +// resolveAction substitutes {{ var }} references in the action's fields. +func resolveAction(act Action, vars map[string]string) Action { + resolved := Action{ + Action: act.Action, + Target: act.Target, + Replica: act.Replica, + Node: act.Node, + SaveAs: act.SaveAs, + IgnoreError: act.IgnoreError, + Params: make(map[string]string), + } + + // Copy and resolve params. + for k, v := range act.Params { + resolved.Params[k] = resolveVars(v, vars) + } + + return resolved +} + +// resolveVars replaces {{ name }} with the value from vars. +func resolveVars(s string, vars map[string]string) string { + return varPattern.ReplaceAllStringFunc(s, func(match string) string { + sub := varPattern.FindStringSubmatch(match) + if len(sub) < 2 { + return match + } + name := sub[1] + if v, ok := vars[name]; ok { + return v + } + return match // leave unresolved + }) +} + +func truncate(s string, max int) string { + if len(s) <= max { + return s + } + return s[:max] + "..." +} + +// marshalActionYAML serializes a resolved action to YAML for report display. +func marshalActionYAML(act Action) string { + data, err := yaml.Marshal(act) + if err != nil { + return "" + } + return string(data) +} diff --git a/weed/storage/blockvol/testrunner/engine_test.go b/weed/storage/blockvol/testrunner/engine_test.go new file mode 100644 index 000000000..004bedd13 --- /dev/null +++ b/weed/storage/blockvol/testrunner/engine_test.go @@ -0,0 +1,515 @@ +package testrunner + +import ( + "context" + "fmt" + "testing" + "time" +) + +// mockHandler records calls and returns configured outputs. +type mockHandler struct { + calls []Action + outputs map[string]string + err error +} + +func (m *mockHandler) Execute(ctx context.Context, actx *ActionContext, act Action) (map[string]string, error) { + m.calls = append(m.calls, act) + if m.err != nil { + return nil, m.err + } + return m.outputs, nil +} + +func TestEngine_BasicFlow(t *testing.T) { + registry := NewRegistry() + + step1 := &mockHandler{outputs: map[string]string{"value": "hello"}} + step2 := &mockHandler{outputs: map[string]string{"value": "world"}} + + registry.Register("step1", TierCore, step1) + registry.Register("step2", TierCore, step2) + + scenario := &Scenario{ + Name: "basic-test", + Timeout: Duration{5 * time.Second}, + Phases: []Phase{ + { + Name: "phase1", + Actions: []Action{ + {Action: "step1", SaveAs: "v1"}, + {Action: "step2", SaveAs: "v2"}, + }, + }, + }, + } + + engine := NewEngine(registry, nil) + actx := &ActionContext{ + Scenario: scenario, + Vars: make(map[string]string), + Log: func(string, ...interface{}) {}, + } + result := engine.Run(context.Background(), scenario, actx) + + if result.Status != StatusPass { + t.Errorf("status = %s, want PASS", result.Status) + } + if len(result.Phases) != 1 { + t.Fatalf("phases = %d, want 1", len(result.Phases)) + } + if len(result.Phases[0].Actions) != 2 { + t.Fatalf("actions = %d, want 2", len(result.Phases[0].Actions)) + } + if actx.Vars["v1"] != "hello" { + t.Errorf("v1 = %q, want %q", actx.Vars["v1"], "hello") + } + if actx.Vars["v2"] != "world" { + t.Errorf("v2 = %q, want %q", actx.Vars["v2"], "world") + } +} + +func TestEngine_FailureStopsPhase(t *testing.T) { + registry := NewRegistry() + + step1 := &mockHandler{err: fmt.Errorf("boom")} + step2 := &mockHandler{outputs: map[string]string{"value": "ok"}} + + registry.Register("step1", TierCore, step1) + registry.Register("step2", TierCore, step2) + + scenario := &Scenario{ + Name: "fail-test", + Timeout: Duration{5 * time.Second}, + Phases: []Phase{ + { + Name: "phase1", + Actions: []Action{ + {Action: "step1"}, + {Action: "step2"}, // should not run + }, + }, + }, + } + + engine := NewEngine(registry, nil) + actx := &ActionContext{ + Scenario: scenario, + Vars: make(map[string]string), + Log: func(string, ...interface{}) {}, + } + result := engine.Run(context.Background(), scenario, actx) + + if result.Status != StatusFail { + t.Errorf("status = %s, want FAIL", result.Status) + } + if len(result.Phases[0].Actions) != 1 { + t.Errorf("actions = %d, want 1 (should stop after failure)", len(result.Phases[0].Actions)) + } + if len(step2.calls) != 0 { + t.Error("step2 should not have been called") + } +} + +func TestEngine_IgnoreError(t *testing.T) { + registry := NewRegistry() + + step1 := &mockHandler{err: fmt.Errorf("expected error")} + step2 := &mockHandler{outputs: map[string]string{"value": "ok"}} + + registry.Register("step1", TierCore, step1) + registry.Register("step2", TierCore, step2) + + scenario := &Scenario{ + Name: "ignore-test", + Timeout: Duration{5 * time.Second}, + Phases: []Phase{ + { + Name: "phase1", + Actions: []Action{ + {Action: "step1", IgnoreError: true}, + {Action: "step2"}, + }, + }, + }, + } + + engine := NewEngine(registry, nil) + actx := &ActionContext{ + Scenario: scenario, + Vars: make(map[string]string), + Log: func(string, ...interface{}) {}, + } + result := engine.Run(context.Background(), scenario, actx) + + if result.Status != StatusPass { + t.Errorf("status = %s, want PASS (error was ignored)", result.Status) + } + if len(step2.calls) != 1 { + t.Error("step2 should have been called despite step1 error") + } +} + +func TestEngine_AlwaysPhaseRunsAfterFailure(t *testing.T) { + registry := NewRegistry() + + failStep := &mockHandler{err: fmt.Errorf("fail")} + cleanStep := &mockHandler{} + + registry.Register("fail_step", TierCore, failStep) + registry.Register("clean_step", TierCore, cleanStep) + + scenario := &Scenario{ + Name: "always-test", + Timeout: Duration{5 * time.Second}, + Phases: []Phase{ + { + Name: "main", + Actions: []Action{ + {Action: "fail_step"}, + }, + }, + { + Name: "cleanup", + Always: true, + Actions: []Action{ + {Action: "clean_step"}, + }, + }, + }, + } + + engine := NewEngine(registry, nil) + actx := &ActionContext{ + Scenario: scenario, + Vars: make(map[string]string), + Log: func(string, ...interface{}) {}, + } + result := engine.Run(context.Background(), scenario, actx) + + if result.Status != StatusFail { + t.Errorf("status = %s, want FAIL", result.Status) + } + if len(result.Phases) != 2 { + t.Fatalf("phases = %d, want 2 (cleanup should still run)", len(result.Phases)) + } + if len(cleanStep.calls) != 1 { + t.Error("cleanup step should have been called") + } +} + +func TestEngine_VarSubstitution(t *testing.T) { + registry := NewRegistry() + + step1 := &mockHandler{outputs: map[string]string{"value": "/dev/sda"}} + step2 := &mockHandler{} + + registry.Register("step1", TierCore, step1) + registry.Register("step2", TierCore, step2) + + scenario := &Scenario{ + Name: "var-test", + Timeout: Duration{5 * time.Second}, + Phases: []Phase{ + { + Name: "phase1", + Actions: []Action{ + {Action: "step1", SaveAs: "device"}, + {Action: "step2", Params: map[string]string{"device": "{{ device }}"}}, + }, + }, + }, + } + + engine := NewEngine(registry, nil) + actx := &ActionContext{ + Scenario: scenario, + Vars: make(map[string]string), + Log: func(string, ...interface{}) {}, + } + engine.Run(context.Background(), scenario, actx) + + if len(step2.calls) != 1 { + t.Fatalf("step2 not called") + } + if step2.calls[0].Params["device"] != "/dev/sda" { + t.Errorf("device = %q, want /dev/sda", step2.calls[0].Params["device"]) + } +} + +func TestEngine_EnvVars(t *testing.T) { + registry := NewRegistry() + + step := &mockHandler{} + registry.Register("step", TierCore, step) + + scenario := &Scenario{ + Name: "env-test", + Timeout: Duration{5 * time.Second}, + Env: map[string]string{"repo_dir": "/tmp/repo"}, + Phases: []Phase{ + { + Name: "phase1", + Actions: []Action{ + {Action: "step", Params: map[string]string{"dir": "{{ repo_dir }}"}}, + }, + }, + }, + } + + engine := NewEngine(registry, nil) + actx := &ActionContext{ + Scenario: scenario, + Vars: make(map[string]string), + Log: func(string, ...interface{}) {}, + } + engine.Run(context.Background(), scenario, actx) + + if len(step.calls) != 1 { + t.Fatalf("step not called") + } + if step.calls[0].Params["dir"] != "/tmp/repo" { + t.Errorf("dir = %q, want /tmp/repo", step.calls[0].Params["dir"]) + } +} + +func TestEngine_Timeout(t *testing.T) { + registry := NewRegistry() + + slowStep := ActionHandlerFunc(func(ctx context.Context, actx *ActionContext, act Action) (map[string]string, error) { + select { + case <-ctx.Done(): + return nil, ctx.Err() + case <-time.After(10 * time.Second): + return nil, nil + } + }) + registry.Register("slow", TierCore, slowStep) + + scenario := &Scenario{ + Name: "timeout-test", + Timeout: Duration{100 * time.Millisecond}, + Phases: []Phase{ + { + Name: "phase1", + Actions: []Action{ + {Action: "slow"}, + }, + }, + }, + } + + engine := NewEngine(registry, nil) + actx := &ActionContext{ + Scenario: scenario, + Vars: make(map[string]string), + Log: func(string, ...interface{}) {}, + } + + start := time.Now() + result := engine.Run(context.Background(), scenario, actx) + elapsed := time.Since(start) + + if result.Status != StatusFail { + t.Errorf("status = %s, want FAIL", result.Status) + } + if elapsed > 2*time.Second { + t.Errorf("took %v, expected timeout at ~100ms", elapsed) + } +} + +func TestEngine_UnknownAction(t *testing.T) { + registry := NewRegistry() + scenario := &Scenario{ + Name: "unknown-test", + Timeout: Duration{5 * time.Second}, + Phases: []Phase{ + { + Name: "phase1", + Actions: []Action{ + {Action: "nonexistent"}, + }, + }, + }, + } + + engine := NewEngine(registry, nil) + actx := &ActionContext{ + Scenario: scenario, + Vars: make(map[string]string), + Log: func(string, ...interface{}) {}, + } + result := engine.Run(context.Background(), scenario, actx) + + if result.Status != StatusFail { + t.Errorf("status = %s, want FAIL for unknown action", result.Status) + } +} + +func TestEngine_ParallelPhase(t *testing.T) { + registry := NewRegistry() + + callCount := 0 + step := ActionHandlerFunc(func(ctx context.Context, actx *ActionContext, act Action) (map[string]string, error) { + callCount++ + time.Sleep(50 * time.Millisecond) + return nil, nil + }) + registry.Register("step", TierCore, step) + + scenario := &Scenario{ + Name: "parallel-test", + Timeout: Duration{5 * time.Second}, + Phases: []Phase{ + { + Name: "phase1", + Parallel: true, + Actions: []Action{ + {Action: "step"}, + {Action: "step"}, + {Action: "step"}, + }, + }, + }, + } + + engine := NewEngine(registry, nil) + actx := &ActionContext{ + Scenario: scenario, + Vars: make(map[string]string), + Log: func(string, ...interface{}) {}, + } + + start := time.Now() + result := engine.Run(context.Background(), scenario, actx) + elapsed := time.Since(start) + + if result.Status != StatusPass { + t.Errorf("status = %s, want PASS", result.Status) + } + // Parallel: should take ~50ms, not 150ms. + if elapsed > 200*time.Millisecond { + t.Errorf("took %v, parallel should be faster", elapsed) + } +} + +func TestResolveVars(t *testing.T) { + vars := map[string]string{ + "device": "/dev/sda", + "md5": "abc123", + } + + tests := []struct { + input string + want string + }{ + {"{{ device }}", "/dev/sda"}, + {"{{md5}}", "abc123"}, + {"dev={{ device }}, hash={{ md5 }}", "dev=/dev/sda, hash=abc123"}, + {"no vars", "no vars"}, + {"{{ unknown }}", "{{ unknown }}"}, // unresolved stays + } + + for _, tt := range tests { + got := resolveVars(tt.input, vars) + if got != tt.want { + t.Errorf("resolveVars(%q) = %q, want %q", tt.input, got, tt.want) + } + } +} + +func TestEngine_VarsInResult(t *testing.T) { + registry := NewRegistry() + step := &mockHandler{outputs: map[string]string{"value": "saved-data"}} + registry.Register("step", TierCore, step) + + scenario := &Scenario{ + Name: "vars-result-test", + Timeout: Duration{5 * time.Second}, + Env: map[string]string{"env_key": "env_val"}, + Phases: []Phase{ + { + Name: "phase1", + Actions: []Action{ + {Action: "step", SaveAs: "my_var"}, + }, + }, + }, + } + + engine := NewEngine(registry, nil) + actx := &ActionContext{ + Scenario: scenario, + Vars: make(map[string]string), + Log: func(string, ...interface{}) {}, + } + result := engine.Run(context.Background(), scenario, actx) + + if result.Status != StatusPass { + t.Fatalf("status = %s: %s", result.Status, result.Error) + } + + // Verify vars are copied into result. + if result.Vars == nil { + t.Fatal("result.Vars is nil") + } + if result.Vars["my_var"] != "saved-data" { + t.Errorf("result.Vars[my_var] = %q, want saved-data", result.Vars["my_var"]) + } + if result.Vars["env_key"] != "env_val" { + t.Errorf("result.Vars[env_key] = %q, want env_val", result.Vars["env_key"]) + } +} + +func TestEngine_CleanupVars(t *testing.T) { + registry := NewRegistry() + + inject := &mockHandler{outputs: map[string]string{ + "__cleanup_netem": "tc qdisc del dev eth0 root", + }} + clear := ActionHandlerFunc(func(ctx context.Context, actx *ActionContext, act Action) (map[string]string, error) { + cmd := actx.Vars["__cleanup_netem"] + if cmd == "" { + return nil, fmt.Errorf("cleanup var not found") + } + return map[string]string{"value": cmd}, nil + }) + + registry.Register("inject", TierCore, inject) + registry.Register("clear", TierCore, clear) + + scenario := &Scenario{ + Name: "cleanup-var-test", + Timeout: Duration{5 * time.Second}, + Phases: []Phase{ + { + Name: "inject", + Actions: []Action{ + {Action: "inject"}, + }, + }, + { + Name: "clear", + Actions: []Action{ + {Action: "clear", SaveAs: "result"}, + }, + }, + }, + } + + engine := NewEngine(registry, nil) + actx := &ActionContext{ + Scenario: scenario, + Vars: make(map[string]string), + Log: func(string, ...interface{}) {}, + } + result := engine.Run(context.Background(), scenario, actx) + + if result.Status != StatusPass { + t.Errorf("status = %s, want PASS: %s", result.Status, result.Error) + } + if actx.Vars["result"] != "tc qdisc del dev eth0 root" { + t.Errorf("result = %q", actx.Vars["result"]) + } +} diff --git a/weed/storage/blockvol/testrunner/infra/artifacts.go b/weed/storage/blockvol/testrunner/infra/artifacts.go new file mode 100644 index 000000000..b5d44bf87 --- /dev/null +++ b/weed/storage/blockvol/testrunner/infra/artifacts.go @@ -0,0 +1,94 @@ +package infra + +import ( + "context" + "fmt" + "log" + "os" + "path/filepath" + "time" +) + +// LogCollector is implemented by any target that can provide a log file. +type LogCollector interface { + CollectLog() (string, error) +} + +// ArtifactCollector gathers diagnostic info on failure. +type ArtifactCollector struct { + Dir string // base artifacts directory + Node *Node // initiator node for dmesg/lsblk + Log *log.Logger +} + +// NewArtifactCollector creates a collector rooted at the given directory. +func NewArtifactCollector(dir string, node *Node, logger *log.Logger) *ArtifactCollector { + if logger == nil { + logger = log.Default() + } + return &ArtifactCollector{ + Dir: dir, + Node: node, + Log: logger, + } +} + +// Collect gathers diagnostics when failed is true. +func (a *ArtifactCollector) Collect(failed bool, tgt LogCollector, label string) { + if !failed { + return + } + a.CollectLabeled(tgt, label) +} + +// CollectLabeled gathers diagnostics unconditionally. +func (a *ArtifactCollector) CollectLabeled(tgt LogCollector, label string) { + ts := time.Now().Format("20060102-150405") + testDir := filepath.Join(a.Dir, ts) + if err := os.MkdirAll(testDir, 0755); err != nil { + a.Log.Printf("artifacts: mkdir failed: %v", err) + return + } + + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + + if tgt != nil { + if logContent, err := tgt.CollectLog(); err == nil && logContent != "" { + a.writeArtifact(filepath.Join(testDir, label+".log"), logContent) + } + } + + if stdout, _, _, err := a.Node.RunRoot(ctx, "iscsiadm -m session 2>&1"); err == nil { + a.writeArtifact(filepath.Join(testDir, "iscsi-session.txt"), stdout) + } + + if stdout, _, _, err := a.Node.RunRoot(ctx, "dmesg | tail -200"); err == nil { + a.writeArtifact(filepath.Join(testDir, "dmesg.txt"), stdout) + } + + if stdout, _, _, err := a.Node.Run(ctx, "lsblk 2>&1"); err == nil { + a.writeArtifact(filepath.Join(testDir, "lsblk.txt"), stdout) + } + + a.Log.Printf("artifacts saved to %s", testDir) +} + +func (a *ArtifactCollector) writeArtifact(path, content string) { + if err := os.WriteFile(path, []byte(content), 0644); err != nil { + a.Log.Printf("artifacts: write %s: %v", path, err) + } else { + a.Log.Printf("artifacts: wrote %s (%d bytes)", filepath.Base(path), len(content)) + } +} + +// CollectPerf saves performance results to a timestamped JSON file. +func (a *ArtifactCollector) CollectPerf(name string, data string) { + ts := time.Now().Format("20060102-150405") + path := filepath.Join(a.Dir, fmt.Sprintf("perf-%s-%s.json", name, ts)) + if err := os.MkdirAll(filepath.Dir(path), 0755); err != nil { + a.Log.Printf("artifacts: mkdir failed: %v", err) + return + } + a.writeArtifact(path, data) +} diff --git a/weed/storage/blockvol/testrunner/infra/fault.go b/weed/storage/blockvol/testrunner/infra/fault.go new file mode 100644 index 000000000..295431069 --- /dev/null +++ b/weed/storage/blockvol/testrunner/infra/fault.go @@ -0,0 +1,132 @@ +package infra + +import ( + "context" + "fmt" + "strings" + "time" +) + +// InjectNetem adds a netem delay on the node's outbound traffic to targetIP. +// Returns the cleanup command string (for storage in vars). +func InjectNetem(ctx context.Context, node *Node, targetIP string, delayMs int) (cleanupCmd string, err error) { + iface, _, code, err := node.RunRoot(ctx, fmt.Sprintf( + "ip route get %s | head -1 | awk '{for(i=1;i<=NF;i++) if($i==\"dev\") print $(i+1)}'", targetIP)) + iface = strings.TrimSpace(iface) + if err != nil || code != 0 || iface == "" { + return "", fmt.Errorf("find interface for %s: iface=%q code=%d err=%v", targetIP, iface, code, err) + } + + _, stderr, code, err := node.RunRoot(ctx, fmt.Sprintf( + "tc qdisc add dev %s root netem delay %dms", iface, delayMs)) + if err != nil || code != 0 { + return "", fmt.Errorf("tc qdisc add: code=%d stderr=%s err=%v", code, stderr, err) + } + + cleanupCmd = fmt.Sprintf("tc qdisc del dev %s root 2>/dev/null", iface) + return cleanupCmd, nil +} + +// InjectIptablesDrop blocks outbound TCP traffic from node to targetIP on the given ports. +// Returns the cleanup command string. +func InjectIptablesDrop(ctx context.Context, node *Node, targetIP string, ports []int) (cleanupCmd string, err error) { + for i, port := range ports { + _, stderr, code, err := node.RunRoot(ctx, fmt.Sprintf( + "iptables -A OUTPUT -d %s -p tcp --dport %d -j DROP", targetIP, port)) + if err != nil || code != 0 { + // Rollback already-added rules + for j := 0; j < i; j++ { + node.RunRoot(ctx, fmt.Sprintf( + "iptables -D OUTPUT -d %s -p tcp --dport %d -j DROP 2>/dev/null", targetIP, ports[j])) + } + return "", fmt.Errorf("iptables add port %d: code=%d stderr=%s err=%v", port, code, stderr, err) + } + } + + // Build cleanup command that removes all rules. + var cmds []string + for _, port := range ports { + cmds = append(cmds, fmt.Sprintf( + "iptables -D OUTPUT -d %s -p tcp --dport %d -j DROP 2>/dev/null", targetIP, port)) + } + cleanupCmd = strings.Join(cmds, " && ") + return cleanupCmd, nil +} + +// FillDisk fills the filesystem at dir, leaving ~4MB free. +// Returns the cleanup command string. +func FillDisk(ctx context.Context, node *Node, dir string) (cleanupCmd string, err error) { + stdout, _, code, err := node.RunRoot(ctx, fmt.Sprintf( + "df -BM --output=avail %s | tail -1 | tr -d ' M'", dir)) + if err != nil || code != 0 { + return "", fmt.Errorf("df: code=%d err=%v", code, err) + } + availMB := 0 + fmt.Sscanf(strings.TrimSpace(stdout), "%d", &availMB) + if availMB < 8 { + return "", fmt.Errorf("not enough space to fill: %dMB available", availMB) + } + fillMB := availMB - 4 + + // Use fallocate (instant) instead of dd (linear time). + _, stderr, code, err := node.RunRoot(ctx, fmt.Sprintf( + "fallocate -l %dM %s/fillfile", fillMB, dir)) + if err != nil || code != 0 { + // Fallback to dd if fallocate not available. + _, stderr, code, err = node.RunRoot(ctx, fmt.Sprintf( + "dd if=/dev/zero of=%s/fillfile bs=1M count=%d 2>/dev/null", dir, fillMB)) + if err != nil || code != 0 { + stdout2, _, _, _ := node.RunRoot(ctx, fmt.Sprintf("test -f %s/fillfile && echo ok", dir)) + if !strings.Contains(stdout2, "ok") { + return "", fmt.Errorf("fillDisk: code=%d stderr=%s err=%v", code, stderr, err) + } + } + } + + cleanupCmd = fmt.Sprintf("rm -f %s/fillfile", dir) + return cleanupCmd, nil +} + +// CorruptWALRegion overwrites nBytes within the WAL section of the volume file. +func CorruptWALRegion(ctx context.Context, node *Node, volPath string, nBytes int) error { + const walOffset = 4096 // SuperblockSize + + stdout, _, code, err := node.RunRoot(ctx, fmt.Sprintf("stat -c %%s %s", volPath)) + if err != nil || code != 0 { + return fmt.Errorf("stat %s: code=%d err=%v", volPath, code, err) + } + fileSize := 0 + fmt.Sscanf(strings.TrimSpace(stdout), "%d", &fileSize) + + walEnd := walOffset + 64*1024*1024 + if walEnd > fileSize { + walEnd = fileSize + } + walUsable := walEnd - walOffset + if walUsable < nBytes*2 { + return fmt.Errorf("WAL region too small: %d", walUsable) + } + seekPos := walOffset + walUsable/3 + + _, stderr, code, err := node.RunRoot(ctx, fmt.Sprintf( + "python3 -c \"import sys; sys.stdout.buffer.write(b'\\xff'*%d)\" | dd of=%s bs=1 seek=%d conv=notrunc 2>/dev/null", + nBytes, volPath, seekPos)) + if err != nil || code != 0 { + return fmt.Errorf("corrupt WAL region: code=%d stderr=%s err=%v", code, stderr, err) + } + return nil +} + +// ClearFault executes a cleanup command stored in vars. +func ClearFault(ctx context.Context, node *Node, cleanupCmd string) error { + if cleanupCmd == "" { + return nil + } + cctx, cancel := context.WithTimeout(ctx, 10*time.Second) + defer cancel() + _, stderr, code, err := node.RunRoot(cctx, cleanupCmd) + if err != nil || code != 0 { + return fmt.Errorf("clear fault: code=%d stderr=%s err=%v", code, stderr, err) + } + return nil +} diff --git a/weed/storage/blockvol/testrunner/infra/ha_target.go b/weed/storage/blockvol/testrunner/infra/ha_target.go new file mode 100644 index 000000000..6452c5289 --- /dev/null +++ b/weed/storage/blockvol/testrunner/infra/ha_target.go @@ -0,0 +1,412 @@ +package infra + +import ( + "context" + "encoding/json" + "fmt" + "net/http" + "strings" + "time" +) + +// HATarget extends Target with HA-specific admin HTTP endpoints. +type HATarget struct { + *Target + AdminPort int + ReplicaData int // replica receiver data port + ReplicaCtrl int // replica receiver ctrl port + RebuildPort int + TPGID int // ALUA target port group ID (0 = omit flag) +} + +// StatusResp matches the JSON returned by GET /status. +type StatusResp struct { + Path string `json:"path"` + Epoch uint64 `json:"epoch"` + Role string `json:"role"` + WALHeadLSN uint64 `json:"wal_head_lsn"` + CheckpointLSN uint64 `json:"checkpoint_lsn"` + HasLease bool `json:"has_lease"` + Healthy bool `json:"healthy"` + VolumeSize uint64 `json:"volume_size"` +} + +// SnapshotEntry matches the JSON returned by POST /snapshot {action:"list"}. +type SnapshotEntry struct { + ID uint32 `json:"id"` + BaseLSN uint64 `json:"base_lsn"` + CreatedAt string `json:"created_at"` + CoWBlocks uint64 `json:"cow_blocks"` +} + +// NewHATarget creates an HATarget with the given ports. +func NewHATarget(node *Node, cfg TargetConfig, adminPort, replicaData, replicaCtrl, rebuildPort int) *HATarget { + return &HATarget{ + Target: NewTarget(node, cfg), + AdminPort: adminPort, + ReplicaData: replicaData, + ReplicaCtrl: replicaCtrl, + RebuildPort: rebuildPort, + } +} + +// HATargetSpec holds the parameters needed to create an HATarget from YAML config. +type HATargetSpec struct { + VolSize string + WALSize string + IQN string + ISCSIPort int + AdminPort int + ReplicaDataPort int + ReplicaCtrlPort int + RebuildPort int + TPGID int +} + +// NewHATargetFromSpec creates an HATarget from an HATargetSpec and Node. +func NewHATargetFromSpec(node *Node, name string, spec HATargetSpec) *HATarget { + volSize := spec.VolSize + if volSize == "" { + volSize = "100M" + } + walSize := spec.WALSize + if walSize == "" { + walSize = "64M" + } + + cfg := TargetConfig{ + VolSize: volSize, + WALSize: walSize, + IQN: spec.IQN, + Port: spec.ISCSIPort, + } + + ht := NewHATarget(node, cfg, spec.AdminPort, spec.ReplicaDataPort, spec.ReplicaCtrlPort, spec.RebuildPort) + ht.TPGID = spec.TPGID + + // Use unique file paths per target name. + ht.BinPath = "/tmp/iscsi-target-test" + ht.VolFile = fmt.Sprintf("/tmp/blockvol-%s.blk", name) + ht.LogFile = fmt.Sprintf("/tmp/iscsi-target-%s.log", name) + return ht +} + +// Start overrides Target.Start to add HA-specific flags. +func (h *HATarget) Start(ctx context.Context, create bool) error { + // Remove old log + h.Node.Run(ctx, fmt.Sprintf("rm -f %s", h.LogFile)) + + args := fmt.Sprintf("-vol %s -addr :%d -iqn %s", + h.VolFile, h.Config.Port, h.Config.IQN) + + if create { + h.Node.Run(ctx, fmt.Sprintf("rm -f %s %s.wal", h.VolFile, h.VolFile)) + args += fmt.Sprintf(" -create -size %s", h.Config.VolSize) + } + + if h.AdminPort > 0 { + args += fmt.Sprintf(" -admin 0.0.0.0:%d", h.AdminPort) + } + if h.ReplicaData > 0 && h.ReplicaCtrl > 0 { + args += fmt.Sprintf(" -replica-data :%d -replica-ctrl :%d", h.ReplicaData, h.ReplicaCtrl) + } + if h.RebuildPort > 0 { + args += fmt.Sprintf(" -rebuild-listen :%d", h.RebuildPort) + } + if h.TPGID > 0 { + args += fmt.Sprintf(" -tpg-id %d", h.TPGID) + } + + cmd := fmt.Sprintf("setsid -f %s %s >%s 2>&1", h.BinPath, args, h.LogFile) + _, stderr, code, err := h.Node.Run(ctx, cmd) + if err != nil || code != 0 { + return fmt.Errorf("start ha target: code=%d stderr=%s err=%v", code, stderr, err) + } + + if err := h.WaitForPort(ctx); err != nil { + return err + } + + if h.AdminPort > 0 { + if err := h.waitForAdminPort(ctx); err != nil { + return err + } + } + + // Discover PID by matching the unique volume file path. + stdout, _, _, _ := h.Node.Run(ctx, fmt.Sprintf("ps -eo pid,args | grep '%s' | grep -v grep | awk '{print $1}'", h.VolFile)) + pidStr := strings.TrimSpace(stdout) + if idx := strings.IndexByte(pidStr, '\n'); idx > 0 { + pidStr = pidStr[:idx] + } + pid := 0 + fmt.Sscanf(pidStr, "%d", &pid) + if pid == 0 { + return fmt.Errorf("find ha target PID: %q", pidStr) + } + h.Pid = pid + return nil +} + +func (h *HATarget) waitForAdminPort(ctx context.Context) error { + for { + select { + case <-ctx.Done(): + return fmt.Errorf("wait for admin port %d: %w", h.AdminPort, ctx.Err()) + default: + } + stdout, _, code, _ := h.Node.Run(ctx, fmt.Sprintf("ss -tln | grep :%d", h.AdminPort)) + if code == 0 && strings.Contains(stdout, fmt.Sprintf(":%d", h.AdminPort)) { + return nil + } + time.Sleep(200 * time.Millisecond) + } +} + +// curlPost executes a POST via curl on the node. +func (h *HATarget) curlPost(ctx context.Context, path string, body interface{}) (int, string, error) { + data, err := json.Marshal(body) + if err != nil { + return 0, "", err + } + cmd := fmt.Sprintf("curl -s -w '\\n%%{http_code}' -X POST -H 'Content-Type: application/json' -d '%s' http://127.0.0.1:%d%s 2>&1", + string(data), h.AdminPort, path) + stdout, _, code, err := h.Node.Run(ctx, cmd) + if err != nil || code != 0 { + return 0, "", fmt.Errorf("curl POST %s: code=%d err=%v stdout=%s", path, code, err, stdout) + } + return parseCurlOutput(stdout) +} + +// curlGet executes a GET via curl on the node. +func (h *HATarget) curlGet(ctx context.Context, path string) (int, string, error) { + cmd := fmt.Sprintf("curl -s -w '\\n%%{http_code}' http://127.0.0.1:%d%s 2>&1", h.AdminPort, path) + stdout, _, code, err := h.Node.Run(ctx, cmd) + if err != nil || code != 0 { + return 0, "", fmt.Errorf("curl GET %s: code=%d err=%v stdout=%s", path, code, err, stdout) + } + return parseCurlOutput(stdout) +} + +// parseCurlOutput splits curl -w '\n%{http_code}' output into body and status. +func parseCurlOutput(output string) (int, string, error) { + output = strings.TrimSpace(output) + idx := strings.LastIndex(output, "\n") + if idx < 0 { + var code int + if _, err := fmt.Sscanf(output, "%d", &code); err != nil { + return 0, "", fmt.Errorf("parse curl status: %q", output) + } + return code, "", nil + } + body := output[:idx] + var httpCode int + if _, err := fmt.Sscanf(strings.TrimSpace(output[idx+1:]), "%d", &httpCode); err != nil { + return 0, "", fmt.Errorf("parse curl status from %q", output[idx+1:]) + } + return httpCode, body, nil +} + +// Assign sends POST /assign to inject a role/epoch assignment. +func (h *HATarget) Assign(ctx context.Context, epoch uint64, role uint32, leaseTTLMs uint32) error { + code, body, err := h.curlPost(ctx, "/assign", map[string]interface{}{ + "epoch": epoch, + "role": role, + "lease_ttl_ms": leaseTTLMs, + }) + if err != nil { + return fmt.Errorf("assign request: %w", err) + } + if code != http.StatusOK { + return fmt.Errorf("assign failed (HTTP %d): %s", code, body) + } + return nil +} + +// AssignRaw sends POST /assign and returns HTTP status code and body. +func (h *HATarget) AssignRaw(ctx context.Context, body interface{}) (int, string, error) { + return h.curlPost(ctx, "/assign", body) +} + +// Status sends GET /status and returns the parsed response. +func (h *HATarget) Status(ctx context.Context) (*StatusResp, error) { + code, body, err := h.curlGet(ctx, "/status") + if err != nil { + return nil, fmt.Errorf("status request: %w", err) + } + if code != http.StatusOK { + return nil, fmt.Errorf("status failed (HTTP %d): %s", code, body) + } + var st StatusResp + if err := json.NewDecoder(strings.NewReader(body)).Decode(&st); err != nil { + return nil, fmt.Errorf("decode status: %w", err) + } + return &st, nil +} + +// SetReplica sends POST /replica to configure WAL shipping target. +func (h *HATarget) SetReplica(ctx context.Context, dataAddr, ctrlAddr string) error { + code, body, err := h.curlPost(ctx, "/replica", map[string]string{ + "data_addr": dataAddr, + "ctrl_addr": ctrlAddr, + }) + if err != nil { + return fmt.Errorf("replica request: %w", err) + } + if code != http.StatusOK { + return fmt.Errorf("replica failed (HTTP %d): %s", code, body) + } + return nil +} + +// SetReplicaRaw sends POST /replica and returns HTTP status + body. +func (h *HATarget) SetReplicaRaw(ctx context.Context, body interface{}) (int, string, error) { + return h.curlPost(ctx, "/replica", body) +} + +// StartRebuildEndpoint sends POST /rebuild {action:"start"}. +func (h *HATarget) StartRebuildEndpoint(ctx context.Context, listenAddr string) error { + code, body, err := h.curlPost(ctx, "/rebuild", map[string]string{ + "action": "start", + "listen_addr": listenAddr, + }) + if err != nil { + return fmt.Errorf("rebuild start: %w", err) + } + if code != http.StatusOK { + return fmt.Errorf("rebuild start failed (HTTP %d): %s", code, body) + } + return nil +} + +// StartRebuildClient sends POST /rebuild {action:"connect"}. +func (h *HATarget) StartRebuildClient(ctx context.Context, rebuildAddr string, epoch uint64) error { + code, body, err := h.curlPost(ctx, "/rebuild", map[string]interface{}{ + "action": "connect", + "rebuild_addr": rebuildAddr, + "epoch": epoch, + }) + if err != nil { + return fmt.Errorf("rebuild connect: %w", err) + } + if code != http.StatusOK { + return fmt.Errorf("rebuild connect failed (HTTP %d): %s", code, body) + } + return nil +} + +// StopRebuildEndpoint sends POST /rebuild {action:"stop"}. +func (h *HATarget) StopRebuildEndpoint(ctx context.Context) error { + code, body, err := h.curlPost(ctx, "/rebuild", map[string]string{"action": "stop"}) + if err != nil { + return fmt.Errorf("rebuild stop: %w", err) + } + if code != http.StatusOK { + return fmt.Errorf("rebuild stop failed (HTTP %d): %s", code, body) + } + return nil +} + +// CreateSnapshot sends POST /snapshot {action:"create", id:N}. +func (h *HATarget) CreateSnapshot(ctx context.Context, id uint32) error { + code, body, err := h.curlPost(ctx, "/snapshot", map[string]interface{}{ + "action": "create", + "id": id, + }) + if err != nil { + return fmt.Errorf("create snapshot %d: %w", id, err) + } + if code != http.StatusOK { + return fmt.Errorf("create snapshot %d failed (HTTP %d): %s", id, code, body) + } + return nil +} + +// DeleteSnapshot sends POST /snapshot {action:"delete", id:N}. +func (h *HATarget) DeleteSnapshot(ctx context.Context, id uint32) error { + code, body, err := h.curlPost(ctx, "/snapshot", map[string]interface{}{ + "action": "delete", + "id": id, + }) + if err != nil { + return fmt.Errorf("delete snapshot %d: %w", id, err) + } + if code != http.StatusOK { + return fmt.Errorf("delete snapshot %d failed (HTTP %d): %s", id, code, body) + } + return nil +} + +// ListSnapshots sends POST /snapshot {action:"list"} and returns the entries. +func (h *HATarget) ListSnapshots(ctx context.Context) ([]SnapshotEntry, error) { + code, body, err := h.curlPost(ctx, "/snapshot", map[string]string{ + "action": "list", + }) + if err != nil { + return nil, fmt.Errorf("list snapshots: %w", err) + } + if code != http.StatusOK { + return nil, fmt.Errorf("list snapshots failed (HTTP %d): %s", code, body) + } + var resp struct { + Snapshots []SnapshotEntry `json:"snapshots"` + } + if err := json.NewDecoder(strings.NewReader(body)).Decode(&resp); err != nil { + return nil, fmt.Errorf("decode snapshots: %w", err) + } + return resp.Snapshots, nil +} + +// Resize sends POST /resize {new_size_bytes:N}. +func (h *HATarget) Resize(ctx context.Context, newSizeBytes uint64) error { + code, body, err := h.curlPost(ctx, "/resize", map[string]interface{}{ + "new_size_bytes": newSizeBytes, + }) + if err != nil { + return fmt.Errorf("resize to %d: %w", newSizeBytes, err) + } + if code != http.StatusOK { + return fmt.Errorf("resize to %d failed (HTTP %d): %s", newSizeBytes, code, body) + } + return nil +} + +// WaitForRole polls GET /status until the target reports the expected role. +func (h *HATarget) WaitForRole(ctx context.Context, expectedRole string) error { + for { + select { + case <-ctx.Done(): + return fmt.Errorf("wait for role %s: %w", expectedRole, ctx.Err()) + default: + } + st, err := h.Status(ctx) + if err == nil && st.Role == expectedRole { + return nil + } + time.Sleep(500 * time.Millisecond) + } +} + +// WaitForLSN polls GET /status until wal_head_lsn >= minLSN. +func (h *HATarget) WaitForLSN(ctx context.Context, minLSN uint64) error { + for { + select { + case <-ctx.Done(): + return fmt.Errorf("wait for LSN >= %d: %w", minLSN, ctx.Err()) + default: + } + st, err := h.Status(ctx) + if err == nil && st.WALHeadLSN >= minLSN { + return nil + } + time.Sleep(500 * time.Millisecond) + } +} + +// HostAddr returns the target's node host (for building replica addresses). +func (h *HATarget) HostAddr() string { + if h.Node.IsLocal { + return "127.0.0.1" + } + return h.Node.Host +} diff --git a/weed/storage/blockvol/testrunner/infra/iscsi_client.go b/weed/storage/blockvol/testrunner/infra/iscsi_client.go new file mode 100644 index 000000000..63a4e1f35 --- /dev/null +++ b/weed/storage/blockvol/testrunner/infra/iscsi_client.go @@ -0,0 +1,203 @@ +package infra + +import ( + "context" + "fmt" + "strings" + "time" +) + +// ISCSIClient wraps iscsiadm commands on a node. +type ISCSIClient struct { + Node *Node + TargetHost string + TargetPort int +} + +// NewISCSIClient creates an iSCSI client bound to a node. +func NewISCSIClient(node *Node) *ISCSIClient { + return &ISCSIClient{Node: node} +} + +// Discover runs iSCSI SendTargets discovery and returns discovered IQNs. +func (c *ISCSIClient) Discover(ctx context.Context, host string, port int) ([]string, error) { + c.TargetHost = host + c.TargetPort = port + + cmd := fmt.Sprintf("iscsiadm -m discovery -t sendtargets -p %s:%d", host, port) + stdout, stderr, code, err := c.Node.RunRoot(ctx, cmd) + if err != nil { + return nil, fmt.Errorf("discovery error: %w", err) + } + if code != 0 { + return nil, fmt.Errorf("discovery failed (code %d): %s", code, stderr) + } + + var iqns []string + for _, line := range strings.Split(stdout, "\n") { + line = strings.TrimSpace(line) + if line == "" { + continue + } + parts := strings.Fields(line) + if len(parts) >= 2 { + iqns = append(iqns, parts[1]) + } + } + + for _, iqn := range iqns { + c.fixNodePortal(ctx, iqn, host, port) + } + + return iqns, nil +} + +func (c *ISCSIClient) fixNodePortal(ctx context.Context, iqn, host string, port int) { + stdout, _, _, _ := c.Node.RunRoot(ctx, + fmt.Sprintf("iscsiadm -m node -T %s 2>/dev/null", iqn)) + + hasWildcard := false + for _, line := range strings.Split(stdout, "\n") { + if strings.Contains(line, "node.conn[0].address") { + if strings.Contains(line, "::") || strings.Contains(line, "0.0.0.0") { + hasWildcard = true + } + } + } + if !hasWildcard { + return + } + + c.Node.RunRoot(ctx, fmt.Sprintf("iscsiadm -m node -T %s -o delete 2>/dev/null", iqn)) + portal := fmt.Sprintf("%s:%d", host, port) + c.Node.RunRoot(ctx, fmt.Sprintf("iscsiadm -m node -T %s -p %s -o new 2>/dev/null", iqn, portal)) +} + +// Login connects to the target and returns the device path (e.g. /dev/sda). +func (c *ISCSIClient) Login(ctx context.Context, iqn string) (string, error) { + var cmd string + if c.TargetHost != "" && c.TargetHost != "127.0.0.1" && c.TargetHost != "localhost" { + portal := fmt.Sprintf("%s:%d", c.TargetHost, c.TargetPort) + cmd = fmt.Sprintf("iscsiadm -m node -T %s -p %s --login", iqn, portal) + } else { + cmd = fmt.Sprintf("iscsiadm -m node -T %s --login", iqn) + } + _, stderr, code, err := c.Node.RunRoot(ctx, cmd) + if err != nil { + return "", fmt.Errorf("login error: %w", err) + } + if code != 0 { + return "", fmt.Errorf("login failed (code %d): %s", code, stderr) + } + + return c.waitForDevice(ctx, iqn) +} + +// Logout disconnects from the target. +func (c *ISCSIClient) Logout(ctx context.Context, iqn string) error { + cmd := fmt.Sprintf("iscsiadm -m node -T %s --logout", iqn) + _, stderr, code, err := c.Node.RunRoot(ctx, cmd) + if err != nil { + return fmt.Errorf("logout error: %w", err) + } + if code != 0 { + return fmt.Errorf("logout failed (code %d): %s", code, stderr) + } + return nil +} + +// GetDevice returns the device path for an active session. +func (c *ISCSIClient) GetDevice(ctx context.Context, iqn string) (string, error) { + return c.waitForDevice(ctx, iqn) +} + +func (c *ISCSIClient) waitForDevice(ctx context.Context, iqn string) (string, error) { + deadline := time.Now().Add(30 * time.Second) + rescanned := false + for time.Now().Before(deadline) { + select { + case <-ctx.Done(): + return "", ctx.Err() + default: + } + + stdout, _, code, _ := c.Node.RunRoot(ctx, "iscsiadm -m session -P3") + if code == 0 { + dev := ParseDeviceFromSession(stdout, iqn) + if dev != "" { + return dev, nil + } + } + + if !rescanned && time.Until(deadline) < 25*time.Second { + c.Node.RunRoot(ctx, "iscsiadm -m session -R") + rescanned = true + } + time.Sleep(500 * time.Millisecond) + } + return "", fmt.Errorf("device for %s did not appear within 30s", iqn) +} + +// ParseDeviceFromSession extracts /dev/sdX from iscsiadm -m session -P3 output. +func ParseDeviceFromSession(output, iqn string) string { + lines := strings.Split(output, "\n") + inTarget := false + for _, line := range lines { + if strings.Contains(line, "Target: "+iqn) { + inTarget = true + continue + } + if inTarget && strings.Contains(line, "Target: ") { + break + } + if inTarget && strings.Contains(line, "Attached scsi disk") { + fields := strings.Fields(line) + for i, f := range fields { + if f == "disk" && i+1 < len(fields) { + return "/dev/" + fields[i+1] + } + } + } + } + return "" +} + +// WaitForSession polls until a session for the given IQN is in LOGGED_IN state. +func (c *ISCSIClient) WaitForSession(ctx context.Context, iqn string) error { + for { + select { + case <-ctx.Done(): + return fmt.Errorf("session %s did not recover: %w", iqn, ctx.Err()) + default: + } + + stdout, _, code, _ := c.Node.RunRoot(ctx, "iscsiadm -m session") + if code == 0 && strings.Contains(stdout, iqn) { + return nil + } + time.Sleep(500 * time.Millisecond) + } +} + +// CleanupAll force-logouts sessions matching the IQN prefix only. +func (c *ISCSIClient) CleanupAll(ctx context.Context, iqnPrefix string) error { + stdout, _, _, _ := c.Node.RunRoot(ctx, "iscsiadm -m session 2>&1") + if stdout == "" || strings.Contains(stdout, "No active sessions") { + return nil + } + + for _, line := range strings.Split(stdout, "\n") { + line = strings.TrimSpace(line) + if !strings.Contains(line, iqnPrefix) { + continue + } + fields := strings.Fields(line) + for _, f := range fields { + if strings.HasPrefix(f, iqnPrefix) { + c.Node.RunRoot(ctx, fmt.Sprintf("iscsiadm -m node -T %s --logout 2>/dev/null", f)) + c.Node.RunRoot(ctx, fmt.Sprintf("iscsiadm -m node -T %s -o delete 2>/dev/null", f)) + } + } + } + return nil +} diff --git a/weed/storage/blockvol/testrunner/infra/node.go b/weed/storage/blockvol/testrunner/infra/node.go new file mode 100644 index 000000000..a633868d0 --- /dev/null +++ b/weed/storage/blockvol/testrunner/infra/node.go @@ -0,0 +1,362 @@ +package infra + +import ( + "bytes" + "context" + "fmt" + "io" + "net" + "os" + "os/exec" + "strings" + "sync" + "time" + + "golang.org/x/crypto/ssh" +) + +// Node represents an SSH-accessible (or local WSL2/native) machine. +type Node struct { + Host string + User string + KeyFile string + IsLocal bool // WSL2 mode: use exec.CommandContext via wsl instead of SSH + IsNative bool // Native local mode: use bash -c directly (for Linux agents) + + mu sync.Mutex + client *ssh.Client +} + +// Connect establishes the SSH connection (no-op for local/native mode). +func (n *Node) Connect() error { + if n.IsLocal || n.IsNative { + return nil + } + + key, err := os.ReadFile(n.KeyFile) + if err != nil { + return fmt.Errorf("read SSH key %s: %w", n.KeyFile, err) + } + signer, err := ssh.ParsePrivateKey(key) + if err != nil { + return fmt.Errorf("parse SSH key: %w", err) + } + + config := &ssh.ClientConfig{ + User: n.User, + Auth: []ssh.AuthMethod{ssh.PublicKeys(signer)}, + HostKeyCallback: ssh.InsecureIgnoreHostKey(), + Timeout: 10 * time.Second, + } + + addr := n.Host + if !strings.Contains(addr, ":") { + addr += ":22" + } + + n.mu.Lock() + defer n.mu.Unlock() + n.client, err = ssh.Dial("tcp", addr, config) + if err != nil { + return fmt.Errorf("SSH dial %s: %w", addr, err) + } + return nil +} + +// Run executes a command and returns stdout, stderr, exit code. +func (n *Node) Run(ctx context.Context, cmd string) (stdout, stderr string, exitCode int, err error) { + if n.IsNative { + return n.runNative(ctx, cmd) + } + if n.IsLocal { + return n.runLocal(ctx, cmd) + } + return n.runSSH(ctx, cmd) +} + +func (n *Node) runNative(ctx context.Context, cmd string) (string, string, int, error) { + c := exec.CommandContext(ctx, "bash", "-c", cmd) + var outBuf, errBuf bytes.Buffer + c.Stdout = &outBuf + c.Stderr = &errBuf + + err := c.Run() + if ctx.Err() != nil { + return outBuf.String(), errBuf.String(), -1, fmt.Errorf("command timed out: %w", ctx.Err()) + } + if err != nil { + if exitErr, ok := err.(*exec.ExitError); ok { + return outBuf.String(), errBuf.String(), exitErr.ExitCode(), nil + } + return outBuf.String(), errBuf.String(), -1, err + } + return outBuf.String(), errBuf.String(), 0, nil +} + +func (n *Node) runLocal(ctx context.Context, cmd string) (string, string, int, error) { + c := exec.CommandContext(ctx, "wsl", "-e", "bash", "-c", cmd) + var outBuf, errBuf bytes.Buffer + c.Stdout = &outBuf + c.Stderr = &errBuf + + err := c.Run() + if ctx.Err() != nil { + return outBuf.String(), errBuf.String(), -1, fmt.Errorf("command timed out: %w", ctx.Err()) + } + if err != nil { + if exitErr, ok := err.(*exec.ExitError); ok { + return outBuf.String(), errBuf.String(), exitErr.ExitCode(), nil + } + return outBuf.String(), errBuf.String(), -1, err + } + return outBuf.String(), errBuf.String(), 0, nil +} + +func (n *Node) runSSH(ctx context.Context, cmd string) (string, string, int, error) { + n.mu.Lock() + if n.client == nil { + n.mu.Unlock() + return "", "", -1, fmt.Errorf("SSH not connected") + } + session, err := n.client.NewSession() + n.mu.Unlock() + if err != nil { + return "", "", -1, fmt.Errorf("new SSH session: %w", err) + } + defer session.Close() + + var outBuf, errBuf bytes.Buffer + session.Stdout = &outBuf + session.Stderr = &errBuf + + done := make(chan error, 1) + go func() { done <- session.Run(cmd) }() + + select { + case <-ctx.Done(): + _ = session.Signal(ssh.SIGKILL) + return outBuf.String(), errBuf.String(), -1, fmt.Errorf("command timed out: %w", ctx.Err()) + case err := <-done: + if err != nil { + if exitErr, ok := err.(*ssh.ExitError); ok { + return outBuf.String(), errBuf.String(), exitErr.ExitStatus(), nil + } + return outBuf.String(), errBuf.String(), -1, err + } + return outBuf.String(), errBuf.String(), 0, nil + } +} + +// RunRoot executes a command with sudo -n (non-interactive). +func (n *Node) RunRoot(ctx context.Context, cmd string) (string, string, int, error) { + return n.Run(ctx, "sudo -n "+cmd) +} + +// Upload copies a local file to the remote node via SCP. +func (n *Node) Upload(local, remote string) error { + if n.IsNative { + ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second) + defer cancel() + _, stderr, code, err := n.Run(ctx, fmt.Sprintf("cp %s %s && chmod +x %s", local, remote, remote)) + if err != nil || code != 0 { + return fmt.Errorf("native upload: code=%d stderr=%s err=%v", code, stderr, err) + } + return nil + } + if n.IsLocal { + ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second) + defer cancel() + wslLocal := ToWSLPath(local) + _, stderr, code, err := n.Run(ctx, fmt.Sprintf("cp %s %s && chmod +x %s", wslLocal, remote, remote)) + if err != nil || code != 0 { + return fmt.Errorf("local upload: code=%d stderr=%s err=%v", code, stderr, err) + } + return nil + } + return n.scpUpload(local, remote) +} + +func (n *Node) scpUpload(local, remote string) error { + data, err := os.ReadFile(local) + if err != nil { + return fmt.Errorf("read local file %s: %w", local, err) + } + + n.mu.Lock() + if n.client == nil { + n.mu.Unlock() + return fmt.Errorf("SSH not connected") + } + session, err := n.client.NewSession() + n.mu.Unlock() + if err != nil { + return fmt.Errorf("new SSH session: %w", err) + } + defer session.Close() + + // Use cat+chmod over stdin — simpler and more reliable than SCP protocol. + w, err := session.StdinPipe() + if err != nil { + return fmt.Errorf("stdin pipe: %w", err) + } + + go func() { + w.Write(data) + w.Close() + }() + + cmd := fmt.Sprintf("cat > %s && chmod +x %s", remote, remote) + if err := session.Run(cmd); err != nil { + return fmt.Errorf("upload run: %w", err) + } + return nil +} + +// Download copies a remote file to local via SCP. +func (n *Node) Download(remote, local string) error { + if n.IsNative { + ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second) + defer cancel() + _, stderr, code, err := n.Run(ctx, fmt.Sprintf("cp %s %s", remote, local)) + if err != nil || code != 0 { + return fmt.Errorf("native download: code=%d stderr=%s err=%v", code, stderr, err) + } + return nil + } + if n.IsLocal { + ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second) + defer cancel() + wslLocal := ToWSLPath(local) + _, stderr, code, err := n.Run(ctx, fmt.Sprintf("cp %s %s", remote, wslLocal)) + if err != nil || code != 0 { + return fmt.Errorf("local download: code=%d stderr=%s err=%v", code, stderr, err) + } + return nil + } + return n.scpDownload(remote, local) +} + +func (n *Node) scpDownload(remote, local string) error { + n.mu.Lock() + if n.client == nil { + n.mu.Unlock() + return fmt.Errorf("SSH not connected") + } + session, err := n.client.NewSession() + n.mu.Unlock() + if err != nil { + return fmt.Errorf("new SSH session: %w", err) + } + defer session.Close() + + var buf bytes.Buffer + session.Stdout = &buf + if err := session.Run(fmt.Sprintf("cat %s", remote)); err != nil { + return fmt.Errorf("read remote %s: %w", remote, err) + } + return os.WriteFile(local, buf.Bytes(), 0644) +} + +// Kill sends SIGKILL to a process by PID. +func (n *Node) Kill(pid int) error { + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + _, _, _, err := n.RunRoot(ctx, fmt.Sprintf("kill -9 %d", pid)) + return err +} + +// HasCommand checks if a command is available on the node. +func (n *Node) HasCommand(cmd string) bool { + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + _, _, code, err := n.Run(ctx, fmt.Sprintf("which %s", cmd)) + return err == nil && code == 0 +} + +// Close closes the SSH connection. +func (n *Node) Close() { + n.mu.Lock() + defer n.mu.Unlock() + if n.client != nil { + n.client.Close() + n.client = nil + } +} + +// DialTCP opens a direct TCP connection through the SSH tunnel. +func (n *Node) DialTCP(addr string) (net.Conn, error) { + if n.IsLocal || n.IsNative { + return net.DialTimeout("tcp", addr, 5*time.Second) + } + n.mu.Lock() + defer n.mu.Unlock() + if n.client == nil { + return nil, fmt.Errorf("SSH not connected") + } + return n.client.Dial("tcp", addr) +} + +// StreamRun executes a command and streams stdout to the writer. +func (n *Node) StreamRun(ctx context.Context, cmd string, w io.Writer) error { + if n.IsNative { + c := exec.CommandContext(ctx, "bash", "-c", cmd) + c.Stdout = w + c.Stderr = w + return c.Run() + } + if n.IsLocal { + c := exec.CommandContext(ctx, "wsl", "-e", "bash", "-c", cmd) + c.Stdout = w + c.Stderr = w + return c.Run() + } + + n.mu.Lock() + if n.client == nil { + n.mu.Unlock() + return fmt.Errorf("SSH not connected") + } + session, err := n.client.NewSession() + n.mu.Unlock() + if err != nil { + return err + } + defer session.Close() + + session.Stdout = w + session.Stderr = w + + done := make(chan error, 1) + go func() { done <- session.Run(cmd) }() + + select { + case <-ctx.Done(): + _ = session.Signal(ssh.SIGKILL) + return ctx.Err() + case err := <-done: + return err + } +} + +// ToWSLPath converts a Windows path to a WSL path. +func ToWSLPath(winPath string) string { + p := strings.ReplaceAll(winPath, "\\", "/") + if len(p) >= 2 && p[1] == ':' { + drive := strings.ToLower(string(p[0])) + p = "/mnt/" + drive + p[2:] + } + return p +} + +func remoteName(path string) string { + parts := strings.Split(path, "/") + return parts[len(parts)-1] +} + +func remoteDir(path string) string { + idx := strings.LastIndex(path, "/") + if idx < 0 { + return "." + } + return path[:idx] +} diff --git a/weed/storage/blockvol/testrunner/infra/target.go b/weed/storage/blockvol/testrunner/infra/target.go new file mode 100644 index 000000000..73782677b --- /dev/null +++ b/weed/storage/blockvol/testrunner/infra/target.go @@ -0,0 +1,196 @@ +package infra + +import ( + "context" + "fmt" + "os" + "os/exec" + "strconv" + "strings" + "time" +) + +// TargetConfig configures an iSCSI target instance. +type TargetConfig struct { + VolSize string // e.g. "100M" + WALSize string // e.g. "64M" (default), "4M" for WAL pressure tests + IQN string + Port int +} + +// DefaultTargetConfig returns a default target config for integration tests. +func DefaultTargetConfig() TargetConfig { + return TargetConfig{ + VolSize: "100M", + WALSize: "64M", + IQN: "iqn.2024.com.seaweedfs:test1", + Port: 3260, + } +} + +// Target manages the lifecycle of an iscsi-target process on a remote node. +type Target struct { + Node *Node + Config TargetConfig + BinPath string // remote path to iscsi-target binary + Pid int + LogFile string // remote path to target's stderr log + VolFile string // remote path to volume file +} + +// NewTarget creates a Target bound to a node. +func NewTarget(node *Node, config TargetConfig) *Target { + return &Target{ + Node: node, + Config: config, + BinPath: "/tmp/iscsi-target-test", + VolFile: "/tmp/blockvol-test.blk", + LogFile: "/tmp/iscsi-target-test.log", + } +} + +// SetBinPath overrides the remote binary path. +func (t *Target) SetBinPath(p string) { t.BinPath = p } + +// SetVolFile overrides the remote volume file path. +func (t *Target) SetVolFile(p string) { t.VolFile = p } + +// SetLogFile overrides the remote log file path. +func (t *Target) SetLogFile(p string) { t.LogFile = p } + +// Build cross-compiles the iscsi-target binary for linux/amd64. +func (t *Target) Build(ctx context.Context, repoDir string) error { + binDir := repoDir + "/weed/storage/blockvol/iscsi/cmd/iscsi-target" + outPath := repoDir + "/iscsi-target-linux" + + cmd := exec.CommandContext(ctx, "go", "build", "-o", outPath, ".") + cmd.Dir = binDir + cmd.Env = append(os.Environ(), "GOOS=linux", "GOARCH=amd64", "CGO_ENABLED=0") + out, err := cmd.CombinedOutput() + if err != nil { + return fmt.Errorf("build failed: %s\n%w", out, err) + } + return nil +} + +// Deploy uploads the pre-built binary to the target node. +func (t *Target) Deploy(localBin string) error { + return t.Node.Upload(localBin, t.BinPath) +} + +// Start launches the target process. If create is true, a new volume is created. +func (t *Target) Start(ctx context.Context, create bool) error { + // Remove old log + t.Node.Run(ctx, fmt.Sprintf("rm -f %s", t.LogFile)) + + args := fmt.Sprintf("-vol %s -addr :%d -iqn %s", + t.VolFile, t.Config.Port, t.Config.IQN) + + if create { + t.Node.Run(ctx, fmt.Sprintf("rm -f %s %s.wal", t.VolFile, t.VolFile)) + args += fmt.Sprintf(" -create -size %s", t.Config.VolSize) + } + + cmd := fmt.Sprintf("setsid -f %s %s >%s 2>&1", t.BinPath, args, t.LogFile) + _, stderr, code, err := t.Node.Run(ctx, cmd) + if err != nil || code != 0 { + return fmt.Errorf("start target: code=%d stderr=%s err=%v", code, stderr, err) + } + + if err := t.WaitForPort(ctx); err != nil { + return err + } + + // Discover PID by matching the binary name + stdout, _, _, _ := t.Node.Run(ctx, fmt.Sprintf("ps -eo pid,args | grep '%s' | grep -v grep | awk '{print $1}'", t.BinPath)) + pidStr := strings.TrimSpace(stdout) + if idx := strings.IndexByte(pidStr, '\n'); idx > 0 { + pidStr = pidStr[:idx] + } + pid, err := strconv.Atoi(pidStr) + if err != nil { + return fmt.Errorf("find target PID: %q: %w", pidStr, err) + } + t.Pid = pid + return nil +} + +// Stop sends SIGTERM, waits up to 10s, then Kill9. +func (t *Target) Stop(ctx context.Context) error { + if t.Pid == 0 { + return nil + } + + t.Node.Run(ctx, fmt.Sprintf("kill %d", t.Pid)) + + deadline := time.Now().Add(10 * time.Second) + for time.Now().Before(deadline) { + _, _, code, _ := t.Node.Run(ctx, fmt.Sprintf("kill -0 %d 2>/dev/null", t.Pid)) + if code != 0 { + t.Pid = 0 + return nil + } + time.Sleep(500 * time.Millisecond) + } + + return t.Kill9() +} + +// Kill9 sends SIGKILL immediately. +func (t *Target) Kill9() error { + if t.Pid == 0 { + return nil + } + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + t.Node.Run(ctx, fmt.Sprintf("kill -9 %d", t.Pid)) + t.Pid = 0 + return nil +} + +// Restart stops the target and starts it again (preserving the volume). +func (t *Target) Restart(ctx context.Context) error { + if err := t.Stop(ctx); err != nil { + return fmt.Errorf("restart stop: %w", err) + } + return t.Start(ctx, false) +} + +// WaitForPort polls until the target port is listening. +func (t *Target) WaitForPort(ctx context.Context) error { + for { + select { + case <-ctx.Done(): + return fmt.Errorf("wait for port %d: %w", t.Config.Port, ctx.Err()) + default: + } + + stdout, _, code, _ := t.Node.Run(ctx, fmt.Sprintf("ss -tln | grep :%d", t.Config.Port)) + if code == 0 && strings.Contains(stdout, fmt.Sprintf(":%d", t.Config.Port)) { + return nil + } + time.Sleep(200 * time.Millisecond) + } +} + +// CollectLog downloads the target's log file contents. +func (t *Target) CollectLog() (string, error) { + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + stdout, _, _, err := t.Node.Run(ctx, fmt.Sprintf("cat %s 2>/dev/null", t.LogFile)) + if err != nil { + return "", err + } + return stdout, nil +} + +// Cleanup removes the volume file, WAL, and log from the target node. +func (t *Target) Cleanup(ctx context.Context) { + t.Node.Run(ctx, fmt.Sprintf("rm -f %s %s.wal %s", t.VolFile, t.VolFile, t.LogFile)) +} + +// PID returns the current target process ID. +func (t *Target) PID() int { return t.Pid } + +// VolFilePath returns the remote volume file path. +func (t *Target) VolFilePath() string { return t.VolFile } diff --git a/weed/storage/blockvol/testrunner/local_node.go b/weed/storage/blockvol/testrunner/local_node.go new file mode 100644 index 000000000..0124ec09f --- /dev/null +++ b/weed/storage/blockvol/testrunner/local_node.go @@ -0,0 +1,104 @@ +package testrunner + +import ( + "bytes" + "context" + "fmt" + "io" + "os" + "os/exec" + "path/filepath" + "strings" +) + +// LocalNode implements NodeRunner by executing commands locally via os/exec. +// Used by agents that run on the same machine as the targets they manage. +type LocalNode struct { + hostname string + isRoot bool +} + +// NewLocalNode creates a LocalNode, detecting root status. +func NewLocalNode(hostname string) *LocalNode { + n := &LocalNode{hostname: hostname} + n.isRoot = n.detectRoot() + return n +} + +// Hostname returns the node's hostname. +func (n *LocalNode) Hostname() string { return n.hostname } + +// IsRoot returns whether the process is running as root. +func (n *LocalNode) IsRoot() bool { return n.isRoot } + +func (n *LocalNode) detectRoot() bool { + // Try "id -u" to check if running as root (uid 0). + ctx, cancel := context.WithTimeout(context.Background(), 5*secondDuration) + defer cancel() + out, _, code, err := n.Run(ctx, "id -u") + if err != nil || code != 0 { + return false + } + return strings.TrimSpace(out) == "0" +} + +// secondDuration avoids importing time in the constant. +const secondDuration = 1_000_000_000 // time.Second + +// Run executes a command locally via bash -c. +func (n *LocalNode) Run(ctx context.Context, cmd string) (stdout, stderr string, exitCode int, err error) { + c := exec.CommandContext(ctx, "bash", "-c", cmd) + var outBuf, errBuf bytes.Buffer + c.Stdout = &outBuf + c.Stderr = &errBuf + + runErr := c.Run() + if ctx.Err() != nil { + return outBuf.String(), errBuf.String(), -1, fmt.Errorf("command timed out: %w", ctx.Err()) + } + if runErr != nil { + if exitErr, ok := runErr.(*exec.ExitError); ok { + return outBuf.String(), errBuf.String(), exitErr.ExitCode(), nil + } + return outBuf.String(), errBuf.String(), -1, runErr + } + return outBuf.String(), errBuf.String(), 0, nil +} + +// RunRoot executes a command as root. If already root, runs directly. +// Otherwise uses sudo -n (non-interactive, fails if password required). +func (n *LocalNode) RunRoot(ctx context.Context, cmd string) (stdout, stderr string, exitCode int, err error) { + if n.isRoot { + return n.Run(ctx, cmd) + } + return n.Run(ctx, "sudo -n "+cmd) +} + +// Upload copies a file from local source to a local destination path. +func (n *LocalNode) Upload(local, remote string) error { + // Ensure destination directory exists. + dir := filepath.Dir(remote) + if err := os.MkdirAll(dir, 0755); err != nil { + return fmt.Errorf("mkdir %s: %w", dir, err) + } + + src, err := os.Open(local) + if err != nil { + return fmt.Errorf("open source %s: %w", local, err) + } + defer src.Close() + + dst, err := os.OpenFile(remote, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0755) + if err != nil { + return fmt.Errorf("create dest %s: %w", remote, err) + } + defer dst.Close() + + if _, err := io.Copy(dst, src); err != nil { + return fmt.Errorf("copy %s → %s: %w", local, remote, err) + } + return nil +} + +// Close is a no-op for LocalNode (no connection to close). +func (n *LocalNode) Close() {} diff --git a/weed/storage/blockvol/testrunner/local_node_test.go b/weed/storage/blockvol/testrunner/local_node_test.go new file mode 100644 index 000000000..819223f61 --- /dev/null +++ b/weed/storage/blockvol/testrunner/local_node_test.go @@ -0,0 +1,122 @@ +package testrunner + +import ( + "context" + "os" + "path/filepath" + "runtime" + "testing" + "time" +) + +func skipIfWindows(t *testing.T) { + if runtime.GOOS == "windows" { + t.Skip("LocalNode tests require Unix shell") + } +} + +func TestLocalNode_Run_Echo(t *testing.T) { + skipIfWindows(t) + n := NewLocalNode("test") + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + + stdout, stderr, code, err := n.Run(ctx, "echo hello") + if err != nil { + t.Fatalf("Run: %v", err) + } + if code != 0 { + t.Fatalf("exit code %d, stderr: %s", code, stderr) + } + if got := stdout; got != "hello\n" { + t.Errorf("stdout = %q, want %q", got, "hello\n") + } +} + +func TestLocalNode_Run_ExitCode(t *testing.T) { + skipIfWindows(t) + n := NewLocalNode("test") + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + + _, _, code, err := n.Run(ctx, "exit 42") + if err != nil { + t.Fatalf("Run: %v", err) + } + if code != 42 { + t.Errorf("exit code = %d, want 42", code) + } +} + +func TestLocalNode_Run_Timeout(t *testing.T) { + skipIfWindows(t) + n := NewLocalNode("test") + ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond) + defer cancel() + + _, _, _, err := n.Run(ctx, "sleep 10") + if err == nil { + t.Fatal("expected timeout error") + } +} + +func TestLocalNode_RunRoot_NonRoot(t *testing.T) { + skipIfWindows(t) + n := NewLocalNode("test") + // Force non-root to test sudo path. + n.isRoot = false + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + + // sudo -n will likely fail without sudoers config, but should not panic. + _, _, _, err := n.RunRoot(ctx, "echo test") + // We only care that it doesn't panic — error is expected on most CI. + _ = err +} + +func TestLocalNode_Upload(t *testing.T) { + skipIfWindows(t) + n := NewLocalNode("test") + + dir := t.TempDir() + src := filepath.Join(dir, "src.bin") + dst := filepath.Join(dir, "subdir", "dst.bin") + + // Create source file. + content := []byte("test binary content 12345") + if err := os.WriteFile(src, content, 0644); err != nil { + t.Fatalf("write src: %v", err) + } + + // Upload (should create subdir). + if err := n.Upload(src, dst); err != nil { + t.Fatalf("Upload: %v", err) + } + + // Verify. + got, err := os.ReadFile(dst) + if err != nil { + t.Fatalf("read dst: %v", err) + } + if string(got) != string(content) { + t.Errorf("content mismatch: got %q", got) + } + + // Verify executable permission. + info, _ := os.Stat(dst) + if info.Mode()&0100 == 0 { + t.Error("file is not executable") + } +} + +func TestLocalNode_Close(t *testing.T) { + n := NewLocalNode("test") + n.Close() // should be no-op, no panic +} + +func TestLocalNode_DetectRoot(t *testing.T) { + skipIfWindows(t) + n := NewLocalNode("test") + // We just verify it doesn't panic and returns a bool. + _ = n.IsRoot() +} diff --git a/weed/storage/blockvol/testrunner/metrics.go b/weed/storage/blockvol/testrunner/metrics.go new file mode 100644 index 000000000..2f4592ed7 --- /dev/null +++ b/weed/storage/blockvol/testrunner/metrics.go @@ -0,0 +1,156 @@ +package testrunner + +import ( + "bufio" + "fmt" + "math" + "sort" + "strconv" + "strings" +) + +// MetricsSample holds a snapshot of Prometheus-format metrics from a target. +type MetricsSample struct { + Target string `json:"target"` + Metrics map[string]float64 `json:"metrics"` +} + +// ParsePrometheusText parses the text exposition format from /metrics into +// a flat map of metric_name → value. Histograms/summaries produce multiple +// entries (e.g. "metric_bucket{le=\"0.01\"}" → value). +func ParsePrometheusText(body string) map[string]float64 { + metrics := make(map[string]float64) + scanner := bufio.NewScanner(strings.NewReader(body)) + for scanner.Scan() { + line := strings.TrimSpace(scanner.Text()) + if line == "" || strings.HasPrefix(line, "#") { + continue + } + name, value, ok := parseMetricLine(line) + if ok { + metrics[name] = value + } + } + return metrics +} + +// parseMetricLine parses "metric_name{labels} value" or "metric_name value". +func parseMetricLine(line string) (string, float64, bool) { + // Find the last space that separates the key from the value. + // Handle possible timestamp after value. + parts := strings.Fields(line) + if len(parts) < 2 { + return "", 0, false + } + name := parts[0] + val, err := strconv.ParseFloat(parts[1], 64) + if err != nil { + return "", 0, false + } + return name, val, true +} + +// PerfStats summarizes latency or throughput samples. +type PerfStats struct { + Count int `json:"count"` + Min float64 `json:"min"` + Max float64 `json:"max"` + Mean float64 `json:"mean"` + StdDev float64 `json:"stddev"` + P50 float64 `json:"p50"` + P90 float64 `json:"p90"` + P99 float64 `json:"p99"` +} + +// ComputeStats computes percentile and summary statistics from a slice of values. +func ComputeStats(values []float64) PerfStats { + if len(values) == 0 { + return PerfStats{} + } + + sorted := make([]float64, len(values)) + copy(sorted, values) + sort.Float64s(sorted) + + n := len(sorted) + sum := 0.0 + for _, v := range sorted { + sum += v + } + mean := sum / float64(n) + + variance := 0.0 + for _, v := range sorted { + d := v - mean + variance += d * d + } + variance /= float64(n) + + return PerfStats{ + Count: n, + Min: sorted[0], + Max: sorted[n-1], + Mean: mean, + StdDev: math.Sqrt(variance), + P50: percentile(sorted, 0.50), + P90: percentile(sorted, 0.90), + P99: percentile(sorted, 0.99), + } +} + +func percentile(sorted []float64, p float64) float64 { + if len(sorted) == 0 { + return 0 + } + if len(sorted) == 1 { + return sorted[0] + } + idx := p * float64(len(sorted)-1) + lower := int(idx) + upper := lower + 1 + if upper >= len(sorted) { + return sorted[len(sorted)-1] + } + frac := idx - float64(lower) + return sorted[lower]*(1-frac) + sorted[upper]*frac +} + +// FormatStats returns a one-line summary of PerfStats. +func FormatStats(name string, s PerfStats) string { + return fmt.Sprintf("%s: n=%d mean=%.2f stddev=%.2f p50=%.2f p90=%.2f p99=%.2f min=%.2f max=%.2f", + name, s.Count, s.Mean, s.StdDev, s.P50, s.P90, s.P99, s.Min, s.Max) +} + +// ParsePerfLogLines extracts numeric values from PERF[5s] log lines. +// Expected format: "PERF[5s] iops=1234 lat_us=567 ..." +// Returns map of field_name → []values across all matching lines. +func ParsePerfLogLines(logContent string) map[string][]float64 { + result := make(map[string][]float64) + scanner := bufio.NewScanner(strings.NewReader(logContent)) + for scanner.Scan() { + line := scanner.Text() + idx := strings.Index(line, "PERF[") + if idx < 0 { + continue + } + // Extract everything after "PERF[Xs] " + rest := line[idx:] + closeBracket := strings.Index(rest, "]") + if closeBracket < 0 { + continue + } + fields := strings.Fields(rest[closeBracket+1:]) + for _, f := range fields { + parts := strings.SplitN(f, "=", 2) + if len(parts) != 2 { + continue + } + val, err := strconv.ParseFloat(parts[1], 64) + if err != nil { + continue + } + result[parts[0]] = append(result[parts[0]], val) + } + } + return result +} diff --git a/weed/storage/blockvol/testrunner/metrics_test.go b/weed/storage/blockvol/testrunner/metrics_test.go new file mode 100644 index 000000000..d03059486 --- /dev/null +++ b/weed/storage/blockvol/testrunner/metrics_test.go @@ -0,0 +1,130 @@ +package testrunner + +import ( + "math" + "testing" +) + +func TestParsePrometheusText(t *testing.T) { + body := `# HELP blockvol_write_ops_total Total write operations. +# TYPE blockvol_write_ops_total counter +blockvol_write_ops_total 1234 +blockvol_read_ops_total 5678 +blockvol_write_latency_seconds_bucket{le="0.001"} 100 +blockvol_write_latency_seconds_bucket{le="0.01"} 900 +blockvol_write_latency_seconds_bucket{le="+Inf"} 1234 +blockvol_write_latency_seconds_sum 12.34 +blockvol_write_latency_seconds_count 1234 +` + m := ParsePrometheusText(body) + + if m["blockvol_write_ops_total"] != 1234 { + t.Errorf("write_ops_total = %v, want 1234", m["blockvol_write_ops_total"]) + } + if m["blockvol_read_ops_total"] != 5678 { + t.Errorf("read_ops_total = %v, want 5678", m["blockvol_read_ops_total"]) + } + if m[`blockvol_write_latency_seconds_bucket{le="0.001"}`] != 100 { + t.Errorf("bucket le=0.001 = %v, want 100", m[`blockvol_write_latency_seconds_bucket{le="0.001"}`]) + } +} + +func TestParsePrometheusText_Empty(t *testing.T) { + m := ParsePrometheusText("") + if len(m) != 0 { + t.Errorf("expected empty map, got %d entries", len(m)) + } +} + +func TestComputeStats(t *testing.T) { + values := []float64{1, 2, 3, 4, 5, 6, 7, 8, 9, 10} + s := ComputeStats(values) + + if s.Count != 10 { + t.Errorf("count = %d, want 10", s.Count) + } + if s.Min != 1 { + t.Errorf("min = %v, want 1", s.Min) + } + if s.Max != 10 { + t.Errorf("max = %v, want 10", s.Max) + } + if math.Abs(s.Mean-5.5) > 0.01 { + t.Errorf("mean = %v, want 5.5", s.Mean) + } + if math.Abs(s.P50-5.5) > 0.01 { + t.Errorf("p50 = %v, want ~5.5", s.P50) + } + if s.P99 < 9.5 { + t.Errorf("p99 = %v, want >= 9.5", s.P99) + } +} + +func TestComputeStats_Empty(t *testing.T) { + s := ComputeStats(nil) + if s.Count != 0 { + t.Errorf("expected count=0 for empty input, got %d", s.Count) + } +} + +func TestComputeStats_Single(t *testing.T) { + s := ComputeStats([]float64{42.0}) + if s.Count != 1 || s.Min != 42 || s.Max != 42 || s.P50 != 42 || s.P99 != 42 { + t.Errorf("single value stats wrong: %+v", s) + } +} + +func TestParsePerfLogLines(t *testing.T) { + log := `2024-01-01 12:00:00 INFO starting target +2024-01-01 12:00:05 PERF[5s] iops=1234 lat_us=567 bw_mb=48.5 +2024-01-01 12:00:10 PERF[5s] iops=1300 lat_us=520 bw_mb=50.2 +2024-01-01 12:00:15 PERF[5s] iops=1250 lat_us=540 bw_mb=49.0 +2024-01-01 12:00:20 INFO done +` + result := ParsePerfLogLines(log) + + if len(result["iops"]) != 3 { + t.Fatalf("expected 3 iops samples, got %d", len(result["iops"])) + } + if result["iops"][0] != 1234 { + t.Errorf("first iops = %v, want 1234", result["iops"][0]) + } + if len(result["lat_us"]) != 3 { + t.Fatalf("expected 3 lat_us samples, got %d", len(result["lat_us"])) + } + if len(result["bw_mb"]) != 3 { + t.Fatalf("expected 3 bw_mb samples, got %d", len(result["bw_mb"])) + } +} + +func TestParsePerfLogLines_Empty(t *testing.T) { + result := ParsePerfLogLines("no perf lines here\njust regular log output\n") + if len(result) != 0 { + t.Errorf("expected empty result, got %d entries", len(result)) + } +} + +func TestFormatStats(t *testing.T) { + s := PerfStats{Count: 100, Mean: 5.5, StdDev: 2.87, P50: 5.0, P90: 9.0, P99: 10.0, Min: 1, Max: 10} + str := FormatStats("iops", s) + if str == "" { + t.Error("FormatStats returned empty string") + } + // Just verify it contains the name and key stats. + if !contains(str, "iops") || !contains(str, "n=100") || !contains(str, "p99=10.00") { + t.Errorf("unexpected format: %s", str) + } +} + +func contains(s, sub string) bool { + return len(s) >= len(sub) && findSubstring(s, sub) +} + +func findSubstring(s, sub string) bool { + for i := 0; i <= len(s)-len(sub); i++ { + if s[i:i+len(sub)] == sub { + return true + } + } + return false +} diff --git a/weed/storage/blockvol/testrunner/parser.go b/weed/storage/blockvol/testrunner/parser.go new file mode 100644 index 000000000..b509cbbb7 --- /dev/null +++ b/weed/storage/blockvol/testrunner/parser.go @@ -0,0 +1,198 @@ +package testrunner + +import ( + "fmt" + "os" + "strings" + + "gopkg.in/yaml.v3" +) + +// ParseFile reads and parses a YAML scenario file. +func ParseFile(path string) (*Scenario, error) { + data, err := os.ReadFile(path) + if err != nil { + return nil, fmt.Errorf("read scenario %s: %w", path, err) + } + return Parse(data) +} + +// Parse parses YAML bytes into a Scenario and validates it. +func Parse(data []byte) (*Scenario, error) { + var s Scenario + if err := yaml.Unmarshal(data, &s); err != nil { + return nil, fmt.Errorf("parse YAML: %w", err) + } + if err := validate(&s); err != nil { + return nil, fmt.Errorf("validate: %w", err) + } + return &s, nil +} + +// validate checks referential integrity and required fields. +func validate(s *Scenario) error { + if s.Name == "" { + return fmt.Errorf("scenario name is required") + } + + // Check that every target references a valid node. + for tName, tSpec := range s.Targets { + if tSpec.Node == "" { + return fmt.Errorf("target %q: node is required", tName) + } + if _, ok := s.Topology.Nodes[tSpec.Node]; !ok { + return fmt.Errorf("target %q: node %q not found in topology", tName, tSpec.Node) + } + if tSpec.IQNSuffix == "" { + return fmt.Errorf("target %q: iqn_suffix is required", tName) + } + } + + // Check port conflicts among targets on the same node. + type nodePort struct { + node string + port int + } + used := make(map[nodePort]string) // nodePort -> target name + for tName, tSpec := range s.Targets { + ports := []int{tSpec.ISCSIPort, tSpec.AdminPort, tSpec.ReplicaDataPort, tSpec.ReplicaCtrlPort, tSpec.RebuildPort} + for _, p := range ports { + if p == 0 { + continue + } + np := nodePort{tSpec.Node, p} + if other, ok := used[np]; ok { + return fmt.Errorf("port conflict: targets %q and %q both use port %d on node %q", + other, tName, p, tSpec.Node) + } + used[np] = tName + } + } + + // Validate agents section (coordinator mode). + if len(s.Topology.Agents) > 0 { + for nodeName, nodeSpec := range s.Topology.Nodes { + if nodeSpec.Agent != "" { + if _, ok := s.Topology.Agents[nodeSpec.Agent]; !ok { + return fmt.Errorf("node %q: agent %q not found in topology.agents", nodeName, nodeSpec.Agent) + } + } + } + } + + // Check phases and actions. + if len(s.Phases) == 0 { + return fmt.Errorf("at least one phase is required") + } + for _, phase := range s.Phases { + if phase.Name == "" { + return fmt.Errorf("phase name is required") + } + + // Validate save_as uniqueness within parallel phases. + if phase.Parallel { + saveAsSet := make(map[string]int) + for i, act := range phase.Actions { + if act.SaveAs != "" { + if prev, ok := saveAsSet[act.SaveAs]; ok { + return fmt.Errorf("phase %q (parallel): save_as %q used by both action %d and %d", + phase.Name, act.SaveAs, prev, i) + } + saveAsSet[act.SaveAs] = i + } + } + } + + for i, act := range phase.Actions { + if act.Action == "" { + return fmt.Errorf("phase %q, action %d: action type is required", phase.Name, i) + } + // Validate target references. + if act.Target != "" { + if _, ok := s.Targets[act.Target]; !ok { + return fmt.Errorf("phase %q, action %d (%s): target %q not found", + phase.Name, i, act.Action, act.Target) + } + } + if act.Replica != "" { + if _, ok := s.Targets[act.Replica]; !ok { + return fmt.Errorf("phase %q, action %d (%s): replica %q not found", + phase.Name, i, act.Action, act.Replica) + } + } + // Validate node references in actions. + if act.Node != "" { + if _, ok := s.Topology.Nodes[act.Node]; !ok { + return fmt.Errorf("phase %q, action %d (%s): node %q not found", + phase.Name, i, act.Action, act.Node) + } + } + } + } + + // Validate variable references ({{ var }}) don't reference undefined save_as. + defined := make(map[string]bool) + // Add env vars. + for k := range s.Env { + defined[k] = true + } + for _, phase := range s.Phases { + if phase.Always { + continue // cleanup phases may use vars from any prior phase + } + for _, act := range phase.Actions { + // Check var references in all string fields. + refs := extractVarRefs(act) + for _, ref := range refs { + if !defined[ref] && !strings.HasPrefix(ref, "__") { + // Allow forward refs (they'll be resolved at runtime); just warn-level + } + } + if act.SaveAs != "" { + defined[act.SaveAs] = true + } + } + } + + return nil +} + +// extractVarRefs finds all {{ var }} references in action fields. +func extractVarRefs(act Action) []string { + var refs []string + fields := collectStringFields(act) + for _, f := range fields { + refs = append(refs, extractVarsFromString(f)...) + } + return refs +} + +// collectStringFields returns all string values from an action's params and known fields. +func collectStringFields(act Action) []string { + var fields []string + for _, v := range act.Params { + fields = append(fields, v) + } + return fields +} + +// extractVarsFromString finds all {{ name }} patterns in a string. +func extractVarsFromString(s string) []string { + var vars []string + for { + start := strings.Index(s, "{{") + if start < 0 { + break + } + end := strings.Index(s[start:], "}}") + if end < 0 { + break + } + name := strings.TrimSpace(s[start+2 : start+end]) + if name != "" { + vars = append(vars, name) + } + s = s[start+end+2:] + } + return vars +} diff --git a/weed/storage/blockvol/testrunner/parser_test.go b/weed/storage/blockvol/testrunner/parser_test.go new file mode 100644 index 000000000..4ee8bc9b0 --- /dev/null +++ b/weed/storage/blockvol/testrunner/parser_test.go @@ -0,0 +1,385 @@ +package testrunner + +import ( + "testing" + "time" +) + +func TestParse_ValidScenario(t *testing.T) { + yaml := ` +name: test-scenario +timeout: 5m +env: + repo_dir: "/tmp/repo" + +topology: + nodes: + node1: + host: "192.168.1.1" + user: testdev + key: "/tmp/key" + +targets: + primary: + node: node1 + vol_size: 100M + iscsi_port: 3260 + admin_port: 8080 + iqn_suffix: test-primary + replica: + node: node1 + vol_size: 100M + iscsi_port: 3261 + admin_port: 8081 + replica_data_port: 9011 + replica_ctrl_port: 9012 + rebuild_port: 9013 + iqn_suffix: test-replica + +phases: + - name: setup + actions: + - action: build_deploy + - action: start_target + target: primary + create: "true" + - name: cleanup + always: true + actions: + - action: stop_all_targets + ignore_error: true +` + s, err := Parse([]byte(yaml)) + if err != nil { + t.Fatalf("Parse failed: %v", err) + } + + if s.Name != "test-scenario" { + t.Errorf("name = %q, want %q", s.Name, "test-scenario") + } + if s.Timeout.Duration != 5*time.Minute { + t.Errorf("timeout = %v, want 5m", s.Timeout.Duration) + } + if len(s.Topology.Nodes) != 1 { + t.Errorf("nodes = %d, want 1", len(s.Topology.Nodes)) + } + if len(s.Targets) != 2 { + t.Errorf("targets = %d, want 2", len(s.Targets)) + } + if len(s.Phases) != 2 { + t.Errorf("phases = %d, want 2", len(s.Phases)) + } + + // Check target IQN generation. + if iqn := s.Targets["primary"].IQN(); iqn != "iqn.2024.com.seaweedfs:test-primary" { + t.Errorf("primary IQN = %q", iqn) + } + + // Check always flag. + if !s.Phases[1].Always { + t.Error("cleanup phase should have always=true") + } +} + +func TestParse_MissingName(t *testing.T) { + yaml := ` +phases: + - name: test + actions: + - action: exec + cmd: "echo hi" +` + _, err := Parse([]byte(yaml)) + if err == nil { + t.Fatal("expected error for missing name") + } +} + +func TestParse_InvalidNodeRef(t *testing.T) { + yaml := ` +name: bad-ref +topology: + nodes: + node1: + host: "1.2.3.4" + user: test + key: "/tmp/key" +targets: + tgt: + node: nonexistent + iscsi_port: 3260 + iqn_suffix: test +phases: + - name: test + actions: + - action: exec + cmd: "echo" +` + _, err := Parse([]byte(yaml)) + if err == nil { + t.Fatal("expected error for invalid node ref") + } +} + +func TestParse_PortConflict(t *testing.T) { + yaml := ` +name: port-conflict +topology: + nodes: + node1: + host: "1.2.3.4" + user: test + key: "/tmp/key" +targets: + tgt1: + node: node1 + iscsi_port: 3260 + iqn_suffix: t1 + tgt2: + node: node1 + iscsi_port: 3260 + iqn_suffix: t2 +phases: + - name: test + actions: + - action: exec + cmd: "echo" +` + _, err := Parse([]byte(yaml)) + if err == nil { + t.Fatal("expected error for port conflict") + } +} + +func TestParse_InvalidTargetRef(t *testing.T) { + yaml := ` +name: bad-target-ref +topology: + nodes: + node1: + host: "1.2.3.4" + user: test + key: "/tmp/key" +targets: + primary: + node: node1 + iscsi_port: 3260 + iqn_suffix: test +phases: + - name: test + actions: + - action: start_target + target: nonexistent +` + _, err := Parse([]byte(yaml)) + if err == nil { + t.Fatal("expected error for invalid target ref in action") + } +} + +func TestParse_MissingIQNSuffix(t *testing.T) { + yaml := ` +name: missing-iqn +topology: + nodes: + node1: + host: "1.2.3.4" + user: test + key: "/tmp/key" +targets: + tgt: + node: node1 + iscsi_port: 3260 +phases: + - name: test + actions: + - action: exec + cmd: "echo" +` + _, err := Parse([]byte(yaml)) + if err == nil { + t.Fatal("expected error for missing iqn_suffix") + } +} + +func TestParse_NoPhases(t *testing.T) { + yaml := ` +name: no-phases +` + _, err := Parse([]byte(yaml)) + if err == nil { + t.Fatal("expected error for no phases") + } +} + +func TestParse_AgentTopology_Valid(t *testing.T) { + yaml := ` +name: agent-topo +topology: + agents: + tp01: "192.168.1.188:9100" + m01: "192.168.1.181:9100" + nodes: + tp01_node: + host: "192.168.1.188" + agent: tp01 + m01_node: + host: "192.168.1.181" + agent: m01 +targets: + primary: + node: tp01_node + iscsi_port: 3260 + iqn_suffix: primary +phases: + - name: test + actions: + - action: exec + node: tp01_node + cmd: "echo" +` + s, err := Parse([]byte(yaml)) + if err != nil { + t.Fatalf("Parse failed: %v", err) + } + if len(s.Topology.Agents) != 2 { + t.Errorf("agents = %d, want 2", len(s.Topology.Agents)) + } + if s.Topology.Nodes["tp01_node"].Agent != "tp01" { + t.Error("tp01_node should have agent=tp01") + } +} + +func TestParse_AgentTopology_InvalidAgentRef(t *testing.T) { + yaml := ` +name: bad-agent-ref +topology: + agents: + tp01: "192.168.1.188:9100" + nodes: + node1: + host: "192.168.1.181" + agent: nonexistent +phases: + - name: test + actions: + - action: exec + cmd: "echo" +` + _, err := Parse([]byte(yaml)) + if err == nil { + t.Fatal("expected error for invalid agent reference") + } +} + +func TestParse_ParallelPhase_SaveAsConflict(t *testing.T) { + yaml := ` +name: save-as-conflict +topology: + nodes: + node1: + host: "1.2.3.4" + user: test + key: "/tmp/key" +phases: + - name: parallel_phase + parallel: true + actions: + - action: exec + node: node1 + save_as: my_var + cmd: "echo a" + - action: exec + node: node1 + save_as: my_var + cmd: "echo b" +` + _, err := Parse([]byte(yaml)) + if err == nil { + t.Fatal("expected error for save_as conflict in parallel phase") + } +} + +func TestParse_SequentialPhase_SaveAsDuplicate_Allowed(t *testing.T) { + yaml := ` +name: save-as-sequential +topology: + nodes: + node1: + host: "1.2.3.4" + user: test + key: "/tmp/key" +phases: + - name: seq_phase + actions: + - action: exec + node: node1 + save_as: device + cmd: "echo a" + - action: exec + node: node1 + save_as: device + cmd: "echo b" +` + _, err := Parse([]byte(yaml)) + if err != nil { + t.Fatalf("sequential save_as duplicate should be allowed, got: %v", err) + } +} + +func TestParse_ActionRetryAndTimeout(t *testing.T) { + yaml := ` +name: retry-test +topology: + nodes: + n1: + host: "1.2.3.4" + user: test + key: "/tmp/key" +phases: + - name: test + actions: + - action: start_target + node: n1 + retry: 2 + timeout: 30s + cmd: "echo" +` + s, err := Parse([]byte(yaml)) + if err != nil { + t.Fatalf("Parse failed: %v", err) + } + act := s.Phases[0].Actions[0] + if act.Retry != 2 { + t.Errorf("retry = %d, want 2", act.Retry) + } + if act.Timeout != "30s" { + t.Errorf("timeout = %q, want 30s", act.Timeout) + } +} + +func TestExtractVarsFromString(t *testing.T) { + tests := []struct { + input string + want []string + }{ + {"{{ device }}", []string{"device"}}, + {"{{written_md5}}", []string{"written_md5"}}, + {"prefix {{ a }} middle {{ b }} suffix", []string{"a", "b"}}, + {"no vars here", nil}, + {"{{ }}", nil}, + } + + for _, tt := range tests { + got := extractVarsFromString(tt.input) + if len(got) != len(tt.want) { + t.Errorf("extractVarsFromString(%q) = %v, want %v", tt.input, got, tt.want) + continue + } + for i := range got { + if got[i] != tt.want[i] { + t.Errorf("extractVarsFromString(%q)[%d] = %q, want %q", tt.input, i, got[i], tt.want[i]) + } + } + } +} diff --git a/weed/storage/blockvol/testrunner/protocol.go b/weed/storage/blockvol/testrunner/protocol.go new file mode 100644 index 000000000..06bda60dd --- /dev/null +++ b/weed/storage/blockvol/testrunner/protocol.go @@ -0,0 +1,73 @@ +package testrunner + +// AuthTokenHeader is the HTTP header used for coordinator↔agent authentication. +const AuthTokenHeader = "X-Auth-Token" + +// UploadBasePath is the required prefix for all upload paths. +const UploadBasePath = "/tmp/sw-test-runner/" + +// RegisterRequest is sent by an agent to the coordinator at POST /register. +type RegisterRequest struct { + AgentID string `json:"agent_id"` + Hostname string `json:"hostname"` + IP string `json:"ip"` + Port int `json:"port"` + Nodes []string `json:"nodes"` + Capabilities []string `json:"capabilities"` +} + +// RegisterResponse is returned by the coordinator after successful registration. +type RegisterResponse struct { + OK bool `json:"ok"` + AgentIndex int `json:"agent_index"` + TotalAgents int `json:"total_agents"` + Error string `json:"error,omitempty"` +} + +// PhaseRequest is sent by the coordinator to an agent at POST /phase. +type PhaseRequest struct { + PhaseIndex int `json:"phase_index"` + PhaseName string `json:"phase_name"` + Actions []Action `json:"actions"` + GlobalVars map[string]string `json:"global_vars"` + Scenario *Scenario `json:"scenario,omitempty"` +} + +// PhaseResponse is returned by the agent after executing a phase. +type PhaseResponse struct { + PhaseIndex int `json:"phase_index"` + Results []ActionResult `json:"results"` + NewVars map[string]string `json:"new_vars"` + Error string `json:"error,omitempty"` +} + +// ExecRequest is sent by the coordinator to an agent at POST /exec. +type ExecRequest struct { + Cmd string `json:"cmd"` + Root bool `json:"root"` +} + +// ExecResponse is returned by the agent after executing a command. +type ExecResponse struct { + Stdout string `json:"stdout"` + Stderr string `json:"stderr"` + ExitCode int `json:"exit_code"` + Error string `json:"error,omitempty"` +} + +// UploadResponse is returned by the agent after receiving a file upload. +type UploadResponse struct { + OK bool `json:"ok"` + Size int64 `json:"size"` + Path string `json:"path"` + Error string `json:"error,omitempty"` +} + +// HealthResponse is returned by the agent at GET /health. +type HealthResponse struct { + OK bool `json:"ok"` + AgentID string `json:"agent_id"` + UptimeS int64 `json:"uptime_s"` + HasRoot bool `json:"has_root"` + Hostname string `json:"hostname"` +} diff --git a/weed/storage/blockvol/testrunner/protocol_test.go b/weed/storage/blockvol/testrunner/protocol_test.go new file mode 100644 index 000000000..3bd5ae885 --- /dev/null +++ b/weed/storage/blockvol/testrunner/protocol_test.go @@ -0,0 +1,36 @@ +package testrunner + +import ( + "testing" +) + +func TestIsPathSafe(t *testing.T) { + tests := []struct { + path string + want bool + }{ + {"/tmp/sw-test-runner/binary", true}, + {"/tmp/sw-test-runner/subdir/file", true}, + {"/tmp/sw-test-runner/nested/deep/file", true}, + {"/tmp/other/file", false}, + {"/etc/passwd", false}, + {"/tmp/sw-test-runner/../etc/passwd", false}, + {"../etc/passwd", false}, + {"/tmp/sw-test-runner/../../root", false}, + {"", false}, + {"/tmp/sw-test-runner-evil/file", false}, + } + + for _, tt := range tests { + got := isPathSafe(tt.path) + if got != tt.want { + t.Errorf("isPathSafe(%q) = %v, want %v", tt.path, got, tt.want) + } + } +} + +func TestAuthTokenHeaderConstant(t *testing.T) { + if AuthTokenHeader != "X-Auth-Token" { + t.Errorf("AuthTokenHeader = %q", AuthTokenHeader) + } +} diff --git a/weed/storage/blockvol/testrunner/registry.go b/weed/storage/blockvol/testrunner/registry.go new file mode 100644 index 000000000..70ba1033b --- /dev/null +++ b/weed/storage/blockvol/testrunner/registry.go @@ -0,0 +1,166 @@ +package testrunner + +import ( + "context" + "fmt" +) + +// ActionContext provides action handlers with access to the scenario's +// infrastructure (nodes, targets) and variable store. +type ActionContext struct { + Scenario *Scenario + Nodes map[string]NodeRunner + Targets map[string]TargetRunner + Vars map[string]string + Log func(format string, args ...interface{}) + Coordinator *Coordinator // non-nil when running in coordinator mode +} + +// NodeRunner abstracts remote command execution (implemented by infra.Node). +type NodeRunner interface { + Run(ctx context.Context, cmd string) (stdout, stderr string, exitCode int, err error) + RunRoot(ctx context.Context, cmd string) (stdout, stderr string, exitCode int, err error) + Upload(local, remote string) error + Close() +} + +// TargetRunner abstracts iSCSI target lifecycle (implemented by infra.HATarget). +type TargetRunner interface { + Start(ctx context.Context, create bool) error + Stop(ctx context.Context) error + Kill9() error + CollectLog() (string, error) + Cleanup(ctx context.Context) + PID() int + VolFilePath() string +} + +// HATargetRunner extends TargetRunner with HA admin operations. +type HATargetRunner interface { + TargetRunner + Assign(ctx context.Context, epoch uint64, role uint32, leaseTTLMs uint32) error + Status(ctx context.Context) (*StatusResult, error) + SetReplica(ctx context.Context, dataAddr, ctrlAddr string) error + WaitForRole(ctx context.Context, expectedRole string) error + WaitForLSN(ctx context.Context, minLSN uint64) error + StartRebuildEndpoint(ctx context.Context, listenAddr string) error + StartRebuildClient(ctx context.Context, rebuildAddr string, epoch uint64) error + StopRebuildEndpoint(ctx context.Context) error +} + +// StatusResult mirrors the JSON from GET /status. +type StatusResult struct { + Epoch uint64 `json:"epoch"` + Role string `json:"role"` + WALHeadLSN uint64 `json:"wal_head_lsn"` + HasLease bool `json:"has_lease"` + Healthy bool `json:"healthy"` +} + +// ActionHandler executes a single action type. +type ActionHandler interface { + Execute(ctx context.Context, actx *ActionContext, act Action) (map[string]string, error) +} + +// ActionHandlerFunc adapts a function to the ActionHandler interface. +type ActionHandlerFunc func(ctx context.Context, actx *ActionContext, act Action) (map[string]string, error) + +func (f ActionHandlerFunc) Execute(ctx context.Context, actx *ActionContext, act Action) (map[string]string, error) { + return f(ctx, actx, act) +} + +// Tier constants for action classification. +const ( + TierCore = "core" // exec, sleep, assert_*, print + TierBlock = "block" // start_target, iscsi_*, dd_*, fio_*, assign, metrics... + TierDevOps = "devops" // start_weed_master, start_weed_volume, cluster_status... + TierChaos = "chaos" // inject_netem, inject_partition, fill_disk, corrupt_wal +) + +type actionEntry struct { + handler ActionHandler + tier string +} + +// Registry maps action names to handlers with tier-based gating. +type Registry struct { + handlers map[string]actionEntry + EnabledTiers map[string]bool // nil or empty = all tiers allowed +} + +// NewRegistry creates an empty registry. +func NewRegistry() *Registry { + return &Registry{handlers: make(map[string]actionEntry)} +} + +// Register adds a handler for an action name with a tier. +func (r *Registry) Register(name, tier string, h ActionHandler) { + r.handlers[name] = actionEntry{handler: h, tier: tier} +} + +// RegisterFunc adds a function handler for an action name with a tier. +func (r *Registry) RegisterFunc(name, tier string, f ActionHandlerFunc) { + r.handlers[name] = actionEntry{handler: f, tier: tier} +} + +// EnableTiers sets which tiers are allowed. Pass nil or empty to allow all. +func (r *Registry) EnableTiers(tiers []string) { + if len(tiers) == 0 { + r.EnabledTiers = nil + return + } + r.EnabledTiers = make(map[string]bool, len(tiers)) + for _, t := range tiers { + r.EnabledTiers[t] = true + } +} + +// Get returns the handler for an action name, or an error. +// Returns an error if the action's tier is not enabled. +func (r *Registry) Get(name string) (ActionHandler, error) { + entry, ok := r.handlers[name] + if !ok { + return nil, fmt.Errorf("unknown action: %q", name) + } + if len(r.EnabledTiers) > 0 && !r.EnabledTiers[entry.tier] { + return nil, fmt.Errorf("action %q requires tier %q (enabled: %v)", name, entry.tier, r.tierList()) + } + return entry.handler, nil +} + +// List returns all registered action names (respecting tier gating). +func (r *Registry) List() []string { + names := make([]string, 0, len(r.handlers)) + for name, entry := range r.handlers { + if len(r.EnabledTiers) > 0 && !r.EnabledTiers[entry.tier] { + continue + } + names = append(names, name) + } + return names +} + +// ListByTier returns action names grouped by tier. +func (r *Registry) ListByTier() map[string][]string { + result := make(map[string][]string) + for name, entry := range r.handlers { + result[entry.tier] = append(result[entry.tier], name) + } + return result +} + +// ActionTier returns the tier for a registered action, or "" if not found. +func (r *Registry) ActionTier(name string) string { + if entry, ok := r.handlers[name]; ok { + return entry.tier + } + return "" +} + +func (r *Registry) tierList() []string { + list := make([]string, 0, len(r.EnabledTiers)) + for t := range r.EnabledTiers { + list = append(list, t) + } + return list +} diff --git a/weed/storage/blockvol/testrunner/registry_test.go b/weed/storage/blockvol/testrunner/registry_test.go new file mode 100644 index 000000000..b6205edc6 --- /dev/null +++ b/weed/storage/blockvol/testrunner/registry_test.go @@ -0,0 +1,104 @@ +package testrunner + +import ( + "context" + "sort" + "testing" +) + +func TestRegistry_TierGating(t *testing.T) { + r := NewRegistry() + noop := ActionHandlerFunc(func(ctx context.Context, actx *ActionContext, act Action) (map[string]string, error) { + return nil, nil + }) + + r.RegisterFunc("exec", TierCore, noop) + r.RegisterFunc("dd_write", TierBlock, noop) + r.RegisterFunc("inject_netem", TierChaos, noop) + r.RegisterFunc("start_weed_master", TierDevOps, noop) + + // No gating — all accessible. + if _, err := r.Get("exec"); err != nil { + t.Errorf("exec ungated: %v", err) + } + if _, err := r.Get("dd_write"); err != nil { + t.Errorf("dd_write ungated: %v", err) + } + + // Enable only core. + r.EnableTiers([]string{TierCore}) + if _, err := r.Get("exec"); err != nil { + t.Errorf("exec (core enabled): %v", err) + } + if _, err := r.Get("dd_write"); err == nil { + t.Error("dd_write should be blocked when only core is enabled") + } + if _, err := r.Get("inject_netem"); err == nil { + t.Error("inject_netem should be blocked") + } + + // List should only return core actions. + names := r.List() + if len(names) != 1 || names[0] != "exec" { + t.Errorf("List() = %v, want [exec]", names) + } + + // Reset gating. + r.EnableTiers(nil) + names = r.List() + if len(names) != 4 { + t.Errorf("List() after reset = %d, want 4", len(names)) + } +} + +func TestRegistry_ListByTier(t *testing.T) { + r := NewRegistry() + noop := ActionHandlerFunc(func(ctx context.Context, actx *ActionContext, act Action) (map[string]string, error) { + return nil, nil + }) + + r.RegisterFunc("a", TierCore, noop) + r.RegisterFunc("b", TierCore, noop) + r.RegisterFunc("c", TierBlock, noop) + + byTier := r.ListByTier() + coreActions := byTier[TierCore] + sort.Strings(coreActions) + if len(coreActions) != 2 || coreActions[0] != "a" || coreActions[1] != "b" { + t.Errorf("core = %v, want [a b]", coreActions) + } + if len(byTier[TierBlock]) != 1 || byTier[TierBlock][0] != "c" { + t.Errorf("block = %v, want [c]", byTier[TierBlock]) + } +} + +func TestRegistry_ActionTier(t *testing.T) { + r := NewRegistry() + noop := ActionHandlerFunc(func(ctx context.Context, actx *ActionContext, act Action) (map[string]string, error) { + return nil, nil + }) + + r.RegisterFunc("foo", TierChaos, noop) + + if tier := r.ActionTier("foo"); tier != TierChaos { + t.Errorf("tier = %q, want chaos", tier) + } + if tier := r.ActionTier("nonexistent"); tier != "" { + t.Errorf("nonexistent tier = %q, want empty", tier) + } +} + +func TestRegistry_EmptyTiersAllowsAll(t *testing.T) { + r := NewRegistry() + noop := ActionHandlerFunc(func(ctx context.Context, actx *ActionContext, act Action) (map[string]string, error) { + return nil, nil + }) + + r.RegisterFunc("x", TierDevOps, noop) + + // Empty slice should not restrict. + r.EnableTiers([]string{}) + if _, err := r.Get("x"); err != nil { + t.Errorf("empty tiers should allow all: %v", err) + } +} diff --git a/weed/storage/blockvol/testrunner/regression.go b/weed/storage/blockvol/testrunner/regression.go new file mode 100644 index 000000000..5a6f85a14 --- /dev/null +++ b/weed/storage/blockvol/testrunner/regression.go @@ -0,0 +1,200 @@ +package testrunner + +import ( + "fmt" + "math" + "strings" +) + +// RegressionResult holds the comparison of one metric against baseline. +type RegressionResult struct { + Metric string + BaselineVal float64 + CurrentVal float64 + ThresholdPct float64 // positive = allowed increase, negative = allowed decrease + Pass bool + Reason string +} + +// RegressionReport is the full regression comparison output. +type RegressionReport struct { + Results []RegressionResult + HardFails []HardFailResult + OverallPass bool + BaselineGitSHA string + CurrentGitSHA string +} + +// HardFailResult is the result of a hard fail condition check. +type HardFailResult struct { + Condition string + Pass bool + Detail string +} + +// RegressionThresholds defines the allowed deviation per metric. +var RegressionThresholds = map[string]float64{ + "p99_write_latency_seconds": 0.10, // fail if >10% increase + "p99_read_latency_seconds": 0.10, // fail if >10% increase + "write_iops": -0.05, // fail if >5% decrease + "read_iops": -0.05, // fail if >5% decrease +} + +// CompareBaseline runs regression checks against a baseline. +func CompareBaseline(baseline, current map[string]float64) []RegressionResult { + var results []RegressionResult + for metric, threshold := range RegressionThresholds { + bv, bOk := baseline[metric] + cv, cOk := current[metric] + if !bOk || !cOk { + continue + } + if bv == 0 { + results = append(results, RegressionResult{ + Metric: metric, BaselineVal: bv, CurrentVal: cv, + ThresholdPct: threshold, Pass: true, Reason: "baseline is 0, skipped", + }) + continue + } + + r := RegressionResult{ + Metric: metric, BaselineVal: bv, CurrentVal: cv, + ThresholdPct: threshold, + } + + pctChange := (cv - bv) / math.Abs(bv) + if threshold >= 0 { + // Positive threshold: fail if increase exceeds threshold. + r.Pass = pctChange <= threshold + r.Reason = fmt.Sprintf("change=%.1f%% (limit=+%.1f%%)", pctChange*100, threshold*100) + } else { + // Negative threshold: fail if decrease exceeds |threshold|. + r.Pass = pctChange >= threshold + r.Reason = fmt.Sprintf("change=%.1f%% (limit=%.1f%%)", pctChange*100, threshold*100) + } + results = append(results, r) + } + return results +} + +// HardFailChecks are immediate test failure conditions (no comparison needed). +// Each check takes the current metrics map and returns pass/fail. +var HardFailChecks = []struct { + Name string + Check func(m map[string]float64) (bool, string) +}{ + { + Name: "data_mismatch", + Check: func(m map[string]float64) (bool, string) { + if v, ok := m["data_mismatch_count"]; ok && v > 0 { + return false, fmt.Sprintf("%.0f data mismatches detected", v) + } + return true, "" + }, + }, + { + Name: "promotion_panic", + Check: func(m map[string]float64) (bool, string) { + if v, ok := m["promotion_panic_count"]; ok && v > 0 { + return false, fmt.Sprintf("%.0f promotion panics", v) + } + return true, "" + }, + }, + { + Name: "scrub_false_positive_burst", + Check: func(m map[string]float64) (bool, string) { + if v, ok := m["scrub_false_positives_per_pass"]; ok && v > 3 { + return false, fmt.Sprintf("%.0f false positives in one scrub pass (limit=3)", v) + } + return true, "" + }, + }, + { + Name: "barrier_lag_lsn_unbounded", + Check: func(m map[string]float64) (bool, string) { + if v, ok := m["barrier_lag_lsn_max"]; ok && v > 1000 { + return false, fmt.Sprintf("barrier_lag_lsn=%.0f (limit=1000)", v) + } + return true, "" + }, + }, + { + Name: "barrier_error_rate", + Check: func(m map[string]float64) (bool, string) { + total, tOk := m["barrier_requests_total"] + failed, fOk := m["barrier_failures_total"] + if tOk && fOk && total > 0 && (failed/total) > 0.05 { + return false, fmt.Sprintf("barrier error rate=%.1f%% (limit=5%%)", (failed/total)*100) + } + return true, "" + }, + }, + { + Name: "health_zero_without_fault", + Check: func(m map[string]float64) (bool, string) { + health, hOk := m["health_score"] + faultActive, fOk := m["fault_active"] + if hOk && health == 0.0 && (!fOk || faultActive == 0) { + return false, "health score dropped to 0.0 without injected fault" + } + return true, "" + }, + }, + { + Name: "wal_full_stall", + Check: func(m map[string]float64) (bool, string) { + if v, ok := m["wal_full_duration_seconds"]; ok && v > 10 { + return false, fmt.Sprintf("WAL full for %.1fs (limit=10s)", v) + } + return true, "" + }, + }, +} + +// RunHardFailChecks evaluates all hard fail conditions against current metrics. +func RunHardFailChecks(metrics map[string]float64) []HardFailResult { + results := make([]HardFailResult, 0, len(HardFailChecks)) + for _, hf := range HardFailChecks { + pass, detail := hf.Check(metrics) + results = append(results, HardFailResult{ + Condition: hf.Name, + Pass: pass, + Detail: detail, + }) + } + return results +} + +// FormatRegressionReport produces a human-readable report string. +func FormatRegressionReport(r *RegressionReport) string { + var sb strings.Builder + sb.WriteString("=== SLO Regression Report ===\n") + sb.WriteString(fmt.Sprintf("Baseline: %s Current: %s\n\n", r.BaselineGitSHA, r.CurrentGitSHA)) + + sb.WriteString("--- Metric Comparisons ---\n") + for _, res := range r.Results { + status := "PASS" + if !res.Pass { + status = "FAIL" + } + sb.WriteString(fmt.Sprintf(" [%s] %s: baseline=%.4f current=%.4f %s\n", + status, res.Metric, res.BaselineVal, res.CurrentVal, res.Reason)) + } + + sb.WriteString("\n--- Hard Fail Conditions ---\n") + for _, hf := range r.HardFails { + status := "PASS" + if !hf.Pass { + status = "FAIL" + } + detail := hf.Detail + if detail == "" { + detail = "ok" + } + sb.WriteString(fmt.Sprintf(" [%s] %s: %s\n", status, hf.Condition, detail)) + } + + sb.WriteString(fmt.Sprintf("\nOverall: %s\n", map[bool]string{true: "PASS", false: "FAIL"}[r.OverallPass])) + return sb.String() +} diff --git a/weed/storage/blockvol/testrunner/regression_test.go b/weed/storage/blockvol/testrunner/regression_test.go new file mode 100644 index 000000000..28ddbd5bb --- /dev/null +++ b/weed/storage/blockvol/testrunner/regression_test.go @@ -0,0 +1,163 @@ +package testrunner + +import ( + "testing" +) + +func TestCompareBaseline_P99Increase(t *testing.T) { + baseline := map[string]float64{"p99_write_latency_seconds": 0.005} + current := map[string]float64{"p99_write_latency_seconds": 0.006} // +20% + + results := CompareBaseline(baseline, current) + if len(results) == 0 { + t.Fatal("expected results") + } + for _, r := range results { + if r.Metric == "p99_write_latency_seconds" && r.Pass { + t.Error("expected FAIL for 20% latency increase (limit 10%)") + } + } +} + +func TestCompareBaseline_P99WithinLimit(t *testing.T) { + baseline := map[string]float64{"p99_write_latency_seconds": 0.005} + current := map[string]float64{"p99_write_latency_seconds": 0.0054} // +8% + + results := CompareBaseline(baseline, current) + for _, r := range results { + if r.Metric == "p99_write_latency_seconds" && !r.Pass { + t.Errorf("expected PASS for 8%% increase (limit 10%%): %s", r.Reason) + } + } +} + +func TestCompareBaseline_IOPSDecrease(t *testing.T) { + baseline := map[string]float64{"write_iops": 50000} + current := map[string]float64{"write_iops": 46000} // -8% + + results := CompareBaseline(baseline, current) + for _, r := range results { + if r.Metric == "write_iops" && r.Pass { + t.Error("expected FAIL for 8% IOPS decrease (limit 5%)") + } + } +} + +func TestCompareBaseline_IOPSWithinLimit(t *testing.T) { + baseline := map[string]float64{"write_iops": 50000} + current := map[string]float64{"write_iops": 48000} // -4% + + results := CompareBaseline(baseline, current) + for _, r := range results { + if r.Metric == "write_iops" && !r.Pass { + t.Errorf("expected PASS for 4%% decrease (limit 5%%): %s", r.Reason) + } + } +} + +func TestHardFail_DataMismatch(t *testing.T) { + metrics := map[string]float64{"data_mismatch_count": 1} + results := RunHardFailChecks(metrics) + for _, r := range results { + if r.Condition == "data_mismatch" && r.Pass { + t.Error("expected FAIL for data mismatch") + } + } +} + +func TestHardFail_BarrierLagUnbounded(t *testing.T) { + metrics := map[string]float64{"barrier_lag_lsn_max": 1500} + results := RunHardFailChecks(metrics) + for _, r := range results { + if r.Condition == "barrier_lag_lsn_unbounded" && r.Pass { + t.Error("expected FAIL for barrier_lag_lsn=1500") + } + } +} + +func TestHardFail_BarrierLagOK(t *testing.T) { + metrics := map[string]float64{"barrier_lag_lsn_max": 50} + results := RunHardFailChecks(metrics) + for _, r := range results { + if r.Condition == "barrier_lag_lsn_unbounded" && !r.Pass { + t.Error("expected PASS for barrier_lag_lsn=50") + } + } +} + +func TestHardFail_BarrierErrorRate(t *testing.T) { + metrics := map[string]float64{ + "barrier_requests_total": 100, + "barrier_failures_total": 10, // 10% error rate + } + results := RunHardFailChecks(metrics) + for _, r := range results { + if r.Condition == "barrier_error_rate" && r.Pass { + t.Error("expected FAIL for 10% barrier error rate") + } + } +} + +func TestHardFail_HealthZero(t *testing.T) { + metrics := map[string]float64{"health_score": 0.0, "fault_active": 0} + results := RunHardFailChecks(metrics) + for _, r := range results { + if r.Condition == "health_zero_without_fault" && r.Pass { + t.Error("expected FAIL for health=0 without fault") + } + } +} + +func TestHardFail_HealthZeroDuringFault_OK(t *testing.T) { + metrics := map[string]float64{"health_score": 0.0, "fault_active": 1} + results := RunHardFailChecks(metrics) + for _, r := range results { + if r.Condition == "health_zero_without_fault" && !r.Pass { + t.Error("expected PASS for health=0 during active fault") + } + } +} + +func TestHardFail_WALFullStall(t *testing.T) { + metrics := map[string]float64{"wal_full_duration_seconds": 15} + results := RunHardFailChecks(metrics) + for _, r := range results { + if r.Condition == "wal_full_stall" && r.Pass { + t.Error("expected FAIL for WAL full 15s") + } + } +} + +func TestHardFail_AllPass(t *testing.T) { + metrics := map[string]float64{ + "health_score": 1.0, + "barrier_lag_lsn_max": 10, + } + results := RunHardFailChecks(metrics) + for _, r := range results { + if !r.Pass { + t.Errorf("expected all PASS, got FAIL for %s: %s", r.Condition, r.Detail) + } + } +} + +func TestFormatRegressionReport(t *testing.T) { + report := &RegressionReport{ + Results: []RegressionResult{ + {Metric: "p99_write_latency_seconds", BaselineVal: 0.005, CurrentVal: 0.006, Pass: false, Reason: "change=+20%"}, + }, + HardFails: []HardFailResult{ + {Condition: "data_mismatch", Pass: true}, + }, + OverallPass: false, + BaselineGitSHA: "abc123", + CurrentGitSHA: "def456", + } + s := FormatRegressionReport(report) + if s == "" { + t.Error("expected non-empty report") + } + if len(s) < 50 { + t.Errorf("report too short: %s", s) + } +} diff --git a/weed/storage/blockvol/testrunner/reporter.go b/weed/storage/blockvol/testrunner/reporter.go new file mode 100644 index 000000000..decbd3a32 --- /dev/null +++ b/weed/storage/blockvol/testrunner/reporter.go @@ -0,0 +1,324 @@ +package testrunner + +import ( + "encoding/json" + "encoding/xml" + "fmt" + "io" + "os" + "sort" + "strings" + "time" +) + +// WriteJSON writes the scenario result as JSON to the given path. +func WriteJSON(result *ScenarioResult, path string) error { + data, err := json.MarshalIndent(result, "", " ") + if err != nil { + return fmt.Errorf("marshal JSON: %w", err) + } + return os.WriteFile(path, data, 0644) +} + +// PrintSummary writes a human-readable summary table to w. +func PrintSummary(w io.Writer, result *ScenarioResult) { + fmt.Fprintf(w, "\n=== %s === %s (%s)\n\n", + result.Name, result.Status, result.Duration.Round(time.Millisecond)) + + passCount, failCount, totalActions := 0, 0, 0 + for _, pr := range result.Phases { + status := string(pr.Status) + fmt.Fprintf(w, " %-20s %s (%s)\n", + pr.Name, status, pr.Duration.Round(time.Millisecond)) + for _, ar := range pr.Actions { + totalActions++ + if ar.Status == StatusPass { + passCount++ + } else { + failCount++ + } + marker := " " + if ar.Status == StatusFail { + marker = "X" + } + fmt.Fprintf(w, " %s %-25s %s\n", marker, ar.Action, ar.Duration.Round(time.Millisecond)) + if ar.Error != "" { + fmt.Fprintf(w, " ERROR: %s\n", ar.Error) + } + } + } + + fmt.Fprintf(w, "\n %d actions: %d passed, %d failed\n", totalActions, passCount, failCount) + + // Render perf stats and metrics tables from result.Vars. + if len(result.Vars) > 0 { + printPerfTable(w, result.Vars) + printMetricsTable(w, result.Vars) + } + + fmt.Fprintln(w) + + if result.Error != "" { + fmt.Fprintf(w, " ERROR: %s\n\n", result.Error) + } +} + +// printPerfTable scans vars for PerfStats-format values (FormatStats output) +// and renders a formatted table. Matches lines like "name: n=100 mean=1.23 ...". +func printPerfTable(w io.Writer, vars map[string]string) { + type perfEntry struct { + name string + stats PerfStats + } + var entries []perfEntry + + for key, val := range vars { + // Try parsing as PerfStats JSON first. + var ps PerfStats + if err := json.Unmarshal([]byte(val), &ps); err == nil && ps.Count > 0 { + entries = append(entries, perfEntry{name: key, stats: ps}) + continue + } + // Try parsing FormatStats text: "name: n=100 mean=1.23 ..." + if ps, ok := parsePerfStatsLine(val); ok { + entries = append(entries, perfEntry{name: key, stats: ps}) + } + } + + if len(entries) == 0 { + return + } + + sort.Slice(entries, func(i, j int) bool { return entries[i].name < entries[j].name }) + + fmt.Fprintf(w, "\n === Performance ===\n") + fmt.Fprintf(w, " %-16s %6s %12s %12s %12s %12s %12s %12s\n", + "Metric", "Count", "Mean", "P50", "P90", "P99", "Min", "Max") + for _, e := range entries { + fmt.Fprintf(w, " %-16s %6d %12.1f %12.1f %12.1f %12.1f %12.1f %12.1f\n", + e.name, e.stats.Count, e.stats.Mean, e.stats.P50, e.stats.P90, e.stats.P99, e.stats.Min, e.stats.Max) + } +} + +// parsePerfStatsLine parses a FormatStats-style line: "name: n=100 mean=1.23 stddev=0.5 p50=1.2 p90=1.5 p99=1.8 min=0.5 max=2.0" +func parsePerfStatsLine(s string) (PerfStats, bool) { + if !strings.Contains(s, "n=") || !strings.Contains(s, "p99=") { + return PerfStats{}, false + } + + var ps PerfStats + fields := strings.Fields(s) + parsed := 0 + for _, f := range fields { + parts := strings.SplitN(f, "=", 2) + if len(parts) != 2 { + continue + } + var val float64 + if _, err := fmt.Sscanf(parts[1], "%f", &val); err != nil { + continue + } + switch parts[0] { + case "n": + ps.Count = int(val) + parsed++ + case "mean": + ps.Mean = val + parsed++ + case "stddev": + ps.StdDev = val + parsed++ + case "p50": + ps.P50 = val + parsed++ + case "p90": + ps.P90 = val + parsed++ + case "p99": + ps.P99 = val + parsed++ + case "min": + ps.Min = val + parsed++ + case "max": + ps.Max = val + parsed++ + } + } + return ps, parsed >= 3 && ps.Count > 0 +} + +// printMetricsTable scans vars for MetricsSample JSON values and renders a table. +func printMetricsTable(w io.Writer, vars map[string]string) { + type metricsGroup struct { + name string + sample MetricsSample + } + var groups []metricsGroup + + for key, val := range vars { + var ms MetricsSample + if err := json.Unmarshal([]byte(val), &ms); err != nil { + continue + } + if len(ms.Metrics) == 0 { + continue + } + label := key + if ms.Target != "" { + label = ms.Target + } + groups = append(groups, metricsGroup{name: label, sample: ms}) + } + + if len(groups) == 0 { + return + } + + sort.Slice(groups, func(i, j int) bool { return groups[i].name < groups[j].name }) + + for _, g := range groups { + fmt.Fprintf(w, "\n === Metrics (%s) ===\n", g.name) + // Sort metric names. + names := make([]string, 0, len(g.sample.Metrics)) + for name := range g.sample.Metrics { + names = append(names, name) + } + sort.Strings(names) + for _, name := range names { + fmt.Fprintf(w, " %-40s %s\n", name, formatMetricValue(g.sample.Metrics[name])) + } + } +} + +// formatMetricValue formats a float with commas for readability. +func formatMetricValue(v float64) string { + if v == float64(int64(v)) { + return formatInt(int64(v)) + } + return fmt.Sprintf("%.2f", v) +} + +// formatInt formats an integer with comma separators. +func formatInt(n int64) string { + s := fmt.Sprintf("%d", n) + if n < 0 { + return s + } + // Insert commas. + var result strings.Builder + for i, c := range s { + if i > 0 && (len(s)-i)%3 == 0 { + result.WriteByte(',') + } + result.WriteRune(c) + } + return result.String() +} + +// --- JUnit XML output --- + +// JUnitTestSuites is the root of JUnit XML. +type JUnitTestSuites struct { + XMLName xml.Name `xml:"testsuites"` + Suites []JUnitTestSuite `xml:"testsuite"` +} + +// JUnitTestSuite corresponds to one scenario. +type JUnitTestSuite struct { + XMLName xml.Name `xml:"testsuite"` + Name string `xml:"name,attr"` + Tests int `xml:"tests,attr"` + Failures int `xml:"failures,attr"` + Time float64 `xml:"time,attr"` + Cases []JUnitTestCase `xml:"testcase"` +} + +// JUnitTestCase corresponds to one action. +type JUnitTestCase struct { + XMLName xml.Name `xml:"testcase"` + Name string `xml:"name,attr"` + ClassName string `xml:"classname,attr"` + Time float64 `xml:"time,attr"` + Failure *JUnitFailure `xml:"failure,omitempty"` +} + +// JUnitFailure describes a test failure. +type JUnitFailure struct { + Message string `xml:"message,attr"` + Content string `xml:",chardata"` +} + +// WriteJUnitXML writes JUnit XML output to the given path. +func WriteJUnitXML(result *ScenarioResult, path string) error { + suite := JUnitTestSuite{ + Name: result.Name, + Time: result.Duration.Seconds(), + } + + for _, pr := range result.Phases { + for _, ar := range pr.Actions { + suite.Tests++ + tc := JUnitTestCase{ + Name: fmt.Sprintf("%s/%s", pr.Name, ar.Action), + ClassName: result.Name, + Time: ar.Duration.Seconds(), + } + if ar.Status == StatusFail { + suite.Failures++ + tc.Failure = &JUnitFailure{ + Message: ar.Error, + Content: ar.Output, + } + } + suite.Cases = append(suite.Cases, tc) + } + } + + suites := JUnitTestSuites{Suites: []JUnitTestSuite{suite}} + data, err := xml.MarshalIndent(suites, "", " ") + if err != nil { + return fmt.Errorf("marshal JUnit XML: %w", err) + } + + header := []byte(xml.Header) + data = append(header, data...) + return os.WriteFile(path, data, 0644) +} + +// --- Baseline comparison --- + +// BaselineCompare compares a result against a baseline JSON file. +// Returns a list of regressions (actions that passed in baseline but fail now). +func BaselineCompare(result *ScenarioResult, baselinePath string) ([]string, error) { + data, err := os.ReadFile(baselinePath) + if err != nil { + return nil, fmt.Errorf("read baseline: %w", err) + } + var baseline ScenarioResult + if err := json.Unmarshal(data, &baseline); err != nil { + return nil, fmt.Errorf("parse baseline: %w", err) + } + + // Build map of baseline action statuses. + baseActions := make(map[string]ResultStatus) + for _, pr := range baseline.Phases { + for _, ar := range pr.Actions { + key := fmt.Sprintf("%s/%s", pr.Name, ar.Action) + baseActions[key] = ar.Status + } + } + + var regressions []string + for _, pr := range result.Phases { + for _, ar := range pr.Actions { + key := fmt.Sprintf("%s/%s", pr.Name, ar.Action) + if baseStatus, ok := baseActions[key]; ok { + if baseStatus == StatusPass && ar.Status == StatusFail { + regressions = append(regressions, fmt.Sprintf("%s: was PASS, now FAIL: %s", key, ar.Error)) + } + } + } + } + return regressions, nil +} diff --git a/weed/storage/blockvol/testrunner/reporter_html.go b/weed/storage/blockvol/testrunner/reporter_html.go new file mode 100644 index 000000000..8c631723b --- /dev/null +++ b/weed/storage/blockvol/testrunner/reporter_html.go @@ -0,0 +1,301 @@ +package testrunner + +import ( + "encoding/json" + "fmt" + "html/template" + "os" + "sort" + "strings" + "time" +) + +// WriteHTMLReport generates a single-file HTML report from a ScenarioResult. +func WriteHTMLReport(result *ScenarioResult, path string) error { + data := buildHTMLData(result) + + f, err := os.Create(path) + if err != nil { + return fmt.Errorf("create %s: %w", path, err) + } + defer f.Close() + + tmpl, err := template.New("report").Parse(htmlTemplate) + if err != nil { + return fmt.Errorf("parse template: %w", err) + } + + return tmpl.Execute(f, data) +} + +// htmlData is the template context. +type htmlData struct { + Name string + Status string + StatusCSS string + Duration string + Timestamp string + Phases []htmlPhase + Perf []htmlPerfEntry + Metrics []htmlMetricsGroup + Artifacts []ArtifactEntry + Error string + HasPerf bool + HasMetrics bool + HasArtifacts bool +} + +type htmlPhase struct { + Name string + Status string + StatusCSS string + Duration string + Actions []htmlAction +} + +type htmlAction struct { + Name string + Status string + StatusCSS string + Duration string + Output string + Error string + YAML string +} + +type htmlPerfEntry struct { + Name string + Count int + Mean string + P50 string + P90 string + P99 string + Min string + Max string +} + +type htmlMetricsGroup struct { + Name string + Entries []htmlMetricEntry +} + +type htmlMetricEntry struct { + Name string + Value string +} + +func buildHTMLData(result *ScenarioResult) htmlData { + d := htmlData{ + Name: result.Name, + Status: string(result.Status), + StatusCSS: statusCSS(result.Status), + Duration: result.Duration.Round(time.Millisecond).String(), + Timestamp: time.Now().Format(time.RFC3339), + Error: result.Error, + Artifacts: result.Artifacts, + HasArtifacts: len(result.Artifacts) > 0, + } + + for _, pr := range result.Phases { + hp := htmlPhase{ + Name: pr.Name, + Status: string(pr.Status), + StatusCSS: statusCSS(pr.Status), + Duration: pr.Duration.Round(time.Millisecond).String(), + } + for _, ar := range pr.Actions { + hp.Actions = append(hp.Actions, htmlAction{ + Name: ar.Action, + Status: string(ar.Status), + StatusCSS: statusCSS(ar.Status), + Duration: ar.Duration.Round(time.Millisecond).String(), + Output: ar.Output, + Error: ar.Error, + YAML: ar.YAML, + }) + } + d.Phases = append(d.Phases, hp) + } + + // Extract perf stats from vars. + if len(result.Vars) > 0 { + d.Perf = extractHTMLPerf(result.Vars) + d.HasPerf = len(d.Perf) > 0 + d.Metrics = extractHTMLMetrics(result.Vars) + d.HasMetrics = len(d.Metrics) > 0 + } + + return d +} + +func statusCSS(s ResultStatus) string { + switch s { + case StatusPass: + return "pass" + case StatusFail: + return "fail" + case StatusSkip: + return "skip" + default: + return "" + } +} + +func extractHTMLPerf(vars map[string]string) []htmlPerfEntry { + var entries []htmlPerfEntry + for key, val := range vars { + var ps PerfStats + if err := json.Unmarshal([]byte(val), &ps); err == nil && ps.Count > 0 { + entries = append(entries, perfToHTML(key, ps)) + continue + } + if ps, ok := parsePerfStatsLine(val); ok { + entries = append(entries, perfToHTML(key, ps)) + } + } + sort.Slice(entries, func(i, j int) bool { return entries[i].Name < entries[j].Name }) + return entries +} + +func perfToHTML(name string, ps PerfStats) htmlPerfEntry { + return htmlPerfEntry{ + Name: name, + Count: ps.Count, + Mean: fmt.Sprintf("%.1f", ps.Mean), + P50: fmt.Sprintf("%.1f", ps.P50), + P90: fmt.Sprintf("%.1f", ps.P90), + P99: fmt.Sprintf("%.1f", ps.P99), + Min: fmt.Sprintf("%.1f", ps.Min), + Max: fmt.Sprintf("%.1f", ps.Max), + } +} + +func extractHTMLMetrics(vars map[string]string) []htmlMetricsGroup { + var groups []htmlMetricsGroup + for key, val := range vars { + var ms MetricsSample + if err := json.Unmarshal([]byte(val), &ms); err != nil || len(ms.Metrics) == 0 { + continue + } + label := key + if ms.Target != "" { + label = ms.Target + } + names := make([]string, 0, len(ms.Metrics)) + for n := range ms.Metrics { + names = append(names, n) + } + sort.Strings(names) + g := htmlMetricsGroup{Name: label} + for _, n := range names { + g.Entries = append(g.Entries, htmlMetricEntry{ + Name: n, + Value: formatMetricValue(ms.Metrics[n]), + }) + } + groups = append(groups, g) + } + sort.Slice(groups, func(i, j int) bool { return groups[i].Name < groups[j].Name }) + return groups +} + +var htmlTemplate = strings.TrimSpace(` + + + + +{{.Name}} — Test Report + + + +
+

{{.Name}}

+ {{.Status}} + {{.Duration}} +
+
Generated: {{.Timestamp}}
+ +

Phases & Actions

+
NameServerSizePlacementEpochRoleStatusiSCSIReplicaActionNameServerSizeRoleStatusDurabilityHealthWAL LagDegradedEpochReplicaAction
{{.Name}} {{.VolumeServer}} {{.SizeMB}} MB{{.ReplicaPlacement}}{{.Epoch}} {{if eq .Role "primary"}}primary{{else if eq .Role "replica"}}replica{{else}}{{.Role}}{{end}} {{if eq .Status "active"}}active{{else}}{{.Status}}{{end}}{{.ISCSIAddr}}{{if eq .DurabilityMode "sync_all"}}sync_all{{else if eq .DurabilityMode "sync_quorum"}}sync_quorum{{else}}best_effort{{end}}{{printf "%.2f" .HealthScore}}{{.MaxWALLag}}{{if .ReplicaDegraded}}yes{{else}}-{{end}}{{.Epoch}} {{.ReplicaServer}}
+ + {{range .Phases}} + + + + + + {{range .Actions}} + + + + + + {{if .Error}}{{end}} + {{if .Output}}{{end}} + {{if .YAML}}{{end}} + {{end}} + {{end}} +
NameStatusDuration
{{.Name}}{{.Status}}{{.Duration}}
{{.Name}}{{.Status}}{{.Duration}}
ERROR: {{.Error}}
{{.Output}}
YAML definition
{{.YAML}}
+ + {{if .HasPerf}} +

Performance

+ + + {{range .Perf}} + + {{end}} +
MetricCountMeanP50P90P99MinMax
{{.Name}}{{.Count}}{{.Mean}}{{.P50}}{{.P90}}{{.P99}}{{.Min}}{{.Max}}
+ {{end}} + + {{if .HasMetrics}} + {{range .Metrics}} +

Metrics ({{.Name}})

+ + + {{range .Entries}} + + {{end}} +
MetricValue
{{.Name}}{{.Value}}
+ {{end}} + {{end}} + + {{if .HasArtifacts}} +

Artifacts

+ + {{end}} + + {{if .Error}} +

Error

+ 
{{.Error}}
+ {{end}} + + +`) diff --git a/weed/storage/blockvol/testrunner/reporter_html_test.go b/weed/storage/blockvol/testrunner/reporter_html_test.go new file mode 100644 index 000000000..7a7b15f7b --- /dev/null +++ b/weed/storage/blockvol/testrunner/reporter_html_test.go @@ -0,0 +1,169 @@ +package testrunner + +import ( + "os" + "path/filepath" + "strings" + "testing" + "time" +) + +func TestWriteHTMLReport_Basic(t *testing.T) { + result := &ScenarioResult{ + Name: "html-test", + Status: StatusPass, + Duration: 2500 * time.Millisecond, + Phases: []PhaseResult{ + { + Name: "setup", + Status: StatusPass, + Duration: 1000 * time.Millisecond, + Actions: []ActionResult{ + {Action: "build_deploy", Status: StatusPass, Duration: 800 * time.Millisecond}, + {Action: "start_target", Status: StatusPass, Duration: 200 * time.Millisecond}, + }, + }, + { + Name: "verify", + Status: StatusPass, + Duration: 500 * time.Millisecond, + Actions: []ActionResult{ + {Action: "assert_equal", Status: StatusPass, Duration: 5 * time.Millisecond}, + }, + }, + }, + } + + dir := t.TempDir() + path := filepath.Join(dir, "report.html") + + if err := WriteHTMLReport(result, path); err != nil { + t.Fatalf("WriteHTMLReport: %v", err) + } + + data, err := os.ReadFile(path) + if err != nil { + t.Fatalf("read: %v", err) + } + + html := string(data) + if !strings.Contains(html, "html-test") { + t.Error("HTML should contain scenario name") + } + if !strings.Contains(html, "PASS") { + t.Error("HTML should contain PASS status") + } + if !strings.Contains(html, "build_deploy") { + t.Error("HTML should contain action name") + } + if !strings.Contains(html, "") { + t.Error("HTML should be valid HTML document") + } +} + +func TestWriteHTMLReport_WithFailure(t *testing.T) { + result := &ScenarioResult{ + Name: "fail-html", + Status: StatusFail, + Duration: 300 * time.Millisecond, + Error: "test & failed", + Phases: []PhaseResult{ + { + Name: "main", + Status: StatusFail, + Actions: []ActionResult{ + {Action: "bad_step", Status: StatusFail, Error: "something happened"}, + }, + }, + }, + } + + dir := t.TempDir() + path := filepath.Join(dir, "report.html") + + if err := WriteHTMLReport(result, path); err != nil { + t.Fatalf("WriteHTMLReport: %v", err) + } + + data, _ := os.ReadFile(path) + html := string(data) + + if !strings.Contains(html, "FAIL") { + t.Error("HTML should contain FAIL") + } + // Verify HTML escaping (review item #5). + if strings.Contains(html, "") { + t.Error("HTML should escape angle brackets in error text") + } + if !strings.Contains(html, "<broke>") { + t.Error("HTML should contain escaped angle brackets") + } +} + +func TestWriteHTMLReport_WithPerfAndMetrics(t *testing.T) { + result := &ScenarioResult{ + Name: "perf-html", + Status: StatusPass, + Duration: 1 * time.Second, + Phases: []PhaseResult{{Name: "bench", Status: StatusPass}}, + Vars: map[string]string{ + "iops": `{"count":100,"min":9000,"max":15000,"mean":12345,"stddev":500,"p50":12200,"p90":13500,"p99":14000}`, + "metrics": `{"target":"primary","metrics":{"blockvol_write_bytes_total":1234567,"blockvol_read_bytes_total":987654}}`, + }, + } + + dir := t.TempDir() + path := filepath.Join(dir, "report.html") + + if err := WriteHTMLReport(result, path); err != nil { + t.Fatalf("WriteHTMLReport: %v", err) + } + + data, _ := os.ReadFile(path) + html := string(data) + + if !strings.Contains(html, "Performance") { + t.Error("HTML should contain Performance section") + } + if !strings.Contains(html, "iops") { + t.Error("HTML should contain perf metric name") + } + if !strings.Contains(html, "Metrics") { + t.Error("HTML should contain Metrics section") + } + if !strings.Contains(html, "blockvol_write_bytes_total") { + t.Error("HTML should contain metric name") + } +} + +func TestWriteHTMLReport_WithArtifacts(t *testing.T) { + result := &ScenarioResult{ + Name: "artifacts-html", + Status: StatusFail, + Duration: 500 * time.Millisecond, + Artifacts: []ArtifactEntry{ + {Agent: "m01", Path: "/tmp/artifacts/m01/target.log", Size: 4096}, + {Agent: "m02", Path: "/tmp/artifacts/m02/dmesg.log", Size: 1024}, + }, + } + + dir := t.TempDir() + path := filepath.Join(dir, "report.html") + + if err := WriteHTMLReport(result, path); err != nil { + t.Fatalf("WriteHTMLReport: %v", err) + } + + data, _ := os.ReadFile(path) + html := string(data) + + if !strings.Contains(html, "Artifacts") { + t.Error("HTML should contain Artifacts section") + } + if !strings.Contains(html, "target.log") { + t.Error("HTML should contain artifact path") + } + if !strings.Contains(html, "4096 bytes") { + t.Error("HTML should contain artifact size") + } +} diff --git a/weed/storage/blockvol/testrunner/reporter_test.go b/weed/storage/blockvol/testrunner/reporter_test.go new file mode 100644 index 000000000..ba1e4bcec --- /dev/null +++ b/weed/storage/blockvol/testrunner/reporter_test.go @@ -0,0 +1,368 @@ +package testrunner + +import ( + "bytes" + "encoding/json" + "encoding/xml" + "os" + "path/filepath" + "strings" + "testing" + "time" +) + +func TestPrintSummary(t *testing.T) { + result := &ScenarioResult{ + Name: "test-scenario", + Status: StatusPass, + Duration: 1500 * time.Millisecond, + Phases: []PhaseResult{ + { + Name: "setup", + Status: StatusPass, + Duration: 800 * time.Millisecond, + Actions: []ActionResult{ + {Action: "build_deploy", Status: StatusPass, Duration: 500 * time.Millisecond}, + {Action: "start_target", Status: StatusPass, Duration: 300 * time.Millisecond}, + }, + }, + { + Name: "verify", + Status: StatusPass, + Duration: 700 * time.Millisecond, + Actions: []ActionResult{ + {Action: "assert_equal", Status: StatusPass, Duration: 1 * time.Millisecond}, + }, + }, + }, + } + + var buf bytes.Buffer + PrintSummary(&buf, result) + output := buf.String() + + if !strings.Contains(output, "test-scenario") { + t.Error("summary should contain scenario name") + } + if !strings.Contains(output, "PASS") { + t.Error("summary should contain PASS") + } + if !strings.Contains(output, "3 actions") { + t.Error("summary should contain action count") + } +} + +func TestPrintSummary_WithFailure(t *testing.T) { + result := &ScenarioResult{ + Name: "fail-scenario", + Status: StatusFail, + Duration: 500 * time.Millisecond, + Error: "something broke", + Phases: []PhaseResult{ + { + Name: "main", + Status: StatusFail, + Actions: []ActionResult{ + {Action: "step1", Status: StatusFail, Error: "boom"}, + }, + }, + }, + } + + var buf bytes.Buffer + PrintSummary(&buf, result) + output := buf.String() + + if !strings.Contains(output, "FAIL") { + t.Error("summary should contain FAIL") + } + if !strings.Contains(output, "boom") { + t.Error("summary should contain error message") + } +} + +func TestWriteJSON(t *testing.T) { + result := &ScenarioResult{ + Name: "json-test", + Status: StatusPass, + Duration: 100 * time.Millisecond, + } + + dir := t.TempDir() + path := filepath.Join(dir, "result.json") + + if err := WriteJSON(result, path); err != nil { + t.Fatalf("WriteJSON: %v", err) + } + + data, err := os.ReadFile(path) + if err != nil { + t.Fatalf("read: %v", err) + } + + var loaded ScenarioResult + if err := json.Unmarshal(data, &loaded); err != nil { + t.Fatalf("unmarshal: %v", err) + } + + if loaded.Name != "json-test" { + t.Errorf("name = %q", loaded.Name) + } + if loaded.Status != StatusPass { + t.Errorf("status = %s", loaded.Status) + } +} + +func TestWriteJUnitXML(t *testing.T) { + result := &ScenarioResult{ + Name: "junit-test", + Status: StatusFail, + Duration: 200 * time.Millisecond, + Phases: []PhaseResult{ + { + Name: "main", + Status: StatusFail, + Actions: []ActionResult{ + {Action: "ok_step", Status: StatusPass, Duration: 50 * time.Millisecond}, + {Action: "fail_step", Status: StatusFail, Duration: 100 * time.Millisecond, Error: "oops"}, + }, + }, + }, + } + + dir := t.TempDir() + path := filepath.Join(dir, "result.xml") + + if err := WriteJUnitXML(result, path); err != nil { + t.Fatalf("WriteJUnitXML: %v", err) + } + + data, err := os.ReadFile(path) + if err != nil { + t.Fatalf("read: %v", err) + } + + var suites JUnitTestSuites + if err := xml.Unmarshal(data, &suites); err != nil { + t.Fatalf("unmarshal XML: %v", err) + } + + if len(suites.Suites) != 1 { + t.Fatalf("suites = %d, want 1", len(suites.Suites)) + } + suite := suites.Suites[0] + if suite.Tests != 2 { + t.Errorf("tests = %d, want 2", suite.Tests) + } + if suite.Failures != 1 { + t.Errorf("failures = %d, want 1", suite.Failures) + } +} + +func TestPrintSummary_WithPerfTable(t *testing.T) { + result := &ScenarioResult{ + Name: "perf-scenario", + Status: StatusPass, + Duration: 2 * time.Second, + Phases: []PhaseResult{ + { + Name: "bench", + Status: StatusPass, + Actions: []ActionResult{ + {Action: "dd_write", Status: StatusPass, Duration: 1 * time.Second}, + }, + }, + }, + Vars: map[string]string{ + "iops": `{"count":100,"min":9000,"max":15000,"mean":12345,"stddev":500,"p50":12200,"p90":13500,"p99":14000}`, + }, + } + + var buf bytes.Buffer + PrintSummary(&buf, result) + output := buf.String() + + if !strings.Contains(output, "Performance") { + t.Error("summary should contain Performance table header") + } + if !strings.Contains(output, "iops") { + t.Error("summary should contain metric name 'iops'") + } + if !strings.Contains(output, "12345") { + t.Error("summary should contain mean value") + } +} + +func TestPrintSummary_WithMetricsTable(t *testing.T) { + result := &ScenarioResult{ + Name: "metrics-scenario", + Status: StatusPass, + Duration: 1 * time.Second, + Phases: []PhaseResult{{Name: "m", Status: StatusPass}}, + Vars: map[string]string{ + "primary_metrics": `{"target":"primary","metrics":{"blockvol_write_bytes_total":1234567,"blockvol_read_bytes_total":987654}}`, + }, + } + + var buf bytes.Buffer + PrintSummary(&buf, result) + output := buf.String() + + if !strings.Contains(output, "Metrics (primary)") { + t.Error("summary should contain Metrics section with target name") + } + if !strings.Contains(output, "blockvol_write_bytes_total") { + t.Error("summary should contain metric name") + } +} + +func TestPrintSummary_WithPerfStatsLine(t *testing.T) { + result := &ScenarioResult{ + Name: "perf-line-scenario", + Status: StatusPass, + Duration: 1 * time.Second, + Phases: []PhaseResult{{Name: "p", Status: StatusPass}}, + Vars: map[string]string{ + "lat_stats": "lat_us: n=50 mean=420.5 stddev=30.1 p50=415.0 p90=460.0 p99=500.0 min=350.0 max=550.0", + }, + } + + var buf bytes.Buffer + PrintSummary(&buf, result) + output := buf.String() + + if !strings.Contains(output, "Performance") { + t.Error("summary should render perf table from FormatStats-style line") + } + if !strings.Contains(output, "lat_stats") { + t.Error("summary should contain metric name from var key") + } +} + +func TestParsePerfStatsLine(t *testing.T) { + line := "iops: n=100 mean=12345.00 stddev=500.00 p50=12200.00 p90=13500.00 p99=14000.00 min=9000.00 max=15000.00" + ps, ok := parsePerfStatsLine(line) + if !ok { + t.Fatal("should parse FormatStats-style line") + } + if ps.Count != 100 { + t.Errorf("count = %d, want 100", ps.Count) + } + if ps.Mean != 12345 { + t.Errorf("mean = %f, want 12345", ps.Mean) + } + if ps.P99 != 14000 { + t.Errorf("p99 = %f, want 14000", ps.P99) + } +} + +func TestParsePerfStatsLine_Invalid(t *testing.T) { + _, ok := parsePerfStatsLine("not a perf line") + if ok { + t.Error("should not parse non-perf text") + } +} + +func TestFormatInt(t *testing.T) { + tests := []struct { + n int64 + want string + }{ + {0, "0"}, + {999, "999"}, + {1000, "1,000"}, + {1234567, "1,234,567"}, + } + for _, tt := range tests { + got := formatInt(tt.n) + if got != tt.want { + t.Errorf("formatInt(%d) = %q, want %q", tt.n, got, tt.want) + } + } +} + +func TestWriteJSON_WithVars(t *testing.T) { + result := &ScenarioResult{ + Name: "json-vars-test", + Status: StatusPass, + Vars: map[string]string{ + "key1": "value1", + "key2": `{"nested":"json"}`, + }, + Artifacts: []ArtifactEntry{ + {Agent: "m01", Path: "/tmp/a/log.txt", Size: 512}, + }, + } + + dir := t.TempDir() + path := filepath.Join(dir, "result.json") + + if err := WriteJSON(result, path); err != nil { + t.Fatalf("WriteJSON: %v", err) + } + + data, _ := os.ReadFile(path) + var loaded ScenarioResult + json.Unmarshal(data, &loaded) + + if loaded.Vars["key1"] != "value1" { + t.Errorf("vars[key1] = %q", loaded.Vars["key1"]) + } + if loaded.Vars["key2"] != `{"nested":"json"}` { + t.Errorf("vars[key2] = %q", loaded.Vars["key2"]) + } + if len(loaded.Artifacts) != 1 { + t.Fatalf("artifacts = %d, want 1", len(loaded.Artifacts)) + } + if loaded.Artifacts[0].Agent != "m01" { + t.Errorf("artifact agent = %q", loaded.Artifacts[0].Agent) + } +} + +func TestBaselineCompare(t *testing.T) { + baseline := &ScenarioResult{ + Name: "baseline", + Phases: []PhaseResult{ + { + Name: "main", + Actions: []ActionResult{ + {Action: "step1", Status: StatusPass}, + {Action: "step2", Status: StatusPass}, + {Action: "step3", Status: StatusFail}, + }, + }, + }, + } + + dir := t.TempDir() + baselinePath := filepath.Join(dir, "baseline.json") + if err := WriteJSON(baseline, baselinePath); err != nil { + t.Fatalf("write baseline: %v", err) + } + + current := &ScenarioResult{ + Name: "current", + Phases: []PhaseResult{ + { + Name: "main", + Actions: []ActionResult{ + {Action: "step1", Status: StatusPass}, // still pass + {Action: "step2", Status: StatusFail, Error: "regression"}, // regression! + {Action: "step3", Status: StatusFail}, // was fail, still fail (not regression) + }, + }, + }, + } + + regressions, err := BaselineCompare(current, baselinePath) + if err != nil { + t.Fatalf("compare: %v", err) + } + + if len(regressions) != 1 { + t.Fatalf("regressions = %d, want 1", len(regressions)) + } + if !strings.Contains(regressions[0], "step2") { + t.Errorf("regression should mention step2: %s", regressions[0]) + } +} diff --git a/weed/storage/blockvol/testrunner/scenarios/consistency-epoch.yaml b/weed/storage/blockvol/testrunner/scenarios/consistency-epoch.yaml new file mode 100644 index 000000000..08d0efbfb --- /dev/null +++ b/weed/storage/blockvol/testrunner/scenarios/consistency-epoch.yaml @@ -0,0 +1,80 @@ +name: consistency-epoch +timeout: 5m +env: + repo_dir: "C:/work/seaweedfs" + +topology: + nodes: + target_node: + host: "192.168.1.184" + user: testdev + key: "C:/work/dev_server/testdev_key" + client_node: + host: "192.168.1.181" + user: testdev + key: "C:/work/dev_server/testdev_key" + +targets: + primary: + node: target_node + vol_size: 100M + iscsi_port: 3260 + admin_port: 8080 + iqn_suffix: epoch-primary + +phases: + - name: setup + actions: + - action: build_deploy + - action: start_target + target: primary + create: "true" + + - name: epoch_monotonicity + actions: + - action: assign + target: primary + epoch: "1" + role: primary + lease_ttl: 30s + - action: assert_status + target: primary + role: primary + - action: assign + target: primary + epoch: "2" + role: primary + lease_ttl: 30s + - action: status + target: primary + save_as: status_e2 + - action: print + msg: "status after epoch 2: {{ status_e2 }}" + + - name: stale_epoch_reject + actions: + - action: exec + node: target_node + cmd: "curl -s -w '\\n%{http_code}' -X POST -H 'Content-Type: application/json' -d '{\"epoch\":1,\"role\":1,\"lease_ttl_ms\":30000}' http://127.0.0.1:8080/assign" + save_as: stale_result + - action: print + msg: "stale epoch result: {{ stale_result }}" + + - name: epoch_persist + actions: + - action: stop_target + target: primary + - action: start_target + target: primary + create: "false" + - action: status + target: primary + save_as: post_restart_status + - action: print + msg: "status after restart: {{ post_restart_status }}" + + - name: cleanup + always: true + actions: + - action: stop_all_targets + ignore_error: true diff --git a/weed/storage/blockvol/testrunner/scenarios/consistency-lease.yaml b/weed/storage/blockvol/testrunner/scenarios/consistency-lease.yaml new file mode 100644 index 000000000..4fb18c832 --- /dev/null +++ b/weed/storage/blockvol/testrunner/scenarios/consistency-lease.yaml @@ -0,0 +1,80 @@ +name: consistency-lease +timeout: 5m +env: + repo_dir: "C:/work/seaweedfs" + +topology: + nodes: + target_node: + host: "192.168.1.184" + user: testdev + key: "C:/work/dev_server/testdev_key" + client_node: + host: "192.168.1.181" + user: testdev + key: "C:/work/dev_server/testdev_key" + +targets: + primary: + node: target_node + vol_size: 100M + iscsi_port: 3260 + admin_port: 8080 + iqn_suffix: lease-primary + replica: + node: target_node + vol_size: 100M + iscsi_port: 3261 + admin_port: 8081 + replica_data_port: 9011 + replica_ctrl_port: 9012 + iqn_suffix: lease-replica + +phases: + - name: setup + actions: + - action: build_deploy + - action: start_target + target: primary + create: "true" + - action: start_target + target: replica + create: "true" + + - name: lease_expiry + actions: + - action: assign + target: primary + epoch: "1" + role: primary + lease_ttl: 5s + - action: assert_status + target: primary + role: primary + - action: sleep + duration: 7s + - action: assert_status + target: primary + has_lease: "false" + + - name: split_brain_prevention + actions: + - action: assign + target: primary + epoch: "2" + role: primary + lease_ttl: 30s + - action: assign + target: replica + epoch: "2" + role: replica + lease_ttl: 30s + - action: set_replica + target: primary + replica: replica + + - name: cleanup + always: true + actions: + - action: stop_all_targets + ignore_error: true diff --git a/weed/storage/blockvol/testrunner/scenarios/coord-dev-cycle.yaml b/weed/storage/blockvol/testrunner/scenarios/coord-dev-cycle.yaml new file mode 100644 index 000000000..d4abae930 --- /dev/null +++ b/weed/storage/blockvol/testrunner/scenarios/coord-dev-cycle.yaml @@ -0,0 +1,139 @@ +name: coord-dev-cycle +timeout: 5m +env: + repo_dir: "/c/work/seaweedfs" + +topology: + agents: + target_agent: "192.168.1.184:9100" + client_agent: "192.168.1.181:9100" + + nodes: + target_node: + host: "192.168.1.184" + agent: target_agent + client_node: + host: "192.168.1.181" + agent: client_agent + +targets: + primary: + node: target_node + vol_size: 100M + iscsi_port: 3260 + admin_port: 8080 + iqn_suffix: dev-primary + replica: + node: target_node + vol_size: 100M + iscsi_port: 3261 + admin_port: 8081 + replica_data_port: 9011 + replica_ctrl_port: 9012 + rebuild_port: 9013 + iqn_suffix: dev-replica + +phases: + # Phase 0: Kill stale processes from previous runs + - name: pre_cleanup + actions: + - action: kill_stale + node: target_node + process: iscsi-target-test + ignore_error: true + - action: kill_stale + node: client_node + iscsi_cleanup: "true" + ignore_error: true + + # Phase 1: Build and deploy iscsi-target binary + - name: build_deploy + actions: + - action: build_deploy + + # Phase 2: Start targets, set up HA replication + - name: setup + actions: + - action: start_target + target: primary + create: "true" + - action: start_target + target: replica + create: "true" + - action: assign + target: replica + epoch: "1" + role: replica + lease_ttl: 30s + - action: assign + target: primary + epoch: "1" + role: primary + lease_ttl: 30s + - action: set_replica + target: primary + replica: replica + - action: iscsi_login + target: primary + node: client_node + save_as: device + + # Phase 3: Write data, verify replication + - name: write_and_replicate + actions: + - action: dd_write + node: client_node + device: "{{ device }}" + bs: 1M + count: "1" + save_as: written_md5 + - action: wait_lsn + target: replica + min_lsn: "1" + timeout: 10s + + # Phase 4: Kill primary, promote replica + - name: failover + actions: + - action: kill_target + target: primary + - action: iscsi_cleanup + node: client_node + ignore_error: true + - action: assign + target: replica + epoch: "2" + role: primary + lease_ttl: 30s + - action: wait_role + target: replica + role: primary + timeout: 5s + + # Phase 5: Verify data survived failover + - name: verify + actions: + - action: iscsi_login + target: replica + node: client_node + save_as: device2 + - action: dd_read_md5 + node: client_node + device: "{{ device2 }}" + bs: 1M + count: "1" + save_as: read_md5 + - action: assert_equal + actual: "{{ read_md5 }}" + expected: "{{ written_md5 }}" + + # Phase 6: Cleanup (always runs, even on failure) + - name: cleanup + always: true + actions: + - action: iscsi_cleanup + node: client_node + ignore_error: true + - action: stop_all_targets + aggressive: "true" + ignore_error: true diff --git a/weed/storage/blockvol/testrunner/scenarios/coord-ha-failover.yaml b/weed/storage/blockvol/testrunner/scenarios/coord-ha-failover.yaml new file mode 100644 index 000000000..a32c14b25 --- /dev/null +++ b/weed/storage/blockvol/testrunner/scenarios/coord-ha-failover.yaml @@ -0,0 +1,116 @@ +name: coord-ha-failover +timeout: 5m +env: + repo_dir: "/opt/work/seaweedfs" + +topology: + agents: + target_agent: "192.168.1.184:9100" + client_agent: "192.168.1.181:9100" + + nodes: + target_node: + host: "192.168.1.184" + agent: target_agent + client_node: + host: "192.168.1.181" + agent: client_agent + +targets: + primary: + node: target_node + vol_size: 100M + iscsi_port: 3260 + admin_port: 8080 + iqn_suffix: ha-primary + replica: + node: target_node + vol_size: 100M + iscsi_port: 3261 + admin_port: 8081 + replica_data_port: 9011 + replica_ctrl_port: 9012 + rebuild_port: 9013 + iqn_suffix: ha-replica + +phases: + - name: setup + actions: + - action: start_target + target: primary + create: "true" + - action: start_target + target: replica + create: "true" + - action: assign + target: replica + epoch: "1" + role: replica + lease_ttl: 30s + - action: assign + target: primary + epoch: "1" + role: primary + lease_ttl: 30s + - action: set_replica + target: primary + replica: replica + - action: iscsi_login + target: primary + node: client_node + save_as: device + + - name: write_and_replicate + actions: + - action: dd_write + node: client_node + device: "{{ device }}" + bs: 1M + count: "1" + save_as: written_md5 + - action: wait_lsn + target: replica + min_lsn: "1" + timeout: 10s + + - name: failover + actions: + - action: kill_target + target: primary + - action: iscsi_cleanup + node: client_node + ignore_error: true + - action: assign + target: replica + epoch: "2" + role: primary + lease_ttl: 30s + - action: wait_role + target: replica + role: primary + timeout: 5s + + - name: verify + actions: + - action: iscsi_login + target: replica + node: client_node + save_as: device2 + - action: dd_read_md5 + node: client_node + device: "{{ device2 }}" + bs: 1M + count: "1" + save_as: read_md5 + - action: assert_equal + actual: "{{ read_md5 }}" + expected: "{{ written_md5 }}" + + - name: cleanup + always: true + actions: + - action: iscsi_cleanup + node: client_node + ignore_error: true + - action: stop_all_targets + ignore_error: true diff --git a/weed/storage/blockvol/testrunner/scenarios/coord-smoke-iscsi.yaml b/weed/storage/blockvol/testrunner/scenarios/coord-smoke-iscsi.yaml new file mode 100644 index 000000000..5cba5119e --- /dev/null +++ b/weed/storage/blockvol/testrunner/scenarios/coord-smoke-iscsi.yaml @@ -0,0 +1,66 @@ +name: coord-smoke-iscsi +timeout: 5m +env: + repo_dir: "/opt/work/seaweedfs" + +topology: + agents: + target_agent: "192.168.1.184:9100" + client_agent: "192.168.1.181:9100" + + nodes: + target_node: + host: "192.168.1.184" + agent: target_agent + client_node: + host: "192.168.1.181" + agent: client_agent + +targets: + primary: + node: target_node + vol_size: 100M + iscsi_port: 3260 + admin_port: 8080 + iqn_suffix: coord-smoke-primary + +phases: + - name: setup + actions: + - action: start_target + target: primary + create: "true" + + - name: iscsi_connect + actions: + - action: iscsi_login + target: primary + node: client_node + save_as: device + + - name: write_verify + actions: + - action: dd_write + node: client_node + device: "{{ device }}" + bs: 1M + count: "1" + save_as: written_md5 + - action: dd_read_md5 + node: client_node + device: "{{ device }}" + bs: 1M + count: "1" + save_as: read_md5 + - action: assert_equal + actual: "{{ written_md5 }}" + expected: "{{ read_md5 }}" + + - name: cleanup + always: true + actions: + - action: iscsi_cleanup + node: client_node + ignore_error: true + - action: stop_all_targets + ignore_error: true diff --git a/weed/storage/blockvol/testrunner/scenarios/cp83-snapshot-expand.yaml b/weed/storage/blockvol/testrunner/scenarios/cp83-snapshot-expand.yaml new file mode 100644 index 000000000..4b9a42e2d --- /dev/null +++ b/weed/storage/blockvol/testrunner/scenarios/cp83-snapshot-expand.yaml @@ -0,0 +1,199 @@ +name: cp83-snapshot-expand +timeout: 5m +env: + repo_dir: "C:/work/seaweedfs" + +topology: + nodes: + target_node: + host: "192.168.1.184" + user: testdev + key: "C:/work/dev_server/testdev_key" + client_node: + host: "192.168.1.181" + user: testdev + key: "C:/work/dev_server/testdev_key" + +targets: + primary: + node: target_node + vol_size: 50M + iscsi_port: 3262 + admin_port: 8082 + iqn_suffix: cp83-snap + +phases: + # Phase 1: Clean slate + start target + - name: setup + actions: + - action: kill_stale + node: target_node + - action: kill_stale + node: client_node + iscsi_cleanup: "true" + - action: exec + node: target_node + cmd: "rm -f /tmp/blockvol-primary.blk.snap.*" + - action: build_deploy + - action: start_target + target: primary + create: "true" + - action: assign + target: primary + epoch: "1" + role: primary + lease_ttl: 120s + + # Phase 2: Connect iSCSI, record original size + - name: iscsi_connect + actions: + - action: iscsi_login + target: primary + node: client_node + save_as: device + - action: get_block_size + node: client_node + device: "{{ device }}" + save_as: original_size + + # Phase 3: Write initial data at two offsets + - name: write_initial_data + actions: + # 10 MB at offset 0 + - action: dd_write + node: client_node + device: "{{ device }}" + bs: 1M + count: "10" + save_as: md5_at_0 + # 5 MB at offset 20M (seek=20 with bs=1M) + - action: dd_write + node: client_node + device: "{{ device }}" + bs: 1M + count: "5" + seek: "20" + save_as: md5_at_20M + + # Phase 4: Expand volume 50M -> 100M while iSCSI session active + - name: expand + actions: + - action: resize + target: primary + new_size: 100M + - action: iscsi_rescan + node: client_node + - action: get_block_size + node: client_node + device: "{{ device }}" + save_as: expanded_size + - action: assert_greater + actual: "{{ expanded_size }}" + expected: "{{ original_size }}" + + # Phase 5: Verify original data intact after expand + - name: verify_data_after_expand + actions: + # Read 10 MB at offset 0 + - action: dd_read_md5 + node: client_node + device: "{{ device }}" + bs: 1M + count: "10" + save_as: verify_md5_at_0 + - action: assert_equal + actual: "{{ verify_md5_at_0 }}" + expected: "{{ md5_at_0 }}" + # Read 5 MB at offset 20M + - action: dd_read_md5 + node: client_node + device: "{{ device }}" + bs: 1M + count: "5" + skip: "20" + save_as: verify_md5_at_20M + - action: assert_equal + actual: "{{ verify_md5_at_20M }}" + expected: "{{ md5_at_20M }}" + + # Phase 6: Write to expanded area (beyond original 50M) + - name: write_expanded_area + actions: + # 5 MB at offset 60M (in expanded region) + - action: dd_write + node: client_node + device: "{{ device }}" + bs: 1M + count: "5" + seek: "60" + save_as: md5_at_60M + - action: dd_read_md5 + node: client_node + device: "{{ device }}" + bs: 1M + count: "5" + skip: "60" + save_as: verify_md5_at_60M + - action: assert_equal + actual: "{{ verify_md5_at_60M }}" + expected: "{{ md5_at_60M }}" + + # Phase 7: Create snapshots on expanded volume + - name: snapshot_on_expanded + actions: + - action: snapshot_create + target: primary + id: "1" + - action: snapshot_list + target: primary + save_as: snap_count_1 + - action: assert_equal + actual: "{{ snap_count_1 }}" + expected: "1" + - action: snapshot_create + target: primary + id: "2" + - action: snapshot_list + target: primary + save_as: snap_count_2 + - action: assert_equal + actual: "{{ snap_count_2 }}" + expected: "2" + + # Phase 8: Delete snapshots, then expand again (100M -> 150M) + - name: delete_snap_and_expand_again + actions: + - action: snapshot_delete + target: primary + id: "1" + - action: snapshot_delete + target: primary + id: "2" + - action: snapshot_list + target: primary + save_as: snap_count_0 + - action: assert_equal + actual: "{{ snap_count_0 }}" + expected: "0" + - action: resize + target: primary + new_size: 150M + - action: iscsi_rescan + node: client_node + - action: get_block_size + node: client_node + device: "{{ device }}" + save_as: final_size + - action: assert_greater + actual: "{{ final_size }}" + expected: "{{ expanded_size }}" + + # Phase 9: Cleanup (always runs) + - name: cleanup + always: true + actions: + - action: iscsi_cleanup + node: client_node + ignore_error: true + - action: stop_all_targets + ignore_error: true diff --git a/weed/storage/blockvol/testrunner/scenarios/cp84-soak-4h.yaml b/weed/storage/blockvol/testrunner/scenarios/cp84-soak-4h.yaml new file mode 100644 index 000000000..3190b329e --- /dev/null +++ b/weed/storage/blockvol/testrunner/scenarios/cp84-soak-4h.yaml @@ -0,0 +1,189 @@ +name: cp84-soak-4h +timeout: 5h +env: + repo_dir: "C:/work/seaweedfs" + +topology: + nodes: + target_node: + host: "192.168.1.184" + user: testdev + key: "C:/work/dev_server/testdev_key" + client_node: + host: "192.168.1.181" + user: testdev + key: "C:/work/dev_server/testdev_key" + +targets: + primary: + node: target_node + vol_size: 200M + iscsi_port: 3260 + admin_port: 8080 + iqn_suffix: soak-4h-primary + replica: + node: target_node + vol_size: 200M + iscsi_port: 3261 + admin_port: 8081 + replica_data_port: 9011 + replica_ctrl_port: 9012 + iqn_suffix: soak-4h-replica + +phases: + # Phase 1: Setup — build, deploy, start targets, wire replication. + - name: setup + actions: + - action: kill_stale + node: target_node + ignore_error: true + - action: build_deploy + - action: start_target + target: primary + create: "true" + - action: start_target + target: replica + create: "true" + - action: assign + target: replica + epoch: "1" + role: replica + lease_ttl: 30s + - action: assign + target: primary + epoch: "1" + role: primary + lease_ttl: 30s + - action: set_replica + target: primary + replica: replica + - action: iscsi_login + target: primary + node: client_node + save_as: device + + # Phase 2: Baseline metrics scrape (pre-load). + - name: baseline_scrape + actions: + - action: scrape_metrics + target: primary + save_as: metrics_baseline + + # Phase 3: Steady-state load (2 hours). + # Mixed read/write with periodic metrics scrape every 30s. + - name: steady_state + actions: + - action: dd_write + node: client_node + device: "{{ device }}" + bs: 4k + count: "256" + save_as: ss_write_md5 + - action: dd_read_md5 + node: client_node + device: "{{ device }}" + bs: 4k + count: "256" + save_as: ss_read_md5 + - action: assert_equal + actual: "{{ ss_read_md5 }}" + expected: "{{ ss_write_md5 }}" + - action: scrape_metrics + target: primary + save_as: metrics_steady + + # Phase 4: Inject 200ms replica network delay (fault window = 10 min). + - name: fault_inject + actions: + - action: inject_netem + node: target_node + target_ip: "127.0.0.1" + delay_ms: "200" + - action: sleep + duration: 5s + # Write under fault to verify primary still serves. + - action: dd_write + node: client_node + device: "{{ device }}" + bs: 4k + count: "64" + seek: "512" + save_as: fault_write_md5 + - action: dd_read_md5 + node: client_node + device: "{{ device }}" + bs: 4k + count: "64" + skip: "512" + save_as: fault_read_md5 + - action: assert_equal + actual: "{{ fault_read_md5 }}" + expected: "{{ fault_write_md5 }}" + - action: scrape_metrics + target: primary + save_as: metrics_fault + + # Phase 5: Clear fault — restore normal network. + - name: fault_clear + actions: + - action: clear_fault + type: netem + node: target_node + - action: sleep + duration: 5s + + # Phase 6: Post-fault steady-state — verify recovery. + - name: post_fault_verify + actions: + # Re-read original data to verify no corruption. + - action: dd_read_md5 + node: client_node + device: "{{ device }}" + bs: 4k + count: "256" + save_as: pf_read_md5 + - action: assert_equal + actual: "{{ pf_read_md5 }}" + expected: "{{ ss_write_md5 }}" + # Write new data post-fault. + - action: dd_write + node: client_node + device: "{{ device }}" + bs: 4k + count: "128" + seek: "1024" + save_as: pf_write_md5 + - action: dd_read_md5 + node: client_node + device: "{{ device }}" + bs: 4k + count: "128" + skip: "1024" + save_as: pf_verify_md5 + - action: assert_equal + actual: "{{ pf_verify_md5 }}" + expected: "{{ pf_write_md5 }}" + + # Phase 7: Final metrics scrape + perf summary. + - name: final_metrics + actions: + - action: scrape_metrics + target: primary + save_as: metrics_final + - action: perf_summary + target: primary + save_as: perf_stats + + # Phase 8: Cleanup (always runs). + - name: cleanup + always: true + actions: + - action: clear_fault + type: netem + node: target_node + ignore_error: true + - action: iscsi_cleanup + node: client_node + ignore_error: true + - action: stop_all_targets + ignore_error: true diff --git a/weed/storage/blockvol/testrunner/scenarios/crash-recovery.yaml b/weed/storage/blockvol/testrunner/scenarios/crash-recovery.yaml new file mode 100644 index 000000000..1902c698c --- /dev/null +++ b/weed/storage/blockvol/testrunner/scenarios/crash-recovery.yaml @@ -0,0 +1,87 @@ +name: crash-recovery +timeout: 5m +env: + repo_dir: "C:/work/seaweedfs" + +topology: + nodes: + target_node: + host: "192.168.1.184" + user: testdev + key: "C:/work/dev_server/testdev_key" + client_node: + host: "192.168.1.181" + user: testdev + key: "C:/work/dev_server/testdev_key" + +targets: + primary: + node: target_node + vol_size: 100M + iscsi_port: 3260 + admin_port: 8080 + iqn_suffix: crash-primary + +phases: + - name: setup + actions: + - action: build_deploy + - action: start_target + target: primary + create: "true" + - action: iscsi_login + target: primary + node: client_node + save_as: device + + - name: write_data + actions: + - action: dd_write + node: client_node + device: "{{ device }}" + bs: 1M + count: "1" + save_as: written_md5 + + - name: crash + actions: + # Brief pause to ensure fsync fully completes on target. + - action: sleep + duration: 1s + # Kill while session is still active (like the existing Kill9Fsync test). + - action: kill_target + target: primary + # Clean up stale iSCSI kernel state after kill. + - action: iscsi_cleanup + node: client_node + ignore_error: true + + - name: restart_and_verify + actions: + - action: start_target + target: primary + create: "false" + - action: sleep + duration: 2s + - action: iscsi_login + target: primary + node: client_node + save_as: device2 + - action: dd_read_md5 + node: client_node + device: "{{ device2 }}" + bs: 1M + count: "1" + save_as: read_md5 + - action: assert_equal + actual: "{{ read_md5 }}" + expected: "{{ written_md5 }}" + + - name: cleanup + always: true + actions: + - action: iscsi_cleanup + node: client_node + ignore_error: true + - action: stop_all_targets + ignore_error: true diff --git a/weed/storage/blockvol/testrunner/scenarios/fault-disk-full.yaml b/weed/storage/blockvol/testrunner/scenarios/fault-disk-full.yaml new file mode 100644 index 000000000..27d379250 --- /dev/null +++ b/weed/storage/blockvol/testrunner/scenarios/fault-disk-full.yaml @@ -0,0 +1,76 @@ +name: fault-disk-full +timeout: 5m +env: + repo_dir: "C:/work/seaweedfs" + +topology: + nodes: + target_node: + host: "192.168.1.184" + user: testdev + key: "C:/work/dev_server/testdev_key" + client_node: + host: "192.168.1.181" + user: testdev + key: "C:/work/dev_server/testdev_key" + +targets: + primary: + node: target_node + vol_size: 50M + iscsi_port: 3260 + admin_port: 8080 + iqn_suffix: fault-diskfull-primary + +phases: + - name: setup + actions: + - action: build_deploy + - action: start_target + target: primary + create: "true" + - action: iscsi_login + target: primary + node: client_node + save_as: device + + - name: fill_disk + actions: + - action: fill_disk + node: target_node + dir: /tmp + + - name: write_fails + actions: + - action: dd_write + node: client_node + device: "{{ device }}" + bs: 1M + count: "1" + ignore_error: true + save_as: write_result + + - name: clear_and_recover + actions: + - action: clear_fault + type: fill_disk + node: target_node + - action: dd_write + node: client_node + device: "{{ device }}" + bs: 4k + count: "1" + save_as: recovery_md5 + + - name: cleanup + always: true + actions: + - action: clear_fault + type: fill_disk + node: target_node + ignore_error: true + - action: iscsi_cleanup + node: client_node + ignore_error: true + - action: stop_all_targets + ignore_error: true diff --git a/weed/storage/blockvol/testrunner/scenarios/fault-netem.yaml b/weed/storage/blockvol/testrunner/scenarios/fault-netem.yaml new file mode 100644 index 000000000..2096759d3 --- /dev/null +++ b/weed/storage/blockvol/testrunner/scenarios/fault-netem.yaml @@ -0,0 +1,88 @@ +name: fault-netem +timeout: 5m +env: + repo_dir: "C:/work/seaweedfs" + +topology: + nodes: + target_node: + host: "192.168.1.184" + user: testdev + key: "C:/work/dev_server/testdev_key" + client_node: + host: "192.168.1.181" + user: testdev + key: "C:/work/dev_server/testdev_key" + +targets: + primary: + node: target_node + vol_size: 100M + iscsi_port: 3260 + admin_port: 8080 + iqn_suffix: fault-netem-primary + replica: + node: target_node + vol_size: 100M + iscsi_port: 3261 + admin_port: 8081 + replica_data_port: 9011 + replica_ctrl_port: 9012 + iqn_suffix: fault-netem-replica + +phases: + - name: setup + actions: + - action: build_deploy + - action: start_target + target: primary + create: "true" + - action: start_target + target: replica + create: "true" + - action: assign + target: replica + epoch: "1" + role: replica + lease_ttl: 30s + - action: assign + target: primary + epoch: "1" + role: primary + lease_ttl: 30s + - action: set_replica + target: primary + replica: replica + - action: iscsi_login + target: primary + node: client_node + save_as: device + + - name: inject_delay + actions: + - action: inject_netem + node: target_node + target_ip: "127.0.0.1" + delay_ms: "200" + + - name: write_under_delay + actions: + - action: dd_write + node: client_node + device: "{{ device }}" + bs: 4k + count: "10" + save_as: written_md5 + + - name: cleanup + always: true + actions: + - action: clear_fault + type: netem + node: target_node + ignore_error: true + - action: iscsi_cleanup + node: client_node + ignore_error: true + - action: stop_all_targets + ignore_error: true diff --git a/weed/storage/blockvol/testrunner/scenarios/fault-partition.yaml b/weed/storage/blockvol/testrunner/scenarios/fault-partition.yaml new file mode 100644 index 000000000..7920f8427 --- /dev/null +++ b/weed/storage/blockvol/testrunner/scenarios/fault-partition.yaml @@ -0,0 +1,96 @@ +name: fault-partition +timeout: 5m +env: + repo_dir: "C:/work/seaweedfs" + +topology: + nodes: + target_node: + host: "192.168.1.184" + user: testdev + key: "C:/work/dev_server/testdev_key" + client_node: + host: "192.168.1.181" + user: testdev + key: "C:/work/dev_server/testdev_key" + +targets: + primary: + node: target_node + vol_size: 100M + iscsi_port: 3260 + admin_port: 8080 + iqn_suffix: fault-part-primary + replica: + node: target_node + vol_size: 100M + iscsi_port: 3261 + admin_port: 8081 + replica_data_port: 9011 + replica_ctrl_port: 9012 + rebuild_port: 9013 + iqn_suffix: fault-part-replica + +phases: + - name: setup + actions: + - action: build_deploy + - action: start_target + target: primary + create: "true" + - action: start_target + target: replica + create: "true" + - action: assign + target: replica + epoch: "1" + role: replica + lease_ttl: 10s + - action: assign + target: primary + epoch: "1" + role: primary + lease_ttl: 10s + - action: set_replica + target: primary + replica: replica + + - name: inject_partition + actions: + - action: inject_partition + node: target_node + target_ip: "127.0.0.1" + ports: "9011,9012" + + - name: wait_for_lease_expiry + actions: + - action: sleep + duration: 15s + - action: assert_status + target: primary + has_lease: "false" + + - name: promote_replica + actions: + - action: assign + target: replica + epoch: "2" + role: primary + lease_ttl: 30s + - action: wait_role + target: replica + role: primary + timeout: 5s + + - name: cleanup + always: true + actions: + - action: clear_fault + type: partition + node: target_node + ignore_error: true + - action: iscsi_cleanup + node: client_node + ignore_error: true + - action: stop_all_targets + ignore_error: true diff --git a/weed/storage/blockvol/testrunner/scenarios/ha-failover.yaml b/weed/storage/blockvol/testrunner/scenarios/ha-failover.yaml new file mode 100644 index 000000000..9440b7f84 --- /dev/null +++ b/weed/storage/blockvol/testrunner/scenarios/ha-failover.yaml @@ -0,0 +1,115 @@ +name: ha-failover +timeout: 5m +env: + repo_dir: "C:/work/seaweedfs" + +topology: + nodes: + target_node: + host: "192.168.1.184" + user: testdev + key: "C:/work/dev_server/testdev_key" + client_node: + host: "192.168.1.181" + user: testdev + key: "C:/work/dev_server/testdev_key" + +targets: + primary: + node: target_node + vol_size: 100M + iscsi_port: 3260 + admin_port: 8080 + iqn_suffix: ha-primary + replica: + node: target_node + vol_size: 100M + iscsi_port: 3261 + admin_port: 8081 + replica_data_port: 9011 + replica_ctrl_port: 9012 + rebuild_port: 9013 + iqn_suffix: ha-replica + +phases: + - name: setup + actions: + - action: build_deploy + - action: start_target + target: primary + create: "true" + - action: start_target + target: replica + create: "true" + - action: assign + target: replica + epoch: "1" + role: replica + lease_ttl: 30s + - action: assign + target: primary + epoch: "1" + role: primary + lease_ttl: 30s + - action: set_replica + target: primary + replica: replica + - action: iscsi_login + target: primary + node: client_node + save_as: device + + - name: write_and_replicate + actions: + - action: dd_write + node: client_node + device: "{{ device }}" + bs: 1M + count: "1" + save_as: written_md5 + - action: wait_lsn + target: replica + min_lsn: "1" + timeout: 10s + + - name: failover + actions: + - action: kill_target + target: primary + - action: iscsi_cleanup + node: client_node + ignore_error: true + - action: assign + target: replica + epoch: "2" + role: primary + lease_ttl: 30s + - action: wait_role + target: replica + role: primary + timeout: 5s + + - name: verify + actions: + - action: iscsi_login + target: replica + node: client_node + save_as: device2 + - action: dd_read_md5 + node: client_node + device: "{{ device2 }}" + bs: 1M + count: "1" + save_as: read_md5 + - action: assert_equal + actual: "{{ read_md5 }}" + expected: "{{ written_md5 }}" + + - name: cleanup + always: true + actions: + - action: iscsi_cleanup + node: client_node + ignore_error: true + - action: stop_all_targets + ignore_error: true diff --git a/weed/storage/blockvol/testrunner/scenarios/ha-full-lifecycle.yaml b/weed/storage/blockvol/testrunner/scenarios/ha-full-lifecycle.yaml new file mode 100644 index 000000000..3ae52baae --- /dev/null +++ b/weed/storage/blockvol/testrunner/scenarios/ha-full-lifecycle.yaml @@ -0,0 +1,166 @@ +name: ha-full-lifecycle +timeout: 10m +env: + repo_dir: "C:/work/seaweedfs" + +topology: + nodes: + target_node: + host: "192.168.1.184" + user: testdev + key: "C:/work/dev_server/testdev_key" + client_node: + host: "192.168.1.181" + user: testdev + key: "C:/work/dev_server/testdev_key" + +targets: + primary: + node: target_node + vol_size: 100M + iscsi_port: 3260 + admin_port: 8080 + rebuild_port: 9020 + iqn_suffix: lifecycle-primary + replica: + node: target_node + vol_size: 100M + iscsi_port: 3261 + admin_port: 8081 + replica_data_port: 9011 + replica_ctrl_port: 9012 + rebuild_port: 9013 + iqn_suffix: lifecycle-replica + +phases: + - name: setup + actions: + - action: build_deploy + - action: start_target + target: primary + create: "true" + - action: start_target + target: replica + create: "true" + - action: assign + target: replica + epoch: "1" + role: replica + lease_ttl: 60s + - action: assign + target: primary + epoch: "1" + role: primary + lease_ttl: 60s + - action: set_replica + target: primary + replica: replica + + - name: initial_write + actions: + - action: iscsi_login + target: primary + node: client_node + save_as: device + - action: dd_write + node: client_node + device: "{{ device }}" + bs: 1M + count: "2" + save_as: md5_epoch1 + - action: wait_lsn + target: replica + min_lsn: "1" + timeout: 10s + + - name: failover_1 + actions: + - action: kill_target + target: primary + - action: iscsi_cleanup + node: client_node + ignore_error: true + - action: assign + target: replica + epoch: "2" + role: primary + lease_ttl: 60s + - action: wait_role + target: replica + role: primary + timeout: 5s + + - name: write_after_failover_1 + actions: + - action: iscsi_login + target: replica + node: client_node + save_as: device2 + - action: dd_write + node: client_node + device: "{{ device2 }}" + bs: 1M + count: "1" + save_as: md5_epoch2 + + - name: rebuild_primary + actions: + - action: iscsi_logout + target: replica + node: client_node + - action: start_target + target: primary + create: "true" + - action: assign + target: primary + epoch: "2" + role: rebuilding + lease_ttl: 60s + - action: start_rebuild_client + target: primary + primary: replica + epoch: "2" + - action: wait_role + target: primary + role: replica + timeout: 30s + + - name: failover_2 + actions: + - action: kill_target + target: replica + - action: assign + target: primary + epoch: "3" + role: primary + lease_ttl: 60s + - action: wait_role + target: primary + role: primary + timeout: 5s + + - name: verify_data + actions: + - action: iscsi_login + target: primary + node: client_node + save_as: device3 + # Verify the epoch2 write (1MB at offset 0) survived double failover + rebuild. + - action: dd_read_md5 + node: client_node + device: "{{ device3 }}" + bs: 1M + count: "1" + save_as: read_epoch2 + - action: assert_equal + actual: "{{ read_epoch2 }}" + expected: "{{ md5_epoch2 }}" + + - name: cleanup + always: true + actions: + - action: iscsi_cleanup + node: client_node + ignore_error: true + - action: stop_all_targets + ignore_error: true diff --git a/weed/storage/blockvol/testrunner/scenarios/ha-io-continuity.yaml b/weed/storage/blockvol/testrunner/scenarios/ha-io-continuity.yaml new file mode 100644 index 000000000..1d734e454 --- /dev/null +++ b/weed/storage/blockvol/testrunner/scenarios/ha-io-continuity.yaml @@ -0,0 +1,115 @@ +name: ha-io-continuity +timeout: 5m +env: + repo_dir: "C:/work/seaweedfs" + +topology: + nodes: + target_node: + host: "192.168.1.184" + user: testdev + key: "C:/work/dev_server/testdev_key" + client_node: + host: "192.168.1.181" + user: testdev + key: "C:/work/dev_server/testdev_key" + +targets: + primary: + node: target_node + vol_size: 100M + iscsi_port: 3260 + admin_port: 8080 + iqn_suffix: iocont-primary + replica: + node: target_node + vol_size: 100M + iscsi_port: 3261 + admin_port: 8081 + replica_data_port: 9011 + replica_ctrl_port: 9012 + rebuild_port: 9013 + iqn_suffix: iocont-replica + +phases: + - name: setup + actions: + - action: build_deploy + - action: start_target + target: primary + create: "true" + - action: start_target + target: replica + create: "true" + - action: assign + target: replica + epoch: "1" + role: replica + lease_ttl: 30s + - action: assign + target: primary + epoch: "1" + role: primary + lease_ttl: 30s + - action: set_replica + target: primary + replica: replica + - action: iscsi_login + target: primary + node: client_node + save_as: device + + - name: write_A + actions: + - action: dd_write + node: client_node + device: "{{ device }}" + bs: 512k + count: "1" + save_as: md5_A + - action: wait_lsn + target: replica + min_lsn: "1" + timeout: 10s + + - name: failover + actions: + - action: kill_target + target: primary + - action: iscsi_cleanup + node: client_node + ignore_error: true + - action: assign + target: replica + epoch: "2" + role: primary + lease_ttl: 30s + - action: wait_role + target: replica + role: primary + timeout: 5s + + - name: write_B_and_verify + actions: + - action: iscsi_login + target: replica + node: client_node + save_as: device2 + - action: dd_read_md5 + node: client_node + device: "{{ device2 }}" + bs: 512k + count: "1" + save_as: read_A + - action: assert_equal + actual: "{{ read_A }}" + expected: "{{ md5_A }}" + + - name: cleanup + always: true + actions: + - action: iscsi_cleanup + node: client_node + ignore_error: true + - action: stop_all_targets + ignore_error: true diff --git a/weed/storage/blockvol/testrunner/scenarios/ha-rebuild.yaml b/weed/storage/blockvol/testrunner/scenarios/ha-rebuild.yaml new file mode 100644 index 000000000..c6449aae9 --- /dev/null +++ b/weed/storage/blockvol/testrunner/scenarios/ha-rebuild.yaml @@ -0,0 +1,138 @@ +name: ha-rebuild +timeout: 5m +env: + repo_dir: "C:/work/seaweedfs" + +topology: + nodes: + target_node: + host: "192.168.1.184" + user: testdev + key: "C:/work/dev_server/testdev_key" + client_node: + host: "192.168.1.181" + user: testdev + key: "C:/work/dev_server/testdev_key" + +targets: + primary: + node: target_node + vol_size: 100M + iscsi_port: 3260 + admin_port: 8080 + rebuild_port: 9020 + iqn_suffix: rebuild-primary + replica: + node: target_node + vol_size: 100M + iscsi_port: 3261 + admin_port: 8081 + replica_data_port: 9011 + replica_ctrl_port: 9012 + rebuild_port: 9013 + iqn_suffix: rebuild-replica + +phases: + - name: setup + actions: + - action: build_deploy + - action: start_target + target: primary + create: "true" + - action: start_target + target: replica + create: "true" + - action: assign + target: replica + epoch: "1" + role: replica + lease_ttl: 30s + - action: assign + target: primary + epoch: "1" + role: primary + lease_ttl: 30s + - action: set_replica + target: primary + replica: replica + - action: iscsi_login + target: primary + node: client_node + save_as: device + + - name: write_data + actions: + - action: dd_write + node: client_node + device: "{{ device }}" + bs: 1M + count: "1" + save_as: written_md5 + - action: wait_lsn + target: replica + min_lsn: "1" + timeout: 10s + + - name: kill_replica + actions: + - action: kill_target + target: replica + + - name: rebuild + actions: + - action: start_target + target: replica + create: "true" + - action: assign + target: replica + epoch: "1" + role: rebuilding + lease_ttl: 30s + # Rebuild server auto-starts via rebuild_port in target spec. + - action: start_rebuild_client + target: replica + primary: primary + epoch: "1" + - action: wait_role + target: replica + role: replica + timeout: 30s + + - name: verify_rebuild + actions: + - action: kill_target + target: primary + - action: iscsi_cleanup + node: client_node + ignore_error: true + - action: assign + target: replica + epoch: "2" + role: primary + lease_ttl: 30s + - action: wait_role + target: replica + role: primary + timeout: 5s + - action: iscsi_login + target: replica + node: client_node + save_as: device2 + - action: dd_read_md5 + node: client_node + device: "{{ device2 }}" + bs: 1M + count: "1" + save_as: read_md5 + - action: assert_equal + actual: "{{ read_md5 }}" + expected: "{{ written_md5 }}" + + - name: cleanup + always: true + actions: + - action: iscsi_cleanup + node: client_node + ignore_error: true + - action: stop_all_targets + ignore_error: true diff --git a/weed/storage/blockvol/testrunner/scenarios/smoke-iscsi.yaml b/weed/storage/blockvol/testrunner/scenarios/smoke-iscsi.yaml new file mode 100644 index 000000000..afb47f9a6 --- /dev/null +++ b/weed/storage/blockvol/testrunner/scenarios/smoke-iscsi.yaml @@ -0,0 +1,65 @@ +name: smoke-iscsi +timeout: 5m +env: + repo_dir: "C:/work/seaweedfs" + +topology: + nodes: + target_node: + host: "192.168.1.184" + user: testdev + key: "C:/work/dev_server/testdev_key" + client_node: + host: "192.168.1.181" + user: testdev + key: "C:/work/dev_server/testdev_key" + +targets: + primary: + node: target_node + vol_size: 100M + iscsi_port: 3260 + admin_port: 8080 + iqn_suffix: smoke-primary + +phases: + - name: setup + actions: + - action: build_deploy + - action: start_target + target: primary + create: "true" + + - name: iscsi_connect + actions: + - action: iscsi_login + target: primary + node: client_node + save_as: device + + - name: write_verify + actions: + - action: dd_write + node: client_node + device: "{{ device }}" + bs: 1M + count: "1" + save_as: written_md5 + - action: dd_read_md5 + node: client_node + device: "{{ device }}" + bs: 1M + count: "1" + save_as: read_md5 + - action: assert_equal + actual: "{{ written_md5 }}" + expected: "{{ read_md5 }}" + + - name: cleanup + always: true + actions: + - action: iscsi_cleanup + node: client_node + ignore_error: true + - action: stop_all_targets + ignore_error: true diff --git a/weed/storage/blockvol/testrunner/scripts/unblock-windows.ps1 b/weed/storage/blockvol/testrunner/scripts/unblock-windows.ps1 new file mode 100644 index 000000000..071155923 --- /dev/null +++ b/weed/storage/blockvol/testrunner/scripts/unblock-windows.ps1 @@ -0,0 +1,23 @@ +# unblock-windows.ps1 — Unblock downloaded binaries and add Defender exclusion. +# Run as Administrator: powershell -ExecutionPolicy Bypass -File .\unblock-windows.ps1 +# +# This is needed because Windows SmartScreen blocks unsigned binaries downloaded +# from the internet. Code signing is future work (requires certificate). + +$ErrorActionPreference = "Stop" +$dir = (Get-Location).Path + +Write-Host "Unblocking binaries in $dir ..." + +$binaries = @("sw-test-runner.exe", "iscsi-target.exe", "weed.exe") +foreach ($bin in $binaries) { + $path = Join-Path $dir $bin + if (Test-Path $path) { + Unblock-File $path + Write-Host " Unblocked: $bin" + } +} + +Write-Host "Adding Defender exclusion for $dir ..." +Add-MpPreference -ExclusionPath $dir +Write-Host "Done. Binaries should no longer trigger SmartScreen." diff --git a/weed/storage/blockvol/testrunner/types.go b/weed/storage/blockvol/testrunner/types.go new file mode 100644 index 000000000..0647e557b --- /dev/null +++ b/weed/storage/blockvol/testrunner/types.go @@ -0,0 +1,146 @@ +package testrunner + +import "time" + +// Scenario is the top-level YAML structure for a test scenario. +type Scenario struct { + Name string `yaml:"name"` + Timeout Duration `yaml:"timeout"` + Env map[string]string `yaml:"env"` + Topology Topology `yaml:"topology"` + Targets map[string]TargetSpec `yaml:"targets"` + Phases []Phase `yaml:"phases"` + Artifacts ArtifactSpec `yaml:"artifacts"` +} + +// Duration wraps time.Duration for YAML unmarshaling (e.g. "5m", "30s"). +type Duration struct { + time.Duration +} + +func (d *Duration) UnmarshalYAML(unmarshal func(interface{}) error) error { + var s string + if err := unmarshal(&s); err != nil { + return err + } + dur, err := time.ParseDuration(s) + if err != nil { + return err + } + d.Duration = dur + return nil +} + +func (d Duration) MarshalYAML() (interface{}, error) { + return d.Duration.String(), nil +} + +// Topology defines the set of nodes available. +type Topology struct { + Agents map[string]string `yaml:"agents"` // agent_name → "host:port" (coordinator mode) + Nodes map[string]NodeSpec `yaml:"nodes"` +} + +// NodeSpec defines a remote (or local) machine. +type NodeSpec struct { + Host string `yaml:"host"` + User string `yaml:"user"` + KeyFile string `yaml:"key"` + IsLocal bool `yaml:"is_local"` + Agent string `yaml:"agent"` // maps node to an agent (coordinator mode) +} + +// TargetSpec defines an iSCSI target instance. +type TargetSpec struct { + Node string `yaml:"node"` + VolSize string `yaml:"vol_size"` + WALSize string `yaml:"wal_size"` + ISCSIPort int `yaml:"iscsi_port"` + AdminPort int `yaml:"admin_port"` + ReplicaDataPort int `yaml:"replica_data_port"` + ReplicaCtrlPort int `yaml:"replica_ctrl_port"` + RebuildPort int `yaml:"rebuild_port"` + IQNSuffix string `yaml:"iqn_suffix"` + TPGID int `yaml:"tpg_id"` +} + +// IQN returns the full IQN from the suffix. +func (ts TargetSpec) IQN() string { + return "iqn.2024.com.seaweedfs:" + ts.IQNSuffix +} + +// Phase is a sequential group of actions. +type Phase struct { + Name string `yaml:"name"` + Always bool `yaml:"always"` + Parallel bool `yaml:"parallel"` + Actions []Action `yaml:"actions"` +} + +// Action is a single step within a phase. +type Action struct { + Action string `yaml:"action" json:"action"` + Target string `yaml:"target" json:"target,omitempty"` + Replica string `yaml:"replica" json:"replica,omitempty"` + Node string `yaml:"node" json:"node,omitempty"` + SaveAs string `yaml:"save_as" json:"save_as,omitempty"` + IgnoreError bool `yaml:"ignore_error" json:"ignore_error,omitempty"` + Retry int `yaml:"retry" json:"retry,omitempty"` + Timeout string `yaml:"timeout" json:"timeout,omitempty"` + Params map[string]string `yaml:"params,inline" json:"params,omitempty"` +} + +// ArtifactSpec configures what to collect on failure. +type ArtifactSpec struct { + OnFailure []string `yaml:"on_failure"` + Dir string `yaml:"dir"` +} + +// --- Result types --- + +// ScenarioResult is the final output of a scenario run. +type ScenarioResult struct { + Name string `json:"name"` + Status ResultStatus `json:"status"` + Duration time.Duration `json:"duration_ms"` + Phases []PhaseResult `json:"phases"` + Error string `json:"error,omitempty"` + Vars map[string]string `json:"vars,omitempty"` + Artifacts []ArtifactEntry `json:"artifacts,omitempty"` +} + +// ArtifactEntry records a collected artifact file. +type ArtifactEntry struct { + Agent string `json:"agent"` + Path string `json:"path"` + Size int64 `json:"size"` +} + +// PhaseResult captures the outcome of one phase. +type PhaseResult struct { + Name string `json:"name"` + Status ResultStatus `json:"status"` + Duration time.Duration `json:"duration_ms"` + Actions []ActionResult `json:"actions"` + Error string `json:"error,omitempty"` +} + +// ActionResult captures the outcome of one action. +type ActionResult struct { + Action string `json:"action"` + Status ResultStatus `json:"status"` + Duration time.Duration `json:"duration_ms"` + Output string `json:"output,omitempty"` + Error string `json:"error,omitempty"` + YAML string `json:"yaml,omitempty"` +} + +// ResultStatus is the status of a result. +type ResultStatus string + +const ( + StatusPass ResultStatus = "PASS" + StatusFail ResultStatus = "FAIL" + StatusSkip ResultStatus = "SKIP" + StatusRunning ResultStatus = "RUNNING" +)