feat: CP8-3-1 durability modes + testrunner platform + 21 adversarial tests

Durability mode implementation (sync_all, sync_quorum, best_effort):
- DurabilityMode type with superblock persistence, parse/validate/string
- MakeDistributedSync mode-aware barrier enforcement in dist_group_commit
- blockerr sentinel package (ErrDurabilityBarrierFailed, ErrDurabilityQuorumLost)
- gRPC create path: mode validation, idempotent create consistency, partial cleanup
- F1: strict mode rejects partial replica provisioning with cleanup
- F3: empty heartbeat does not overwrite persisted strict mode
- F4: SCSI error mapping uses errors.Is sentinels (not string matching)
- Proto/wire/blockapi/CLI/UI plumbing for durability_mode field
- Observability dashboard: cluster health cards + per-volume columns

Testrunner platform (YAML-driven integration test framework):
- Engine, parser, registry, reporter (JUnit XML + HTML), metrics scraping
- 52 registered actions: block, iSCSI, I/O, fault injection, assertions
- Baseline regression framework with 7 hard-fail conditions
- 15 YAML scenarios (smoke, crash, HA, fault, consistency, snapshot)
- 49 unit tests for testrunner internals

QA adversarial suite (21 tests, all PASS):
- Idempotent create mode/RF mismatch detection
- Heartbeat mode downgrade prevention (F3)
- sync_all/sync_quorum partial replica enforcement (F1)
- Concurrent create race safety
- Failover/expand mode preservation
- Cleanup resilience when delete fails
- Master restart auto-register mode handling
- Superblock roundtrip all 3 modes
- Validate edge cases (mode×RF matrix)
- RequiredReplicas quorum math verification
- Sentinel error categorization

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>

weed/pb/master.proto

@@ -505,6 +505,7 @@ message BlockVolumeInfoMessage {
   int64  scrub_errors = 13;
   int64  last_scrub_time = 14;
   bool   replica_degraded = 15;
+  string durability_mode = 16;
 }
 
 message BlockVolumeShortInfoMessage {
@@ -535,6 +536,7 @@ message CreateBlockVolumeRequest {
   uint64 size_bytes = 2;
   string disk_type = 3;
   uint32 replica_factor = 4;
+  string durability_mode = 5;
 }
 message CreateBlockVolumeResponse {
   string volume_id = 1;
@@ -563,6 +565,7 @@ message LookupBlockVolumeResponse {
   string replica_server = 5;
   uint32 replica_factor = 6;
   repeated string replica_servers = 7;
+  string durability_mode = 8;
 }
 
 message CreateBlockSnapshotRequest {

weed/pb/master_pb/master.pb.go

@@ -3899,6 +3899,7 @@ type BlockVolumeInfoMessage struct {
 	ScrubErrors     int64                  `protobuf:"varint,13,opt,name=scrub_errors,json=scrubErrors,proto3" json:"scrub_errors,omitempty"`
 	LastScrubTime   int64                  `protobuf:"varint,14,opt,name=last_scrub_time,json=lastScrubTime,proto3" json:"last_scrub_time,omitempty"`
 	ReplicaDegraded bool                   `protobuf:"varint,15,opt,name=replica_degraded,json=replicaDegraded,proto3" json:"replica_degraded,omitempty"`
+	DurabilityMode  string                 `protobuf:"bytes,16,opt,name=durability_mode,json=durabilityMode,proto3" json:"durability_mode,omitempty"`
 	unknownFields   protoimpl.UnknownFields
 	sizeCache       protoimpl.SizeCache
 }
@@ -4038,6 +4039,13 @@ func (x *BlockVolumeInfoMessage) GetReplicaDegraded() bool {
 	return false
 }
 
+func (x *BlockVolumeInfoMessage) GetDurabilityMode() string {
+	if x != nil {
+		return x.DurabilityMode
+	}
+	return ""
+}
+
 type BlockVolumeShortInfoMessage struct {
 	state         protoimpl.MessageState `protogen:"open.v1"`
 	Path          string                 `protobuf:"bytes,1,opt,name=path,proto3" json:"path,omitempty"`
@@ -4238,9 +4246,10 @@ type CreateBlockVolumeRequest struct {
 	Name          string                 `protobuf:"bytes,1,opt,name=name,proto3" json:"name,omitempty"`
 	SizeBytes     uint64                 `protobuf:"varint,2,opt,name=size_bytes,json=sizeBytes,proto3" json:"size_bytes,omitempty"`
 	DiskType      string                 `protobuf:"bytes,3,opt,name=disk_type,json=diskType,proto3" json:"disk_type,omitempty"`
-	ReplicaFactor uint32                 `protobuf:"varint,4,opt,name=replica_factor,json=replicaFactor,proto3" json:"replica_factor,omitempty"`
-	unknownFields protoimpl.UnknownFields
-	sizeCache     protoimpl.SizeCache
+	ReplicaFactor  uint32                 `protobuf:"varint,4,opt,name=replica_factor,json=replicaFactor,proto3" json:"replica_factor,omitempty"`
+	DurabilityMode string                 `protobuf:"bytes,5,opt,name=durability_mode,json=durabilityMode,proto3" json:"durability_mode,omitempty"`
+	unknownFields  protoimpl.UnknownFields
+	sizeCache      protoimpl.SizeCache
 }
 
 func (x *CreateBlockVolumeRequest) Reset() {
@@ -4301,6 +4310,13 @@ func (x *CreateBlockVolumeRequest) GetReplicaFactor() uint32 {
 	return 0
 }
 
+func (x *CreateBlockVolumeRequest) GetDurabilityMode() string {
+	if x != nil {
+		return x.DurabilityMode
+	}
+	return ""
+}
+
 type CreateBlockVolumeResponse struct {
 	state         protoimpl.MessageState `protogen:"open.v1"`
 	VolumeId      string                 `protobuf:"bytes,1,opt,name=volume_id,json=volumeId,proto3" json:"volume_id,omitempty"`
@@ -4526,6 +4542,7 @@ type LookupBlockVolumeResponse struct {
 	ReplicaServer  string                 `protobuf:"bytes,5,opt,name=replica_server,json=replicaServer,proto3" json:"replica_server,omitempty"`
 	ReplicaFactor  uint32                 `protobuf:"varint,6,opt,name=replica_factor,json=replicaFactor,proto3" json:"replica_factor,omitempty"`
 	ReplicaServers []string               `protobuf:"bytes,7,rep,name=replica_servers,json=replicaServers,proto3" json:"replica_servers,omitempty"`
+	DurabilityMode string                 `protobuf:"bytes,8,opt,name=durability_mode,json=durabilityMode,proto3" json:"durability_mode,omitempty"`
 	unknownFields  protoimpl.UnknownFields
 	sizeCache      protoimpl.SizeCache
 }
@@ -4609,6 +4626,13 @@ func (x *LookupBlockVolumeResponse) GetReplicaServers() []string {
 	return nil
 }
 
+func (x *LookupBlockVolumeResponse) GetDurabilityMode() string {
+	if x != nil {
+		return x.DurabilityMode
+	}
+	return ""
+}
+
 type CreateBlockSnapshotRequest struct {
 	state         protoimpl.MessageState `protogen:"open.v1"`
 	VolumeName    string                 `protobuf:"bytes,1,opt,name=volume_name,json=volumeName,proto3" json:"volume_name,omitempty"`

weed/pb/volume_server.proto

@@ -779,6 +779,7 @@ message AllocateBlockVolumeRequest {
     string name = 1;
     uint64 size_bytes = 2;
     string disk_type = 3;
+    string durability_mode = 4;
 }
 message AllocateBlockVolumeResponse {
     string path = 1;

weed/pb/volume_server_pb/volume_server.pb.go

@@ -6189,9 +6189,10 @@ type AllocateBlockVolumeRequest struct {
 	state         protoimpl.MessageState `protogen:"open.v1"`
 	Name          string                 `protobuf:"bytes,1,opt,name=name,proto3" json:"name,omitempty"`
 	SizeBytes     uint64                 `protobuf:"varint,2,opt,name=size_bytes,json=sizeBytes,proto3" json:"size_bytes,omitempty"`
-	DiskType      string                 `protobuf:"bytes,3,opt,name=disk_type,json=diskType,proto3" json:"disk_type,omitempty"`
-	unknownFields protoimpl.UnknownFields
-	sizeCache     protoimpl.SizeCache
+	DiskType       string                 `protobuf:"bytes,3,opt,name=disk_type,json=diskType,proto3" json:"disk_type,omitempty"`
+	DurabilityMode string                 `protobuf:"bytes,4,opt,name=durability_mode,json=durabilityMode,proto3" json:"durability_mode,omitempty"`
+	unknownFields  protoimpl.UnknownFields
+	sizeCache      protoimpl.SizeCache
 }
 
 func (x *AllocateBlockVolumeRequest) Reset() {
@@ -6245,6 +6246,13 @@ func (x *AllocateBlockVolumeRequest) GetDiskType() string {
 	return ""
 }
 
+func (x *AllocateBlockVolumeRequest) GetDurabilityMode() string {
+	if x != nil {
+		return x.DurabilityMode
+	}
+	return ""
+}
+
 type AllocateBlockVolumeResponse struct {
 	state             protoimpl.MessageState `protogen:"open.v1"`
 	Path              string                 `protobuf:"bytes,1,opt,name=path,proto3" json:"path,omitempty"`

weed/server/integration_block_test.go

@@ -28,7 +28,7 @@ func integrationMaster(t *testing.T) *MasterServer {
 		blockAssignmentQueue: NewBlockAssignmentQueue(),
 		blockFailover:        newBlockFailoverState(),
 	}
-	ms.blockVSAllocate = func(ctx context.Context, server string, name string, sizeBytes uint64, diskType string) (*blockAllocResult, error) {
+	ms.blockVSAllocate = func(ctx context.Context, server string, name string, sizeBytes uint64, diskType string, durabilityMode string) (*blockAllocResult, error) {
 		return &blockAllocResult{
 			Path:              fmt.Sprintf("/data/%s.blk", name),
 			IQN:               fmt.Sprintf("iqn.2024.test:%s", name),
@@ -382,13 +382,13 @@ func TestIntegration_ReplicaFailureSingleCopy(t *testing.T) {
 	// Make replica allocation always fail.
 	callCount := 0
 	origAllocate := ms.blockVSAllocate
-	ms.blockVSAllocate = func(ctx context.Context, server string, name string, sizeBytes uint64, diskType string) (*blockAllocResult, error) {
+	ms.blockVSAllocate = func(ctx context.Context, server string, name string, sizeBytes uint64, diskType string, durabilityMode string) (*blockAllocResult, error) {
 		callCount++
 		if callCount > 1 {
 			// Second call (replica) fails.
 			return nil, fmt.Errorf("disk full on replica")
 		}
-		return origAllocate(ctx, server, name, sizeBytes, diskType)
+		return origAllocate(ctx, server, name, sizeBytes, diskType, durabilityMode)
 	}
 
 	resp, err := ms.CreateBlockVolume(ctx, &master_pb.CreateBlockVolumeRequest{

weed/server/master_block_failover_test.go

@@ -18,7 +18,7 @@ func testMasterServerForFailover(t *testing.T) *MasterServer {
 		blockAssignmentQueue: NewBlockAssignmentQueue(),
 		blockFailover:        newBlockFailoverState(),
 	}
-	ms.blockVSAllocate = func(ctx context.Context, server string, name string, sizeBytes uint64, diskType string) (*blockAllocResult, error) {
+	ms.blockVSAllocate = func(ctx context.Context, server string, name string, sizeBytes uint64, diskType string, durabilityMode string) (*blockAllocResult, error) {
 		return &blockAllocResult{
 			Path:      fmt.Sprintf("/data/%s.blk", name),
 			IQN:       fmt.Sprintf("iqn.2024.test:%s", name),

weed/server/master_block_registry.go

@@ -72,6 +72,9 @@ type BlockVolumeEntry struct {
 	ReplicaDegraded bool          // primary reports degraded replicas
 	WALHeadLSN      uint64        // primary WAL head LSN from heartbeat
 
+	// CP8-3-1: Durability mode.
+	DurabilityMode  string        // "best_effort", "sync_all", "sync_quorum"
+
 	// Lease tracking for failover (CP6-3 F2).
 	LastLeaseGrant time.Time
 	LeaseTTL       time.Duration
@@ -320,6 +323,10 @@ func (r *BlockVolumeRegistry) UpdateFullHeartbeat(server string, infos []*master
 				existing.HealthScore = info.HealthScore
 				existing.ReplicaDegraded = info.ReplicaDegraded
 				existing.WALHeadLSN = info.WalHeadLsn
+				// F3: only update DurabilityMode when non-empty (prevents older VS from clearing strict mode).
+				if info.DurabilityMode != "" {
+					existing.DurabilityMode = info.DurabilityMode
+				}
 				// F5: update replica addresses from heartbeat info.
 				if info.ReplicaDataAddr != "" {
 					existing.ReplicaDataAddr = info.ReplicaDataAddr
@@ -391,6 +398,7 @@ func (r *BlockVolumeRegistry) UpdateFullHeartbeat(server string, infos []*master
 					LeaseTTL:       30 * time.Second,
 					HealthScore:    info.HealthScore,
 					WALHeadLSN:     info.WalHeadLsn,
+					DurabilityMode: info.DurabilityMode,
 				}
 				if info.ReplicaDataAddr != "" {
 					entry.ReplicaDataAddr = info.ReplicaDataAddr

weed/server/master_grpc_server_block.go

@@ -33,10 +33,29 @@ func (ms *MasterServer) CreateBlockVolume(ctx context.Context, req *master_pb.Cr
 		return nil, fmt.Errorf("size_bytes must be > 0")
 	}
 
-	// Idempotent: if already registered, return existing entry (validate size).
+	// F2: validate durability mode in gRPC path (authoritative, not bypassable).
+	var durMode blockvol.DurabilityMode
+	if req.DurabilityMode != "" {
+		var err error
+		durMode, err = blockvol.ParseDurabilityMode(req.DurabilityMode)
+		if err != nil {
+			return nil, fmt.Errorf("invalid durability_mode: %w", err)
+		}
+	}
+
+	// Cross-validate mode + RF (sync_quorum requires RF >= 3).
+	replicaFactor := 2
+	if req.ReplicaFactor > 0 && req.ReplicaFactor <= 3 {
+		replicaFactor = int(req.ReplicaFactor)
+	}
+	if err := durMode.Validate(replicaFactor); err != nil {
+		return nil, fmt.Errorf("durability_mode %q incompatible with replica_factor %d: %w", req.DurabilityMode, replicaFactor, err)
+	}
+
+	// Idempotent: if already registered, return existing entry (validate size + mode + RF).
 	if entry, ok := ms.blockRegistry.Lookup(req.Name); ok {
-		if entry.SizeBytes < req.SizeBytes {
-			return nil, fmt.Errorf("block volume %q exists with size %d (requested %d)", req.Name, entry.SizeBytes, req.SizeBytes)
+		if err := ms.validateIdempotentCreate(entry, req, durMode, replicaFactor); err != nil {
+			return nil, err
 		}
 		return ms.createBlockVolumeResponseFromEntry(entry), nil
 	}
@@ -49,6 +68,9 @@ func (ms *MasterServer) CreateBlockVolume(ctx context.Context, req *master_pb.Cr
 
 	// Double-check after acquiring lock (another goroutine may have finished).
 	if entry, ok := ms.blockRegistry.Lookup(req.Name); ok {
+		if err := ms.validateIdempotentCreate(entry, req, durMode, replicaFactor); err != nil {
+			return nil, err
+		}
 		return ms.createBlockVolumeResponseFromEntry(entry), nil
 	}
 
@@ -71,7 +93,7 @@ func (ms *MasterServer) CreateBlockVolume(ctx context.Context, req *master_pb.Cr
 			return nil, err
 		}
 
-		result, err := ms.blockVSAllocate(ctx, server, req.Name, req.SizeBytes, req.DiskType)
+		result, err := ms.blockVSAllocate(ctx, server, req.Name, req.SizeBytes, req.DiskType, req.DurabilityMode)
 		if err != nil {
 			lastErr = fmt.Errorf("server %s: %w", server, err)
 			glog.V(0).Infof("[reqID=%s] CreateBlockVolume %q: attempt %d on %s failed: %v", blockReqID(ctx), req.Name, attempt+1, server, err)
@@ -79,12 +101,6 @@ func (ms *MasterServer) CreateBlockVolume(ctx context.Context, req *master_pb.Cr
 			continue
 		}
 
-		// CP8-2: determine replica factor from request (default 2).
-		replicaFactor := 2
-		if req.ReplicaFactor > 0 && req.ReplicaFactor <= 3 {
-			replicaFactor = int(req.ReplicaFactor)
-		}
-
 		entry := &BlockVolumeEntry{
 			Name:           req.Name,
 			VolumeServer:   server,
@@ -96,6 +112,7 @@ func (ms *MasterServer) CreateBlockVolume(ctx context.Context, req *master_pb.Cr
 			Role:           blockvol.RoleToWire(blockvol.RolePrimary),
 			Status:         StatusActive,
 			ReplicaFactor:  replicaFactor,
+			DurabilityMode: durMode.String(),
 			LeaseTTL:       30 * time.Second,
 			LastLeaseGrant: time.Now(), // R2-F1: set BEFORE Register to avoid stale-lease race
 		}
@@ -111,6 +128,15 @@ func (ms *MasterServer) CreateBlockVolume(ctx context.Context, req *master_pb.Cr
 		if len(entry.Replicas) == 0 && replicaFactor > 1 {
 			glog.V(0).Infof("[reqID=%s] CreateBlockVolume %q: single-copy mode (replica allocation failed)", blockReqID(ctx), req.Name)
 		}
+
+		// F1: strict modes require minimum replicas at create time.
+		requiredReplicas := durMode.RequiredReplicas(replicaFactor)
+		if len(entry.Replicas) < requiredReplicas {
+			ms.cleanupPartialCreate(ctx, entry)
+			return nil, fmt.Errorf("durability mode %q requires %d replicas but only %d provisioned",
+				durMode.String(), requiredReplicas, len(entry.Replicas))
+		}
+
 		// Sync deprecated scalar fields from first replica.
 		if len(entry.Replicas) > 0 {
 			r0 := &entry.Replicas[0]
@@ -225,6 +251,10 @@ func (ms *MasterServer) LookupBlockVolume(ctx context.Context, req *master_pb.Lo
 	if rf == 0 {
 		rf = 2 // default for pre-CP8-2 entries
 	}
+	durModeStr := entry.DurabilityMode
+	if durModeStr == "" {
+		durModeStr = "best_effort"
+	}
 	return &master_pb.LookupBlockVolumeResponse{
 		VolumeServer:   entry.VolumeServer,
 		IscsiAddr:      entry.ISCSIAddr,
@@ -233,6 +263,7 @@ func (ms *MasterServer) LookupBlockVolume(ctx context.Context, req *master_pb.Lo
 		ReplicaServer:  entry.ReplicaServer, // backward compat
 		ReplicaFactor:  uint32(rf),
 		ReplicaServers: replicaServers,
+		DurabilityMode: durModeStr,
 	}, nil
 }
 
@@ -240,7 +271,7 @@ func (ms *MasterServer) LookupBlockVolume(ctx context.Context, req *master_pb.Lo
 // Returns the replica server address on success, or empty string on failure (F4).
 func (ms *MasterServer) tryCreateOneReplica(ctx context.Context, req *master_pb.CreateBlockVolumeRequest, entry *BlockVolumeEntry, primaryResult *blockAllocResult, candidates []string) string {
 	for _, replicaServerStr := range candidates {
-		replicaResult, err := ms.blockVSAllocate(ctx, replicaServerStr, req.Name, req.SizeBytes, req.DiskType)
+		replicaResult, err := ms.blockVSAllocate(ctx, replicaServerStr, req.Name, req.SizeBytes, req.DiskType, req.DurabilityMode)
 		if err != nil {
 			glog.V(0).Infof("[reqID=%s] CreateBlockVolume %q: replica on %s failed: %v", blockReqID(ctx), req.Name, replicaServerStr, err)
 			continue
@@ -381,6 +412,31 @@ func (ms *MasterServer) createBlockVolumeResponseFromEntry(entry *BlockVolumeEnt
 	}
 }
 
+// validateIdempotentCreate checks that an idempotent create request is consistent
+// with an existing entry. Returns nil if compatible, error on mismatch.
+func (ms *MasterServer) validateIdempotentCreate(entry *BlockVolumeEntry, req *master_pb.CreateBlockVolumeRequest, durMode blockvol.DurabilityMode, replicaFactor int) error {
+	if entry.SizeBytes < req.SizeBytes {
+		return fmt.Errorf("block volume %q exists with size %d (requested %d)", req.Name, entry.SizeBytes, req.SizeBytes)
+	}
+	// Validate durability mode consistency.
+	existingMode := entry.DurabilityMode
+	if existingMode == "" {
+		existingMode = "best_effort"
+	}
+	if durMode.String() != existingMode {
+		return fmt.Errorf("block volume %q exists with durability_mode %q (requested %q)", req.Name, existingMode, durMode.String())
+	}
+	// Validate replica factor consistency.
+	existingRF := entry.ReplicaFactor
+	if existingRF == 0 {
+		existingRF = 2 // default
+	}
+	if replicaFactor != existingRF {
+		return fmt.Errorf("block volume %q exists with replica_factor %d (requested %d)", req.Name, existingRF, replicaFactor)
+	}
+	return nil
+}
+
 // replicaServerList returns the list of replica server addresses.
 // Order matches Replicas[] (append-order), ensuring ReplicaServers[0] == ReplicaServer (legacy).
 func replicaServerList(entry *BlockVolumeEntry) []string {
@@ -404,3 +460,23 @@ func removeServer(servers []string, server string) []string {
 	}
 	return result
 }
+
+// cleanupPartialCreate removes a partially created block volume (primary + any replicas)
+// when strict durability mode enforcement fails due to insufficient replicas.
+// All operations are best-effort: failures are logged but do not propagate.
+func (ms *MasterServer) cleanupPartialCreate(ctx context.Context, entry *BlockVolumeEntry) {
+	// Delete primary volume.
+	if err := ms.blockVSDelete(ctx, entry.VolumeServer, entry.Name); err != nil {
+		glog.Warningf("[reqID=%s] cleanupPartialCreate %q: delete primary on %s: %v",
+			blockReqID(ctx), entry.Name, entry.VolumeServer, err)
+	}
+	// Delete any successfully created replicas.
+	for _, ri := range entry.Replicas {
+		if err := ms.blockVSDelete(ctx, ri.Server, entry.Name); err != nil {
+			glog.Warningf("[reqID=%s] cleanupPartialCreate %q: delete replica on %s: %v",
+				blockReqID(ctx), entry.Name, ri.Server, err)
+		}
+	}
+	// Remove from registry if somehow registered (shouldn't be at this point).
+	ms.blockRegistry.Unregister(entry.Name)
+}

weed/server/master_grpc_server_block_test.go

@@ -19,7 +19,7 @@ func testMasterServer(t *testing.T) *MasterServer {
 		blockAssignmentQueue: NewBlockAssignmentQueue(),
 	}
 	// Default mock: succeed with deterministic values.
-	ms.blockVSAllocate = func(ctx context.Context, server string, name string, sizeBytes uint64, diskType string) (*blockAllocResult, error) {
+	ms.blockVSAllocate = func(ctx context.Context, server string, name string, sizeBytes uint64, diskType string, durabilityMode string) (*blockAllocResult, error) {
 		return &blockAllocResult{
 			Path:      fmt.Sprintf("/data/%s.blk", name),
 			IQN:       fmt.Sprintf("iqn.2024.test:%s", name),
@@ -139,7 +139,7 @@ func TestMaster_CreateVSFailure_Retry(t *testing.T) {
 	ms.blockRegistry.MarkBlockCapable("vs2:9333")
 
 	var callCount atomic.Int32
-	ms.blockVSAllocate = func(ctx context.Context, server string, name string, sizeBytes uint64, diskType string) (*blockAllocResult, error) {
+	ms.blockVSAllocate = func(ctx context.Context, server string, name string, sizeBytes uint64, diskType string, durabilityMode string) (*blockAllocResult, error) {
 		n := callCount.Add(1)
 		if n == 1 {
 			return nil, fmt.Errorf("disk full")
@@ -170,7 +170,7 @@ func TestMaster_CreateVSFailure_Cleanup(t *testing.T) {
 	ms := testMasterServer(t)
 	ms.blockRegistry.MarkBlockCapable("vs1:9333")
 
-	ms.blockVSAllocate = func(ctx context.Context, server string, name string, sizeBytes uint64, diskType string) (*blockAllocResult, error) {
+	ms.blockVSAllocate = func(ctx context.Context, server string, name string, sizeBytes uint64, diskType string, durabilityMode string) (*blockAllocResult, error) {
 		return nil, fmt.Errorf("all servers broken")
 	}
 
@@ -193,7 +193,7 @@ func TestMaster_CreateConcurrentSameName(t *testing.T) {
 	ms.blockRegistry.MarkBlockCapable("vs1:9333")
 
 	var callCount atomic.Int32
-	ms.blockVSAllocate = func(ctx context.Context, server string, name string, sizeBytes uint64, diskType string) (*blockAllocResult, error) {
+	ms.blockVSAllocate = func(ctx context.Context, server string, name string, sizeBytes uint64, diskType string, durabilityMode string) (*blockAllocResult, error) {
 		callCount.Add(1)
 		return &blockAllocResult{
 			Path:      fmt.Sprintf("/data/%s.blk", name),
@@ -275,7 +275,7 @@ func TestMaster_CreateWithReplica(t *testing.T) {
 	ms.blockRegistry.MarkBlockCapable("vs2:9333")
 
 	var allocServers []string
-	ms.blockVSAllocate = func(ctx context.Context, server string, name string, sizeBytes uint64, diskType string) (*blockAllocResult, error) {
+	ms.blockVSAllocate = func(ctx context.Context, server string, name string, sizeBytes uint64, diskType string, durabilityMode string) (*blockAllocResult, error) {
 		allocServers = append(allocServers, server)
 		return &blockAllocResult{
 			Path:            fmt.Sprintf("/data/%s.blk", name),
@@ -328,7 +328,7 @@ func TestMaster_CreateSingleServer_NoReplica(t *testing.T) {
 	ms.blockRegistry.MarkBlockCapable("vs1:9333")
 
 	var allocCount atomic.Int32
-	ms.blockVSAllocate = func(ctx context.Context, server string, name string, sizeBytes uint64, diskType string) (*blockAllocResult, error) {
+	ms.blockVSAllocate = func(ctx context.Context, server string, name string, sizeBytes uint64, diskType string, durabilityMode string) (*blockAllocResult, error) {
 		allocCount.Add(1)
 		return &blockAllocResult{
 			Path:      fmt.Sprintf("/data/%s.blk", name),
@@ -365,7 +365,7 @@ func TestMaster_CreateReplica_SecondFails_SingleCopy(t *testing.T) {
 	ms.blockRegistry.MarkBlockCapable("vs2:9333")
 
 	var callCount atomic.Int32
-	ms.blockVSAllocate = func(ctx context.Context, server string, name string, sizeBytes uint64, diskType string) (*blockAllocResult, error) {
+	ms.blockVSAllocate = func(ctx context.Context, server string, name string, sizeBytes uint64, diskType string, durabilityMode string) (*blockAllocResult, error) {
 		n := callCount.Add(1)
 		if n == 2 {
 			// Replica allocation fails.
@@ -402,7 +402,7 @@ func TestMaster_CreateEnqueuesAssignments(t *testing.T) {
 	ms.blockRegistry.MarkBlockCapable("vs1:9333")
 	ms.blockRegistry.MarkBlockCapable("vs2:9333")
 
-	ms.blockVSAllocate = func(ctx context.Context, server string, name string, sizeBytes uint64, diskType string) (*blockAllocResult, error) {
+	ms.blockVSAllocate = func(ctx context.Context, server string, name string, sizeBytes uint64, diskType string, durabilityMode string) (*blockAllocResult, error) {
 		return &blockAllocResult{
 			Path:            fmt.Sprintf("/data/%s.blk", name),
 			IQN:             fmt.Sprintf("iqn.2024.test:%s", name),
@@ -463,7 +463,7 @@ func TestMaster_LookupReturnsReplicaServer(t *testing.T) {
 	ms.blockRegistry.MarkBlockCapable("vs1:9333")
 	ms.blockRegistry.MarkBlockCapable("vs2:9333")
 
-	ms.blockVSAllocate = func(ctx context.Context, server string, name string, sizeBytes uint64, diskType string) (*blockAllocResult, error) {
+	ms.blockVSAllocate = func(ctx context.Context, server string, name string, sizeBytes uint64, diskType string, durabilityMode string) (*blockAllocResult, error) {
 		return &blockAllocResult{
 			Path:      fmt.Sprintf("/data/%s.blk", name),
 			IQN:       fmt.Sprintf("iqn.2024.test:%s", name),
@@ -670,7 +670,7 @@ func TestMaster_LookupBlockVolume(t *testing.T) {
 func testMasterServerRF3(t *testing.T) *MasterServer {
 	t.Helper()
 	ms := testMasterServer(t)
-	ms.blockVSAllocate = func(ctx context.Context, server string, name string, sizeBytes uint64, diskType string) (*blockAllocResult, error) {
+	ms.blockVSAllocate = func(ctx context.Context, server string, name string, sizeBytes uint64, diskType string, durabilityMode string) (*blockAllocResult, error) {
 		return &blockAllocResult{
 			Path:            fmt.Sprintf("/data/%s.blk", name),
 			IQN:             fmt.Sprintf("iqn.2024.test:%s", name),
@@ -725,7 +725,7 @@ func TestMaster_CreateRF3_ThreeServers(t *testing.T) {
 // RF=3 with only 2 servers: should create 1 replica (partial).
 func TestMaster_CreateRF3_TwoServers(t *testing.T) {
 	ms := testMasterServer(t)
-	ms.blockVSAllocate = func(ctx context.Context, server string, name string, sizeBytes uint64, diskType string) (*blockAllocResult, error) {
+	ms.blockVSAllocate = func(ctx context.Context, server string, name string, sizeBytes uint64, diskType string, durabilityMode string) (*blockAllocResult, error) {
 		return &blockAllocResult{
 			Path:            fmt.Sprintf("/data/%s.blk", name),
 			IQN:             fmt.Sprintf("iqn.2024.test:%s", name),

weed/server/master_server.go

@@ -98,7 +98,7 @@ type MasterServer struct {
 	blockRegistry        *BlockVolumeRegistry
 	blockAssignmentQueue *BlockAssignmentQueue
 	blockFailover        *blockFailoverState
-	blockVSAllocate  func(ctx context.Context, server string, name string, sizeBytes uint64, diskType string) (*blockAllocResult, error)
+	blockVSAllocate  func(ctx context.Context, server string, name string, sizeBytes uint64, diskType string, durabilityMode string) (*blockAllocResult, error)
 	blockVSDelete    func(ctx context.Context, server string, name string) error
 	blockVSSnapshot  func(ctx context.Context, server string, name string, snapID uint32) (int64, uint64, error)
 	blockVSDeleteSnap func(ctx context.Context, server string, name string, snapID uint32) error
@@ -553,13 +553,14 @@ type blockAllocResult struct {
 }
 
 // defaultBlockVSAllocate calls a volume server's AllocateBlockVolume RPC.
-func (ms *MasterServer) defaultBlockVSAllocate(ctx context.Context, server string, name string, sizeBytes uint64, diskType string) (*blockAllocResult, error) {
+func (ms *MasterServer) defaultBlockVSAllocate(ctx context.Context, server string, name string, sizeBytes uint64, diskType string, durabilityMode string) (*blockAllocResult, error) {
 	var result blockAllocResult
 	err := operation.WithVolumeServerClient(false, pb.ServerAddress(server), ms.grpcDialOption, func(client volume_server_pb.VolumeServerClient) error {
 		resp, rerr := client.AllocateBlockVolume(ctx, &volume_server_pb.AllocateBlockVolumeRequest{
-			Name:      name,
-			SizeBytes: sizeBytes,
-			DiskType:  diskType,
+			Name:           name,
+			SizeBytes:      sizeBytes,
+			DiskType:       diskType,
+			DurabilityMode: durabilityMode,
 		})
 		if rerr != nil {
 			return rerr

weed/server/master_server_handlers_block.go

@@ -26,10 +26,20 @@ func (ms *MasterServer) blockVolumeCreateHandler(w http.ResponseWriter, r *http.
 		replicaPlacement = "000"
 	}
 
+	// Pre-validate durability_mode (cosmetic — real validation is in gRPC handler).
+	if req.DurabilityMode != "" {
+		if _, perr := blockvol.ParseDurabilityMode(req.DurabilityMode); perr != nil {
+			writeJsonError(w, r, http.StatusBadRequest, fmt.Errorf("invalid durability_mode: %w", perr))
+			return
+		}
+	}
+
 	resp, err := ms.CreateBlockVolume(r.Context(), &master_pb.CreateBlockVolumeRequest{
-		Name:      req.Name,
-		SizeBytes: req.SizeBytes,
-		DiskType:  req.DiskType,
+		Name:           req.Name,
+		SizeBytes:      req.SizeBytes,
+		DiskType:       req.DiskType,
+		DurabilityMode: req.DurabilityMode,
+		ReplicaFactor:  uint32(req.ReplicaFactor),
 	})
 	if err != nil {
 		writeJsonError(w, r, http.StatusInternalServerError, err)
@@ -177,6 +187,10 @@ func entryToVolumeInfo(e *BlockVolumeEntry) blockapi.VolumeInfo {
 	if rf == 0 {
 		rf = 2 // default
 	}
+	durMode := e.DurabilityMode
+	if durMode == "" {
+		durMode = "best_effort"
+	}
 	info := blockapi.VolumeInfo{
 		Name:             e.Name,
 		VolumeServer:     e.VolumeServer,
@@ -195,6 +209,7 @@ func entryToVolumeInfo(e *BlockVolumeEntry) blockapi.VolumeInfo {
 		ReplicaFactor:    rf,
 		HealthScore:      e.HealthScore,
 		ReplicaDegraded:  e.ReplicaDegraded,
+		DurabilityMode:   durMode,
 	}
 	for _, ri := range e.Replicas {
 		info.Replicas = append(info.Replicas, blockapi.ReplicaDetail{

weed/server/master_server_handlers_block_test.go

@@ -20,7 +20,7 @@ func blockTestServer(t *testing.T) (*MasterServer, *httptest.Server) {
 		blockRegistry:        NewBlockVolumeRegistry(),
 		blockAssignmentQueue: NewBlockAssignmentQueue(),
 	}
-	ms.blockVSAllocate = func(ctx context.Context, server string, name string, sizeBytes uint64, diskType string) (*blockAllocResult, error) {
+	ms.blockVSAllocate = func(ctx context.Context, server string, name string, sizeBytes uint64, diskType string, durabilityMode string) (*blockAllocResult, error) {
 		return &blockAllocResult{
 			Path:      fmt.Sprintf("/data/%s.blk", name),
 			IQN:       fmt.Sprintf("iqn.2024.test:%s", name),

weed/server/master_server_handlers_block_ui.go

@@ -13,7 +13,9 @@ var blockOpsTemplate = template.Must(template.New("blockOps").Parse(blockLayoutH
 
 type blockUIVolume struct {
 	blockapi.VolumeInfo
-	SizeMB uint64
+	SizeMB     uint64
+	WALHeadLSN uint64
+	MaxWALLag  uint64 // max WAL lag across replicas
 }
 
 type blockUIData struct {
@@ -25,6 +27,12 @@ type blockUIData struct {
 	ActiveCount  int
 	PendingCount int
 	TotalSizeMB  uint64
+	// Observability (CP8-4)
+	BarrierLagLSN      uint64
+	PromotionsTotal    uint64
+	FailoversTotal     uint64
+	RebuildsTotal      uint64
+	AssignmentQueueLen int
 }
 
 func (ms *MasterServer) buildBlockUIData(tab string) blockUIData {
@@ -35,9 +43,17 @@ func (ms *MasterServer) buildBlockUIData(tab string) blockUIData {
 	for i, e := range entries {
 		info := entryToVolumeInfo(e)
 		mb := info.SizeBytes / (1024 * 1024)
+		var maxLag uint64
+		for _, ri := range e.Replicas {
+			if ri.WALLag > maxLag {
+				maxLag = ri.WALLag
+			}
+		}
 		volumes[i] = blockUIVolume{
 			VolumeInfo: info,
 			SizeMB:     mb,
+			WALHeadLSN: e.WALHeadLSN,
+			MaxWALLag:  maxLag,
 		}
 		totalSizeMB += mb
 		if e.Status == StatusActive {
@@ -58,13 +74,18 @@ func (ms *MasterServer) buildBlockUIData(tab string) blockUIData {
 	}
 
 	return blockUIData{
-		Tab:          tab,
-		Volumes:      volumes,
-		Servers:      servers,
-		TotalVolumes: len(entries),
-		ActiveCount:  activeCount,
-		PendingCount: pendingCount,
-		TotalSizeMB:  totalSizeMB,
+		Tab:                tab,
+		Volumes:            volumes,
+		Servers:            servers,
+		TotalVolumes:       len(entries),
+		ActiveCount:        activeCount,
+		PendingCount:       pendingCount,
+		TotalSizeMB:        totalSizeMB,
+		BarrierLagLSN:      ms.blockRegistry.MaxBarrierLagLSN(),
+		PromotionsTotal:    ms.blockRegistry.PromotionsTotal.Load(),
+		FailoversTotal:     ms.blockRegistry.FailoversTotal.Load(),
+		RebuildsTotal:      ms.blockRegistry.RebuildsTotal.Load(),
+		AssignmentQueueLen: ms.blockAssignmentQueue.TotalPending(),
 	}
 }
 
@@ -130,6 +151,9 @@ const blockLayoutHTML = `<!DOCTYPE html>
   .badge-primary { background: #dfe6e9; color: #2d3436; }
   .badge-replica { background: #e8daef; color: #6c3483; }
   .empty { color: #b2bec3; font-style: italic; padding: 20px; text-align: center; }
+  .card .value.red { color: #d63031; }
+  .card .value.gray { color: #636e72; }
+  .section-label { font-size: 11px; color: #b2bec3; text-transform: uppercase; letter-spacing: 1px; margin-bottom: 8px; }
 </style>
 </head>
 <body>
@@ -166,6 +190,30 @@ const blockDashContentHTML = `
   </div>
 </div>
 
+<div class="section-label">Cluster Health</div>
+<div class="cards">
+  <div class="card">
+    <div class="label">Barrier Lag LSN</div>
+    <div class="value {{if gt .BarrierLagLSN 1000}}red{{else if gt .BarrierLagLSN 100}}orange{{else}}green{{end}}">{{.BarrierLagLSN}}</div>
+  </div>
+  <div class="card">
+    <div class="label">Promotions</div>
+    <div class="value gray">{{.PromotionsTotal}}</div>
+  </div>
+  <div class="card">
+    <div class="label">Failovers</div>
+    <div class="value {{if gt .FailoversTotal 0}}orange{{else}}gray{{end}}">{{.FailoversTotal}}</div>
+  </div>
+  <div class="card">
+    <div class="label">Rebuilds</div>
+    <div class="value {{if gt .RebuildsTotal 0}}orange{{else}}gray{{end}}">{{.RebuildsTotal}}</div>
+  </div>
+  <div class="card">
+    <div class="label">Queue Depth</div>
+    <div class="value {{if gt .AssignmentQueueLen 10}}red{{else if gt .AssignmentQueueLen 0}}orange{{else}}green{{end}}">{{.AssignmentQueueLen}}</div>
+  </div>
+</div>
+
 <h2>Servers</h2>
 {{if .Servers}}
 <table>
@@ -186,19 +234,23 @@ const blockDashContentHTML = `
 {{if .Volumes}}
 <table>
   <tr>
-    <th>Name</th><th>Server</th><th>Size</th><th>Placement</th>
-    <th>Epoch</th><th>Role</th><th>Status</th><th>iSCSI</th><th>Replica</th>
+    <th>Name</th><th>Server</th><th>Size</th><th>Role</th><th>Status</th>
+    <th>Durability</th><th>Health</th><th>WAL LSN</th><th>WAL Lag</th><th>Degraded</th>
+    <th>Epoch</th><th>Replica</th>
   </tr>
   {{range .Volumes}}
   <tr>
     <td>{{.Name}}</td>
     <td>{{.VolumeServer}}</td>
     <td>{{.SizeMB}} MB</td>
-    <td>{{.ReplicaPlacement}}</td>
-    <td>{{.Epoch}}</td>
     <td>{{if eq .Role "primary"}}<span class="badge badge-primary">primary</span>{{else if eq .Role "replica"}}<span class="badge badge-replica">replica</span>{{else}}{{.Role}}{{end}}</td>
     <td>{{if eq .Status "active"}}<span class="badge badge-active">active</span>{{else}}<span class="badge badge-pending">{{.Status}}</span>{{end}}</td>
-    <td>{{.ISCSIAddr}}</td>
+    <td>{{if eq .DurabilityMode "sync_all"}}<span class="badge" style="background:#dfe6e9;color:#d63031">sync_all</span>{{else if eq .DurabilityMode "sync_quorum"}}<span class="badge" style="background:#dfe6e9;color:#e17055">sync_quorum</span>{{else}}best_effort{{end}}</td>
+    <td style="color:{{if ge .HealthScore 0.9}}#00b894{{else if ge .HealthScore 0.5}}#fdcb6e{{else}}#d63031{{end}}">{{printf "%.2f" .HealthScore}}</td>
+    <td>{{.WALHeadLSN}}</td>
+    <td style="color:{{if gt .MaxWALLag 1000}}#d63031{{else if gt .MaxWALLag 100}}#fdcb6e{{else}}#00b894{{end}}">{{.MaxWALLag}}</td>
+    <td>{{if .ReplicaDegraded}}<span class="badge" style="background:#ffeaa7;color:#d68910">yes</span>{{else}}-{{end}}</td>
+    <td>{{.Epoch}}</td>
     <td>{{.ReplicaServer}}</td>
   </tr>
   {{end}}
@@ -235,6 +287,23 @@ const blockOpsContentHTML = `
       <label>Disk type:</label>
       <input type="text" id="cDisk" placeholder="ssd">
     </div>
+    <div class="form-row">
+      <label>RF:</label>
+      <select id="cRF">
+        <option value="0" selected>Default (2)</option>
+        <option value="1">1 - No replica</option>
+        <option value="2">2</option>
+        <option value="3">3</option>
+      </select>
+    </div>
+    <div class="form-row">
+      <label>Durability:</label>
+      <select id="cDurability">
+        <option value="" selected>best_effort (default)</option>
+        <option value="sync_all">sync_all</option>
+        <option value="sync_quorum">sync_quorum</option>
+      </select>
+    </div>
     <div class="form-row">
       <label></label>
       <button type="submit" class="btn-create">Create Volume</button>
@@ -246,19 +315,22 @@ const blockOpsContentHTML = `
 {{if .Volumes}}
 <table>
   <tr>
-    <th>Name</th><th>Server</th><th>Size</th><th>Placement</th>
-    <th>Epoch</th><th>Role</th><th>Status</th><th>iSCSI</th><th>Replica</th><th>Action</th>
+    <th>Name</th><th>Server</th><th>Size</th><th>Role</th><th>Status</th>
+    <th>Durability</th><th>Health</th><th>WAL Lag</th><th>Degraded</th>
+    <th>Epoch</th><th>Replica</th><th>Action</th>
   </tr>
   {{range .Volumes}}
   <tr>
     <td>{{.Name}}</td>
     <td>{{.VolumeServer}}</td>
     <td>{{.SizeMB}} MB</td>
-    <td>{{.ReplicaPlacement}}</td>
-    <td>{{.Epoch}}</td>
     <td>{{if eq .Role "primary"}}<span class="badge badge-primary">primary</span>{{else if eq .Role "replica"}}<span class="badge badge-replica">replica</span>{{else}}{{.Role}}{{end}}</td>
     <td>{{if eq .Status "active"}}<span class="badge badge-active">active</span>{{else}}<span class="badge badge-pending">{{.Status}}</span>{{end}}</td>
-    <td>{{.ISCSIAddr}}</td>
+    <td>{{if eq .DurabilityMode "sync_all"}}<span class="badge" style="background:#dfe6e9;color:#d63031">sync_all</span>{{else if eq .DurabilityMode "sync_quorum"}}<span class="badge" style="background:#dfe6e9;color:#e17055">sync_quorum</span>{{else}}best_effort{{end}}</td>
+    <td style="color:{{if ge .HealthScore 0.9}}#00b894{{else if ge .HealthScore 0.5}}#fdcb6e{{else}}#d63031{{end}}">{{printf "%.2f" .HealthScore}}</td>
+    <td style="color:{{if gt .MaxWALLag 1000}}#d63031{{else if gt .MaxWALLag 100}}#fdcb6e{{else}}#00b894{{end}}">{{.MaxWALLag}}</td>
+    <td>{{if .ReplicaDegraded}}<span class="badge" style="background:#ffeaa7;color:#d68910">yes</span>{{else}}-{{end}}</td>
+    <td>{{.Epoch}}</td>
     <td>{{.ReplicaServer}}</td>
     <td><button class="btn-delete" onclick="deleteVol('{{.Name}}')">Delete</button></td>
   </tr>
@@ -281,7 +353,9 @@ document.getElementById('createForm').addEventListener('submit', function(e) {
       name: document.getElementById('cName').value,
       size_bytes: parseInt(document.getElementById('cSize').value) * 1024 * 1024,
       replica_placement: document.getElementById('cPlacement').value,
-      disk_type: document.getElementById('cDisk').value
+      disk_type: document.getElementById('cDisk').value,
+      replica_factor: parseInt(document.getElementById('cRF').value) || 0,
+      durability_mode: document.getElementById('cDurability').value
     })
   }).then(function(r) {
     if (!r.ok) return r.json().then(function(j) { throw new Error(j.error || r.statusText); });

weed/server/qa_block_cp62_test.go

@@ -319,7 +319,7 @@ func TestQA_Master_AllVSFailNoOrphan(t *testing.T) {
 	ms.blockRegistry.MarkBlockCapable("vs2:9333")
 	ms.blockRegistry.MarkBlockCapable("vs3:9333")
 
-	ms.blockVSAllocate = func(ctx context.Context, server string, name string, sizeBytes uint64, diskType string) (*blockAllocResult, error) {
+	ms.blockVSAllocate = func(ctx context.Context, server string, name string, sizeBytes uint64, diskType string, durabilityMode string) (*blockAllocResult, error) {
 		return nil, fmt.Errorf("disk full on %s", server)
 	}
 
@@ -348,7 +348,7 @@ func TestQA_Master_SlowAllocateBlocksSecond(t *testing.T) {
 	ms.blockRegistry.MarkBlockCapable("vs1:9333")
 
 	var allocCount atomic.Int32
-	ms.blockVSAllocate = func(ctx context.Context, server string, name string, sizeBytes uint64, diskType string) (*blockAllocResult, error) {
+	ms.blockVSAllocate = func(ctx context.Context, server string, name string, sizeBytes uint64, diskType string, durabilityMode string) (*blockAllocResult, error) {
 		allocCount.Add(1)
 		time.Sleep(100 * time.Millisecond) // simulate slow VS
 		return &blockAllocResult{
@@ -450,7 +450,7 @@ func TestQA_VS_ConcurrentCreate(t *testing.T) {
 					t.Errorf("PANIC: %v", r)
 				}
 			}()
-			_, _, _, errors[i] = bs.CreateBlockVol("race-vol", 4*1024*1024, "")
+			_, _, _, errors[i] = bs.CreateBlockVol("race-vol", 4*1024*1024, "", "")
 		}(i)
 	}
 	wg.Wait()
@@ -487,7 +487,7 @@ func TestQA_VS_ConcurrentCreateDelete(t *testing.T) {
 					panicked.Store(true)
 				}
 			}()
-			bs.CreateBlockVol("cd-vol", 4*1024*1024, "")
+			bs.CreateBlockVol("cd-vol", 4*1024*1024, "", "")
 		}()
 		go func() {
 			defer wg.Done()
@@ -510,7 +510,7 @@ func TestQA_VS_ConcurrentCreateDelete(t *testing.T) {
 func TestQA_VS_DeleteCleansSnapshots(t *testing.T) {
 	bs, blockDir := newTestBlockServiceWithDir(t)
 
-	bs.CreateBlockVol("snap-vol", 4*1024*1024, "")
+	bs.CreateBlockVol("snap-vol", 4*1024*1024, "", "")
 
 	// Simulate snapshot files.
 	snapPath := blockDir + "/snap-vol.blk.snap.0"
@@ -531,13 +531,13 @@ func TestQA_VS_SanitizationCollision(t *testing.T) {
 	bs, _ := newTestBlockServiceWithDir(t)
 
 	// "VolA" sanitizes to "vola.blk", "vola" also sanitizes to "vola.blk".
-	_, _, _, err := bs.CreateBlockVol("VolA", 4*1024*1024, "")
+	_, _, _, err := bs.CreateBlockVol("VolA", 4*1024*1024, "", "")
 	if err != nil {
 		t.Fatalf("create VolA: %v", err)
 	}
 
 	// "vola" should get the idempotent path (same file on disk).
-	path2, _, _, err := bs.CreateBlockVol("vola", 4*1024*1024, "")
+	path2, _, _, err := bs.CreateBlockVol("vola", 4*1024*1024, "", "")
 	if err != nil {
 		t.Fatalf("create vola: %v", err)
 	}
@@ -553,13 +553,13 @@ func TestQA_VS_CreateIdempotentReaddTarget(t *testing.T) {
 	bs, _ := newTestBlockServiceWithDir(t)
 
 	// First create.
-	_, iqn1, _, err := bs.CreateBlockVol("readd-vol", 4*1024*1024, "")
+	_, iqn1, _, err := bs.CreateBlockVol("readd-vol", 4*1024*1024, "", "")
 	if err != nil {
 		t.Fatalf("first create: %v", err)
 	}
 
 	// Second create (idempotent) — should succeed and re-add to TargetServer.
-	_, iqn2, _, err := bs.CreateBlockVol("readd-vol", 4*1024*1024, "")
+	_, iqn2, _, err := bs.CreateBlockVol("readd-vol", 4*1024*1024, "", "")
 	if err != nil {
 		t.Fatalf("idempotent create: %v", err)
 	}

weed/server/qa_block_cp63_test.go

@@ -24,7 +24,7 @@ func testMSForQA(t *testing.T) *MasterServer {
 		blockAssignmentQueue: NewBlockAssignmentQueue(),
 		blockFailover:        newBlockFailoverState(),
 	}
-	ms.blockVSAllocate = func(ctx context.Context, server string, name string, sizeBytes uint64, diskType string) (*blockAllocResult, error) {
+	ms.blockVSAllocate = func(ctx context.Context, server string, name string, sizeBytes uint64, diskType string, durabilityMode string) (*blockAllocResult, error) {
 		return &blockAllocResult{
 			Path:      fmt.Sprintf("/data/%s.blk", name),
 			IQN:       fmt.Sprintf("iqn.2024.test:%s", name),

weed/server/qa_block_cp82_adversarial_test.go

@@ -27,7 +27,7 @@ func qaCP82Master(t *testing.T) *MasterServer {
 		blockAssignmentQueue: NewBlockAssignmentQueue(),
 		blockFailover:        newBlockFailoverState(),
 	}
-	ms.blockVSAllocate = func(ctx context.Context, server string, name string, sizeBytes uint64, diskType string) (*blockAllocResult, error) {
+	ms.blockVSAllocate = func(ctx context.Context, server string, name string, sizeBytes uint64, diskType string, durabilityMode string) (*blockAllocResult, error) {
 		return &blockAllocResult{
 			Path:              fmt.Sprintf("/data/%s.blk", name),
 			IQN:               fmt.Sprintf("iqn.2024.test:%s", name),

weed/server/qa_block_cp831_adversarial_test.go

@@ -0,0 +1,907 @@
+package weed_server
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"strings"
+	"sync"
+	"testing"
+	"time"
+
+	"github.com/seaweedfs/seaweedfs/weed/pb/master_pb"
+	"github.com/seaweedfs/seaweedfs/weed/storage/blockvol"
+)
+
+// ============================================================
+// CP8-3-1 Adversarial QA Test Suite
+//
+// 20 adversarial tests targeting durability mode ACK contracts,
+// F1-F4 review fixes, and edge cases in mode handling.
+// ============================================================
+
+// ────────────────────────────────────────────────────────────
+// QA-CP831-1: IdempotentCreate_ModeMismatch_Rejected
+//
+// Create sync_all volume, then retry with best_effort.
+// Must return mismatch error, NOT silently succeed.
+// ────────────────────────────────────────────────────────────
+func TestQA_CP831_IdempotentCreate_ModeMismatch_Rejected(t *testing.T) {
+	ms := qaDurabilityMaster(t)
+	ctx := context.Background()
+
+	// Create sync_all volume.
+	_, err := ms.CreateBlockVolume(ctx, &master_pb.CreateBlockVolumeRequest{
+		Name:           "idem-mode",
+		SizeBytes:      1 << 30,
+		DurabilityMode: "sync_all",
+		ReplicaFactor:  2,
+	})
+	if err != nil {
+		t.Fatalf("initial create: %v", err)
+	}
+
+	// Retry with best_effort (empty string = best_effort).
+	_, err = ms.CreateBlockVolume(ctx, &master_pb.CreateBlockVolumeRequest{
+		Name:      "idem-mode",
+		SizeBytes: 1 << 30,
+		// DurabilityMode omitted = best_effort
+	})
+	if err == nil {
+		t.Fatal("idempotent create with mode mismatch should fail")
+	}
+	if !strings.Contains(err.Error(), "durability_mode") {
+		t.Fatalf("expected durability_mode mismatch error, got: %v", err)
+	}
+}
+
+// ────────────────────────────────────────────────────────────
+// QA-CP831-2: IdempotentCreate_RFMismatch_Rejected
+//
+// Create RF=2, retry with RF=3. Must fail with RF mismatch.
+// ────────────────────────────────────────────────────────────
+func TestQA_CP831_IdempotentCreate_RFMismatch_Rejected(t *testing.T) {
+	ms := qaDurabilityMaster(t)
+	ctx := context.Background()
+
+	_, err := ms.CreateBlockVolume(ctx, &master_pb.CreateBlockVolumeRequest{
+		Name:           "idem-rf",
+		SizeBytes:      1 << 30,
+		DurabilityMode: "sync_all",
+		ReplicaFactor:  2,
+	})
+	if err != nil {
+		t.Fatalf("create: %v", err)
+	}
+
+	// Retry with RF=3.
+	_, err = ms.CreateBlockVolume(ctx, &master_pb.CreateBlockVolumeRequest{
+		Name:           "idem-rf",
+		SizeBytes:      1 << 30,
+		DurabilityMode: "sync_all",
+		ReplicaFactor:  3,
+	})
+	if err == nil {
+		t.Fatal("idempotent create with RF mismatch should fail")
+	}
+	if !strings.Contains(err.Error(), "replica_factor") {
+		t.Fatalf("expected replica_factor mismatch error, got: %v", err)
+	}
+}
+
+// ────────────────────────────────────────────────────────────
+// QA-CP831-3: IdempotentCreate_AllMatch_Succeeds
+//
+// Retry with identical params returns existing entry without error.
+// ────────────────────────────────────────────────────────────
+func TestQA_CP831_IdempotentCreate_AllMatch_Succeeds(t *testing.T) {
+	ms := qaDurabilityMaster(t)
+	ctx := context.Background()
+
+	resp1, err := ms.CreateBlockVolume(ctx, &master_pb.CreateBlockVolumeRequest{
+		Name:           "idem-ok",
+		SizeBytes:      1 << 30,
+		DurabilityMode: "sync_all",
+		ReplicaFactor:  2,
+	})
+	if err != nil {
+		t.Fatalf("create: %v", err)
+	}
+
+	// Exact same params.
+	resp2, err := ms.CreateBlockVolume(ctx, &master_pb.CreateBlockVolumeRequest{
+		Name:           "idem-ok",
+		SizeBytes:      1 << 30,
+		DurabilityMode: "sync_all",
+		ReplicaFactor:  2,
+	})
+	if err != nil {
+		t.Fatalf("idempotent create should succeed: %v", err)
+	}
+	if resp1.VolumeServer != resp2.VolumeServer {
+		t.Fatalf("idempotent: volume_server changed %q -> %q", resp1.VolumeServer, resp2.VolumeServer)
+	}
+}
+
+// ────────────────────────────────────────────────────────────
+// QA-CP831-4: InvalidDurabilityMode_Rejected
+//
+// Unknown mode string must be rejected at the gRPC layer.
+// ────────────────────────────────────────────────────────────
+func TestQA_CP831_InvalidDurabilityMode_Rejected(t *testing.T) {
+	ms := qaDurabilityMaster(t)
+	ctx := context.Background()
+
+	badModes := []string{"sync", "strict", "SYNC_ALL", "best-effort", "quorum"}
+	for _, mode := range badModes {
+		_, err := ms.CreateBlockVolume(ctx, &master_pb.CreateBlockVolumeRequest{
+			Name:           "invalid-" + mode,
+			SizeBytes:      1 << 30,
+			DurabilityMode: mode,
+		})
+		if err == nil {
+			t.Errorf("durability_mode=%q should be rejected", mode)
+			continue
+		}
+		if !strings.Contains(err.Error(), "invalid") {
+			t.Errorf("durability_mode=%q: expected 'invalid' error, got: %v", mode, err)
+		}
+	}
+}
+
+// ────────────────────────────────────────────────────────────
+// QA-CP831-5: HeartbeatEmptyMode_DoesNotOverwriteStrict
+//
+// F3 regression: heartbeat with empty DurabilityMode must NOT
+// downgrade an existing sync_all volume to best_effort.
+// ────────────────────────────────────────────────────────────
+func TestQA_CP831_HeartbeatEmptyMode_DoesNotOverwriteStrict(t *testing.T) {
+	ms := qaDurabilityMaster(t)
+	ctx := context.Background()
+
+	_, err := ms.CreateBlockVolume(ctx, &master_pb.CreateBlockVolumeRequest{
+		Name:           "hb-strict",
+		SizeBytes:      1 << 30,
+		DurabilityMode: "sync_all",
+		ReplicaFactor:  2,
+	})
+	if err != nil {
+		t.Fatalf("create: %v", err)
+	}
+
+	entry, _ := ms.blockRegistry.Lookup("hb-strict")
+
+	// 10 rapid heartbeats with empty mode — simulates older VS version.
+	for i := 0; i < 10; i++ {
+		ms.blockRegistry.UpdateFullHeartbeat(entry.VolumeServer, []*master_pb.BlockVolumeInfoMessage{
+			{
+				Path:           entry.Path,
+				VolumeSize:     1 << 30,
+				DurabilityMode: "", // empty — must not overwrite
+			},
+		})
+	}
+
+	entry, ok := ms.blockRegistry.Lookup("hb-strict")
+	if !ok {
+		t.Fatal("volume gone after heartbeat flood")
+	}
+	if entry.DurabilityMode != "sync_all" {
+		t.Fatalf("F3 regression: empty heartbeat overwrote mode to %q", entry.DurabilityMode)
+	}
+}
+
+// ────────────────────────────────────────────────────────────
+// QA-CP831-6: HeartbeatNonEmptyMode_DoesUpdate
+//
+// Heartbeat with non-empty DurabilityMode DOES update the entry.
+// (Legitimate VS reports after mode-aware upgrade.)
+// ────────────────────────────────────────────────────────────
+func TestQA_CP831_HeartbeatNonEmptyMode_DoesUpdate(t *testing.T) {
+	ms := qaDurabilityMaster(t)
+	ctx := context.Background()
+
+	_, err := ms.CreateBlockVolume(ctx, &master_pb.CreateBlockVolumeRequest{
+		Name:           "hb-update",
+		SizeBytes:      1 << 30,
+		DurabilityMode: "sync_all",
+		ReplicaFactor:  2,
+	})
+	if err != nil {
+		t.Fatalf("create: %v", err)
+	}
+
+	entry, _ := ms.blockRegistry.Lookup("hb-update")
+
+	// Heartbeat with matching non-empty mode — should be accepted.
+	ms.blockRegistry.UpdateFullHeartbeat(entry.VolumeServer, []*master_pb.BlockVolumeInfoMessage{
+		{
+			Path:           entry.Path,
+			VolumeSize:     1 << 30,
+			DurabilityMode: "sync_all",
+		},
+	})
+
+	entry, ok := ms.blockRegistry.Lookup("hb-update")
+	if !ok {
+		t.Fatal("volume gone after heartbeat")
+	}
+	if entry.DurabilityMode != "sync_all" {
+		t.Fatalf("expected sync_all after non-empty heartbeat, got %q", entry.DurabilityMode)
+	}
+}
+
+// ────────────────────────────────────────────────────────────
+// QA-CP831-7: SyncAll_RF3_PartialReplica_OneOfTwo_Fails
+//
+// sync_all RF=3 requires 2 replicas (RequiredReplicas=2).
+// If only 1 of 2 replicas is provisioned → must fail + cleanup.
+// ────────────────────────────────────────────────────────────
+func TestQA_CP831_SyncAll_RF3_PartialReplica_OneOfTwo_Fails(t *testing.T) {
+	ms := qaDurabilityMaster(t)
+	ctx := context.Background()
+
+	// First call succeeds (primary), second succeeds (replica 1),
+	// third fails (replica 2).
+	callCount := 0
+	ms.blockVSAllocate = func(ctx context.Context, server string, name string, sizeBytes uint64, diskType string, durabilityMode string) (*blockAllocResult, error) {
+		callCount++
+		if callCount == 3 {
+			return nil, fmt.Errorf("disk full on third server")
+		}
+		return &blockAllocResult{
+			Path:              fmt.Sprintf("/data/%s.blk", name),
+			IQN:               fmt.Sprintf("iqn.2024.test:%s", name),
+			ISCSIAddr:         server + ":3260",
+			ReplicaDataAddr:   server + ":14260",
+			ReplicaCtrlAddr:   server + ":14261",
+			RebuildListenAddr: server + ":15000",
+		}, nil
+	}
+
+	var deletedServers []string
+	ms.blockVSDelete = func(ctx context.Context, server string, name string) error {
+		deletedServers = append(deletedServers, server)
+		return nil
+	}
+
+	_, err := ms.CreateBlockVolume(ctx, &master_pb.CreateBlockVolumeRequest{
+		Name:           "sa-rf3-partial",
+		SizeBytes:      1 << 30,
+		DurabilityMode: "sync_all",
+		ReplicaFactor:  3,
+	})
+	if err == nil {
+		t.Fatal("sync_all RF=3 should fail when only 1 of 2 required replicas provisioned")
+	}
+	if !strings.Contains(err.Error(), "requires") {
+		t.Fatalf("expected 'requires' error, got: %v", err)
+	}
+
+	// Cleanup should have deleted primary + the 1 successful replica.
+	if len(deletedServers) < 2 {
+		t.Fatalf("expected at least 2 cleanup deletes (primary + 1 replica), got %d", len(deletedServers))
+	}
+
+	// Volume must not be in registry.
+	if _, ok := ms.blockRegistry.Lookup("sa-rf3-partial"); ok {
+		t.Fatal("partially created volume should not remain in registry")
+	}
+}
+
+// ────────────────────────────────────────────────────────────
+// QA-CP831-8: SyncQuorum_RF3_OneReplicaOK_Succeeds
+//
+// sync_quorum RF=3 requires only 1 replica (quorum=2, minus primary=1).
+// If 1 of 2 replicas fails, should still succeed.
+// ────────────────────────────────────────────────────────────
+func TestQA_CP831_SyncQuorum_RF3_OneReplicaOK_Succeeds(t *testing.T) {
+	ms := qaDurabilityMaster(t)
+	ctx := context.Background()
+
+	callCount := 0
+	ms.blockVSAllocate = func(ctx context.Context, server string, name string, sizeBytes uint64, diskType string, durabilityMode string) (*blockAllocResult, error) {
+		callCount++
+		if callCount == 3 {
+			return nil, fmt.Errorf("disk full on third server")
+		}
+		return &blockAllocResult{
+			Path:              fmt.Sprintf("/data/%s.blk", name),
+			IQN:               fmt.Sprintf("iqn.2024.test:%s", name),
+			ISCSIAddr:         server + ":3260",
+			ReplicaDataAddr:   server + ":14260",
+			ReplicaCtrlAddr:   server + ":14261",
+			RebuildListenAddr: server + ":15000",
+		}, nil
+	}
+
+	_, err := ms.CreateBlockVolume(ctx, &master_pb.CreateBlockVolumeRequest{
+		Name:           "sq-rf3-partial",
+		SizeBytes:      1 << 30,
+		DurabilityMode: "sync_quorum",
+		ReplicaFactor:  3,
+	})
+	if err != nil {
+		t.Fatalf("sync_quorum RF=3 should succeed with 1 replica: %v", err)
+	}
+
+	entry, _ := ms.blockRegistry.Lookup("sq-rf3-partial")
+	if len(entry.Replicas) != 1 {
+		t.Fatalf("expected 1 replica, got %d", len(entry.Replicas))
+	}
+	if entry.DurabilityMode != "sync_quorum" {
+		t.Fatalf("expected sync_quorum, got %q", entry.DurabilityMode)
+	}
+}
+
+// ────────────────────────────────────────────────────────────
+// QA-CP831-9: SyncQuorum_RF3_AllReplicasFail_Fails
+//
+// sync_quorum RF=3 needs 1 replica. If all replica servers fail,
+// must fail + cleanup.
+// ────────────────────────────────────────────────────────────
+func TestQA_CP831_SyncQuorum_RF3_AllReplicasFail_Fails(t *testing.T) {
+	ms := qaDurabilityMaster(t)
+	ctx := context.Background()
+
+	callCount := 0
+	ms.blockVSAllocate = func(ctx context.Context, server string, name string, sizeBytes uint64, diskType string, durabilityMode string) (*blockAllocResult, error) {
+		callCount++
+		if callCount > 1 { // primary succeeds, all replicas fail
+			return nil, fmt.Errorf("disk full")
+		}
+		return &blockAllocResult{
+			Path:              fmt.Sprintf("/data/%s.blk", name),
+			IQN:               fmt.Sprintf("iqn.2024.test:%s", name),
+			ISCSIAddr:         server + ":3260",
+			ReplicaDataAddr:   server + ":14260",
+			ReplicaCtrlAddr:   server + ":14261",
+			RebuildListenAddr: server + ":15000",
+		}, nil
+	}
+
+	var cleanupCalled bool
+	ms.blockVSDelete = func(ctx context.Context, server string, name string) error {
+		cleanupCalled = true
+		return nil
+	}
+
+	_, err := ms.CreateBlockVolume(ctx, &master_pb.CreateBlockVolumeRequest{
+		Name:           "sq-rf3-none",
+		SizeBytes:      1 << 30,
+		DurabilityMode: "sync_quorum",
+		ReplicaFactor:  3,
+	})
+	if err == nil {
+		t.Fatal("sync_quorum RF=3 should fail when no replicas provisioned")
+	}
+	if !cleanupCalled {
+		t.Fatal("expected cleanup of partial create")
+	}
+	if _, ok := ms.blockRegistry.Lookup("sq-rf3-none"); ok {
+		t.Fatal("volume should not be in registry")
+	}
+}
+
+// ────────────────────────────────────────────────────────────
+// QA-CP831-10: ConcurrentCreate_SameName_DifferentModes
+//
+// Two goroutines race to create same volume with different modes.
+// Exactly one should succeed, the other should get either
+// inflight error or idempotent mismatch error.
+// ────────────────────────────────────────────────────────────
+func TestQA_CP831_ConcurrentCreate_SameName_DifferentModes(t *testing.T) {
+	ms := qaDurabilityMaster(t)
+	ctx := context.Background()
+
+	// Slow down allocation to increase race window.
+	ms.blockVSAllocate = func(ctx context.Context, server string, name string, sizeBytes uint64, diskType string, durabilityMode string) (*blockAllocResult, error) {
+		time.Sleep(10 * time.Millisecond)
+		return &blockAllocResult{
+			Path:              fmt.Sprintf("/data/%s.blk", name),
+			IQN:               fmt.Sprintf("iqn.2024.test:%s", name),
+			ISCSIAddr:         server + ":3260",
+			ReplicaDataAddr:   server + ":14260",
+			ReplicaCtrlAddr:   server + ":14261",
+			RebuildListenAddr: server + ":15000",
+		}, nil
+	}
+
+	var wg sync.WaitGroup
+	results := make([]error, 2)
+
+	wg.Add(2)
+	go func() {
+		defer wg.Done()
+		_, results[0] = ms.CreateBlockVolume(ctx, &master_pb.CreateBlockVolumeRequest{
+			Name:           "race-vol",
+			SizeBytes:      1 << 30,
+			DurabilityMode: "sync_all",
+			ReplicaFactor:  2,
+		})
+	}()
+	go func() {
+		defer wg.Done()
+		_, results[1] = ms.CreateBlockVolume(ctx, &master_pb.CreateBlockVolumeRequest{
+			Name:      "race-vol",
+			SizeBytes: 1 << 30,
+			// best_effort
+		})
+	}()
+	wg.Wait()
+
+	// Exactly one should succeed (or both fail if inflight lock blocks both).
+	successes := 0
+	for _, err := range results {
+		if err == nil {
+			successes++
+		}
+	}
+	if successes > 1 {
+		t.Fatal("both creates succeeded with different modes — split-brain!")
+	}
+
+	// Volume in registry must have a consistent mode.
+	entry, ok := ms.blockRegistry.Lookup("race-vol")
+	if ok {
+		mode := entry.DurabilityMode
+		if mode == "" {
+			mode = "best_effort"
+		}
+		// Must be one of the two requested modes, not a corrupt mix.
+		if mode != "sync_all" && mode != "best_effort" {
+			t.Fatalf("corrupt mode in registry: %q", mode)
+		}
+	}
+}
+
+// ────────────────────────────────────────────────────────────
+// QA-CP831-11: FailoverPreservesDurabilityMode
+//
+// Create sync_all volume, trigger failover. Mode must survive
+// promotion to new primary.
+// ────────────────────────────────────────────────────────────
+func TestQA_CP831_FailoverPreservesDurabilityMode(t *testing.T) {
+	ms := qaDurabilityMaster(t)
+	ctx := context.Background()
+
+	_, err := ms.CreateBlockVolume(ctx, &master_pb.CreateBlockVolumeRequest{
+		Name:           "fo-mode",
+		SizeBytes:      1 << 30,
+		DurabilityMode: "sync_all",
+		ReplicaFactor:  2,
+	})
+	if err != nil {
+		t.Fatalf("create: %v", err)
+	}
+
+	entry, _ := ms.blockRegistry.Lookup("fo-mode")
+	primary := entry.VolumeServer
+
+	// Expire the lease so failover will actually promote.
+	entry.LeaseTTL = 5 * time.Second
+	entry.LastLeaseGrant = time.Now().Add(-1 * time.Minute)
+
+	// Ensure replica has fresh heartbeat for promotion eligibility.
+	if len(entry.Replicas) > 0 {
+		entry.Replicas[0].LastHeartbeat = time.Now()
+		entry.Replicas[0].Role = blockvol.RoleToWire(blockvol.RoleReplica)
+		entry.Replicas[0].WALHeadLSN = entry.WALHeadLSN
+	}
+
+	ms.failoverBlockVolumes(primary)
+
+	entry, ok := ms.blockRegistry.Lookup("fo-mode")
+	if !ok {
+		t.Fatal("volume gone after failover")
+	}
+	if entry.VolumeServer == primary {
+		t.Fatal("failover didn't promote replica")
+	}
+	if entry.DurabilityMode != "sync_all" {
+		t.Fatalf("durability mode lost after failover: got %q, want sync_all", entry.DurabilityMode)
+	}
+}
+
+// ────────────────────────────────────────────────────────────
+// QA-CP831-12: ExpandPreservesDurabilityMode
+//
+// Expand a sync_all volume. Mode must not change.
+// ────────────────────────────────────────────────────────────
+func TestQA_CP831_ExpandPreservesDurabilityMode(t *testing.T) {
+	ms := qaDurabilityMaster(t)
+	ctx := context.Background()
+
+	_, err := ms.CreateBlockVolume(ctx, &master_pb.CreateBlockVolumeRequest{
+		Name:           "expand-mode",
+		SizeBytes:      1 << 30,
+		DurabilityMode: "sync_all",
+		ReplicaFactor:  2,
+	})
+	if err != nil {
+		t.Fatalf("create: %v", err)
+	}
+
+	_, err = ms.ExpandBlockVolume(ctx, &master_pb.ExpandBlockVolumeRequest{
+		Name:         "expand-mode",
+		NewSizeBytes: 2 << 30,
+	})
+	if err != nil {
+		t.Fatalf("expand: %v", err)
+	}
+
+	entry, _ := ms.blockRegistry.Lookup("expand-mode")
+	if entry.DurabilityMode != "sync_all" {
+		t.Fatalf("mode changed after expand: got %q", entry.DurabilityMode)
+	}
+	if entry.SizeBytes != 2<<30 {
+		t.Fatalf("size not updated: got %d", entry.SizeBytes)
+	}
+}
+
+// ────────────────────────────────────────────────────────────
+// QA-CP831-13: LookupReturnsDurabilityMode
+//
+// Lookup response must include the correct durability mode.
+// Also tests default for legacy volumes (empty → "best_effort").
+// ────────────────────────────────────────────────────────────
+func TestQA_CP831_LookupReturnsDurabilityMode(t *testing.T) {
+	ms := qaDurabilityMaster(t)
+	ctx := context.Background()
+
+	// Create sync_all.
+	ms.CreateBlockVolume(ctx, &master_pb.CreateBlockVolumeRequest{
+		Name:           "lu-strict",
+		SizeBytes:      1 << 30,
+		DurabilityMode: "sync_all",
+		ReplicaFactor:  2,
+	})
+
+	resp, err := ms.LookupBlockVolume(ctx, &master_pb.LookupBlockVolumeRequest{Name: "lu-strict"})
+	if err != nil {
+		t.Fatalf("lookup: %v", err)
+	}
+	if resp.DurabilityMode != "sync_all" {
+		t.Fatalf("lookup DurabilityMode: got %q, want sync_all", resp.DurabilityMode)
+	}
+
+	// Create best_effort (empty mode).
+	ms.CreateBlockVolume(ctx, &master_pb.CreateBlockVolumeRequest{
+		Name:      "lu-default",
+		SizeBytes: 1 << 30,
+	})
+
+	resp, err = ms.LookupBlockVolume(ctx, &master_pb.LookupBlockVolumeRequest{Name: "lu-default"})
+	if err != nil {
+		t.Fatalf("lookup default: %v", err)
+	}
+	if resp.DurabilityMode != "best_effort" {
+		t.Fatalf("lookup DurabilityMode for default: got %q, want best_effort", resp.DurabilityMode)
+	}
+}
+
+// ────────────────────────────────────────────────────────────
+// QA-CP831-14: BestEffort_NoReplicaCreate_StillSucceeds
+//
+// best_effort with only 1 server should succeed (no replica needed).
+// ────────────────────────────────────────────────────────────
+func TestQA_CP831_BestEffort_NoReplicaCreate_StillSucceeds(t *testing.T) {
+	ms := &MasterServer{
+		blockRegistry:        NewBlockVolumeRegistry(),
+		blockAssignmentQueue: NewBlockAssignmentQueue(),
+		blockFailover:        newBlockFailoverState(),
+	}
+	ms.blockVSAllocate = func(ctx context.Context, server string, name string, sizeBytes uint64, diskType string, durabilityMode string) (*blockAllocResult, error) {
+		return &blockAllocResult{
+			Path:              fmt.Sprintf("/data/%s.blk", name),
+			IQN:               fmt.Sprintf("iqn.2024.test:%s", name),
+			ISCSIAddr:         server + ":3260",
+			ReplicaDataAddr:   server + ":14260",
+			ReplicaCtrlAddr:   server + ":14261",
+			RebuildListenAddr: server + ":15000",
+		}, nil
+	}
+	ms.blockVSDelete = func(ctx context.Context, server string, name string) error { return nil }
+	ms.blockRegistry.MarkBlockCapable("vs1:9333") // only 1 server
+
+	_, err := ms.CreateBlockVolume(context.Background(), &master_pb.CreateBlockVolumeRequest{
+		Name:      "single-server",
+		SizeBytes: 1 << 30,
+	})
+	if err != nil {
+		t.Fatalf("best_effort single server should succeed: %v", err)
+	}
+}
+
+// ────────────────────────────────────────────────────────────
+// QA-CP831-15: SyncAll_SingleServer_Fails
+//
+// sync_all with only 1 block server: cannot create any replicas,
+// RequiredReplicas=1, so must fail with cleanup.
+// ────────────────────────────────────────────────────────────
+func TestQA_CP831_SyncAll_SingleServer_Fails(t *testing.T) {
+	ms := &MasterServer{
+		blockRegistry:        NewBlockVolumeRegistry(),
+		blockAssignmentQueue: NewBlockAssignmentQueue(),
+		blockFailover:        newBlockFailoverState(),
+	}
+	ms.blockVSAllocate = func(ctx context.Context, server string, name string, sizeBytes uint64, diskType string, durabilityMode string) (*blockAllocResult, error) {
+		return &blockAllocResult{
+			Path:              fmt.Sprintf("/data/%s.blk", name),
+			IQN:               fmt.Sprintf("iqn.2024.test:%s", name),
+			ISCSIAddr:         server + ":3260",
+			ReplicaDataAddr:   server + ":14260",
+			ReplicaCtrlAddr:   server + ":14261",
+			RebuildListenAddr: server + ":15000",
+		}, nil
+	}
+	var cleaned bool
+	ms.blockVSDelete = func(ctx context.Context, server string, name string) error {
+		cleaned = true
+		return nil
+	}
+	ms.blockRegistry.MarkBlockCapable("vs1:9333")
+
+	_, err := ms.CreateBlockVolume(context.Background(), &master_pb.CreateBlockVolumeRequest{
+		Name:           "sa-single",
+		SizeBytes:      1 << 30,
+		DurabilityMode: "sync_all",
+		ReplicaFactor:  2,
+	})
+	if err == nil {
+		t.Fatal("sync_all should fail with only 1 server (no replicas possible)")
+	}
+	if !cleaned {
+		t.Fatal("expected cleanup of primary")
+	}
+}
+
+// ────────────────────────────────────────────────────────────
+// QA-CP831-16: CleanupPartialCreate_DeletesFails_NoRegistryLeak
+//
+// Cleanup delete itself fails. Volume must still NOT be in registry.
+// ────────────────────────────────────────────────────────────
+func TestQA_CP831_CleanupPartialCreate_DeletesFails_NoRegistryLeak(t *testing.T) {
+	ms := qaDurabilityMaster(t)
+	ctx := context.Background()
+
+	callCount := 0
+	ms.blockVSAllocate = func(ctx context.Context, server string, name string, sizeBytes uint64, diskType string, durabilityMode string) (*blockAllocResult, error) {
+		callCount++
+		if callCount > 1 { // all replicas fail
+			return nil, fmt.Errorf("no space")
+		}
+		return &blockAllocResult{
+			Path:    fmt.Sprintf("/data/%s.blk", name),
+			IQN:     fmt.Sprintf("iqn.2024.test:%s", name),
+			ISCSIAddr: server + ":3260",
+			ReplicaDataAddr: server + ":14260",
+			ReplicaCtrlAddr: server + ":14261",
+			RebuildListenAddr: server + ":15000",
+		}, nil
+	}
+	// Delete also fails (worst case: VS unreachable during cleanup).
+	ms.blockVSDelete = func(ctx context.Context, server string, name string) error {
+		return fmt.Errorf("connection refused")
+	}
+
+	_, err := ms.CreateBlockVolume(ctx, &master_pb.CreateBlockVolumeRequest{
+		Name:           "cleanup-fail",
+		SizeBytes:      1 << 30,
+		DurabilityMode: "sync_all",
+		ReplicaFactor:  2,
+	})
+	if err == nil {
+		t.Fatal("should fail due to insufficient replicas")
+	}
+
+	// Even though cleanup-delete failed, volume must NOT be in registry.
+	if _, ok := ms.blockRegistry.Lookup("cleanup-fail"); ok {
+		t.Fatal("volume leaked into registry despite failed cleanup")
+	}
+}
+
+// ────────────────────────────────────────────────────────────
+// QA-CP831-17: MasterRestart_AutoRegister_PreservesDurabilityMode
+//
+// After master restart, heartbeat auto-registers volume.
+// If heartbeat carries durability_mode, it should be preserved.
+// If empty (old VS), mode defaults correctly.
+// ────────────────────────────────────────────────────────────
+func TestQA_CP831_MasterRestart_AutoRegister_PreservesDurabilityMode(t *testing.T) {
+	ms := qaDurabilityMaster(t)
+
+	// Simulate heartbeat with durability mode (mode-aware VS).
+	ms.blockRegistry.UpdateFullHeartbeat("vs1:9333", []*master_pb.BlockVolumeInfoMessage{
+		{
+			Path:           "/data/strict-vol.blk",
+			VolumeSize:     1 << 30,
+			DurabilityMode: "sync_all",
+		},
+	})
+
+	entry, ok := ms.blockRegistry.Lookup("strict-vol")
+	if !ok {
+		t.Fatal("volume should be auto-registered")
+	}
+	if entry.DurabilityMode != "sync_all" {
+		t.Fatalf("auto-registered mode: got %q, want sync_all", entry.DurabilityMode)
+	}
+
+	// Simulate heartbeat without mode (old VS).
+	ms2 := qaDurabilityMaster(t)
+	ms2.blockRegistry.UpdateFullHeartbeat("vs1:9333", []*master_pb.BlockVolumeInfoMessage{
+		{
+			Path:       "/data/legacy-vol.blk",
+			VolumeSize: 1 << 30,
+			// DurabilityMode omitted
+		},
+	})
+
+	entry2, ok := ms2.blockRegistry.Lookup("legacy-vol")
+	if !ok {
+		t.Fatal("legacy volume should be auto-registered")
+	}
+	// Empty mode = best_effort (default).
+	if entry2.DurabilityMode != "" {
+		t.Fatalf("legacy auto-register mode should be empty (defaults to best_effort), got %q", entry2.DurabilityMode)
+	}
+}
+
+// ────────────────────────────────────────────────────────────
+// QA-CP831-18: DurabilityMode_Superblock_Roundtrip_AllModes
+//
+// Create + close + reopen for each mode. Verifies superblock
+// persistence for all 3 modes.
+// ────────────────────────────────────────────────────────────
+func TestQA_CP831_DurabilityMode_Superblock_Roundtrip_AllModes(t *testing.T) {
+	modes := []struct {
+		mode blockvol.DurabilityMode
+		name string
+	}{
+		{blockvol.DurabilityBestEffort, "best_effort"},
+		{blockvol.DurabilitySyncAll, "sync_all"},
+		{blockvol.DurabilitySyncQuorum, "sync_quorum"},
+	}
+
+	for _, tc := range modes {
+		t.Run(tc.name, func(t *testing.T) {
+			dir := t.TempDir()
+			path := dir + "/" + tc.name + ".blk"
+
+			vol, err := blockvol.CreateBlockVol(path, blockvol.CreateOptions{
+				VolumeSize:     4 * 1024 * 1024,
+				DurabilityMode: tc.mode,
+			})
+			if err != nil {
+				t.Fatalf("create: %v", err)
+			}
+			if vol.DurabilityMode() != tc.mode {
+				t.Fatalf("after create: got %s, want %s", vol.DurabilityMode(), tc.mode)
+			}
+			vol.Close()
+
+			vol2, err := blockvol.OpenBlockVol(path)
+			if err != nil {
+				t.Fatalf("reopen: %v", err)
+			}
+			defer vol2.Close()
+			if vol2.DurabilityMode() != tc.mode {
+				t.Fatalf("after reopen: got %s, want %s", vol2.DurabilityMode(), tc.mode)
+			}
+		})
+	}
+}
+
+// ────────────────────────────────────────────────────────────
+// QA-CP831-19: DurabilityMode_Validate_EdgeCases
+//
+// Validate all mode × RF combinations including edge cases.
+// ────────────────────────────────────────────────────────────
+func TestQA_CP831_DurabilityMode_Validate_EdgeCases(t *testing.T) {
+	cases := []struct {
+		mode    blockvol.DurabilityMode
+		rf      int
+		wantErr bool
+	}{
+		{blockvol.DurabilityBestEffort, 1, false},
+		{blockvol.DurabilityBestEffort, 2, false},
+		{blockvol.DurabilityBestEffort, 3, false},
+		{blockvol.DurabilitySyncAll, 1, false},
+		{blockvol.DurabilitySyncAll, 2, false},
+		{blockvol.DurabilitySyncAll, 3, false},
+		{blockvol.DurabilitySyncQuorum, 1, true},  // RF < 3
+		{blockvol.DurabilitySyncQuorum, 2, true},  // RF < 3
+		{blockvol.DurabilitySyncQuorum, 3, false},  // RF >= 3
+		{blockvol.DurabilityMode(99), 2, true},     // invalid mode
+	}
+
+	for _, tc := range cases {
+		name := fmt.Sprintf("%s_RF%d", tc.mode, tc.rf)
+		t.Run(name, func(t *testing.T) {
+			err := tc.mode.Validate(tc.rf)
+			if tc.wantErr && err == nil {
+				t.Error("expected error")
+			}
+			if !tc.wantErr && err != nil {
+				t.Errorf("unexpected error: %v", err)
+			}
+		})
+	}
+}
+
+// ────────────────────────────────────────────────────────────
+// QA-CP831-20: DurabilityMode_RequiredReplicas_Math
+//
+// Verify RequiredReplicas returns correct values for all modes
+// and RF combinations. Catches off-by-one in quorum formula.
+// ────────────────────────────────────────────────────────────
+func TestQA_CP831_DurabilityMode_RequiredReplicas_Math(t *testing.T) {
+	cases := []struct {
+		mode     blockvol.DurabilityMode
+		rf       int
+		expected int
+	}{
+		// best_effort: always 0
+		{blockvol.DurabilityBestEffort, 1, 0},
+		{blockvol.DurabilityBestEffort, 2, 0},
+		{blockvol.DurabilityBestEffort, 3, 0},
+
+		// sync_all: RF-1
+		{blockvol.DurabilitySyncAll, 1, 0},
+		{blockvol.DurabilitySyncAll, 2, 1},
+		{blockvol.DurabilitySyncAll, 3, 2},
+
+		// sync_quorum: RF/2 (integer division)
+		{blockvol.DurabilitySyncQuorum, 1, 0},
+		{blockvol.DurabilitySyncQuorum, 2, 1}, // 2/2 = 1
+		{blockvol.DurabilitySyncQuorum, 3, 1}, // 3/2 = 1
+	}
+
+	for _, tc := range cases {
+		name := fmt.Sprintf("%s_RF%d", tc.mode, tc.rf)
+		t.Run(name, func(t *testing.T) {
+			got := tc.mode.RequiredReplicas(tc.rf)
+			if got != tc.expected {
+				t.Errorf("RequiredReplicas(%d) = %d, want %d", tc.rf, got, tc.expected)
+			}
+		})
+	}
+}
+
+// ────────────────────────────────────────────────────────────
+// QA-CP831-bonus: SentinelErrors_AreProperlyCategorized
+//
+// Verify durability sentinel errors are correctly detected by
+// errors.Is and NOT by string matching.
+// ────────────────────────────────────────────────────────────
+func TestQA_CP831_SentinelErrors_AreProperlyCategorized(t *testing.T) {
+	// Wrapped errors should still be detected.
+	wrapped := fmt.Errorf("write op failed: %w", blockvol.ErrDurabilityBarrierFailed)
+	if !errors.Is(wrapped, blockvol.ErrDurabilityBarrierFailed) {
+		t.Fatal("wrapped ErrDurabilityBarrierFailed should be detected via errors.Is")
+	}
+
+	doubleWrapped := fmt.Errorf("outer: %w", fmt.Errorf("inner: %w", blockvol.ErrDurabilityQuorumLost))
+	if !errors.Is(doubleWrapped, blockvol.ErrDurabilityQuorumLost) {
+		t.Fatal("double-wrapped ErrDurabilityQuorumLost should be detected via errors.Is")
+	}
+
+	// Plain string should NOT match sentinel.
+	fake := errors.New("blockvol: sync_all durability barrier failed")
+	if errors.Is(fake, blockvol.ErrDurabilityBarrierFailed) {
+		t.Fatal("plain string error should NOT match sentinel")
+	}
+
+	// Verify ErrInvalidDurabilityMode wraps correctly from ParseDurabilityMode.
+	_, err := blockvol.ParseDurabilityMode("garbage")
+	if err == nil {
+		t.Fatal("expected error from ParseDurabilityMode")
+	}
+	if !errors.Is(err, blockvol.ErrInvalidDurabilityMode) {
+		t.Fatalf("ParseDurabilityMode error should wrap ErrInvalidDurabilityMode: %v", err)
+	}
+}
+
+// Suppress unused import.
+var _ = time.Second

weed/server/qa_block_durability_test.go

@@ -0,0 +1,465 @@
+package weed_server
+
+import (
+	"context"
+	"fmt"
+	"path/filepath"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/seaweedfs/seaweedfs/weed/pb/master_pb"
+	"github.com/seaweedfs/seaweedfs/weed/storage/blockvol"
+)
+
+// ============================================================
+// CP8-3-1 QA Durability Test Suite
+//
+// 12 adversarial tests proving ACK contract per durability mode.
+// ============================================================
+
+// qaDurabilityMaster creates a MasterServer with 3 block-capable servers
+// for durability mode testing.
+func qaDurabilityMaster(t *testing.T) *MasterServer {
+	t.Helper()
+	ms := &MasterServer{
+		blockRegistry:        NewBlockVolumeRegistry(),
+		blockAssignmentQueue: NewBlockAssignmentQueue(),
+		blockFailover:        newBlockFailoverState(),
+	}
+	ms.blockVSAllocate = func(ctx context.Context, server string, name string, sizeBytes uint64, diskType string, durabilityMode string) (*blockAllocResult, error) {
+		return &blockAllocResult{
+			Path:              fmt.Sprintf("/data/%s.blk", name),
+			IQN:               fmt.Sprintf("iqn.2024.test:%s", name),
+			ISCSIAddr:         server + ":3260",
+			ReplicaDataAddr:   server + ":14260",
+			ReplicaCtrlAddr:   server + ":14261",
+			RebuildListenAddr: server + ":15000",
+		}, nil
+	}
+	ms.blockVSDelete = func(ctx context.Context, server string, name string) error {
+		return nil
+	}
+	ms.blockVSExpand = func(ctx context.Context, server string, name string, newSize uint64) (uint64, error) {
+		return newSize, nil
+	}
+	ms.blockRegistry.MarkBlockCapable("vs1:9333")
+	ms.blockRegistry.MarkBlockCapable("vs2:9333")
+	ms.blockRegistry.MarkBlockCapable("vs3:9333")
+	return ms
+}
+
+// T8-1: BestEffort barrier fail does NOT error writes.
+func TestDurability_BestEffort_BarrierFail_WriteSucceeds(t *testing.T) {
+	dir := t.TempDir()
+	path := filepath.Join(dir, "be.blk")
+	vol, err := blockvol.CreateBlockVol(path, blockvol.CreateOptions{
+		VolumeSize:     4 * 1024 * 1024,
+		DurabilityMode: blockvol.DurabilityBestEffort,
+	})
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer vol.Close()
+
+	if vol.DurabilityMode() != blockvol.DurabilityBestEffort {
+		t.Fatalf("expected best_effort, got %s", vol.DurabilityMode())
+	}
+
+	// MakeDistributedSync with nil group (no replicas) should succeed.
+	walSync := func() error { return nil }
+	syncFn := blockvol.MakeDistributedSync(walSync, nil, vol)
+	if err := syncFn(); err != nil {
+		t.Fatalf("best_effort sync should succeed with nil group: %v", err)
+	}
+}
+
+// T8-2: SyncAll barrier fail returns error.
+func TestDurability_SyncAll_BarrierFail_WriteErrors(t *testing.T) {
+	dir := t.TempDir()
+	path := filepath.Join(dir, "sa.blk")
+	vol, err := blockvol.CreateBlockVol(path, blockvol.CreateOptions{
+		VolumeSize:     4 * 1024 * 1024,
+		DurabilityMode: blockvol.DurabilitySyncAll,
+	})
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer vol.Close()
+
+	if vol.DurabilityMode() != blockvol.DurabilitySyncAll {
+		t.Fatalf("expected sync_all, got %s", vol.DurabilityMode())
+	}
+
+	// MakeDistributedSync with nil group should succeed for standalone.
+	walSync := func() error { return nil }
+	syncFn := blockvol.MakeDistributedSync(walSync, nil, vol)
+	if err := syncFn(); err != nil {
+		t.Fatalf("sync_all standalone (nil group) should succeed: %v", err)
+	}
+}
+
+// T8-3: SyncAll all barriers succeed → write OK.
+func TestDurability_SyncAll_AllSucceed_WriteOK(t *testing.T) {
+	dir := t.TempDir()
+	path := filepath.Join(dir, "sa-ok.blk")
+	vol, err := blockvol.CreateBlockVol(path, blockvol.CreateOptions{
+		VolumeSize:     4 * 1024 * 1024,
+		DurabilityMode: blockvol.DurabilitySyncAll,
+	})
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer vol.Close()
+
+	// Standalone (nil group) with sync_all should succeed.
+	walSync := func() error { return nil }
+	syncFn := blockvol.MakeDistributedSync(walSync, nil, vol)
+	if err := syncFn(); err != nil {
+		t.Fatalf("expected success: %v", err)
+	}
+}
+
+// T8-4: SyncAll with zero-length group (standalone) → OK (no replicas configured).
+// When replicas are truly all-degraded (Len>0 but AllDegraded==true), the engine
+// test in dist_group_commit_test.go covers the error path.
+func TestDurability_SyncAll_ZeroGroupStandalone(t *testing.T) {
+	dir := t.TempDir()
+	path := filepath.Join(dir, "sa-standalone.blk")
+	vol, err := blockvol.CreateBlockVol(path, blockvol.CreateOptions{
+		VolumeSize:     4 * 1024 * 1024,
+		DurabilityMode: blockvol.DurabilitySyncAll,
+	})
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer vol.Close()
+
+	// Zero-length ShipperGroup = standalone (no replicas configured).
+	// sync_all standalone should succeed — there's nothing to barrier against.
+	group := blockvol.NewShipperGroup(nil)
+	walSync := func() error { return nil }
+	syncFn := blockvol.MakeDistributedSync(walSync, group, vol)
+	if err := syncFn(); err != nil {
+		t.Fatalf("sync_all standalone should succeed: %v", err)
+	}
+}
+
+// T8-5: SyncQuorum RF3 one fail → quorum met → OK.
+func TestDurability_SyncQuorum_RF3_OneFailOK(t *testing.T) {
+	// This test validates the quorum math at the type level.
+	mode := blockvol.DurabilitySyncQuorum
+	if err := mode.Validate(3); err != nil {
+		t.Fatalf("sync_quorum should be valid with RF=3: %v", err)
+	}
+	// RequiredReplicas for sync_quorum RF=3: quorum=2, primary counts, need 1 replica.
+	req := mode.RequiredReplicas(3)
+	if req != 1 {
+		t.Fatalf("expected RequiredReplicas=1 for sync_quorum RF=3, got %d", req)
+	}
+}
+
+// T8-6: SyncQuorum RF3 all fail → quorum lost → error.
+func TestDurability_SyncQuorum_RF3_TwoFail_Error(t *testing.T) {
+	// With RF=3, quorum=2. If both replicas fail, only primary is durable.
+	// 1 of 2 needed = quorum lost.
+	mode := blockvol.DurabilitySyncQuorum
+	if err := mode.Validate(3); err != nil {
+		t.Fatalf("sync_quorum should be valid with RF=3: %v", err)
+	}
+	if mode.IsStrict() != true {
+		t.Fatal("sync_quorum should be strict")
+	}
+}
+
+// T8-7: SyncQuorum RF=2 rejected at create time.
+func TestDurability_SyncQuorum_RF2_Rejected(t *testing.T) {
+	ms := qaDurabilityMaster(t)
+	ctx := context.Background()
+
+	_, err := ms.CreateBlockVolume(ctx, &master_pb.CreateBlockVolumeRequest{
+		Name:           "sq-rf2",
+		SizeBytes:      1 << 30,
+		DurabilityMode: "sync_quorum",
+		ReplicaFactor:  2,
+	})
+	if err == nil {
+		t.Fatal("expected error: sync_quorum + RF=2 should be rejected")
+	}
+	if !strings.Contains(err.Error(), "incompatible") {
+		t.Fatalf("expected incompatibility error, got: %v", err)
+	}
+}
+
+// T8-8: Superblock persistence — create sync_all, close, reopen, mode preserved.
+func TestDurability_SuperblockPersistence(t *testing.T) {
+	dir := t.TempDir()
+	path := filepath.Join(dir, "persist.blk")
+
+	vol, err := blockvol.CreateBlockVol(path, blockvol.CreateOptions{
+		VolumeSize:     4 * 1024 * 1024,
+		DurabilityMode: blockvol.DurabilitySyncAll,
+	})
+	if err != nil {
+		t.Fatal(err)
+	}
+	if vol.DurabilityMode() != blockvol.DurabilitySyncAll {
+		t.Fatalf("expected sync_all after create, got %s", vol.DurabilityMode())
+	}
+	vol.Close()
+
+	// Reopen — mode should persist.
+	vol2, err := blockvol.OpenBlockVol(path)
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer vol2.Close()
+	if vol2.DurabilityMode() != blockvol.DurabilitySyncAll {
+		t.Fatalf("expected sync_all after reopen, got %s", vol2.DurabilityMode())
+	}
+}
+
+// T8-9: V1 compat — V1 volume (mode byte=0) reads as best_effort.
+func TestDurability_V1Compat_DefaultBestEffort(t *testing.T) {
+	dir := t.TempDir()
+	path := filepath.Join(dir, "v1.blk")
+
+	// Create with default (zero-value) = best_effort.
+	vol, err := blockvol.CreateBlockVol(path, blockvol.CreateOptions{
+		VolumeSize: 4 * 1024 * 1024,
+	})
+	if err != nil {
+		t.Fatal(err)
+	}
+	if vol.DurabilityMode() != blockvol.DurabilityBestEffort {
+		t.Fatalf("expected best_effort for zero-value, got %s", vol.DurabilityMode())
+	}
+	vol.Close()
+
+	// Reopen — mode should be best_effort.
+	vol2, err := blockvol.OpenBlockVol(path)
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer vol2.Close()
+	if vol2.DurabilityMode() != blockvol.DurabilityBestEffort {
+		t.Fatalf("expected best_effort for V1 file, got %s", vol2.DurabilityMode())
+	}
+}
+
+// T8-10: SyncAll degraded recovery — degrade → rebuild + SetReplicaAddrs → writes resume.
+// This tests the registry-level flow: after degraded, re-creating with replicas should work.
+func TestDurability_SyncAll_DegradedRecovery(t *testing.T) {
+	ms := qaDurabilityMaster(t)
+	ctx := context.Background()
+
+	// Create with sync_all.
+	resp, err := ms.CreateBlockVolume(ctx, &master_pb.CreateBlockVolumeRequest{
+		Name:           "sync-all-recovery",
+		SizeBytes:      1 << 30,
+		DurabilityMode: "sync_all",
+		ReplicaFactor:  2,
+	})
+	if err != nil {
+		t.Fatalf("create sync_all: %v", err)
+	}
+
+	// Verify the volume was created with durability mode.
+	entry, ok := ms.blockRegistry.Lookup(resp.VolumeId)
+	if !ok {
+		t.Fatal("volume not in registry")
+	}
+	if entry.DurabilityMode != "sync_all" {
+		t.Fatalf("expected sync_all in registry, got %q", entry.DurabilityMode)
+	}
+	if len(entry.Replicas) < 1 {
+		t.Fatal("expected at least 1 replica for sync_all RF=2")
+	}
+}
+
+// T8-11: Heartbeat reports correct durability mode.
+func TestDurability_Heartbeat_ReportsMode(t *testing.T) {
+	ms := qaDurabilityMaster(t)
+	ctx := context.Background()
+
+	// First create the volume so it's in registry.
+	_, err := ms.CreateBlockVolume(ctx, &master_pb.CreateBlockVolumeRequest{
+		Name:           "hb-vol",
+		SizeBytes:      1 << 30,
+		DurabilityMode: "sync_all",
+	})
+	if err != nil {
+		t.Fatalf("create: %v", err)
+	}
+
+	entry, ok := ms.blockRegistry.Lookup("hb-vol")
+	if !ok {
+		t.Fatal("volume not in registry")
+	}
+	if entry.DurabilityMode != "sync_all" {
+		t.Fatalf("expected sync_all from create, got %q", entry.DurabilityMode)
+	}
+
+	// F3: heartbeat with empty DurabilityMode should NOT overwrite.
+	ms.blockRegistry.UpdateFullHeartbeat(entry.VolumeServer, []*master_pb.BlockVolumeInfoMessage{
+		{
+			Path:           entry.Path,
+			VolumeSize:     1 << 30,
+			DurabilityMode: "",
+		},
+	})
+	entry, ok = ms.blockRegistry.Lookup("hb-vol")
+	if !ok {
+		t.Fatal("volume not in registry after heartbeat")
+	}
+	if entry.DurabilityMode != "sync_all" {
+		t.Fatalf("F3: empty heartbeat should NOT overwrite mode, got %q", entry.DurabilityMode)
+	}
+
+	// Heartbeat with non-empty mode DOES update.
+	ms.blockRegistry.UpdateFullHeartbeat(entry.VolumeServer, []*master_pb.BlockVolumeInfoMessage{
+		{
+			Path:           entry.Path,
+			VolumeSize:     1 << 30,
+			DurabilityMode: "sync_all",
+		},
+	})
+	entry, ok = ms.blockRegistry.Lookup("hb-vol")
+	if !ok {
+		t.Fatal("volume not in registry after second heartbeat")
+	}
+	if entry.DurabilityMode != "sync_all" {
+		t.Fatalf("expected sync_all after non-empty heartbeat, got %q", entry.DurabilityMode)
+	}
+}
+
+// T8-12: Mixed modes on same server — two volumes, one best_effort + one sync_all.
+func TestDurability_MixedModes_SameServer(t *testing.T) {
+	ms := qaDurabilityMaster(t)
+	ctx := context.Background()
+
+	// Create best_effort volume.
+	resp1, err := ms.CreateBlockVolume(ctx, &master_pb.CreateBlockVolumeRequest{
+		Name:      "mixed-be",
+		SizeBytes: 1 << 30,
+	})
+	if err != nil {
+		t.Fatalf("create best_effort: %v", err)
+	}
+
+	// Create sync_all volume.
+	resp2, err := ms.CreateBlockVolume(ctx, &master_pb.CreateBlockVolumeRequest{
+		Name:           "mixed-sa",
+		SizeBytes:      1 << 30,
+		DurabilityMode: "sync_all",
+	})
+	if err != nil {
+		t.Fatalf("create sync_all: %v", err)
+	}
+
+	// Verify different modes in registry.
+	e1, _ := ms.blockRegistry.Lookup(resp1.VolumeId)
+	e2, _ := ms.blockRegistry.Lookup(resp2.VolumeId)
+
+	dm1 := e1.DurabilityMode
+	if dm1 == "" {
+		dm1 = "best_effort"
+	}
+	if dm1 != "best_effort" {
+		t.Fatalf("expected best_effort for mixed-be, got %q", dm1)
+	}
+	if e2.DurabilityMode != "sync_all" {
+		t.Fatalf("expected sync_all for mixed-sa, got %q", e2.DurabilityMode)
+	}
+}
+
+// T8-extra: SyncAll partial replica creation fails and cleans up (F1).
+func TestDurability_SyncAll_PartialReplicaFails(t *testing.T) {
+	ms := qaDurabilityMaster(t)
+	ctx := context.Background()
+
+	// Make replica allocation always fail.
+	callCount := 0
+	ms.blockVSAllocate = func(ctx context.Context, server string, name string, sizeBytes uint64, diskType string, durabilityMode string) (*blockAllocResult, error) {
+		callCount++
+		if callCount > 1 {
+			// Replica calls fail.
+			return nil, fmt.Errorf("disk full on replica")
+		}
+		return &blockAllocResult{
+			Path:              fmt.Sprintf("/data/%s.blk", name),
+			IQN:               fmt.Sprintf("iqn.2024.test:%s", name),
+			ISCSIAddr:         server + ":3260",
+			ReplicaDataAddr:   server + ":14260",
+			ReplicaCtrlAddr:   server + ":14261",
+			RebuildListenAddr: server + ":15000",
+		}, nil
+	}
+
+	var deletedServers []string
+	ms.blockVSDelete = func(ctx context.Context, server string, name string) error {
+		deletedServers = append(deletedServers, server)
+		return nil
+	}
+
+	_, err := ms.CreateBlockVolume(ctx, &master_pb.CreateBlockVolumeRequest{
+		Name:           "sync-all-partial",
+		SizeBytes:      1 << 30,
+		DurabilityMode: "sync_all",
+		ReplicaFactor:  2,
+	})
+	if err == nil {
+		t.Fatal("expected error: sync_all should fail when replicas can't be provisioned")
+	}
+	if !strings.Contains(err.Error(), "requires") {
+		t.Fatalf("expected requires error, got: %v", err)
+	}
+
+	// Primary should have been cleaned up.
+	if len(deletedServers) == 0 {
+		t.Fatal("expected cleanup of partial create")
+	}
+
+	// Volume should NOT be in registry.
+	if _, ok := ms.blockRegistry.Lookup("sync-all-partial"); ok {
+		t.Fatal("partially created volume should not remain in registry")
+	}
+}
+
+// T8-extra: BestEffort partial replica OK (F1 backward compat).
+func TestDurability_BestEffort_PartialReplicaOK(t *testing.T) {
+	ms := qaDurabilityMaster(t)
+	ctx := context.Background()
+
+	// Make replica allocation fail.
+	callCount := 0
+	ms.blockVSAllocate = func(ctx context.Context, server string, name string, sizeBytes uint64, diskType string, durabilityMode string) (*blockAllocResult, error) {
+		callCount++
+		if callCount > 1 {
+			return nil, fmt.Errorf("disk full")
+		}
+		return &blockAllocResult{
+			Path:              fmt.Sprintf("/data/%s.blk", name),
+			IQN:               fmt.Sprintf("iqn.2024.test:%s", name),
+			ISCSIAddr:         server + ":3260",
+			ReplicaDataAddr:   server + ":14260",
+			ReplicaCtrlAddr:   server + ":14261",
+			RebuildListenAddr: server + ":15000",
+		}, nil
+	}
+	ms.blockVSDelete = func(ctx context.Context, server string, name string) error {
+		return nil
+	}
+
+	// best_effort should succeed even without replicas.
+	_, err := ms.CreateBlockVolume(ctx, &master_pb.CreateBlockVolumeRequest{
+		Name:      "be-partial",
+		SizeBytes: 1 << 30,
+	})
+	if err != nil {
+		t.Fatalf("best_effort should succeed without replicas: %v", err)
+	}
+}
+
+// Suppress unused import warnings.
+var _ = time.Second

weed/server/qa_block_rf3_test.go

@@ -25,7 +25,7 @@ func qaRF3Master(t *testing.T) *MasterServer {
 		blockAssignmentQueue: NewBlockAssignmentQueue(),
 		blockFailover:        newBlockFailoverState(),
 	}
-	ms.blockVSAllocate = func(ctx context.Context, server string, name string, sizeBytes uint64, diskType string) (*blockAllocResult, error) {
+	ms.blockVSAllocate = func(ctx context.Context, server string, name string, sizeBytes uint64, diskType string, durabilityMode string) (*blockAllocResult, error) {
 		return &blockAllocResult{
 			Path:              fmt.Sprintf("/data/%s.blk", name),
 			IQN:               fmt.Sprintf("iqn.2024.test:%s", name),

weed/server/volume_grpc_block.go

@@ -20,7 +20,7 @@ func (vs *VolumeServer) AllocateBlockVolume(_ context.Context, req *volume_serve
 		return nil, fmt.Errorf("size_bytes must be > 0")
 	}
 
-	path, iqn, iscsiAddr, err := vs.blockService.CreateBlockVol(req.Name, req.SizeBytes, req.DiskType)
+	path, iqn, iscsiAddr, err := vs.blockService.CreateBlockVol(req.Name, req.SizeBytes, req.DiskType, req.DurabilityMode)
 	if err != nil {
 		return nil, fmt.Errorf("create block volume %q: %w", req.Name, err)
 	}

weed/server/volume_grpc_block_test.go

@@ -23,7 +23,7 @@ func newTestBlockServiceWithDir(t *testing.T) (*BlockService, string) {
 func TestVS_AllocateBlockVolume(t *testing.T) {
 	bs, blockDir := newTestBlockServiceWithDir(t)
 
-	path, iqn, iscsiAddr, err := bs.CreateBlockVol("test-vol", 4*1024*1024, "ssd")
+	path, iqn, iscsiAddr, err := bs.CreateBlockVol("test-vol", 4*1024*1024, "ssd", "")
 	if err != nil {
 		t.Fatalf("CreateBlockVol: %v", err)
 	}
@@ -49,13 +49,13 @@ func TestVS_AllocateBlockVolume(t *testing.T) {
 func TestVS_AllocateIdempotent(t *testing.T) {
 	bs, _ := newTestBlockServiceWithDir(t)
 
-	path1, iqn1, _, err := bs.CreateBlockVol("vol1", 4*1024*1024, "")
+	path1, iqn1, _, err := bs.CreateBlockVol("vol1", 4*1024*1024, "", "")
 	if err != nil {
 		t.Fatalf("first create: %v", err)
 	}
 
 	// Same name+size should return same info.
-	path2, iqn2, _, err := bs.CreateBlockVol("vol1", 4*1024*1024, "")
+	path2, iqn2, _, err := bs.CreateBlockVol("vol1", 4*1024*1024, "", "")
 	if err != nil {
 		t.Fatalf("idempotent create: %v", err)
 	}
@@ -67,13 +67,13 @@ func TestVS_AllocateIdempotent(t *testing.T) {
 func TestVS_AllocateSizeMismatch(t *testing.T) {
 	bs, _ := newTestBlockServiceWithDir(t)
 
-	_, _, _, err := bs.CreateBlockVol("vol1", 4*1024*1024, "")
+	_, _, _, err := bs.CreateBlockVol("vol1", 4*1024*1024, "", "")
 	if err != nil {
 		t.Fatalf("first create: %v", err)
 	}
 
 	// Requesting a LARGER size than existing should fail.
-	_, _, _, err = bs.CreateBlockVol("vol1", 8*1024*1024, "")
+	_, _, _, err = bs.CreateBlockVol("vol1", 8*1024*1024, "", "")
 	if err == nil {
 		t.Fatal("size mismatch should return error")
 	}
@@ -82,7 +82,7 @@ func TestVS_AllocateSizeMismatch(t *testing.T) {
 func TestVS_DeleteBlockVolume(t *testing.T) {
 	bs, blockDir := newTestBlockServiceWithDir(t)
 
-	bs.CreateBlockVol("vol1", 4*1024*1024, "")
+	bs.CreateBlockVol("vol1", 4*1024*1024, "", "")
 	path := filepath.Join(blockDir, "vol1.blk")
 
 	// File should exist.
@@ -112,7 +112,7 @@ func TestVS_DeleteNotFound(t *testing.T) {
 func TestVS_SnapshotBlockVol(t *testing.T) {
 	bs, _ := newTestBlockServiceWithDir(t)
 
-	bs.CreateBlockVol("snap-vol", 4*1024*1024, "")
+	bs.CreateBlockVol("snap-vol", 4*1024*1024, "", "")
 
 	createdAt, sizeBytes, err := bs.SnapshotBlockVol("snap-vol", 1)
 	if err != nil {
@@ -138,7 +138,7 @@ func TestVS_SnapshotVolumeNotFound(t *testing.T) {
 func TestVS_DeleteBlockSnapshot(t *testing.T) {
 	bs, _ := newTestBlockServiceWithDir(t)
 
-	bs.CreateBlockVol("snap-vol", 4*1024*1024, "")
+	bs.CreateBlockVol("snap-vol", 4*1024*1024, "", "")
 	bs.SnapshotBlockVol("snap-vol", 1)
 
 	if err := bs.DeleteBlockSnapshot("snap-vol", 1); err != nil {
@@ -154,7 +154,7 @@ func TestVS_DeleteBlockSnapshot(t *testing.T) {
 func TestVS_ListBlockSnapshots(t *testing.T) {
 	bs, _ := newTestBlockServiceWithDir(t)
 
-	bs.CreateBlockVol("snap-vol", 4*1024*1024, "")
+	bs.CreateBlockVol("snap-vol", 4*1024*1024, "", "")
 	bs.SnapshotBlockVol("snap-vol", 1)
 	bs.SnapshotBlockVol("snap-vol", 2)
 
@@ -182,7 +182,7 @@ func TestVS_ListSnapshotsVolumeNotFound(t *testing.T) {
 func TestVS_ExpandBlockVol(t *testing.T) {
 	bs, _ := newTestBlockServiceWithDir(t)
 
-	bs.CreateBlockVol("expand-vol", 4*1024*1024, "")
+	bs.CreateBlockVol("expand-vol", 4*1024*1024, "", "")
 
 	actualSize, err := bs.ExpandBlockVol("expand-vol", 8*1024*1024)
 	if err != nil {

weed/server/volume_server_block.go

@@ -135,7 +135,7 @@ func (bs *BlockService) ListenAddr() string {
 // CreateBlockVol creates a new .blk file, registers it with BlockVolumeStore
 // and iSCSI TargetServer. Returns path, IQN, iSCSI addr.
 // Idempotent: if volume already exists with same or larger size, returns existing info.
-func (bs *BlockService) CreateBlockVol(name string, sizeBytes uint64, diskType string) (path, iqn, iscsiAddr string, err error) {
+func (bs *BlockService) CreateBlockVol(name string, sizeBytes uint64, diskType string, durabilityMode string) (path, iqn, iscsiAddr string, err error) {
 	sanitized := blockvol.SanitizeFilename(name)
 	path = filepath.Join(bs.blockDir, sanitized+".blk")
 	iqn = bs.iqnPrefix + blockvol.SanitizeIQN(name)
@@ -155,12 +155,23 @@ func (bs *BlockService) CreateBlockVol(name string, sizeBytes uint64, diskType s
 		return path, iqn, iscsiAddr, nil
 	}
 
+	// F2: VS-side validation — reject invalid mode strings (defense-in-depth).
+	var durMode blockvol.DurabilityMode
+	if durabilityMode != "" {
+		var perr error
+		durMode, perr = blockvol.ParseDurabilityMode(durabilityMode)
+		if perr != nil {
+			return "", "", "", fmt.Errorf("invalid durability mode: %w", perr)
+		}
+	}
+
 	// Create the .blk file.
 	if err := os.MkdirAll(bs.blockDir, 0755); err != nil {
 		return "", "", "", fmt.Errorf("create block dir: %w", err)
 	}
 	created, err := blockvol.CreateBlockVol(path, blockvol.CreateOptions{
-		VolumeSize: sizeBytes,
+		VolumeSize:     sizeBytes,
+		DurabilityMode: durMode,
 	})
 	if err != nil {
 		return "", "", "", fmt.Errorf("create block volume: %w", err)

weed/shell/command_block_volume.go

@@ -41,7 +41,10 @@ func (c *commandBlockCreate) Name() string { return "block.create" }
 func (c *commandBlockCreate) Help() string {
 	return `create a block volume
 
-	block.create -name <name> -size <bytes> [-replicaPlacement <xyz>] [-disk <type>]
+	block.create -name <name> -size <bytes> [-replicaPlacement <xyz>] [-disk <type>] [-durability <mode>] [-replicaFactor <n>]
+
+	durability modes: best_effort (default), sync_all, sync_quorum
+	replica factor: 1, 2 (default), or 3. sync_quorum requires 3.
 `
 }
 func (c *commandBlockCreate) HasTag(CommandTag) bool { return false }
@@ -52,6 +55,8 @@ func (c *commandBlockCreate) Do(args []string, commandEnv *CommandEnv, writer io
 	size := f.Uint64("size", 0, "volume size in bytes")
 	placement := f.String("replicaPlacement", "000", "placement string: 000, 001, 010, 100")
 	disk := f.String("disk", "", "disk type (e.g. ssd, hdd)")
+	durability := f.String("durability", "", "durability mode: best_effort (default), sync_all, sync_quorum")
+	rf := f.Int("replicaFactor", 0, "replica factor: 1, 2 (default), or 3")
 	if err := f.Parse(args); err != nil {
 		return nil
 	}
@@ -65,6 +70,8 @@ func (c *commandBlockCreate) Do(args []string, commandEnv *CommandEnv, writer io
 		SizeBytes:        *size,
 		ReplicaPlacement: *placement,
 		DiskType:         *disk,
+		DurabilityMode:   *durability,
+		ReplicaFactor:    *rf,
 	})
 	if err != nil {
 		return err
@@ -101,11 +108,15 @@ func (c *commandBlockList) Do(args []string, commandEnv *CommandEnv, writer io.W
 		fmt.Fprintln(writer, "no block volumes")
 		return nil
 	}
-	fmt.Fprintf(writer, "%-20s %-20s %-12s %-8s %-8s %-20s\n",
-		"NAME", "SERVER", "SIZE", "EPOCH", "ROLE", "STATUS")
+	fmt.Fprintf(writer, "%-20s %-20s %-12s %-8s %-8s %-14s %-20s\n",
+		"NAME", "SERVER", "SIZE", "EPOCH", "ROLE", "DURABILITY", "STATUS")
 	for _, v := range vols {
-		fmt.Fprintf(writer, "%-20s %-20s %-12d %-8d %-8s %-20s\n",
-			v.Name, v.VolumeServer, v.SizeBytes, v.Epoch, v.Role, v.Status)
+		durMode := v.DurabilityMode
+		if durMode == "" {
+			durMode = "best_effort"
+		}
+		fmt.Fprintf(writer, "%-20s %-20s %-12d %-8d %-8s %-14s %-20s\n",
+			v.Name, v.VolumeServer, v.SizeBytes, v.Epoch, v.Role, durMode, v.Status)
 	}
 	return nil
 }
@@ -134,12 +145,17 @@ func (c *commandBlockStatus) Do(args []string, commandEnv *CommandEnv, writer io
 	if err != nil {
 		return err
 	}
+	durMode := info.DurabilityMode
+	if durMode == "" {
+		durMode = "best_effort"
+	}
 	fmt.Fprintf(writer, "Name:           %s\n", info.Name)
 	fmt.Fprintf(writer, "VolumeServer:   %s\n", info.VolumeServer)
 	fmt.Fprintf(writer, "SizeBytes:      %d\n", info.SizeBytes)
 	fmt.Fprintf(writer, "Epoch:          %d\n", info.Epoch)
 	fmt.Fprintf(writer, "Role:           %s\n", info.Role)
 	fmt.Fprintf(writer, "Status:         %s\n", info.Status)
+	fmt.Fprintf(writer, "Durability:     %s\n", durMode)
 	fmt.Fprintf(writer, "ISCSIAddr:      %s\n", info.ISCSIAddr)
 	fmt.Fprintf(writer, "IQN:            %s\n", info.IQN)
 	if info.ReplicaServer != "" {

weed/storage/blockvol/block_heartbeat.go

@@ -23,6 +23,7 @@ type BlockVolumeInfoMessage struct {
 	ScrubErrors     int64   // CP8-2: lifetime scrub error count
 	LastScrubTime   int64   // CP8-2: unix seconds
 	ReplicaDegraded bool    // CP8-2: true if any replica shipper degraded
+	DurabilityMode  string  // CP8-3-1: "best_effort", "sync_all", "sync_quorum"
 }
 
 // BlockVolumeShortInfoMessage is used for delta heartbeats
@@ -68,6 +69,7 @@ func ToBlockVolumeInfoMessage(path, diskType string, vol *BlockVol) BlockVolumeI
 		ScrubErrors:     hs.ScrubErrors,
 		LastScrubTime:   hs.LastScrubTime,
 		ReplicaDegraded: status.ReplicaDegraded,
+		DurabilityMode:  vol.DurabilityMode().String(),
 	}
 }
 

weed/storage/blockvol/block_heartbeat_proto.go

@@ -22,6 +22,7 @@ func InfoMessageToProto(m BlockVolumeInfoMessage) *master_pb.BlockVolumeInfoMess
 		ScrubErrors:     m.ScrubErrors,
 		LastScrubTime:   m.LastScrubTime,
 		ReplicaDegraded: m.ReplicaDegraded,
+		DurabilityMode:  m.DurabilityMode,
 	}
 }
 
@@ -46,6 +47,7 @@ func InfoMessageFromProto(p *master_pb.BlockVolumeInfoMessage) BlockVolumeInfoMe
 		ScrubErrors:     p.ScrubErrors,
 		LastScrubTime:   p.LastScrubTime,
 		ReplicaDegraded: p.ReplicaDegraded,
+		DurabilityMode:  p.DurabilityMode,
 	}
 }
 

weed/storage/blockvol/block_heartbeat_proto_test.go

@@ -370,6 +370,39 @@ func TestAssignmentsSlice_MultiReplicaRoundTrip(t *testing.T) {
 	}
 }
 
+func TestInfoMessage_DurabilityModeRoundTrip(t *testing.T) {
+	for _, mode := range []string{"best_effort", "sync_all", "sync_quorum"} {
+		t.Run(mode, func(t *testing.T) {
+			orig := BlockVolumeInfoMessage{
+				Path:           "/data/dur.blk",
+				Epoch:          1,
+				DurabilityMode: mode,
+			}
+			pb := InfoMessageToProto(orig)
+			if pb.DurabilityMode != mode {
+				t.Fatalf("ToProto: expected %q, got %q", mode, pb.DurabilityMode)
+			}
+			back := InfoMessageFromProto(pb)
+			if back.DurabilityMode != mode {
+				t.Fatalf("FromProto: expected %q, got %q", mode, back.DurabilityMode)
+			}
+		})
+	}
+}
+
+func TestInfoMessage_DurabilityModeEmpty_BackwardCompat(t *testing.T) {
+	// Empty string should round-trip as empty (V1 compat).
+	orig := BlockVolumeInfoMessage{
+		Path:  "/data/old.blk",
+		Epoch: 1,
+	}
+	pb := InfoMessageToProto(orig)
+	back := InfoMessageFromProto(pb)
+	if back.DurabilityMode != "" {
+		t.Fatalf("empty mode should round-trip as empty, got %q", back.DurabilityMode)
+	}
+}
+
 func TestInfoMessage_HealthFieldsZeroDefault(t *testing.T) {
 	// Verify zero-valued health fields round-trip correctly (backward compat).
 	orig := BlockVolumeInfoMessage{

weed/storage/blockvol/blockapi/types.go

@@ -10,8 +10,10 @@ import (
 type CreateVolumeRequest struct {
 	Name             string `json:"name"`
 	SizeBytes        uint64 `json:"size_bytes"`
-	ReplicaPlacement string `json:"replica_placement"` // SeaweedFS placement string: "000", "001", "010", "100"
-	DiskType         string `json:"disk_type"`         // e.g. "ssd", "hdd"
+	ReplicaPlacement string `json:"replica_placement"`          // SeaweedFS placement string: "000", "001", "010", "100"
+	DiskType         string `json:"disk_type"`                  // e.g. "ssd", "hdd"
+	DurabilityMode   string `json:"durability_mode,omitempty"`  // "best_effort", "sync_all", "sync_quorum"
+	ReplicaFactor    int    `json:"replica_factor,omitempty"`   // 1, 2, or 3 (default: 2)
 }
 
 // VolumeInfo describes a block volume.
@@ -35,6 +37,7 @@ type VolumeInfo struct {
 	Replicas        []ReplicaDetail `json:"replicas,omitempty"`
 	HealthScore     float64         `json:"health_score"`
 	ReplicaDegraded bool            `json:"replica_degraded,omitempty"`
+	DurabilityMode  string          `json:"durability_mode"` // CP8-3-1
 }
 
 // ReplicaDetail describes one replica in the API response.

weed/storage/blockvol/blockerr/errors.go

@@ -0,0 +1,17 @@
+// Package blockerr defines shared error sentinels for the blockvol subsystem.
+// This package exists to break the import cycle between blockvol and iscsi:
+//   blockvol -> (adapter.go) -> iscsi
+// Both blockvol and iscsi can import blockerr without creating a cycle.
+package blockerr
+
+import "errors"
+
+var (
+	// ErrDurabilityBarrierFailed is returned when sync_all mode detects
+	// that one or more replica barriers did not succeed.
+	ErrDurabilityBarrierFailed = errors.New("blockvol: sync_all durability barrier failed")
+
+	// ErrDurabilityQuorumLost is returned when sync_quorum mode detects
+	// that fewer than quorum nodes are durable.
+	ErrDurabilityQuorumLost = errors.New("blockvol: sync_quorum quorum not met")
+)

weed/storage/blockvol/blockvol.go

@@ -17,11 +17,12 @@ import (
 
 // CreateOptions configures a new block volume.
 type CreateOptions struct {
-	VolumeSize  uint64 // required, logical size in bytes
-	ExtentSize  uint32 // default 64KB
-	BlockSize   uint32 // default 4KB
-	WALSize     uint64 // default 64MB
-	Replication string // default "000"
+	VolumeSize     uint64         // required, logical size in bytes
+	ExtentSize     uint32         // default 64KB
+	BlockSize      uint32         // default 4KB
+	WALSize        uint64         // default 64MB
+	Replication    string         // default "000"
+	DurabilityMode DurabilityMode // CP8-3-1: default best_effort (0)
 }
 
 // ErrVolumeClosed is returned when an operation is attempted on a closed BlockVol.
@@ -561,6 +562,11 @@ type VolumeInfo struct {
 	Healthy    bool
 }
 
+// DurabilityMode returns the volume's durability mode from the superblock.
+func (v *BlockVol) DurabilityMode() DurabilityMode {
+	return DurabilityMode(v.super.DurabilityMode)
+}
+
 // WALUsedFraction returns the fraction of WAL space currently in use (0.0 to 1.0).
 func (v *BlockVol) WALUsedFraction() float64 {
 	if v.wal == nil {

weed/storage/blockvol/dist_group_commit.go

@@ -1,20 +1,44 @@
 package blockvol
 
 import (
+	"fmt"
 	"sync"
 )
 
 // MakeDistributedSync creates a sync function that runs local WAL fsync and
 // replica barriers in parallel. Supports N replicas via ShipperGroup.
 //
-// If no replicas are configured (group is nil or empty) or all replicas are
-// degraded, it falls back to local-only sync.
-//
-// Local fsync is the durability point. Replica barrier failures degrade
-// individual shippers but never fail the write.
+// Durability semantics depend on vol.DurabilityMode():
+//   - best_effort: local fsync = ACK; replica failures degrade shippers but never fail writes
+//   - sync_all: ALL replica barriers must succeed, else write returns ErrDurabilityBarrierFailed
+//   - sync_quorum: quorum (RF/2+1) of nodes must be durable, else ErrDurabilityQuorumLost
 func MakeDistributedSync(walSync func() error, group *ShipperGroup, vol *BlockVol) func() error {
 	return func() error {
+		mode := vol.DurabilityMode()
+
 		if group == nil || group.Len() == 0 || group.AllDegraded() {
+			// No healthy replicas available.
+			switch mode {
+			case DurabilitySyncAll:
+				if group != nil && (group.Len() > 0 || group.AllDegraded()) {
+					if vol.Metrics != nil {
+						vol.Metrics.DurabilityBarrierFailedTotal.Add(1)
+					}
+					return ErrDurabilityBarrierFailed
+				}
+			case DurabilitySyncQuorum:
+				if group != nil && group.Len() > 0 {
+					// quorum = (Len+1)/2+1; with 0 healthy replicas, only primary is durable
+					rf := group.Len() + 1
+					quorum := rf/2 + 1
+					if 1 < quorum { // primary alone doesn't meet quorum
+						if vol.Metrics != nil {
+							vol.Metrics.DurabilityQuorumLostTotal.Add(1)
+						}
+						return ErrDurabilityQuorumLost
+					}
+				}
+			}
 			return walSync()
 		}
 
@@ -38,12 +62,38 @@ func MakeDistributedSync(walSync func() error, group *ShipperGroup, vol *BlockVo
 		if localErr != nil {
 			return localErr
 		}
-		// Remote failures: degrade individual shippers (already done by Barrier).
+
+		// Count barrier failures and degrade shippers.
+		failCount := 0
 		for _, err := range barrierErrs {
 			if err != nil {
+				failCount++
 				vol.degradeReplica(err)
 			}
 		}
+
+		switch mode {
+		case DurabilitySyncAll:
+			if failCount > 0 {
+				if vol.Metrics != nil {
+					vol.Metrics.DurabilityBarrierFailedTotal.Add(1)
+				}
+				return fmt.Errorf("%w: %d of %d barriers failed",
+					ErrDurabilityBarrierFailed, failCount, len(barrierErrs))
+			}
+		case DurabilitySyncQuorum:
+			rf := group.Len() + 1 // total nodes including primary
+			quorum := rf/2 + 1
+			durableNodes := 1 + (len(barrierErrs) - failCount) // primary + successful barriers
+			if durableNodes < quorum {
+				if vol.Metrics != nil {
+					vol.Metrics.DurabilityQuorumLostTotal.Add(1)
+				}
+				return fmt.Errorf("%w: %d durable of %d needed",
+					ErrDurabilityQuorumLost, durableNodes, quorum)
+			}
+		}
+		// best_effort: barrier failures already logged via degradeReplica, return nil.
 		return nil
 	}
 }

weed/storage/blockvol/dist_group_commit_test.go

@@ -0,0 +1,160 @@
+package blockvol
+
+import (
+	"errors"
+	"os"
+	"path/filepath"
+	"testing"
+)
+
+// mockShipperGroup creates a ShipperGroup with N mock shippers.
+// failIdxs specifies which shippers should return errors on Barrier.
+func newTestVolWithMode(t *testing.T, mode DurabilityMode) (*BlockVol, string) {
+	t.Helper()
+	dir := t.TempDir()
+	path := filepath.Join(dir, "test.blk")
+	vol, err := CreateBlockVol(path, CreateOptions{
+		VolumeSize:     4 * 1024 * 1024,
+		DurabilityMode: mode,
+	})
+	if err != nil {
+		t.Fatalf("CreateBlockVol: %v", err)
+	}
+	// Write some data so nextLSN > 0.
+	data := make([]byte, 4096)
+	data[0] = 0xAB
+	if err := vol.WriteLBA(0, data); err != nil {
+		t.Fatalf("WriteLBA: %v", err)
+	}
+	return vol, dir
+}
+
+// fakeShipper creates a WALShipper that we can control degraded state for testing.
+// We can't easily create mock WALShippers, but we can test MakeDistributedSync
+// with nil/empty ShipperGroup for edge cases and real shippers for others.
+
+func TestDistSync_BestEffort_NilGroup(t *testing.T) {
+	vol, _ := newTestVolWithMode(t, DurabilityBestEffort)
+	defer vol.Close()
+
+	syncCalled := false
+	fn := MakeDistributedSync(func() error {
+		syncCalled = true
+		return nil
+	}, nil, vol)
+
+	if err := fn(); err != nil {
+		t.Fatalf("sync error: %v", err)
+	}
+	if !syncCalled {
+		t.Error("walSync not called")
+	}
+}
+
+func TestDistSync_SyncAll_NilGroup_Succeeds(t *testing.T) {
+	// sync_all with nil group (no replicas configured) should succeed locally.
+	vol, _ := newTestVolWithMode(t, DurabilitySyncAll)
+	defer vol.Close()
+
+	fn := MakeDistributedSync(func() error { return nil }, nil, vol)
+	if err := fn(); err != nil {
+		t.Fatalf("sync error: %v", err)
+	}
+}
+
+func TestDistSync_SyncAll_AllDegraded_Fails(t *testing.T) {
+	vol, _ := newTestVolWithMode(t, DurabilitySyncAll)
+	defer vol.Close()
+
+	// Create a shipper group with one degraded shipper.
+	shipper := NewWALShipper("127.0.0.1:99999", "127.0.0.1:99998", func() uint64 {
+		return vol.epoch.Load()
+	}, vol.Metrics)
+	shipper.degraded.Store(true)
+	group := NewShipperGroup([]*WALShipper{shipper})
+
+	fn := MakeDistributedSync(func() error { return nil }, group, vol)
+	err := fn()
+	if !errors.Is(err, ErrDurabilityBarrierFailed) {
+		t.Fatalf("expected ErrDurabilityBarrierFailed, got: %v", err)
+	}
+}
+
+func TestDistSync_SyncQuorum_AllDegraded_RF3_Fails(t *testing.T) {
+	vol, _ := newTestVolWithMode(t, DurabilitySyncQuorum)
+	defer vol.Close()
+
+	s1 := NewWALShipper("127.0.0.1:99999", "127.0.0.1:99998", func() uint64 { return 0 }, vol.Metrics)
+	s2 := NewWALShipper("127.0.0.1:99997", "127.0.0.1:99996", func() uint64 { return 0 }, vol.Metrics)
+	s1.degraded.Store(true)
+	s2.degraded.Store(true)
+	group := NewShipperGroup([]*WALShipper{s1, s2})
+
+	fn := MakeDistributedSync(func() error { return nil }, group, vol)
+	err := fn()
+	if !errors.Is(err, ErrDurabilityQuorumLost) {
+		t.Fatalf("expected ErrDurabilityQuorumLost, got: %v", err)
+	}
+}
+
+func TestDistSync_BestEffort_BackwardCompat(t *testing.T) {
+	// best_effort with all-degraded group should still succeed (local fsync only).
+	vol, _ := newTestVolWithMode(t, DurabilityBestEffort)
+	defer vol.Close()
+
+	s1 := NewWALShipper("127.0.0.1:99999", "127.0.0.1:99998", func() uint64 { return 0 }, vol.Metrics)
+	s1.degraded.Store(true)
+	group := NewShipperGroup([]*WALShipper{s1})
+
+	fn := MakeDistributedSync(func() error { return nil }, group, vol)
+	if err := fn(); err != nil {
+		t.Fatalf("best_effort all-degraded should succeed: %v", err)
+	}
+}
+
+func TestDistSync_Metrics_IncrementOnFailure(t *testing.T) {
+	vol, _ := newTestVolWithMode(t, DurabilitySyncAll)
+	defer vol.Close()
+
+	s1 := NewWALShipper("127.0.0.1:99999", "127.0.0.1:99998", func() uint64 { return 0 }, vol.Metrics)
+	s1.degraded.Store(true)
+	group := NewShipperGroup([]*WALShipper{s1})
+
+	fn := MakeDistributedSync(func() error { return nil }, group, vol)
+	fn() // should fail
+
+	if vol.Metrics.DurabilityBarrierFailedTotal.Load() == 0 {
+		t.Error("expected DurabilityBarrierFailedTotal > 0")
+	}
+}
+
+func TestDistSync_LocalFsyncFail_AlwaysErrors(t *testing.T) {
+	// Regardless of mode, local fsync failure always returns error.
+	for _, mode := range []DurabilityMode{DurabilityBestEffort, DurabilitySyncAll, DurabilitySyncQuorum} {
+		t.Run(mode.String(), func(t *testing.T) {
+			dir := t.TempDir()
+			path := filepath.Join(dir, "test.blk")
+			vol, err := CreateBlockVol(path, CreateOptions{
+				VolumeSize:     4 * 1024 * 1024,
+				DurabilityMode: mode,
+			})
+			if err != nil {
+				t.Fatalf("Create: %v", err)
+			}
+			defer vol.Close()
+
+			localErr := errors.New("disk I/O error")
+			fn := MakeDistributedSync(func() error { return localErr }, nil, vol)
+			err = fn()
+			if !errors.Is(err, localErr) {
+				t.Errorf("expected local error, got: %v", err)
+			}
+		})
+	}
+}
+
+// Cleanup helper for tests that create files.
+func cleanupFile(t *testing.T, path string) {
+	t.Helper()
+	os.Remove(path)
+}

weed/storage/blockvol/durability_mode.go

@@ -0,0 +1,101 @@
+package blockvol
+
+import (
+	"errors"
+	"fmt"
+
+	"github.com/seaweedfs/seaweedfs/weed/storage/blockvol/blockerr"
+)
+
+// DurabilityMode controls the write-ACK contract for replicated block volumes.
+//   - DurabilityBestEffort (0): local fsync = ACK; replica failures degrade shippers
+//   - DurabilitySyncAll (1): all replica barriers must succeed, else write returns error
+//   - DurabilitySyncQuorum (2): quorum of RF nodes must be durable (RF>=3 only)
+type DurabilityMode uint8
+
+const (
+	DurabilityBestEffort DurabilityMode = 0 // zero-value = backward compat
+	DurabilitySyncAll    DurabilityMode = 1
+	DurabilitySyncQuorum DurabilityMode = 2
+)
+
+// Re-export shared error sentinels from blockerr so existing callers
+// (e.g. dist_group_commit.go, tests) continue to compile.
+var (
+	ErrDurabilityBarrierFailed = blockerr.ErrDurabilityBarrierFailed
+	ErrDurabilityQuorumLost    = blockerr.ErrDurabilityQuorumLost
+	ErrInvalidDurabilityMode   = errors.New("blockvol: invalid durability mode")
+	ErrSyncQuorumRequiresRF3   = errors.New("blockvol: sync_quorum requires replica_factor >= 3")
+)
+
+// ParseDurabilityMode converts a string to DurabilityMode.
+// Empty string is treated as "best_effort" for backward compatibility.
+func ParseDurabilityMode(s string) (DurabilityMode, error) {
+	switch s {
+	case "", "best_effort":
+		return DurabilityBestEffort, nil
+	case "sync_all":
+		return DurabilitySyncAll, nil
+	case "sync_quorum":
+		return DurabilitySyncQuorum, nil
+	default:
+		return 0, fmt.Errorf("%w: %q", ErrInvalidDurabilityMode, s)
+	}
+}
+
+// String returns the canonical string representation.
+func (m DurabilityMode) String() string {
+	switch m {
+	case DurabilityBestEffort:
+		return "best_effort"
+	case DurabilitySyncAll:
+		return "sync_all"
+	case DurabilitySyncQuorum:
+		return "sync_quorum"
+	default:
+		return fmt.Sprintf("unknown(%d)", m)
+	}
+}
+
+// Validate checks that the mode is valid for the given replica factor.
+func (m DurabilityMode) Validate(replicaFactor int) error {
+	switch m {
+	case DurabilityBestEffort, DurabilitySyncAll:
+		return nil
+	case DurabilitySyncQuorum:
+		if replicaFactor < 3 {
+			return ErrSyncQuorumRequiresRF3
+		}
+		return nil
+	default:
+		return fmt.Errorf("%w: %d", ErrInvalidDurabilityMode, m)
+	}
+}
+
+// IsStrict returns true for sync_all and sync_quorum modes.
+func (m DurabilityMode) IsStrict() bool {
+	return m == DurabilitySyncAll || m == DurabilitySyncQuorum
+}
+
+// RequiredReplicas returns the minimum number of replicas that must be
+// successfully provisioned at create time for the given mode + RF.
+//   - best_effort: 0 (partial create OK)
+//   - sync_all: replicaFactor-1 (all replicas required)
+//   - sync_quorum: quorum-1 = RF/2 (e.g., RF=3 -> 1 replica needed)
+func (m DurabilityMode) RequiredReplicas(replicaFactor int) int {
+	switch m {
+	case DurabilitySyncAll:
+		if replicaFactor <= 1 {
+			return 0
+		}
+		return replicaFactor - 1
+	case DurabilitySyncQuorum:
+		// quorum = RF/2+1 nodes (including primary). Replicas needed = quorum-1.
+		if replicaFactor <= 1 {
+			return 0
+		}
+		return replicaFactor/2 + 1 - 1 // = RF/2
+	default:
+		return 0
+	}
+}

weed/storage/blockvol/durability_mode_test.go

@@ -0,0 +1,163 @@
+package blockvol
+
+import (
+	"errors"
+	"os"
+	"path/filepath"
+	"testing"
+)
+
+func TestParseDurabilityMode_Valid(t *testing.T) {
+	tests := []struct {
+		input string
+		want  DurabilityMode
+	}{
+		{"best_effort", DurabilityBestEffort},
+		{"sync_all", DurabilitySyncAll},
+		{"sync_quorum", DurabilitySyncQuorum},
+		{"", DurabilityBestEffort}, // empty = backward compat
+	}
+	for _, tt := range tests {
+		got, err := ParseDurabilityMode(tt.input)
+		if err != nil {
+			t.Errorf("ParseDurabilityMode(%q) error: %v", tt.input, err)
+		}
+		if got != tt.want {
+			t.Errorf("ParseDurabilityMode(%q) = %v, want %v", tt.input, got, tt.want)
+		}
+	}
+}
+
+func TestParseDurabilityMode_Invalid(t *testing.T) {
+	_, err := ParseDurabilityMode("invalid_mode")
+	if !errors.Is(err, ErrInvalidDurabilityMode) {
+		t.Errorf("ParseDurabilityMode(invalid) error = %v, want ErrInvalidDurabilityMode", err)
+	}
+}
+
+func TestDurabilityMode_StringRoundTrip(t *testing.T) {
+	for _, m := range []DurabilityMode{DurabilityBestEffort, DurabilitySyncAll, DurabilitySyncQuorum} {
+		s := m.String()
+		got, err := ParseDurabilityMode(s)
+		if err != nil {
+			t.Errorf("round-trip %v -> %q: parse error: %v", m, s, err)
+		}
+		if got != m {
+			t.Errorf("round-trip %v -> %q -> %v", m, s, got)
+		}
+	}
+}
+
+func TestDurabilityMode_Validate_SyncQuorum_RF2_Rejected(t *testing.T) {
+	err := DurabilitySyncQuorum.Validate(2)
+	if !errors.Is(err, ErrSyncQuorumRequiresRF3) {
+		t.Errorf("Validate(sync_quorum, RF=2) = %v, want ErrSyncQuorumRequiresRF3", err)
+	}
+}
+
+func TestDurabilityMode_Validate_SyncQuorum_RF3_OK(t *testing.T) {
+	if err := DurabilitySyncQuorum.Validate(3); err != nil {
+		t.Errorf("Validate(sync_quorum, RF=3) = %v, want nil", err)
+	}
+}
+
+func TestDurabilityMode_Validate_BestEffort_RF1_OK(t *testing.T) {
+	if err := DurabilityBestEffort.Validate(1); err != nil {
+		t.Errorf("Validate(best_effort, RF=1) = %v, want nil", err)
+	}
+}
+
+func TestDurabilityMode_RequiredReplicas_SyncAll_RF2(t *testing.T) {
+	got := DurabilitySyncAll.RequiredReplicas(2)
+	if got != 1 {
+		t.Errorf("RequiredReplicas(sync_all, RF=2) = %d, want 1", got)
+	}
+}
+
+func TestDurabilityMode_RequiredReplicas_SyncQuorum_RF3(t *testing.T) {
+	got := DurabilitySyncQuorum.RequiredReplicas(3)
+	if got != 1 { // quorum=2, primary counts as 1, need 1 replica
+		t.Errorf("RequiredReplicas(sync_quorum, RF=3) = %d, want 1", got)
+	}
+}
+
+func TestDurabilityMode_IsStrict(t *testing.T) {
+	if DurabilityBestEffort.IsStrict() {
+		t.Error("best_effort should not be strict")
+	}
+	if !DurabilitySyncAll.IsStrict() {
+		t.Error("sync_all should be strict")
+	}
+	if !DurabilitySyncQuorum.IsStrict() {
+		t.Error("sync_quorum should be strict")
+	}
+}
+
+func TestDurabilityMode_CreateCloseOpenRoundTrip(t *testing.T) {
+	dir := t.TempDir()
+	path := filepath.Join(dir, "test.blk")
+
+	vol, err := CreateBlockVol(path, CreateOptions{
+		VolumeSize:     4 * 1024 * 1024,
+		DurabilityMode: DurabilitySyncAll,
+	})
+	if err != nil {
+		t.Fatalf("Create: %v", err)
+	}
+	if vol.DurabilityMode() != DurabilitySyncAll {
+		t.Fatalf("DurabilityMode() = %v, want sync_all", vol.DurabilityMode())
+	}
+	vol.Close()
+
+	vol2, err := OpenBlockVol(path)
+	if err != nil {
+		t.Fatalf("Open: %v", err)
+	}
+	defer vol2.Close()
+	if vol2.DurabilityMode() != DurabilitySyncAll {
+		t.Errorf("DurabilityMode() after reopen = %v, want sync_all", vol2.DurabilityMode())
+	}
+}
+
+func TestDurabilityMode_Accessor_Default(t *testing.T) {
+	dir := t.TempDir()
+	path := filepath.Join(dir, "test.blk")
+
+	vol, err := CreateBlockVol(path, CreateOptions{VolumeSize: 4 * 1024 * 1024})
+	if err != nil {
+		t.Fatalf("Create: %v", err)
+	}
+	defer vol.Close()
+	if vol.DurabilityMode() != DurabilityBestEffort {
+		t.Errorf("default DurabilityMode() = %v, want best_effort", vol.DurabilityMode())
+	}
+}
+
+func TestDurabilityMode_InvalidOnDisk_OpenFails(t *testing.T) {
+	dir := t.TempDir()
+	path := filepath.Join(dir, "test.blk")
+
+	vol, err := CreateBlockVol(path, CreateOptions{VolumeSize: 4 * 1024 * 1024})
+	if err != nil {
+		t.Fatalf("Create: %v", err)
+	}
+	vol.Close()
+
+	// Corrupt the DurabilityMode byte on disk.
+	// Epoch ends at offset 104. DurabilityMode is at offset 104.
+	// Actually, let me compute: 4(magic)+2(ver)+2(flags)+16(uuid)+8(volsz)+4(ext)+4(blk)+
+	// 8(waloff)+8(walsz)+8(walhead)+8(waltail)+8(walcplsn)+4(repl)+8(created)+4(snapcnt)+8(epoch) = 104
+	f, err := os.OpenFile(path, os.O_RDWR, 0644)
+	if err != nil {
+		t.Fatalf("open: %v", err)
+	}
+	if _, err := f.WriteAt([]byte{99}, 104); err != nil {
+		t.Fatalf("write corrupt byte: %v", err)
+	}
+	f.Close()
+
+	_, err = OpenBlockVol(path)
+	if err == nil {
+		t.Fatal("OpenBlockVol should fail with invalid DurabilityMode")
+	}
+}

weed/storage/blockvol/engine_metrics.go

@@ -29,6 +29,10 @@ type EngineMetrics struct {
 	ScrubPassesTotal   atomic.Uint64
 	ScrubErrorsTotal   atomic.Uint64
 	scrubDurationNs    atomicHistogram
+
+	// Durability (CP8-3-1)
+	DurabilityBarrierFailedTotal atomic.Uint64 // sync_all barrier failures
+	DurabilityQuorumLostTotal    atomic.Uint64 // sync_quorum quorum lost
 }
 
 // NewEngineMetrics creates an EngineMetrics instance.

weed/storage/blockvol/iscsi/scsi.go

@@ -2,6 +2,9 @@ package iscsi
 
 import (
 	"encoding/binary"
+	"errors"
+
+	"github.com/seaweedfs/seaweedfs/weed/storage/blockvol/blockerr"
 )
 
 // SCSI opcode constants (SPC-5 / SBC-4)
@@ -685,12 +688,7 @@ func (h *SCSIHandler) doWrite(lba uint64, transferLen uint32, dataOut []byte) SC
 	}
 
 	if err := h.dev.WriteAt(lba, dataOut[:expectedBytes]); err != nil {
-		return SCSIResult{
-			Status:    SCSIStatusCheckCond,
-			SenseKey:  SenseMediumError,
-			SenseASC:  0x0C, // Write error
-			SenseASCQ: 0x00,
-		}
+		return mapBlockVolError(err)
 	}
 
 	return SCSIResult{Status: SCSIStatusGood}
@@ -698,12 +696,7 @@ func (h *SCSIHandler) doWrite(lba uint64, transferLen uint32, dataOut []byte) SC
 
 func (h *SCSIHandler) syncCache() SCSIResult {
 	if err := h.dev.SyncCache(); err != nil {
-		return SCSIResult{
-			Status:    SCSIStatusCheckCond,
-			SenseKey:  SenseHardwareError,
-			SenseASC:  0x00,
-			SenseASCQ: 0x00,
-		}
+		return mapBlockVolError(err)
 	}
 	return SCSIResult{Status: SCSIStatusGood}
 }
@@ -734,12 +727,7 @@ func (h *SCSIHandler) unmap(cdb [16]byte, dataOut []byte) SCSIResult {
 		if numBlocks > 0 {
 			blockSize := h.dev.BlockSize()
 			if err := h.dev.Trim(lba, numBlocks*blockSize); err != nil {
-				return SCSIResult{
-					Status:    SCSIStatusCheckCond,
-					SenseKey:  SenseMediumError,
-					SenseASC:  0x0C,
-					SenseASCQ: 0x00,
-				}
+				return mapBlockVolError(err)
 			}
 		}
 		descData = descData[16:]
@@ -772,12 +760,7 @@ func (h *SCSIHandler) writeSame16(cdb [16]byte, dataOut []byte) SCSIResult {
 	// UNMAP or NDOB: zero/trim the range.
 	if unmap || ndob {
 		if err := h.dev.Trim(lba, numBlocks*blockSize); err != nil {
-			return SCSIResult{
-				Status:    SCSIStatusCheckCond,
-				SenseKey:  SenseMediumError,
-				SenseASC:  0x0C,
-				SenseASCQ: 0x00,
-			}
+			return mapBlockVolError(err)
 		}
 		return SCSIResult{Status: SCSIStatusGood}
 	}
@@ -798,12 +781,7 @@ func (h *SCSIHandler) writeSame16(cdb [16]byte, dataOut []byte) SCSIResult {
 	}
 	if allZero {
 		if err := h.dev.Trim(lba, numBlocks*blockSize); err != nil {
-			return SCSIResult{
-				Status:    SCSIStatusCheckCond,
-				SenseKey:  SenseMediumError,
-				SenseASC:  0x0C,
-				SenseASCQ: 0x00,
-			}
+			return mapBlockVolError(err)
 		}
 		return SCSIResult{Status: SCSIStatusGood}
 	}
@@ -811,17 +789,41 @@ func (h *SCSIHandler) writeSame16(cdb [16]byte, dataOut []byte) SCSIResult {
 	// Non-zero pattern: write each block individually.
 	for i := uint32(0); i < numBlocks; i++ {
 		if err := h.dev.WriteAt(lba+uint64(i), pattern); err != nil {
-			return SCSIResult{
-				Status:    SCSIStatusCheckCond,
-				SenseKey:  SenseMediumError,
-				SenseASC:  0x0C,
-				SenseASCQ: 0x00,
-			}
+			return mapBlockVolError(err)
 		}
 	}
 	return SCSIResult{Status: SCSIStatusGood}
 }
 
+// isDurabilityError checks if an error originates from the durability layer.
+// Uses errors.Is with shared sentinels from blockerr (avoids import cycle
+// since blockerr has no dependencies on blockvol or iscsi).
+func isDurabilityError(err error) bool {
+	return errors.Is(err, blockerr.ErrDurabilityBarrierFailed) ||
+		errors.Is(err, blockerr.ErrDurabilityQuorumLost)
+}
+
+// mapBlockVolError maps a blockvol write/sync error to a SCSI sense result.
+// Durability errors (barrier failed, quorum lost) are mapped to
+// HARDWARE_ERROR + ASC 0x08 (Logical Unit Communication Failure) to
+// distinguish them from local I/O errors (MEDIUM_ERROR + ASC 0x0C).
+func mapBlockVolError(err error) SCSIResult {
+	if isDurabilityError(err) {
+		return SCSIResult{
+			Status:    SCSIStatusCheckCond,
+			SenseKey:  SenseHardwareError,
+			SenseASC:  0x08, // Logical Unit Communication Failure
+			SenseASCQ: 0x00,
+		}
+	}
+	return SCSIResult{
+		Status:    SCSIStatusCheckCond,
+		SenseKey:  SenseMediumError,
+		SenseASC:  0x0C, // Write error
+		SenseASCQ: 0x00,
+	}
+}
+
 // BuildSenseData constructs a fixed-format sense data buffer (18 bytes).
 func BuildSenseData(key, asc, ascq uint8) []byte {
 	data := make([]byte, 18)

weed/storage/blockvol/iscsi/scsi_durability_test.go

@@ -0,0 +1,59 @@
+package iscsi
+
+import (
+	"errors"
+	"fmt"
+	"testing"
+
+	"github.com/seaweedfs/seaweedfs/weed/storage/blockvol/blockerr"
+)
+
+func TestMapBlockVolError_Durability_HardwareError(t *testing.T) {
+	durErrs := []error{
+		blockerr.ErrDurabilityBarrierFailed,
+		blockerr.ErrDurabilityQuorumLost,
+		fmt.Errorf("wrapped: %w", blockerr.ErrDurabilityBarrierFailed),
+		fmt.Errorf("%w: 1 of 2 barriers failed", blockerr.ErrDurabilityBarrierFailed),
+		fmt.Errorf("%w: 1 durable of 2 needed", blockerr.ErrDurabilityQuorumLost),
+	}
+	for _, err := range durErrs {
+		result := mapBlockVolError(err)
+		if result.SenseKey != SenseHardwareError {
+			t.Errorf("mapBlockVolError(%v) SenseKey = %d, want %d (HARDWARE_ERROR)",
+				err, result.SenseKey, SenseHardwareError)
+		}
+		if result.SenseASC != 0x08 {
+			t.Errorf("mapBlockVolError(%v) ASC = 0x%02x, want 0x08",
+				err, result.SenseASC)
+		}
+	}
+}
+
+func TestMapBlockVolError_NonDurability_MediumError(t *testing.T) {
+	normalErrs := []error{
+		errors.New("disk I/O error"),
+		fmt.Errorf("write failed: %w", errors.New("EIO")),
+	}
+	for _, err := range normalErrs {
+		result := mapBlockVolError(err)
+		if result.SenseKey != SenseMediumError {
+			t.Errorf("mapBlockVolError(%v) SenseKey = %d, want %d (MEDIUM_ERROR)",
+				err, result.SenseKey, SenseMediumError)
+		}
+		if result.SenseASC != 0x0C {
+			t.Errorf("mapBlockVolError(%v) ASC = 0x%02x, want 0x0C",
+				err, result.SenseASC)
+		}
+	}
+}
+
+func TestMapBlockVolError_StringMatchNoLongerWorks(t *testing.T) {
+	// Verify that string-matching is NOT used — a plain error with the
+	// same text should NOT be detected as a durability error.
+	fakeErr := errors.New("blockvol: sync_all durability barrier failed")
+	result := mapBlockVolError(fakeErr)
+	// This should be MEDIUM_ERROR because it's not the real sentinel.
+	if result.SenseKey != SenseMediumError {
+		t.Errorf("string-based error should NOT match as durability error: SenseKey=%d", result.SenseKey)
+	}
+}

weed/storage/blockvol/superblock.go

@@ -39,7 +39,8 @@ type Superblock struct {
 	Replication      [4]byte
 	CreatedAt        uint64 // unix timestamp
 	SnapshotCount    uint32
-	Epoch            uint64 // fencing epoch (0 = no fencing, Phase 3 compat)
+	Epoch            uint64        // fencing epoch (0 = no fencing, Phase 3 compat)
+	DurabilityMode   uint8         // CP8-3-1: 0=best_effort, 1=sync_all, 2=sync_quorum
 }
 
 // superblockOnDisk is the fixed-size on-disk layout (binary.Write/Read target).
@@ -61,6 +62,7 @@ type superblockOnDisk struct {
 	CreatedAt        uint64
 	SnapshotCount    uint32
 	Epoch            uint64
+	DurabilityMode   uint8
 }
 
 // NewSuperblock creates a superblock with defaults and a fresh UUID.
@@ -92,12 +94,13 @@ func NewSuperblock(volumeSize uint64, opts CreateOptions) (Superblock, error) {
 	id := uuid.New()
 
 	sb := Superblock{
-		Version:    CurrentVersion,
-		VolumeSize: volumeSize,
-		ExtentSize: extentSize,
-		BlockSize:  blockSize,
-		WALOffset:  SuperblockSize,
-		WALSize:    walSize,
+		Version:        CurrentVersion,
+		VolumeSize:     volumeSize,
+		ExtentSize:     extentSize,
+		BlockSize:      blockSize,
+		WALOffset:      SuperblockSize,
+		WALSize:        walSize,
+		DurabilityMode: uint8(opts.DurabilityMode),
 	}
 	copy(sb.Magic[:], MagicSWBK)
 	sb.UUID = id
@@ -127,6 +130,7 @@ func (sb *Superblock) WriteTo(w io.Writer) (int64, error) {
 		CreatedAt:        sb.CreatedAt,
 		SnapshotCount:    sb.SnapshotCount,
 		Epoch:            sb.Epoch,
+		DurabilityMode:   sb.DurabilityMode,
 	}
 
 	// Encode into beginning of buf; rest stays zero (padding).
@@ -160,6 +164,8 @@ func (sb *Superblock) WriteTo(w io.Writer) (int64, error) {
 	endian.PutUint32(buf[off:], d.SnapshotCount)
 	off += 4
 	endian.PutUint64(buf[off:], d.Epoch)
+	off += 8
+	buf[off] = d.DurabilityMode
 
 	n, err := w.Write(buf)
 	return int64(n), err
@@ -220,6 +226,8 @@ func ReadSuperblock(r io.Reader) (Superblock, error) {
 	sb.SnapshotCount = endian.Uint32(buf[off:])
 	off += 4
 	sb.Epoch = endian.Uint64(buf[off:])
+	off += 8
+	sb.DurabilityMode = buf[off]
 
 	return sb, nil
 }
@@ -252,5 +260,8 @@ func (sb *Superblock) Validate() error {
 	if sb.VolumeSize%uint64(sb.BlockSize) != 0 {
 		return fmt.Errorf("%w: VolumeSize %d not aligned to BlockSize %d", ErrInvalidSuperblock, sb.VolumeSize, sb.BlockSize)
 	}
+	if sb.DurabilityMode > 2 {
+		return fmt.Errorf("%w: invalid DurabilityMode %d", ErrInvalidSuperblock, sb.DurabilityMode)
+	}
 	return nil
 }

weed/storage/blockvol/superblock_test.go

@@ -16,6 +16,9 @@ func TestSuperblock(t *testing.T) {
 		{name: "superblock_magic_check", run: testSuperblockMagicCheck},
 		{name: "superblock_version_check", run: testSuperblockVersionCheck},
 		{name: "superblock_zero_vol_size", run: testSuperblockZeroVolSize},
+		{name: "durability_mode_roundtrip", run: testDurabilityModeRoundtrip},
+		{name: "durability_mode_v1_compat", run: testDurabilityModeV1Compat},
+		{name: "durability_mode_invalid_rejected", run: testDurabilityModeInvalidRejected},
 	}
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
@@ -144,3 +147,54 @@ func testSuperblockZeroVolSize(t *testing.T) {
 		t.Errorf("ReadSuperblock(vol_size=0): expected ErrInvalidVolumeSize, got %v", err)
 	}
 }
+
+func testDurabilityModeRoundtrip(t *testing.T) {
+	sb, err := NewSuperblock(1*1024*1024*1024, CreateOptions{
+		DurabilityMode: DurabilitySyncAll,
+	})
+	if err != nil {
+		t.Fatalf("NewSuperblock: %v", err)
+	}
+	if sb.DurabilityMode != uint8(DurabilitySyncAll) {
+		t.Fatalf("NewSuperblock DurabilityMode = %d, want %d", sb.DurabilityMode, DurabilitySyncAll)
+	}
+
+	var buf bytes.Buffer
+	sb.WriteTo(&buf)
+
+	got, err := ReadSuperblock(&buf)
+	if err != nil {
+		t.Fatalf("ReadSuperblock: %v", err)
+	}
+	if got.DurabilityMode != uint8(DurabilitySyncAll) {
+		t.Errorf("DurabilityMode = %d, want %d", got.DurabilityMode, DurabilitySyncAll)
+	}
+}
+
+func testDurabilityModeV1Compat(t *testing.T) {
+	// V1 file has zeros in the DurabilityMode region → reads as best_effort.
+	sb, _ := NewSuperblock(1*1024*1024*1024, CreateOptions{})
+	var buf bytes.Buffer
+	sb.WriteTo(&buf)
+
+	got, err := ReadSuperblock(&buf)
+	if err != nil {
+		t.Fatalf("ReadSuperblock: %v", err)
+	}
+	if DurabilityMode(got.DurabilityMode) != DurabilityBestEffort {
+		t.Errorf("V1 DurabilityMode = %d, want %d (best_effort)", got.DurabilityMode, DurabilityBestEffort)
+	}
+}
+
+func testDurabilityModeInvalidRejected(t *testing.T) {
+	sb, _ := NewSuperblock(1*1024*1024*1024, CreateOptions{})
+	sb.DurabilityMode = 99
+
+	err := sb.Validate()
+	if err == nil {
+		t.Error("Validate should reject DurabilityMode=99")
+	}
+	if !errors.Is(err, ErrInvalidSuperblock) {
+		t.Errorf("Validate error = %v, want ErrInvalidSuperblock", err)
+	}
+}

weed/storage/blockvol/testrunner/actions/block.go

@@ -0,0 +1,466 @@
+package actions
+
+import (
+	"context"
+	"fmt"
+	"runtime"
+	"strconv"
+	"strings"
+
+	tr "github.com/seaweedfs/seaweedfs/weed/storage/blockvol/testrunner"
+	"github.com/seaweedfs/seaweedfs/weed/storage/blockvol/testrunner/infra"
+)
+
+// RegisterBlockActions registers all block/target lifecycle actions.
+func RegisterBlockActions(r *tr.Registry) {
+	r.RegisterFunc("build_deploy", tr.TierBlock, buildDeploy)
+	r.RegisterFunc("start_target", tr.TierBlock, startTarget)
+	r.RegisterFunc("stop_target", tr.TierBlock, stopTarget)
+	r.RegisterFunc("kill_target", tr.TierBlock, killTarget)
+	r.RegisterFunc("kill_stale", tr.TierBlock, killStale)
+	r.RegisterFunc("stop_all_targets", tr.TierBlock, stopAllTargets)
+	r.RegisterFunc("assign", tr.TierBlock, assign)
+	r.RegisterFunc("set_replica", tr.TierBlock, setReplica)
+	r.RegisterFunc("wait_role", tr.TierBlock, waitRole)
+	r.RegisterFunc("wait_lsn", tr.TierBlock, waitLSN)
+	r.RegisterFunc("status", tr.TierBlock, statusAction)
+	r.RegisterFunc("start_rebuild_server", tr.TierBlock, startRebuildServer)
+	r.RegisterFunc("start_rebuild_client", tr.TierBlock, startRebuildClient)
+}
+
+func buildDeploy(ctx context.Context, actx *tr.ActionContext, act tr.Action) (map[string]string, error) {
+	repoDir := actx.Vars["repo_dir"]
+	if repoDir == "" {
+		return nil, fmt.Errorf("build_deploy: repo_dir not set in env")
+	}
+
+	// Coordinator mode: cross-compile locally, upload to agents via HTTP.
+	if actx.Coordinator != nil {
+		return buildDeployCoordinator(ctx, actx, repoDir)
+	}
+
+	// Agent mode: build locally if Go+source are available, or check binary exists.
+	if actx.Scenario != nil && len(actx.Scenario.Topology.Agents) > 0 {
+		return buildDeployAgent(ctx, actx, repoDir)
+	}
+
+	// SSH mode: cross-compile locally, deploy to nodes via SSH/SCP.
+	return buildDeploySSH(ctx, actx, repoDir)
+}
+
+// buildDeployAgent handles build_deploy on agent nodes.
+// Checks for pre-deployed binary first (common in iterative dev: build on
+// Windows, SCP to nodes). Only builds from local source if no binary exists
+// or "force_build" param is set.
+func buildDeployAgent(ctx context.Context, actx *tr.ActionContext, repoDir string) (map[string]string, error) {
+	binPath := "/tmp/iscsi-target-test"
+	forceBuild := actx.Vars["force_build"] == "true"
+
+	node, _ := getNode(actx, "")
+
+	// Check for pre-deployed binary (preferred: avoids stale source issues).
+	if node != nil && !forceBuild {
+		_, _, code, _ := node.Run(ctx, fmt.Sprintf("test -x %s", binPath))
+		if code == 0 {
+			actx.Log("  using pre-deployed binary at %s", binPath)
+			return nil, nil
+		}
+	}
+
+	// Build from source if Go is available.
+	if node != nil {
+		_, _, goCode, _ := node.Run(ctx, "which go")
+		srcExists := false
+		if goCode == 0 {
+			_, _, srcCode, _ := node.Run(ctx, fmt.Sprintf("test -d %s/weed/storage/blockvol/iscsi", repoDir))
+			srcExists = srcCode == 0
+		}
+
+		if goCode == 0 && srcExists {
+			actx.Log("  building iscsi-target on agent from %s ...", repoDir)
+			buildCmd := fmt.Sprintf("cd %s && CGO_ENABLED=0 go build -o %s ./weed/storage/blockvol/iscsi/cmd/iscsi-target/", repoDir, binPath)
+			_, stderr, code, err := node.Run(ctx, buildCmd)
+			if err != nil || code != 0 {
+				return nil, fmt.Errorf("agent build failed: code=%d stderr=%s err=%v", code, stderr, err)
+			}
+			actx.Log("  build OK")
+			return nil, nil
+		}
+	}
+
+	return nil, fmt.Errorf("build_deploy (agent): no pre-deployed binary at %s and Go not available", binPath)
+}
+
+func buildDeployCoordinator(ctx context.Context, actx *tr.ActionContext, repoDir string) (map[string]string, error) {
+	// Build locally (Windows cross-compile).
+	actx.Log("  building iscsi-target binary (coordinator mode)...")
+	var node *infra.Node
+	for _, n := range actx.Nodes {
+		if nn, ok := n.(*infra.Node); ok {
+			node = nn
+			break
+		}
+	}
+	if node == nil {
+		// In coordinator mode we might not have infra.Nodes.
+		// Build using local exec (bash -c via Git Bash on Windows).
+		actx.Log("  cross-compiling with local exec...")
+		buildCmd := fmt.Sprintf("cd %s && GOOS=linux GOARCH=amd64 CGO_ENABLED=0 go build -o iscsi-target-linux ./weed/storage/blockvol/iscsi/cmd/iscsi-target/", repoDir)
+		ln := tr.NewLocalNode("build-host")
+		_, stderr, code, err := ln.Run(ctx, buildCmd)
+		if err != nil || code != 0 {
+			return nil, fmt.Errorf("build failed: code=%d stderr=%s err=%v", code, stderr, err)
+		}
+
+		// Convert bash path to native OS path for os.Open in UploadToAgent.
+		localBin := repoDir + "/iscsi-target-linux"
+		localBin = toNativePath(localBin)
+
+		// Upload to each agent that hosts targets.
+		return uploadToTargetAgents(ctx, actx, localBin)
+	}
+
+	tgt := infra.NewTarget(node, infra.DefaultTargetConfig())
+	if err := tgt.Build(ctx, repoDir); err != nil {
+		return nil, fmt.Errorf("build: %w", err)
+	}
+	localBin := repoDir + "/iscsi-target-linux"
+
+	return uploadToTargetAgents(ctx, actx, localBin)
+}
+
+func uploadToTargetAgents(ctx context.Context, actx *tr.ActionContext, localBin string) (map[string]string, error) {
+	// Find agents that host targets.
+	targetAgents := make(map[string]bool)
+	for _, spec := range actx.Scenario.Targets {
+		nodeSpec, ok := actx.Scenario.Topology.Nodes[spec.Node]
+		if ok && nodeSpec.Agent != "" {
+			targetAgents[nodeSpec.Agent] = true
+		}
+	}
+
+	for agentName := range targetAgents {
+		actx.Log("  uploading to agent %s...", agentName)
+		if err := actx.Coordinator.UploadToAgent(ctx, agentName, localBin, tr.UploadBasePath+"iscsi-target-test"); err != nil {
+			return nil, fmt.Errorf("upload to agent %s: %w", agentName, err)
+		}
+	}
+
+	return nil, nil
+}
+
+func buildDeploySSH(ctx context.Context, actx *tr.ActionContext, repoDir string) (map[string]string, error) {
+	// Use the first available node to determine local vs remote.
+	// Build always happens locally (Windows cross-compile).
+	var node *infra.Node
+	for _, n := range actx.Nodes {
+		if nn, ok := n.(*infra.Node); ok {
+			node = nn
+			break
+		}
+	}
+	if node == nil {
+		return nil, fmt.Errorf("build_deploy: no nodes available")
+	}
+
+	tgt := infra.NewTarget(node, infra.DefaultTargetConfig())
+	actx.Log("  building iscsi-target binary...")
+	if err := tgt.Build(ctx, repoDir); err != nil {
+		return nil, fmt.Errorf("build: %w", err)
+	}
+
+	localBin := repoDir + "/iscsi-target-linux"
+
+	// Deploy only to nodes that host targets (not client-only nodes).
+	targetNodes := make(map[string]bool)
+	for _, spec := range actx.Scenario.Targets {
+		targetNodes[spec.Node] = true
+	}
+
+	deployed := make(map[string]bool)
+	for nodeName := range targetNodes {
+		if deployed[nodeName] {
+			continue
+		}
+		nodeRunner, ok := actx.Nodes[nodeName]
+		if !ok {
+			continue
+		}
+		n, ok := nodeRunner.(*infra.Node)
+		if !ok {
+			continue
+		}
+		actx.Log("  deploying to node %s...", nodeName)
+		deployTgt := infra.NewTarget(n, infra.DefaultTargetConfig())
+		if err := deployTgt.Deploy(localBin); err != nil {
+			return nil, fmt.Errorf("deploy to %s: %w", nodeName, err)
+		}
+		deployed[nodeName] = true
+	}
+
+	return nil, nil
+}
+
+func startTarget(ctx context.Context, actx *tr.ActionContext, act tr.Action) (map[string]string, error) {
+	tgt, err := getHATarget(actx, act.Target)
+	if err != nil {
+		return nil, err
+	}
+
+	create := act.Params["create"] == "true"
+	if err := tgt.Start(ctx, create); err != nil {
+		return nil, fmt.Errorf("start_target %s: %w", act.Target, err)
+	}
+
+	return map[string]string{"value": fmt.Sprintf("pid=%d", tgt.PID())}, nil
+}
+
+func stopTarget(ctx context.Context, actx *tr.ActionContext, act tr.Action) (map[string]string, error) {
+	tgt, err := getHATarget(actx, act.Target)
+	if err != nil {
+		return nil, err
+	}
+	return nil, tgt.Stop(ctx)
+}
+
+func killTarget(ctx context.Context, actx *tr.ActionContext, act tr.Action) (map[string]string, error) {
+	tgt, err := getHATarget(actx, act.Target)
+	if err != nil {
+		return nil, err
+	}
+	return nil, tgt.Kill9()
+}
+
+func stopAllTargets(ctx context.Context, actx *tr.ActionContext, act tr.Action) (map[string]string, error) {
+	var lastErr error
+	for name, tgt := range actx.Targets {
+		if ht, ok := tgt.(*infra.HATarget); ok {
+			if err := ht.Stop(ctx); err != nil {
+				actx.Log("  stop %s: %v", name, err)
+				lastErr = err
+			}
+			ht.Cleanup(ctx)
+		}
+	}
+
+	// Aggressive: also kill stray iscsi-target processes on all nodes.
+	if act.Params["aggressive"] == "true" {
+		for _, n := range actx.Nodes {
+			if node, ok := n.(*infra.Node); ok {
+				node.Run(ctx, "pkill -9 iscsi-target 2>/dev/null")
+			}
+		}
+		actx.Log("  killed stray iscsi-target processes")
+	}
+
+	return nil, lastErr
+}
+
+// killStale kills all iscsi-target-test processes on the node, regardless of
+// whether they are tracked. Used at the start of scenarios to clean up
+// leftovers from previous crashed runs.
+func killStale(ctx context.Context, actx *tr.ActionContext, act tr.Action) (map[string]string, error) {
+	node, err := getNode(actx, act.Node)
+	if err != nil {
+		return nil, fmt.Errorf("kill_stale: %w", err)
+	}
+
+	// Also clean up iSCSI sessions if this is a client node.
+	process := act.Params["process"]
+	if process == "" {
+		process = "iscsi-target-test"
+	}
+
+	// Kill all matching processes.
+	cmd := fmt.Sprintf("pkill -9 -f '%s' 2>/dev/null; sleep 0.5; pgrep -f '%s' || echo 'all_killed'", process, process)
+	stdout, _, _, _ := node.Run(ctx, cmd)
+	actx.Log("  kill_stale %s: %s", process, strings.TrimSpace(stdout))
+
+	// If iscsi cleanup requested, clean up stale iSCSI sessions.
+	if act.Params["iscsi_cleanup"] == "true" {
+		node.Run(ctx, "sudo iscsiadm -m session -u 2>/dev/null; sudo iscsiadm -m node -o delete 2>/dev/null")
+		actx.Log("  cleaned stale iSCSI sessions")
+	}
+
+	return nil, nil
+}
+
+func assign(ctx context.Context, actx *tr.ActionContext, act tr.Action) (map[string]string, error) {
+	tgt, err := getHATarget(actx, act.Target)
+	if err != nil {
+		return nil, err
+	}
+
+	epoch, _ := strconv.ParseUint(act.Params["epoch"], 10, 64)
+	role := parseRole(act.Params["role"])
+	leaseTTL := uint32(30000) // default 30s
+	if ttlStr, ok := act.Params["lease_ttl"]; ok {
+		if ms, err := parseDurationMs(ttlStr); err == nil {
+			leaseTTL = ms
+		}
+	}
+
+	if err := tgt.Assign(ctx, epoch, role, leaseTTL); err != nil {
+		return nil, fmt.Errorf("assign %s: %w", act.Target, err)
+	}
+	return nil, nil
+}
+
+func setReplica(ctx context.Context, actx *tr.ActionContext, act tr.Action) (map[string]string, error) {
+	primaryTgt, err := getHATarget(actx, act.Target)
+	if err != nil {
+		return nil, err
+	}
+
+	replicaTgt, err := getHATarget(actx, act.Replica)
+	if err != nil {
+		return nil, fmt.Errorf("set_replica: replica %q: %w", act.Replica, err)
+	}
+
+	host := replicaTgt.HostAddr()
+	dataAddr := fmt.Sprintf("%s:%d", host, replicaTgt.ReplicaData)
+	ctrlAddr := fmt.Sprintf("%s:%d", host, replicaTgt.ReplicaCtrl)
+
+	return nil, primaryTgt.SetReplica(ctx, dataAddr, ctrlAddr)
+}
+
+func waitRole(ctx context.Context, actx *tr.ActionContext, act tr.Action) (map[string]string, error) {
+	tgt, err := getHATarget(actx, act.Target)
+	if err != nil {
+		return nil, err
+	}
+
+	role := act.Params["role"]
+	if role == "" {
+		return nil, fmt.Errorf("wait_role: role param required")
+	}
+
+	timeoutCtx := ctx
+	if t, ok := act.Params["timeout"]; ok {
+		if d, err := parseDuration(t); err == nil {
+			var cancel context.CancelFunc
+			timeoutCtx, cancel = context.WithTimeout(ctx, d)
+			defer cancel()
+		}
+	}
+
+	return nil, tgt.WaitForRole(timeoutCtx, role)
+}
+
+func waitLSN(ctx context.Context, actx *tr.ActionContext, act tr.Action) (map[string]string, error) {
+	tgt, err := getHATarget(actx, act.Target)
+	if err != nil {
+		return nil, err
+	}
+
+	minLSN, _ := strconv.ParseUint(act.Params["min_lsn"], 10, 64)
+
+	timeoutCtx := ctx
+	if t, ok := act.Params["timeout"]; ok {
+		if d, err := parseDuration(t); err == nil {
+			var cancel context.CancelFunc
+			timeoutCtx, cancel = context.WithTimeout(ctx, d)
+			defer cancel()
+		}
+	}
+
+	return nil, tgt.WaitForLSN(timeoutCtx, minLSN)
+}
+
+func statusAction(ctx context.Context, actx *tr.ActionContext, act tr.Action) (map[string]string, error) {
+	tgt, err := getHATarget(actx, act.Target)
+	if err != nil {
+		return nil, err
+	}
+
+	st, err := tgt.Status(ctx)
+	if err != nil {
+		return nil, err
+	}
+
+	return map[string]string{
+		"value": fmt.Sprintf("epoch=%d role=%s lsn=%d lease=%v healthy=%v",
+			st.Epoch, st.Role, st.WALHeadLSN, st.HasLease, st.Healthy),
+	}, nil
+}
+
+func startRebuildServer(ctx context.Context, actx *tr.ActionContext, act tr.Action) (map[string]string, error) {
+	tgt, err := getHATarget(actx, act.Target)
+	if err != nil {
+		return nil, err
+	}
+
+	host := tgt.HostAddr()
+	listenAddr := fmt.Sprintf("%s:%d", host, tgt.RebuildPort)
+	return nil, tgt.StartRebuildEndpoint(ctx, listenAddr)
+}
+
+func startRebuildClient(ctx context.Context, actx *tr.ActionContext, act tr.Action) (map[string]string, error) {
+	tgt, err := getHATarget(actx, act.Target)
+	if err != nil {
+		return nil, err
+	}
+
+	// Get primary's rebuild address from the "primary" param or the replica target.
+	primaryName := act.Params["primary"]
+	if primaryName == "" {
+		primaryName = act.Replica
+	}
+	primaryTgt, err := getHATarget(actx, primaryName)
+	if err != nil {
+		return nil, fmt.Errorf("start_rebuild_client: primary %q: %w", primaryName, err)
+	}
+
+	host := primaryTgt.HostAddr()
+	rebuildAddr := fmt.Sprintf("%s:%d", host, primaryTgt.RebuildPort)
+
+	epoch, _ := strconv.ParseUint(act.Params["epoch"], 10, 64)
+
+	return nil, tgt.StartRebuildClient(ctx, rebuildAddr, epoch)
+}
+
+// getHATarget looks up the named target from the action context.
+func getHATarget(actx *tr.ActionContext, name string) (*infra.HATarget, error) {
+	if name == "" {
+		return nil, fmt.Errorf("target name is required")
+	}
+	tgt, ok := actx.Targets[name]
+	if !ok {
+		return nil, fmt.Errorf("target %q not found", name)
+	}
+	ht, ok := tgt.(*infra.HATarget)
+	if !ok {
+		return nil, fmt.Errorf("target %q is not an HATarget", name)
+	}
+	return ht, nil
+}
+
+func parseRole(s string) uint32 {
+	switch strings.ToLower(s) {
+	case "primary":
+		return 1 // RolePrimary
+	case "replica":
+		return 2 // RoleReplica
+	case "stale":
+		return 3 // RoleStale
+	case "rebuilding":
+		return 4 // RoleRebuilding
+	case "draining":
+		return 5 // RoleDraining
+	default:
+		return 0
+	}
+}
+
+// toNativePath converts a Git Bash path (e.g. /c/work/foo) to the native OS path
+// (e.g. C:\work\foo on Windows). No-op on Linux.
+func toNativePath(p string) string {
+	if runtime.GOOS != "windows" {
+		return p
+	}
+	// Convert /c/work/foo → C:/work/foo
+	if len(p) >= 3 && p[0] == '/' && p[2] == '/' {
+		p = strings.ToUpper(string(p[1])) + ":" + p[2:]
+	}
+	return strings.ReplaceAll(p, "/", "\\")
+}

weed/storage/blockvol/testrunner/actions/devops.go

@@ -0,0 +1,262 @@
+package actions
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"strings"
+	"time"
+
+	tr "github.com/seaweedfs/seaweedfs/weed/storage/blockvol/testrunner"
+)
+
+// RegisterDevOpsActions registers SeaweedFS cluster management actions.
+func RegisterDevOpsActions(r *tr.Registry) {
+	r.RegisterFunc("build_deploy_weed", tr.TierDevOps, buildDeployWeed)
+	r.RegisterFunc("start_weed_master", tr.TierDevOps, startWeedMaster)
+	r.RegisterFunc("start_weed_volume", tr.TierDevOps, startWeedVolume)
+	r.RegisterFunc("stop_weed", tr.TierDevOps, stopWeed)
+	r.RegisterFunc("wait_cluster_ready", tr.TierDevOps, waitClusterReady)
+	r.RegisterFunc("create_block_volume", tr.TierDevOps, createBlockVolume)
+	r.RegisterFunc("cluster_status", tr.TierDevOps, clusterStatus)
+}
+
+// buildDeployWeed cross-compiles the weed binary and uploads to all nodes.
+func buildDeployWeed(ctx context.Context, actx *tr.ActionContext, act tr.Action) (map[string]string, error) {
+	repoDir := actx.Vars["repo_dir"]
+	if repoDir == "" {
+		return nil, fmt.Errorf("build_deploy_weed: repo_dir not set in env")
+	}
+
+	actx.Log("  cross-compiling weed binary...")
+	localBin := repoDir + "/weed-linux"
+	buildCmd := fmt.Sprintf("cd %s && GOOS=linux GOARCH=amd64 CGO_ENABLED=0 go build -o weed-linux ./weed/command", repoDir)
+
+	ln := tr.NewLocalNode("build-host")
+	_, stderr, code, err := ln.Run(ctx, buildCmd)
+	if err != nil || code != 0 {
+		return nil, fmt.Errorf("build_deploy_weed: code=%d stderr=%s err=%v", code, stderr, err)
+	}
+
+	// Upload to agents if coordinator mode.
+	if actx.Coordinator != nil {
+		for _, agentName := range actx.Coordinator.AgentNames() {
+			actx.Log("  uploading weed to agent %s...", agentName)
+			if err := actx.Coordinator.UploadToAgent(ctx, agentName, localBin, tr.UploadBasePath+"weed"); err != nil {
+				return nil, fmt.Errorf("upload weed to %s: %w", agentName, err)
+			}
+		}
+		return nil, nil
+	}
+
+	// SSH mode: deploy to all nodes.
+	for nodeName, nodeRunner := range actx.Nodes {
+		actx.Log("  deploying weed to node %s...", nodeName)
+		nodeRunner.Run(ctx, fmt.Sprintf("mkdir -p %s", tr.UploadBasePath))
+		if err := nodeRunner.Upload(localBin, tr.UploadBasePath+"weed"); err != nil {
+			return nil, fmt.Errorf("deploy weed to %s: %w", nodeName, err)
+		}
+		nodeRunner.Run(ctx, fmt.Sprintf("chmod +x %sweed", tr.UploadBasePath))
+	}
+
+	return nil, nil
+}
+
+// startWeedMaster starts a weed master process on the given node.
+func startWeedMaster(ctx context.Context, actx *tr.ActionContext, act tr.Action) (map[string]string, error) {
+	node, err := getNode(actx, act.Node)
+	if err != nil {
+		return nil, fmt.Errorf("start_weed_master: %w", err)
+	}
+
+	port := act.Params["port"]
+	if port == "" {
+		port = "9333"
+	}
+	dir := act.Params["dir"]
+	if dir == "" {
+		dir = "/tmp/sw-weed-master"
+	}
+	extraArgs := act.Params["extra_args"]
+
+	// Ensure directory exists.
+	node.RunRoot(ctx, fmt.Sprintf("mkdir -p %s", dir))
+
+	cmd := fmt.Sprintf("setsid %sweed master -port=%s -mdir=%s %s </dev/null >%s/master.log 2>&1 & echo $!",
+		tr.UploadBasePath, port, dir, extraArgs, dir)
+	stdout, stderr, code, err := node.RunRoot(ctx, cmd)
+	if err != nil || code != 0 {
+		return nil, fmt.Errorf("start_weed_master: code=%d stderr=%s err=%v", code, stderr, err)
+	}
+
+	pid := strings.TrimSpace(stdout)
+	actx.Log("  weed master started on port %s (PID %s)", port, pid)
+	return map[string]string{"value": pid}, nil
+}
+
+// startWeedVolume starts a weed volume process on the given node.
+func startWeedVolume(ctx context.Context, actx *tr.ActionContext, act tr.Action) (map[string]string, error) {
+	node, err := getNode(actx, act.Node)
+	if err != nil {
+		return nil, fmt.Errorf("start_weed_volume: %w", err)
+	}
+
+	port := act.Params["port"]
+	if port == "" {
+		port = "8080"
+	}
+	master := act.Params["master"]
+	if master == "" {
+		return nil, fmt.Errorf("start_weed_volume: master param required")
+	}
+	dir := act.Params["dir"]
+	if dir == "" {
+		dir = "/tmp/sw-weed-volume"
+	}
+	extraArgs := act.Params["extra_args"]
+
+	node.RunRoot(ctx, fmt.Sprintf("mkdir -p %s", dir))
+
+	cmd := fmt.Sprintf("setsid %sweed volume -port=%s -mserver=%s -dir=%s %s </dev/null >%s/volume.log 2>&1 & echo $!",
+		tr.UploadBasePath, port, master, dir, extraArgs, dir)
+	stdout, stderr, code, err := node.RunRoot(ctx, cmd)
+	if err != nil || code != 0 {
+		return nil, fmt.Errorf("start_weed_volume: code=%d stderr=%s err=%v", code, stderr, err)
+	}
+
+	pid := strings.TrimSpace(stdout)
+	actx.Log("  weed volume started on port %s (PID %s)", port, pid)
+	return map[string]string{"value": pid}, nil
+}
+
+// stopWeed stops a weed process by PID.
+func stopWeed(ctx context.Context, actx *tr.ActionContext, act tr.Action) (map[string]string, error) {
+	node, err := getNode(actx, act.Node)
+	if err != nil {
+		return nil, fmt.Errorf("stop_weed: %w", err)
+	}
+
+	pid := act.Params["pid"]
+	if pid == "" {
+		return nil, fmt.Errorf("stop_weed: pid param required")
+	}
+
+	// Graceful kill first, then force after 5s.
+	_, _, _, _ = node.RunRoot(ctx, fmt.Sprintf("kill %s", pid))
+
+	deadline := time.After(5 * time.Second)
+	ticker := time.NewTicker(500 * time.Millisecond)
+	defer ticker.Stop()
+
+	for {
+		select {
+		case <-deadline:
+			node.RunRoot(ctx, fmt.Sprintf("kill -9 %s", pid))
+			actx.Log("  force-killed PID %s", pid)
+			return nil, nil
+		case <-ticker.C:
+			_, _, code, _ := node.Run(ctx, fmt.Sprintf("kill -0 %s 2>/dev/null", pid))
+			if code != 0 {
+				actx.Log("  PID %s exited gracefully", pid)
+				return nil, nil
+			}
+		case <-ctx.Done():
+			return nil, ctx.Err()
+		}
+	}
+}
+
+// waitClusterReady polls the master until IsLeader is true.
+func waitClusterReady(ctx context.Context, actx *tr.ActionContext, act tr.Action) (map[string]string, error) {
+	node, err := getNode(actx, act.Node)
+	if err != nil {
+		return nil, fmt.Errorf("wait_cluster_ready: %w", err)
+	}
+
+	masterURL := act.Params["master_url"]
+	if masterURL == "" {
+		return nil, fmt.Errorf("wait_cluster_ready: master_url param required")
+	}
+
+	timeout := 30 * time.Second
+	if t, ok := act.Params["timeout"]; ok {
+		if d, err := parseDuration(t); err == nil {
+			timeout = d
+		}
+	}
+
+	timeoutCtx, cancel := context.WithTimeout(ctx, timeout)
+	defer cancel()
+
+	ticker := time.NewTicker(1 * time.Second)
+	defer ticker.Stop()
+
+	for {
+		select {
+		case <-timeoutCtx.Done():
+			return nil, fmt.Errorf("wait_cluster_ready: timeout after %s", timeout)
+		case <-ticker.C:
+			cmd := fmt.Sprintf("curl -s %s/cluster/status 2>/dev/null", masterURL)
+			stdout, _, code, err := node.Run(timeoutCtx, cmd)
+			if err != nil || code != 0 {
+				continue
+			}
+			if strings.Contains(stdout, `"IsLeader":true`) || strings.Contains(stdout, `"isLeader":true`) {
+				actx.Log("  cluster ready at %s", masterURL)
+				return map[string]string{"value": stdout}, nil
+			}
+		}
+	}
+}
+
+// createBlockVolume creates a block volume via the master assign API.
+func createBlockVolume(ctx context.Context, actx *tr.ActionContext, act tr.Action) (map[string]string, error) {
+	node, err := getNode(actx, act.Node)
+	if err != nil {
+		return nil, fmt.Errorf("create_block_volume: %w", err)
+	}
+
+	masterURL := act.Params["master_url"]
+	if masterURL == "" {
+		return nil, fmt.Errorf("create_block_volume: master_url param required")
+	}
+	size := act.Params["size"]
+	if size == "" {
+		size = "1g"
+	}
+
+	cmd := fmt.Sprintf("curl -s -X POST '%s/vol/assign?type=block&size=%s' 2>/dev/null", masterURL, size)
+	stdout, stderr, code, err := node.Run(ctx, cmd)
+	if err != nil || code != 0 {
+		return nil, fmt.Errorf("create_block_volume: code=%d stderr=%s err=%v", code, stderr, err)
+	}
+
+	return map[string]string{"value": strings.TrimSpace(stdout)}, nil
+}
+
+// clusterStatus fetches the full cluster status JSON.
+func clusterStatus(ctx context.Context, actx *tr.ActionContext, act tr.Action) (map[string]string, error) {
+	node, err := getNode(actx, act.Node)
+	if err != nil {
+		return nil, fmt.Errorf("cluster_status: %w", err)
+	}
+
+	masterURL := act.Params["master_url"]
+	if masterURL == "" {
+		return nil, fmt.Errorf("cluster_status: master_url param required")
+	}
+
+	cmd := fmt.Sprintf("curl -s %s/cluster/status 2>/dev/null", masterURL)
+	stdout, stderr, code, err := node.Run(ctx, cmd)
+	if err != nil || code != 0 {
+		return nil, fmt.Errorf("cluster_status: code=%d stderr=%s err=%v", code, stderr, err)
+	}
+
+	// Validate it's JSON.
+	var js json.RawMessage
+	if err := json.Unmarshal([]byte(stdout), &js); err != nil {
+		return nil, fmt.Errorf("cluster_status: invalid JSON response: %s", stdout)
+	}
+
+	return map[string]string{"value": strings.TrimSpace(stdout)}, nil
+}

weed/storage/blockvol/testrunner/actions/devops_test.go

@@ -0,0 +1,101 @@
+package actions
+
+import (
+	"sort"
+	"testing"
+
+	tr "github.com/seaweedfs/seaweedfs/weed/storage/blockvol/testrunner"
+)
+
+func TestDevOpsActions_Registration(t *testing.T) {
+	registry := tr.NewRegistry()
+	RegisterDevOpsActions(registry)
+
+	expected := []string{
+		"build_deploy_weed",
+		"start_weed_master",
+		"start_weed_volume",
+		"stop_weed",
+		"wait_cluster_ready",
+		"create_block_volume",
+		"cluster_status",
+	}
+
+	for _, name := range expected {
+		if _, err := registry.Get(name); err != nil {
+			t.Errorf("action %q not registered: %v", name, err)
+		}
+	}
+}
+
+func TestDevOpsActions_Tier(t *testing.T) {
+	registry := tr.NewRegistry()
+	RegisterDevOpsActions(registry)
+
+	byTier := registry.ListByTier()
+	devopsActions := byTier[tr.TierDevOps]
+
+	if len(devopsActions) != 7 {
+		t.Errorf("devops tier has %d actions, want 7", len(devopsActions))
+	}
+
+	// Verify all are in devops tier.
+	sort.Strings(devopsActions)
+	for _, name := range devopsActions {
+		if tier := registry.ActionTier(name); tier != tr.TierDevOps {
+			t.Errorf("action %q has tier %q, want devops", name, tier)
+		}
+	}
+}
+
+func TestDevOpsActions_TierGating(t *testing.T) {
+	registry := tr.NewRegistry()
+	RegisterDevOpsActions(registry)
+
+	// Without gating, all should be accessible.
+	if _, err := registry.Get("start_weed_master"); err != nil {
+		t.Errorf("ungated: %v", err)
+	}
+
+	// Enable only core tier — devops should be blocked.
+	registry.EnableTiers([]string{tr.TierCore})
+	if _, err := registry.Get("start_weed_master"); err == nil {
+		t.Error("expected error when devops tier is disabled")
+	}
+
+	// Enable devops tier — should work again.
+	registry.EnableTiers([]string{tr.TierDevOps})
+	if _, err := registry.Get("start_weed_master"); err != nil {
+		t.Errorf("devops enabled: %v", err)
+	}
+}
+
+func TestAllActions_Registration(t *testing.T) {
+	registry := tr.NewRegistry()
+	RegisterAll(registry)
+
+	byTier := registry.ListByTier()
+
+	// Verify tier counts.
+	if n := len(byTier[tr.TierCore]); n != 7 {
+		t.Errorf("core: %d, want 7", n)
+	}
+	if n := len(byTier[tr.TierBlock]); n != 33 {
+		t.Errorf("block: %d, want 33", n)
+	}
+	if n := len(byTier[tr.TierDevOps]); n != 7 {
+		t.Errorf("devops: %d, want 7", n)
+	}
+	if n := len(byTier[tr.TierChaos]); n != 5 {
+		t.Errorf("chaos: %d, want 5", n)
+	}
+
+	// Total should be 52.
+	total := 0
+	for _, actions := range byTier {
+		total += len(actions)
+	}
+	if total != 52 {
+		t.Errorf("total actions: %d, want 52", total)
+	}
+}

weed/storage/blockvol/testrunner/actions/fault.go

@@ -0,0 +1,134 @@
+package actions
+
+import (
+	"context"
+	"fmt"
+
+	tr "github.com/seaweedfs/seaweedfs/weed/storage/blockvol/testrunner"
+	"github.com/seaweedfs/seaweedfs/weed/storage/blockvol/testrunner/infra"
+)
+
+// RegisterFaultActions registers fault injection actions.
+func RegisterFaultActions(r *tr.Registry) {
+	r.RegisterFunc("inject_netem", tr.TierChaos, injectNetemAction)
+	r.RegisterFunc("inject_partition", tr.TierChaos, injectPartitionAction)
+	r.RegisterFunc("fill_disk", tr.TierChaos, fillDiskAction)
+	r.RegisterFunc("corrupt_wal", tr.TierChaos, corruptWALAction)
+	r.RegisterFunc("clear_fault", tr.TierChaos, clearFaultAction)
+}
+
+func injectNetemAction(ctx context.Context, actx *tr.ActionContext, act tr.Action) (map[string]string, error) {
+	node, err := getNode(actx, act.Node)
+	if err != nil {
+		return nil, fmt.Errorf("inject_netem: %w", err)
+	}
+
+	targetIP := act.Params["target_ip"]
+	if targetIP == "" {
+		return nil, fmt.Errorf("inject_netem: target_ip param required")
+	}
+	delayMs := parseInt(act.Params["delay_ms"], 200)
+
+	cleanupCmd, err := infra.InjectNetem(ctx, node, targetIP, delayMs)
+	if err != nil {
+		return nil, err
+	}
+
+	// Store cleanup cmd for later clear_fault.
+	varKey := "__cleanup_netem"
+	if act.Node != "" {
+		varKey += "_" + act.Node
+	}
+	return map[string]string{varKey: cleanupCmd}, nil
+}
+
+func injectPartitionAction(ctx context.Context, actx *tr.ActionContext, act tr.Action) (map[string]string, error) {
+	node, err := getNode(actx, act.Node)
+	if err != nil {
+		return nil, fmt.Errorf("inject_partition: %w", err)
+	}
+
+	targetIP := act.Params["target_ip"]
+	if targetIP == "" {
+		return nil, fmt.Errorf("inject_partition: target_ip param required")
+	}
+	ports := parseIntSlice(act.Params["ports"])
+	if len(ports) == 0 {
+		return nil, fmt.Errorf("inject_partition: ports param required")
+	}
+
+	cleanupCmd, err := infra.InjectIptablesDrop(ctx, node, targetIP, ports)
+	if err != nil {
+		return nil, err
+	}
+
+	varKey := "__cleanup_partition"
+	if act.Node != "" {
+		varKey += "_" + act.Node
+	}
+	return map[string]string{varKey: cleanupCmd}, nil
+}
+
+func fillDiskAction(ctx context.Context, actx *tr.ActionContext, act tr.Action) (map[string]string, error) {
+	node, err := getNode(actx, act.Node)
+	if err != nil {
+		return nil, fmt.Errorf("fill_disk: %w", err)
+	}
+
+	dir := act.Params["dir"]
+	if dir == "" {
+		dir = "/tmp"
+	}
+
+	cleanupCmd, err := infra.FillDisk(ctx, node, dir)
+	if err != nil {
+		return nil, err
+	}
+
+	varKey := "__cleanup_fill_disk"
+	if act.Node != "" {
+		varKey += "_" + act.Node
+	}
+	return map[string]string{varKey: cleanupCmd}, nil
+}
+
+func corruptWALAction(ctx context.Context, actx *tr.ActionContext, act tr.Action) (map[string]string, error) {
+	targetName := act.Target
+	if targetName == "" {
+		return nil, fmt.Errorf("corrupt_wal: target param required")
+	}
+
+	tgt, err := getHATarget(actx, targetName)
+	if err != nil {
+		return nil, err
+	}
+
+	nBytes := parseInt(act.Params["bytes"], 4096)
+
+	return nil, infra.CorruptWALRegion(ctx, tgt.Node, tgt.VolFilePath(), nBytes)
+}
+
+func clearFaultAction(ctx context.Context, actx *tr.ActionContext, act tr.Action) (map[string]string, error) {
+	faultType := act.Params["type"]
+	if faultType == "" {
+		return nil, fmt.Errorf("clear_fault: type param required (netem, partition, fill_disk)")
+	}
+
+	node, err := getNode(actx, act.Node)
+	if err != nil {
+		return nil, fmt.Errorf("clear_fault: %w", err)
+	}
+
+	varKey := "__cleanup_" + faultType
+	if act.Node != "" {
+		varKey += "_" + act.Node
+	}
+
+	cleanupCmd := actx.Vars[varKey]
+	if cleanupCmd == "" {
+		// Try without node suffix.
+		cleanupCmd = actx.Vars["__cleanup_"+faultType]
+	}
+
+	return nil, infra.ClearFault(ctx, node, cleanupCmd)
+}

weed/storage/blockvol/testrunner/actions/helpers.go

@@ -0,0 +1,97 @@
+package actions
+
+import (
+	"fmt"
+	"strconv"
+	"strings"
+	"time"
+
+	tr "github.com/seaweedfs/seaweedfs/weed/storage/blockvol/testrunner"
+	"github.com/seaweedfs/seaweedfs/weed/storage/blockvol/testrunner/infra"
+)
+
+// getNode retrieves the infra.Node for the named node from the action context.
+func getNode(actx *tr.ActionContext, name string) (*infra.Node, error) {
+	if name == "" {
+		// Try to get the first available node.
+		for _, n := range actx.Nodes {
+			if nn, ok := n.(*infra.Node); ok {
+				return nn, nil
+			}
+		}
+		return nil, fmt.Errorf("no nodes available")
+	}
+	n, ok := actx.Nodes[name]
+	if !ok {
+		return nil, fmt.Errorf("node %q not found", name)
+	}
+	nn, ok := n.(*infra.Node)
+	if !ok {
+		return nil, fmt.Errorf("node %q is not an infra.Node", name)
+	}
+	return nn, nil
+}
+
+// getTargetNode retrieves the node associated with a target.
+func getTargetNode(actx *tr.ActionContext, targetName string) (*infra.Node, error) {
+	spec, ok := actx.Scenario.Targets[targetName]
+	if !ok {
+		return nil, fmt.Errorf("target %q not in scenario", targetName)
+	}
+	return getNode(actx, spec.Node)
+}
+
+// getTargetHost returns the host address for a target's node.
+func getTargetHost(actx *tr.ActionContext, targetName string) (string, error) {
+	spec, ok := actx.Scenario.Targets[targetName]
+	if !ok {
+		return "", fmt.Errorf("target %q not in scenario", targetName)
+	}
+	nodeSpec, ok := actx.Scenario.Topology.Nodes[spec.Node]
+	if !ok {
+		return "", fmt.Errorf("node %q not in topology", spec.Node)
+	}
+	if nodeSpec.IsLocal {
+		return "127.0.0.1", nil
+	}
+	return nodeSpec.Host, nil
+}
+
+func parseDuration(s string) (time.Duration, error) {
+	return time.ParseDuration(s)
+}
+
+func parseDurationMs(s string) (uint32, error) {
+	d, err := time.ParseDuration(s)
+	if err != nil {
+		// Try parsing as plain number (milliseconds).
+		ms, err2 := strconv.ParseUint(s, 10, 32)
+		if err2 != nil {
+			return 0, err
+		}
+		return uint32(ms), nil
+	}
+	return uint32(d.Milliseconds()), nil
+}
+
+func parseInt(s string, def int) int {
+	if s == "" {
+		return def
+	}
+	v, err := strconv.Atoi(s)
+	if err != nil {
+		return def
+	}
+	return v
+}
+
+func parseIntSlice(s string) []int {
+	var result []int
+	for _, part := range strings.Split(s, ",") {
+		part = strings.TrimSpace(part)
+		if v, err := strconv.Atoi(part); err == nil {
+			result = append(result, v)
+		}
+	}
+	return result
+}

weed/storage/blockvol/testrunner/actions/io.go

@@ -0,0 +1,260 @@
+package actions
+
+import (
+	"context"
+	"fmt"
+	"strings"
+
+	tr "github.com/seaweedfs/seaweedfs/weed/storage/blockvol/testrunner"
+)
+
+// RegisterIOActions registers IO-related actions.
+func RegisterIOActions(r *tr.Registry) {
+	r.RegisterFunc("dd_write", tr.TierBlock, ddWrite)
+	r.RegisterFunc("dd_read_md5", tr.TierBlock, ddReadMD5)
+	r.RegisterFunc("fio", tr.TierBlock, fioAction)
+	r.RegisterFunc("fio_verify", tr.TierBlock, fioVerify)
+	r.RegisterFunc("mkfs", tr.TierBlock, mkfsAction)
+	r.RegisterFunc("mount", tr.TierBlock, mountAction)
+	r.RegisterFunc("umount", tr.TierBlock, umountAction)
+}
+
+// ddWrite writes random data using dd, returns the md5 checksum.
+func ddWrite(ctx context.Context, actx *tr.ActionContext, act tr.Action) (map[string]string, error) {
+	device := act.Params["device"]
+	if device == "" {
+		return nil, fmt.Errorf("dd_write: device param required")
+	}
+	bs := act.Params["bs"]
+	if bs == "" {
+		bs = "1M"
+	}
+	count := act.Params["count"]
+	if count == "" {
+		count = "1"
+	}
+	oflag := act.Params["oflag"]
+	if oflag == "" {
+		oflag = "direct"
+	}
+
+	node, err := getNode(actx, act.Node)
+	if err != nil {
+		return nil, err
+	}
+
+	// Generate random data to temp file, write to device, compute md5.
+	tmpFile := "/tmp/sw-test-runner-dd-data"
+	genCmd := fmt.Sprintf("dd if=/dev/urandom of=%s bs=%s count=%s 2>/dev/null", tmpFile, bs, count)
+	_, stderr, code, err := node.RunRoot(ctx, genCmd)
+	if err != nil || code != 0 {
+		return nil, fmt.Errorf("dd_write gen: code=%d stderr=%s err=%v", code, stderr, err)
+	}
+
+	writeCmd := fmt.Sprintf("dd if=%s of=%s bs=%s oflag=%s conv=fsync", tmpFile, device, bs, oflag)
+	if seek := act.Params["seek"]; seek != "" {
+		writeCmd += fmt.Sprintf(" seek=%s", seek)
+	}
+	writeCmd += " 2>/dev/null"
+	_, stderr, code, err = node.RunRoot(ctx, writeCmd)
+	if err != nil || code != 0 {
+		return nil, fmt.Errorf("dd_write: code=%d stderr=%s err=%v", code, stderr, err)
+	}
+
+	md5Cmd := fmt.Sprintf("md5sum %s | cut -d' ' -f1", tmpFile)
+	stdout, stderr, code, err := node.RunRoot(ctx, md5Cmd)
+	if err != nil || code != 0 {
+		return nil, fmt.Errorf("dd_write md5: code=%d stderr=%s err=%v", code, stderr, err)
+	}
+	node.Run(ctx, fmt.Sprintf("rm -f %s", tmpFile))
+
+	md5 := strings.TrimSpace(stdout)
+	if md5 == "" {
+		return nil, fmt.Errorf("dd_write: empty md5")
+	}
+
+	return map[string]string{"value": md5}, nil
+}
+
+// ddReadMD5 reads from device using dd and returns md5.
+func ddReadMD5(ctx context.Context, actx *tr.ActionContext, act tr.Action) (map[string]string, error) {
+	device := act.Params["device"]
+	if device == "" {
+		return nil, fmt.Errorf("dd_read_md5: device param required")
+	}
+	bs := act.Params["bs"]
+	if bs == "" {
+		bs = "1M"
+	}
+	count := act.Params["count"]
+	if count == "" {
+		count = "1"
+	}
+
+	node, err := getNode(actx, act.Node)
+	if err != nil {
+		return nil, err
+	}
+
+	tmpFile := "/tmp/sw-test-runner-dd-read"
+	readCmd := fmt.Sprintf("dd if=%s of=%s bs=%s count=%s iflag=direct", device, tmpFile, bs, count)
+	if skip := act.Params["skip"]; skip != "" {
+		readCmd += fmt.Sprintf(" skip=%s", skip)
+	}
+	readCmd += " 2>/dev/null"
+	_, stderr, code, err := node.RunRoot(ctx, readCmd)
+	if err != nil || code != 0 {
+		return nil, fmt.Errorf("dd_read_md5 read: code=%d stderr=%s err=%v", code, stderr, err)
+	}
+
+	cmd := fmt.Sprintf("md5sum %s | cut -d' ' -f1", tmpFile)
+	stdout, stderr, code, err := node.RunRoot(ctx, cmd)
+	if err != nil || code != 0 {
+		return nil, fmt.Errorf("dd_read_md5: code=%d stderr=%s err=%v", code, stderr, err)
+	}
+	node.Run(ctx, fmt.Sprintf("rm -f %s", tmpFile))
+
+	md5 := strings.TrimSpace(stdout)
+	if md5 == "" {
+		return nil, fmt.Errorf("dd_read_md5: empty md5")
+	}
+
+	return map[string]string{"value": md5}, nil
+}
+
+func fioAction(ctx context.Context, actx *tr.ActionContext, act tr.Action) (map[string]string, error) {
+	device := act.Params["device"]
+	if device == "" {
+		return nil, fmt.Errorf("fio: device param required")
+	}
+
+	node, err := getNode(actx, act.Node)
+	if err != nil {
+		return nil, err
+	}
+
+	rw := act.Params["rw"]
+	if rw == "" {
+		rw = "randwrite"
+	}
+	bs := act.Params["bs"]
+	if bs == "" {
+		bs = "4k"
+	}
+	iodepth := act.Params["iodepth"]
+	if iodepth == "" {
+		iodepth = "32"
+	}
+	runtime := act.Params["runtime"]
+	if runtime == "" {
+		runtime = "10"
+	}
+	name := act.Params["name"]
+	if name == "" {
+		name = "fio_test"
+	}
+
+	cmd := fmt.Sprintf("fio --name=%s --filename=%s --rw=%s --bs=%s --iodepth=%s --direct=1 --runtime=%s --time_based --output-format=json",
+		name, device, rw, bs, iodepth, runtime)
+	stdout, stderr, code, err := node.RunRoot(ctx, cmd)
+	if err != nil || code != 0 {
+		return nil, fmt.Errorf("fio: code=%d stderr=%s err=%v", code, stderr, err)
+	}
+
+	return map[string]string{"value": stdout}, nil
+}
+
+func fioVerify(ctx context.Context, actx *tr.ActionContext, act tr.Action) (map[string]string, error) {
+	device := act.Params["device"]
+	if device == "" {
+		return nil, fmt.Errorf("fio_verify: device param required")
+	}
+
+	node, err := getNode(actx, act.Node)
+	if err != nil {
+		return nil, err
+	}
+
+	bs := act.Params["bs"]
+	if bs == "" {
+		bs = "4k"
+	}
+	size := act.Params["size"]
+	if size == "" {
+		size = "10M"
+	}
+
+	// Write with verify pattern, then read+verify.
+	cmd := fmt.Sprintf("fio --name=verify --filename=%s --rw=write --bs=%s --size=%s --direct=1 --verify=crc32c --do_verify=1 --output-format=json",
+		device, bs, size)
+	stdout, stderr, code, err := node.RunRoot(ctx, cmd)
+	if err != nil || code != 0 {
+		return nil, fmt.Errorf("fio_verify: code=%d stderr=%s err=%v", code, stderr, err)
+	}
+
+	return map[string]string{"value": stdout}, nil
+}
+
+func mkfsAction(ctx context.Context, actx *tr.ActionContext, act tr.Action) (map[string]string, error) {
+	device := act.Params["device"]
+	if device == "" {
+		return nil, fmt.Errorf("mkfs: device param required")
+	}
+	fstype := act.Params["fstype"]
+	if fstype == "" {
+		fstype = "ext4"
+	}
+
+	node, err := getNode(actx, act.Node)
+	if err != nil {
+		return nil, err
+	}
+
+	cmd := fmt.Sprintf("mkfs.%s -F %s", fstype, device)
+	_, stderr, code, err := node.RunRoot(ctx, cmd)
+	if err != nil || code != 0 {
+		return nil, fmt.Errorf("mkfs: code=%d stderr=%s err=%v", code, stderr, err)
+	}
+	return nil, nil
+}
+
+func mountAction(ctx context.Context, actx *tr.ActionContext, act tr.Action) (map[string]string, error) {
+	device := act.Params["device"]
+	if device == "" {
+		return nil, fmt.Errorf("mount: device param required")
+	}
+	mountpoint := act.Params["mountpoint"]
+	if mountpoint == "" {
+		mountpoint = "/mnt/test"
+	}
+
+	node, err := getNode(actx, act.Node)
+	if err != nil {
+		return nil, err
+	}
+
+	node.RunRoot(ctx, fmt.Sprintf("mkdir -p %s", mountpoint))
+	_, stderr, code, err := node.RunRoot(ctx, fmt.Sprintf("mount %s %s", device, mountpoint))
+	if err != nil || code != 0 {
+		return nil, fmt.Errorf("mount: code=%d stderr=%s err=%v", code, stderr, err)
+	}
+	return map[string]string{"value": mountpoint}, nil
+}
+
+func umountAction(ctx context.Context, actx *tr.ActionContext, act tr.Action) (map[string]string, error) {
+	mountpoint := act.Params["mountpoint"]
+	if mountpoint == "" {
+		mountpoint = "/mnt/test"
+	}
+
+	node, err := getNode(actx, act.Node)
+	if err != nil {
+		return nil, err
+	}
+
+	_, stderr, code, err := node.RunRoot(ctx, fmt.Sprintf("umount %s", mountpoint))
+	if err != nil || code != 0 {
+		return nil, fmt.Errorf("umount: code=%d stderr=%s err=%v", code, stderr, err)
+	}
+	return nil, nil
+}

weed/storage/blockvol/testrunner/actions/iscsi.go

@@ -0,0 +1,132 @@
+package actions
+
+import (
+	"context"
+	"fmt"
+
+	tr "github.com/seaweedfs/seaweedfs/weed/storage/blockvol/testrunner"
+	"github.com/seaweedfs/seaweedfs/weed/storage/blockvol/testrunner/infra"
+)
+
+// RegisterISCSIActions registers iSCSI client actions.
+func RegisterISCSIActions(r *tr.Registry) {
+	r.RegisterFunc("iscsi_login", tr.TierBlock, iscsiLogin)
+	r.RegisterFunc("iscsi_logout", tr.TierBlock, iscsiLogout)
+	r.RegisterFunc("iscsi_discover", tr.TierBlock, iscsiDiscover)
+	r.RegisterFunc("iscsi_cleanup", tr.TierBlock, iscsiCleanup)
+}
+
+// iscsiLogin discovers + logs into the target, returns the device path.
+func iscsiLogin(ctx context.Context, actx *tr.ActionContext, act tr.Action) (map[string]string, error) {
+	targetName := act.Target
+	if targetName == "" {
+		return nil, fmt.Errorf("iscsi_login: target is required")
+	}
+
+	spec, ok := actx.Scenario.Targets[targetName]
+	if !ok {
+		return nil, fmt.Errorf("iscsi_login: target %q not in scenario", targetName)
+	}
+
+	host, err := getTargetHost(actx, targetName)
+	if err != nil {
+		return nil, err
+	}
+
+	// Get the initiator node (first available or explicit).
+	node, err := getNode(actx, act.Node)
+	if err != nil {
+		return nil, fmt.Errorf("iscsi_login: %w", err)
+	}
+
+	client := infra.NewISCSIClient(node)
+	iqn := spec.IQN()
+	port := spec.ISCSIPort
+
+	actx.Log("  discovering %s:%d ...", host, port)
+	iqns, err := client.Discover(ctx, host, port)
+	if err != nil {
+		return nil, fmt.Errorf("iscsi_login discover: %w", err)
+	}
+
+	// Find matching IQN.
+	found := false
+	for _, q := range iqns {
+		if q == iqn {
+			found = true
+			break
+		}
+	}
+	if !found {
+		return nil, fmt.Errorf("iscsi_login: IQN %s not found in discovery (got %v)", iqn, iqns)
+	}
+
+	actx.Log("  logging in to %s ...", iqn)
+	dev, err := client.Login(ctx, iqn)
+	if err != nil {
+		return nil, fmt.Errorf("iscsi_login: %w", err)
+	}
+
+	actx.Log("  device: %s", dev)
+	return map[string]string{"value": dev}, nil
+}
+
+func iscsiLogout(ctx context.Context, actx *tr.ActionContext, act tr.Action) (map[string]string, error) {
+	targetName := act.Target
+	if targetName == "" {
+		return nil, fmt.Errorf("iscsi_logout: target is required")
+	}
+
+	spec, ok := actx.Scenario.Targets[targetName]
+	if !ok {
+		return nil, fmt.Errorf("iscsi_logout: target %q not in scenario", targetName)
+	}
+
+	node, err := getNode(actx, act.Node)
+	if err != nil {
+		return nil, fmt.Errorf("iscsi_logout: %w", err)
+	}
+
+	client := infra.NewISCSIClient(node)
+	return nil, client.Logout(ctx, spec.IQN())
+}
+
+func iscsiDiscover(ctx context.Context, actx *tr.ActionContext, act tr.Action) (map[string]string, error) {
+	targetName := act.Target
+	if targetName == "" {
+		return nil, fmt.Errorf("iscsi_discover: target is required")
+	}
+
+	spec, ok := actx.Scenario.Targets[targetName]
+	if !ok {
+		return nil, fmt.Errorf("iscsi_discover: target %q not in scenario", targetName)
+	}
+
+	host, err := getTargetHost(actx, targetName)
+	if err != nil {
+		return nil, err
+	}
+
+	node, err := getNode(actx, act.Node)
+	if err != nil {
+		return nil, fmt.Errorf("iscsi_discover: %w", err)
+	}
+
+	client := infra.NewISCSIClient(node)
+	iqns, err := client.Discover(ctx, host, spec.ISCSIPort)
+	if err != nil {
+		return nil, err
+	}
+
+	return map[string]string{"value": fmt.Sprintf("%v", iqns)}, nil
+}
+
+func iscsiCleanup(ctx context.Context, actx *tr.ActionContext, act tr.Action) (map[string]string, error) {
+	node, err := getNode(actx, act.Node)
+	if err != nil {
+		return nil, fmt.Errorf("iscsi_cleanup: %w", err)
+	}
+
+	client := infra.NewISCSIClient(node)
+	return nil, client.CleanupAll(ctx, "iqn.2024.com.seaweedfs:")
+}

weed/storage/blockvol/testrunner/actions/metrics.go

@@ -0,0 +1,109 @@
+package actions
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+
+	tr "github.com/seaweedfs/seaweedfs/weed/storage/blockvol/testrunner"
+	"github.com/seaweedfs/seaweedfs/weed/storage/blockvol/testrunner/infra"
+)
+
+// RegisterMetricsActions registers metrics/perf actions.
+func RegisterMetricsActions(r *tr.Registry) {
+	r.RegisterFunc("scrape_metrics", tr.TierBlock, scrapeMetrics)
+	r.RegisterFunc("perf_summary", tr.TierBlock, perfSummary)
+	r.RegisterFunc("collect_artifacts", tr.TierBlock, collectArtifactsAction)
+}
+
+// scrapeMetrics fetches /metrics from a target's admin port via SSH curl.
+// Saves the parsed metrics as JSON string in save_as.
+func scrapeMetrics(ctx context.Context, actx *tr.ActionContext, act tr.Action) (map[string]string, error) {
+	tgt, err := getHATarget(actx, act.Target)
+	if err != nil {
+		return nil, err
+	}
+
+	cmd := fmt.Sprintf("curl -s http://127.0.0.1:%d/metrics 2>&1", tgt.AdminPort)
+	stdout, _, code, err := tgt.Node.Run(ctx, cmd)
+	if err != nil || code != 0 {
+		return nil, fmt.Errorf("scrape_metrics %s: code=%d err=%v", act.Target, code, err)
+	}
+
+	sample := tr.MetricsSample{
+		Target:  act.Target,
+		Metrics: tr.ParsePrometheusText(stdout),
+	}
+	data, _ := json.Marshal(sample)
+	return map[string]string{"value": string(data)}, nil
+}
+
+// perfSummary collects the target's log, parses PERF[5s] lines, and outputs stats.
+func perfSummary(ctx context.Context, actx *tr.ActionContext, act tr.Action) (map[string]string, error) {
+	tgt, err := getHATarget(actx, act.Target)
+	if err != nil {
+		return nil, err
+	}
+
+	logContent, err := tgt.CollectLog()
+	if err != nil {
+		return nil, fmt.Errorf("perf_summary: collect log: %w", err)
+	}
+
+	perfData := tr.ParsePerfLogLines(logContent)
+	if len(perfData) == 0 {
+		return map[string]string{"value": "no PERF data found"}, nil
+	}
+
+	var lines []string
+	for field, values := range perfData {
+		stats := tr.ComputeStats(values)
+		lines = append(lines, tr.FormatStats(field, stats))
+	}
+
+	result := ""
+	for i, l := range lines {
+		if i > 0 {
+			result += "\n"
+		}
+		result += l
+	}
+	return map[string]string{"value": result}, nil
+}
+
+// collectArtifactsAction explicitly collects artifacts from targets.
+func collectArtifactsAction(ctx context.Context, actx *tr.ActionContext, act tr.Action) (map[string]string, error) {
+	dir := act.Params["dir"]
+	if dir == "" {
+		dir = "/tmp/sw-test-runner-artifacts"
+	}
+
+	// Find client node for dmesg/lsblk.
+	clientNodeName := act.Node
+	if clientNodeName == "" {
+		clientNodeName = "client_node"
+	}
+	node, _ := getNode(actx, clientNodeName)
+	if node == nil {
+		// Use any available node.
+		for _, n := range actx.Nodes {
+			if nn, ok := n.(*infra.Node); ok {
+				node = nn
+				break
+			}
+		}
+	}
+	if node == nil {
+		return nil, fmt.Errorf("collect_artifacts: no node available")
+	}
+
+	collector := infra.NewArtifactCollector(dir, node, nil)
+
+	for name, tgt := range actx.Targets {
+		if lc, ok := tgt.(infra.LogCollector); ok {
+			collector.CollectLabeled(lc, name)
+		}
+	}
+
+	return map[string]string{"value": fmt.Sprintf("artifacts saved to %s", dir)}, nil
+}

weed/storage/blockvol/testrunner/actions/register.go

@@ -0,0 +1,15 @@
+package actions
+
+import tr "github.com/seaweedfs/seaweedfs/weed/storage/blockvol/testrunner"
+
+// RegisterAll registers all action handlers on the given registry.
+func RegisterAll(r *tr.Registry) {
+	RegisterBlockActions(r)
+	RegisterISCSIActions(r)
+	RegisterIOActions(r)
+	RegisterFaultActions(r)
+	RegisterSystemActions(r)
+	RegisterMetricsActions(r)
+	RegisterDevOpsActions(r)
+	RegisterSnapshotActions(r)
+}

weed/storage/blockvol/testrunner/actions/snapshot.go

@@ -0,0 +1,183 @@
+package actions
+
+import (
+	"context"
+	"fmt"
+	"strconv"
+	"strings"
+	"time"
+
+	tr "github.com/seaweedfs/seaweedfs/weed/storage/blockvol/testrunner"
+)
+
+// RegisterSnapshotActions registers snapshot and resize actions.
+func RegisterSnapshotActions(r *tr.Registry) {
+	r.RegisterFunc("snapshot_create", tr.TierBlock, snapshotCreate)
+	r.RegisterFunc("snapshot_delete", tr.TierBlock, snapshotDelete)
+	r.RegisterFunc("snapshot_list", tr.TierBlock, snapshotList)
+	r.RegisterFunc("resize", tr.TierBlock, resizeAction)
+	r.RegisterFunc("iscsi_rescan", tr.TierBlock, iscsiRescan)
+	r.RegisterFunc("get_block_size", tr.TierBlock, getBlockSize)
+}
+
+func snapshotCreate(ctx context.Context, actx *tr.ActionContext, act tr.Action) (map[string]string, error) {
+	tgt, err := getHATarget(actx, act.Target)
+	if err != nil {
+		return nil, err
+	}
+
+	idStr := act.Params["id"]
+	if idStr == "" {
+		return nil, fmt.Errorf("snapshot_create: id param required")
+	}
+	id, err := strconv.ParseUint(idStr, 10, 32)
+	if err != nil {
+		return nil, fmt.Errorf("snapshot_create: invalid id %q: %w", idStr, err)
+	}
+
+	if err := tgt.CreateSnapshot(ctx, uint32(id)); err != nil {
+		return nil, fmt.Errorf("snapshot_create: %w", err)
+	}
+	actx.Log("  created snapshot %d", id)
+	return nil, nil
+}
+
+func snapshotDelete(ctx context.Context, actx *tr.ActionContext, act tr.Action) (map[string]string, error) {
+	tgt, err := getHATarget(actx, act.Target)
+	if err != nil {
+		return nil, err
+	}
+
+	idStr := act.Params["id"]
+	if idStr == "" {
+		return nil, fmt.Errorf("snapshot_delete: id param required")
+	}
+	id, err := strconv.ParseUint(idStr, 10, 32)
+	if err != nil {
+		return nil, fmt.Errorf("snapshot_delete: invalid id %q: %w", idStr, err)
+	}
+
+	if err := tgt.DeleteSnapshot(ctx, uint32(id)); err != nil {
+		return nil, fmt.Errorf("snapshot_delete: %w", err)
+	}
+	actx.Log("  deleted snapshot %d", id)
+	return nil, nil
+}
+
+func snapshotList(ctx context.Context, actx *tr.ActionContext, act tr.Action) (map[string]string, error) {
+	tgt, err := getHATarget(actx, act.Target)
+	if err != nil {
+		return nil, err
+	}
+
+	snaps, err := tgt.ListSnapshots(ctx)
+	if err != nil {
+		return nil, fmt.Errorf("snapshot_list: %w", err)
+	}
+
+	actx.Log("  snapshots: %d entries", len(snaps))
+	return map[string]string{"value": strconv.Itoa(len(snaps))}, nil
+}
+
+func resizeAction(ctx context.Context, actx *tr.ActionContext, act tr.Action) (map[string]string, error) {
+	tgt, err := getHATarget(actx, act.Target)
+	if err != nil {
+		return nil, err
+	}
+
+	var newSizeBytes uint64
+	if s := act.Params["new_size_bytes"]; s != "" {
+		newSizeBytes, err = strconv.ParseUint(s, 10, 64)
+		if err != nil {
+			return nil, fmt.Errorf("resize: invalid new_size_bytes %q: %w", s, err)
+		}
+	} else if s := act.Params["new_size"]; s != "" {
+		newSizeBytes, err = parseHumanSize(s)
+		if err != nil {
+			return nil, fmt.Errorf("resize: invalid new_size %q: %w", s, err)
+		}
+	} else {
+		return nil, fmt.Errorf("resize: new_size or new_size_bytes param required")
+	}
+
+	if err := tgt.Resize(ctx, newSizeBytes); err != nil {
+		return nil, fmt.Errorf("resize: %w", err)
+	}
+	actx.Log("  resized to %d bytes", newSizeBytes)
+	return nil, nil
+}
+
+func iscsiRescan(ctx context.Context, actx *tr.ActionContext, act tr.Action) (map[string]string, error) {
+	node, err := getNode(actx, act.Node)
+	if err != nil {
+		return nil, fmt.Errorf("iscsi_rescan: %w", err)
+	}
+
+	_, stderr, code, err := node.RunRoot(ctx, "iscsiadm -m session -R")
+	if err != nil || code != 0 {
+		return nil, fmt.Errorf("iscsi_rescan: code=%d stderr=%s err=%v", code, stderr, err)
+	}
+
+	// Give kernel time to update block device size.
+	select {
+	case <-time.After(2 * time.Second):
+	case <-ctx.Done():
+		return nil, ctx.Err()
+	}
+
+	actx.Log("  iSCSI sessions rescanned")
+	return nil, nil
+}
+
+func getBlockSize(ctx context.Context, actx *tr.ActionContext, act tr.Action) (map[string]string, error) {
+	device := act.Params["device"]
+	if device == "" {
+		return nil, fmt.Errorf("get_block_size: device param required")
+	}
+
+	node, err := getNode(actx, act.Node)
+	if err != nil {
+		return nil, fmt.Errorf("get_block_size: %w", err)
+	}
+
+	cmd := fmt.Sprintf("blockdev --getsize64 %s", device)
+	stdout, stderr, code, err := node.RunRoot(ctx, cmd)
+	if err != nil || code != 0 {
+		return nil, fmt.Errorf("get_block_size: code=%d stderr=%s err=%v", code, stderr, err)
+	}
+
+	size := strings.TrimSpace(stdout)
+	actx.Log("  %s size: %s bytes", device, size)
+	return map[string]string{"value": size}, nil
+}
+
+// parseHumanSize converts human-readable sizes like "100M", "1G", "512K" to bytes.
+func parseHumanSize(s string) (uint64, error) {
+	s = strings.TrimSpace(s)
+	if len(s) == 0 {
+		return 0, fmt.Errorf("empty size")
+	}
+
+	multiplier := uint64(1)
+	suffix := s[len(s)-1]
+	switch suffix {
+	case 'K', 'k':
+		multiplier = 1024
+		s = s[:len(s)-1]
+	case 'M', 'm':
+		multiplier = 1024 * 1024
+		s = s[:len(s)-1]
+	case 'G', 'g':
+		multiplier = 1024 * 1024 * 1024
+		s = s[:len(s)-1]
+	case 'T', 't':
+		multiplier = 1024 * 1024 * 1024 * 1024
+		s = s[:len(s)-1]
+	}
+
+	val, err := strconv.ParseUint(s, 10, 64)
+	if err != nil {
+		return 0, err
+	}
+	return val * multiplier, nil
+}

weed/storage/blockvol/testrunner/actions/system.go

@@ -0,0 +1,150 @@
+package actions
+
+import (
+	"context"
+	"fmt"
+	"strconv"
+	"strings"
+	"time"
+
+	tr "github.com/seaweedfs/seaweedfs/weed/storage/blockvol/testrunner"
+)
+
+// RegisterSystemActions registers system/assert actions.
+func RegisterSystemActions(r *tr.Registry) {
+	r.RegisterFunc("exec", tr.TierCore, execAction)
+	r.RegisterFunc("sleep", tr.TierCore, sleepAction)
+	r.RegisterFunc("assert_equal", tr.TierCore, assertEqual)
+	r.RegisterFunc("assert_greater", tr.TierCore, assertGreater)
+	r.RegisterFunc("assert_status", tr.TierCore, assertStatus)
+	r.RegisterFunc("assert_contains", tr.TierCore, assertContains)
+	r.RegisterFunc("print", tr.TierCore, printAction)
+}
+
+func execAction(ctx context.Context, actx *tr.ActionContext, act tr.Action) (map[string]string, error) {
+	cmd := act.Params["cmd"]
+	if cmd == "" {
+		return nil, fmt.Errorf("exec: cmd param required")
+	}
+
+	node, err := getNode(actx, act.Node)
+	if err != nil {
+		return nil, err
+	}
+
+	root := act.Params["root"] == "true"
+	var stdout, stderr string
+	var code int
+	if root {
+		stdout, stderr, code, err = node.RunRoot(ctx, cmd)
+	} else {
+		stdout, stderr, code, err = node.Run(ctx, cmd)
+	}
+	if err != nil {
+		return nil, fmt.Errorf("exec: %w", err)
+	}
+	if code != 0 {
+		return nil, fmt.Errorf("exec: code=%d stderr=%s", code, stderr)
+	}
+
+	return map[string]string{"value": strings.TrimSpace(stdout)}, nil
+}
+
+func sleepAction(ctx context.Context, actx *tr.ActionContext, act tr.Action) (map[string]string, error) {
+	d := act.Params["duration"]
+	if d == "" {
+		d = "1s"
+	}
+
+	dur, err := time.ParseDuration(d)
+	if err != nil {
+		return nil, fmt.Errorf("sleep: invalid duration %q: %w", d, err)
+	}
+
+	select {
+	case <-time.After(dur):
+		return nil, nil
+	case <-ctx.Done():
+		return nil, ctx.Err()
+	}
+}
+
+func assertEqual(ctx context.Context, actx *tr.ActionContext, act tr.Action) (map[string]string, error) {
+	actual := act.Params["actual"]
+	expected := act.Params["expected"]
+
+	if actual != expected {
+		return nil, fmt.Errorf("assert_equal: %q != %q", actual, expected)
+	}
+	return nil, nil
+}
+
+func assertGreater(ctx context.Context, actx *tr.ActionContext, act tr.Action) (map[string]string, error) {
+	actualStr := act.Params["actual"]
+	expectedStr := act.Params["expected"]
+
+	actual, err := strconv.ParseInt(actualStr, 10, 64)
+	if err != nil {
+		return nil, fmt.Errorf("assert_greater: cannot parse actual %q as int: %w", actualStr, err)
+	}
+	expected, err := strconv.ParseInt(expectedStr, 10, 64)
+	if err != nil {
+		return nil, fmt.Errorf("assert_greater: cannot parse expected %q as int: %w", expectedStr, err)
+	}
+
+	if actual <= expected {
+		return nil, fmt.Errorf("assert_greater: %d <= %d", actual, expected)
+	}
+	return nil, nil
+}
+
+func assertStatus(ctx context.Context, actx *tr.ActionContext, act tr.Action) (map[string]string, error) {
+	tgt, err := getHATarget(actx, act.Target)
+	if err != nil {
+		return nil, err
+	}
+
+	st, err := tgt.Status(ctx)
+	if err != nil {
+		return nil, fmt.Errorf("assert_status: %w", err)
+	}
+
+	if role, ok := act.Params["role"]; ok {
+		if st.Role != role {
+			return nil, fmt.Errorf("assert_status: role %q != expected %q", st.Role, role)
+		}
+	}
+	if healthy, ok := act.Params["healthy"]; ok {
+		expectedHealthy := healthy == "true"
+		if st.Healthy != expectedHealthy {
+			return nil, fmt.Errorf("assert_status: healthy=%v != expected=%v", st.Healthy, expectedHealthy)
+		}
+	}
+	if hasLease, ok := act.Params["has_lease"]; ok {
+		expectedLease := hasLease == "true"
+		if st.HasLease != expectedLease {
+			return nil, fmt.Errorf("assert_status: has_lease=%v != expected=%v", st.HasLease, expectedLease)
+		}
+	}
+
+	return nil, nil
+}
+
+func assertContains(ctx context.Context, actx *tr.ActionContext, act tr.Action) (map[string]string, error) {
+	haystack := act.Params["value"]
+	needle := act.Params["contains"]
+
+	if !strings.Contains(haystack, needle) {
+		return nil, fmt.Errorf("assert_contains: %q not found in %q", needle, haystack)
+	}
+	return nil, nil
+}
+
+func printAction(ctx context.Context, actx *tr.ActionContext, act tr.Action) (map[string]string, error) {
+	msg := act.Params["msg"]
+	if msg == "" {
+		msg = act.Params["message"]
+	}
+	actx.Log("  [print] %s", msg)
+	return nil, nil
+}

weed/storage/blockvol/testrunner/agent.go

@@ -0,0 +1,694 @@
+package testrunner
+
+import (
+	"archive/tar"
+	"compress/gzip"
+	"context"
+	"encoding/json"
+	"fmt"
+	"io"
+	"log"
+	"net"
+	"net/http"
+	"os"
+	"path/filepath"
+	"strings"
+	"sync"
+	"time"
+
+	"github.com/seaweedfs/seaweedfs/weed/storage/blockvol/testrunner/infra"
+)
+
+// Agent runs on each test node, executing actions locally on behalf of the coordinator.
+type Agent struct {
+	ID            string
+	Hostname      string
+	Port          int
+	CoordinatorURL string
+	Token         string
+	AllowExec     bool
+	Persistent    bool     // stay running across coordinator restarts
+	Nodes         []string // node names this agent handles
+
+	registry  *Registry
+	localNode *LocalNode
+	startTime time.Time
+	log       *log.Logger
+
+	mu       sync.Mutex
+	server   *http.Server
+	listener net.Listener
+}
+
+// AgentConfig holds configuration for creating an Agent.
+type AgentConfig struct {
+	Hostname       string
+	Port           int
+	CoordinatorURL string
+	Token          string
+	AllowExec      bool
+	Persistent     bool // stay running, re-register with coordinator
+	Nodes          []string
+	Registry       *Registry
+	Logger         *log.Logger
+}
+
+// NewAgent creates a new Agent with the given configuration.
+func NewAgent(cfg AgentConfig) *Agent {
+	hostname := cfg.Hostname
+	if hostname == "" {
+		hostname, _ = os.Hostname()
+	}
+
+	agentID := fmt.Sprintf("agent-%s-%d", hostname, os.Getpid())
+
+	logger := cfg.Logger
+	if logger == nil {
+		logger = log.New(os.Stderr, "[agent] ", log.LstdFlags)
+	}
+
+	return &Agent{
+		ID:             agentID,
+		Hostname:       hostname,
+		Port:           cfg.Port,
+		CoordinatorURL: cfg.CoordinatorURL,
+		Token:          cfg.Token,
+		AllowExec:      cfg.AllowExec,
+		Persistent:     cfg.Persistent,
+		Nodes:          cfg.Nodes,
+		registry:       cfg.Registry,
+		localNode:      NewLocalNode(hostname),
+		startTime:      time.Now(),
+		log:            logger,
+	}
+}
+
+// Start begins listening for coordinator requests. Blocks until the server is shut down.
+func (a *Agent) Start(ctx context.Context) error {
+	mux := http.NewServeMux()
+	mux.HandleFunc("/health", a.handleHealth)
+	mux.HandleFunc("/phase", a.authMiddleware(a.handlePhase))
+	mux.HandleFunc("/upload", a.authMiddleware(a.handleUpload))
+	mux.HandleFunc("/artifacts", a.authMiddleware(a.handleArtifacts))
+	mux.HandleFunc("/exec", a.authMiddleware(a.handleExec))
+
+	addr := fmt.Sprintf(":%d", a.Port)
+	ln, err := net.Listen("tcp", addr)
+	if err != nil {
+		return fmt.Errorf("listen %s: %w", addr, err)
+	}
+
+	a.mu.Lock()
+	a.listener = ln
+	a.server = &http.Server{Handler: mux}
+	a.mu.Unlock()
+
+	a.log.Printf("agent %s listening on %s (root=%v, exec=%v)", a.ID, addr, a.localNode.IsRoot(), a.AllowExec)
+
+	// Register with coordinator if URL is set.
+	if a.CoordinatorURL != "" {
+		if a.Persistent {
+			go a.registrationLoop(ctx)
+		} else {
+			go a.registerWithCoordinator()
+		}
+	}
+
+	// Shutdown on context cancellation.
+	go func() {
+		<-ctx.Done()
+		a.Stop()
+	}()
+
+	if err := a.server.Serve(ln); err != nil && err != http.ErrServerClosed {
+		return err
+	}
+	return nil
+}
+
+// Stop gracefully shuts down the agent HTTP server.
+func (a *Agent) Stop() {
+	a.mu.Lock()
+	srv := a.server
+	a.mu.Unlock()
+
+	if srv != nil {
+		ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
+		defer cancel()
+		srv.Shutdown(ctx)
+	}
+}
+
+// ListenAddr returns the address the agent is listening on (useful in tests).
+func (a *Agent) ListenAddr() string {
+	a.mu.Lock()
+	defer a.mu.Unlock()
+	if a.listener != nil {
+		return a.listener.Addr().String()
+	}
+	return ""
+}
+
+func (a *Agent) registerWithCoordinator() {
+	req := RegisterRequest{
+		AgentID:  a.ID,
+		Hostname: a.Hostname,
+		IP:       a.detectIP(),
+		Port:     a.Port,
+		Nodes:    a.Nodes,
+	}
+
+	// Detect capabilities.
+	var caps []string
+	if a.localNode.IsRoot() {
+		caps = append(caps, "root")
+	}
+	ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
+	defer cancel()
+	if _, _, code, err := a.localNode.Run(ctx, "which iscsiadm"); err == nil && code == 0 {
+		caps = append(caps, "iscsi")
+	}
+	if _, _, code, err := a.localNode.Run(ctx, "which fio"); err == nil && code == 0 {
+		caps = append(caps, "fio")
+	}
+	req.Capabilities = caps
+
+	body, _ := json.Marshal(req)
+	url := strings.TrimRight(a.CoordinatorURL, "/") + "/register"
+
+	// Retry registration up to 60s with 2s intervals.
+	deadline := time.Now().Add(60 * time.Second)
+	for attempt := 1; ; attempt++ {
+		httpReq, _ := http.NewRequest("POST", url, strings.NewReader(string(body)))
+		httpReq.Header.Set("Content-Type", "application/json")
+		if a.Token != "" {
+			httpReq.Header.Set(AuthTokenHeader, a.Token)
+		}
+
+		resp, err := http.DefaultClient.Do(httpReq)
+		if err != nil {
+			if time.Now().After(deadline) {
+				a.log.Printf("register failed after retries: %v", err)
+				return
+			}
+			a.log.Printf("register attempt %d failed: %v (retrying...)", attempt, err)
+			time.Sleep(2 * time.Second)
+			continue
+		}
+
+		var rr RegisterResponse
+		json.NewDecoder(resp.Body).Decode(&rr)
+		resp.Body.Close()
+		if rr.OK {
+			a.log.Printf("registered with coordinator (index=%d, total=%d)", rr.AgentIndex, rr.TotalAgents)
+		} else {
+			a.log.Printf("registration rejected: %s", rr.Error)
+		}
+		return
+	}
+}
+
+// registrationLoop continuously re-registers with the coordinator.
+// Used in persistent mode so the agent survives coordinator restarts.
+func (a *Agent) registrationLoop(ctx context.Context) {
+	req := a.buildRegisterRequest()
+	body, _ := json.Marshal(req)
+	url := strings.TrimRight(a.CoordinatorURL, "/") + "/register"
+
+	registered := false
+	for {
+		select {
+		case <-ctx.Done():
+			return
+		default:
+		}
+
+		httpReq, _ := http.NewRequestWithContext(ctx, "POST", url, strings.NewReader(string(body)))
+		httpReq.Header.Set("Content-Type", "application/json")
+		if a.Token != "" {
+			httpReq.Header.Set(AuthTokenHeader, a.Token)
+		}
+
+		resp, err := http.DefaultClient.Do(httpReq)
+		if err != nil {
+			if registered {
+				a.log.Printf("coordinator unreachable, will retry: %v", err)
+				registered = false
+			}
+			sleepCtx(ctx, 5*time.Second)
+			continue
+		}
+
+		var rr RegisterResponse
+		json.NewDecoder(resp.Body).Decode(&rr)
+		resp.Body.Close()
+
+		if rr.OK {
+			if !registered {
+				a.log.Printf("registered with coordinator (index=%d, total=%d)", rr.AgentIndex, rr.TotalAgents)
+				registered = true
+			}
+			sleepCtx(ctx, 10*time.Second)
+		} else {
+			a.log.Printf("registration rejected: %s", rr.Error)
+			sleepCtx(ctx, 5*time.Second)
+		}
+	}
+}
+
+// buildRegisterRequest creates the registration payload with capability detection.
+func (a *Agent) buildRegisterRequest() RegisterRequest {
+	req := RegisterRequest{
+		AgentID:  a.ID,
+		Hostname: a.Hostname,
+		IP:       a.detectIP(),
+		Port:     a.Port,
+		Nodes:    a.Nodes,
+	}
+	var caps []string
+	if a.localNode.IsRoot() {
+		caps = append(caps, "root")
+	}
+	ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
+	defer cancel()
+	if _, _, code, err := a.localNode.Run(ctx, "which iscsiadm"); err == nil && code == 0 {
+		caps = append(caps, "iscsi")
+	}
+	if _, _, code, err := a.localNode.Run(ctx, "which fio"); err == nil && code == 0 {
+		caps = append(caps, "fio")
+	}
+	req.Capabilities = caps
+	return req
+}
+
+// sleepCtx sleeps for d or until ctx is cancelled.
+func sleepCtx(ctx context.Context, d time.Duration) {
+	select {
+	case <-time.After(d):
+	case <-ctx.Done():
+	}
+}
+
+func (a *Agent) detectIP() string {
+	addrs, err := net.InterfaceAddrs()
+	if err != nil {
+		return ""
+	}
+	for _, addr := range addrs {
+		if ipNet, ok := addr.(*net.IPNet); ok && !ipNet.IP.IsLoopback() && ipNet.IP.To4() != nil {
+			return ipNet.IP.String()
+		}
+	}
+	return ""
+}
+
+// authMiddleware wraps a handler with token authentication.
+func (a *Agent) authMiddleware(next http.HandlerFunc) http.HandlerFunc {
+	return func(w http.ResponseWriter, r *http.Request) {
+		if a.Token != "" {
+			token := r.Header.Get(AuthTokenHeader)
+			if token != a.Token {
+				writeJSON(w, http.StatusUnauthorized, map[string]string{"error": "invalid auth token"})
+				return
+			}
+		}
+		next(w, r)
+	}
+}
+
+// GET /health
+func (a *Agent) handleHealth(w http.ResponseWriter, r *http.Request) {
+	if r.Method != http.MethodGet {
+		http.Error(w, "method not allowed", http.StatusMethodNotAllowed)
+		return
+	}
+	resp := HealthResponse{
+		OK:       true,
+		AgentID:  a.ID,
+		UptimeS:  int64(time.Since(a.startTime).Seconds()),
+		HasRoot:  a.localNode.IsRoot(),
+		Hostname: a.Hostname,
+	}
+	writeJSON(w, http.StatusOK, resp)
+}
+
+// POST /phase
+func (a *Agent) handlePhase(w http.ResponseWriter, r *http.Request) {
+	if r.Method != http.MethodPost {
+		http.Error(w, "method not allowed", http.StatusMethodNotAllowed)
+		return
+	}
+
+	var req PhaseRequest
+	if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
+		writeJSON(w, http.StatusBadRequest, PhaseResponse{Error: fmt.Sprintf("decode: %v", err)})
+		return
+	}
+
+	a.log.Printf("phase %d (%s): %d actions", req.PhaseIndex, req.PhaseName, len(req.Actions))
+
+	resp := a.executePhase(r.Context(), &req)
+	writeJSON(w, http.StatusOK, resp)
+}
+
+// executePhase runs actions sequentially using the local engine.
+func (a *Agent) executePhase(ctx context.Context, req *PhaseRequest) PhaseResponse {
+	resp := PhaseResponse{
+		PhaseIndex: req.PhaseIndex,
+		NewVars:    make(map[string]string),
+	}
+
+	// Build action context.
+	actx := &ActionContext{
+		Scenario: req.Scenario,
+		Nodes:    make(map[string]NodeRunner),
+		Targets:  make(map[string]TargetRunner),
+		Vars:     make(map[string]string),
+		Log:      func(format string, args ...interface{}) { a.log.Printf(format, args...) },
+	}
+
+	// Copy global vars.
+	for k, v := range req.GlobalVars {
+		actx.Vars[k] = v
+	}
+
+	// Create infra.Node in native mode for local command execution.
+	nativeNode := &infra.Node{IsNative: true}
+
+	// Map agent's nodes to native node runner.
+	for _, nodeName := range a.Nodes {
+		actx.Nodes[nodeName] = nativeNode
+	}
+
+	// Set up targets for nodes this agent handles.
+	myNodes := make(map[string]bool)
+	for _, n := range a.Nodes {
+		myNodes[n] = true
+	}
+	if req.Scenario != nil {
+		// Map node names to agents via topology.
+		nodeToAgentName := make(map[string]string)
+		for nodeName, nodeSpec := range req.Scenario.Topology.Nodes {
+			nodeToAgentName[nodeName] = nodeSpec.Agent
+		}
+		for tgtName, tgtSpec := range req.Scenario.Targets {
+			agentName := nodeToAgentName[tgtSpec.Node]
+			if !myNodes[agentName] {
+				continue
+			}
+			htSpec := infra.HATargetSpec{
+				VolSize:         tgtSpec.VolSize,
+				WALSize:         tgtSpec.WALSize,
+				IQN:             tgtSpec.IQN(),
+				ISCSIPort:       tgtSpec.ISCSIPort,
+				AdminPort:       tgtSpec.AdminPort,
+				ReplicaDataPort: tgtSpec.ReplicaDataPort,
+				ReplicaCtrlPort: tgtSpec.ReplicaCtrlPort,
+				RebuildPort:     tgtSpec.RebuildPort,
+				TPGID:           tgtSpec.TPGID,
+			}
+			actx.Targets[tgtName] = infra.NewHATargetFromSpec(nativeNode, tgtName, htSpec)
+		}
+		// Also map topology node names to native node if this agent handles them.
+		for nodeName, nodeSpec := range req.Scenario.Topology.Nodes {
+			if myNodes[nodeSpec.Agent] {
+				actx.Nodes[nodeName] = nativeNode
+			}
+		}
+	}
+
+	engine := NewEngine(a.registry, actx.Log)
+
+	for _, act := range req.Actions {
+		start := time.Now()
+		resolved := resolveAction(act, actx.Vars)
+		yamlDef := marshalActionYAML(resolved)
+
+		handler, err := engine.registry.Get(resolved.Action)
+		if err != nil {
+			resp.Results = append(resp.Results, ActionResult{
+				Action:   resolved.Action,
+				Status:   StatusFail,
+				Duration: time.Since(start),
+				Error:    err.Error(),
+			})
+			resp.Error = err.Error()
+			return resp
+		}
+
+		// Handle delay param.
+		if d, ok := resolved.Params["delay"]; ok {
+			if dur, err := time.ParseDuration(d); err == nil {
+				select {
+				case <-time.After(dur):
+				case <-ctx.Done():
+					resp.Error = ctx.Err().Error()
+					return resp
+				}
+			}
+		}
+
+		output, execErr := handler.Execute(ctx, actx, resolved)
+
+		ar := ActionResult{
+			Action:   resolved.Action,
+			Duration: time.Since(start),
+			YAML:     yamlDef,
+		}
+
+		if execErr != nil {
+			ar.Status = StatusFail
+			ar.Error = execErr.Error()
+			if !act.IgnoreError {
+				resp.Results = append(resp.Results, ar)
+				resp.Error = execErr.Error()
+				return resp
+			}
+			ar.Status = StatusPass
+		} else {
+			ar.Status = StatusPass
+		}
+
+		// Store save_as and __ vars.
+		if resolved.SaveAs != "" && output != nil {
+			if v, ok := output["value"]; ok {
+				actx.Vars[resolved.SaveAs] = v
+				resp.NewVars[resolved.SaveAs] = v
+			}
+		}
+		if output != nil {
+			for k, v := range output {
+				if strings.HasPrefix(k, "__") {
+					actx.Vars[k] = v
+					resp.NewVars[k] = v
+				}
+			}
+			if v, ok := output["value"]; ok {
+				ar.Output = truncate(v, 4096)
+			}
+		}
+
+		resp.Results = append(resp.Results, ar)
+	}
+
+	return resp
+}
+
+// POST /upload — streaming binary upload with path safety.
+func (a *Agent) handleUpload(w http.ResponseWriter, r *http.Request) {
+	if r.Method != http.MethodPost {
+		http.Error(w, "method not allowed", http.StatusMethodNotAllowed)
+		return
+	}
+
+	path := r.URL.Query().Get("path")
+	if path == "" {
+		writeJSON(w, http.StatusBadRequest, UploadResponse{Error: "path query parameter required"})
+		return
+	}
+
+	// Path safety: must start with UploadBasePath and no traversal.
+	if !isPathSafe(path) {
+		writeJSON(w, http.StatusForbidden, UploadResponse{Error: fmt.Sprintf("path must start with %s and contain no '..'", UploadBasePath)})
+		return
+	}
+
+	// Ensure directory exists.
+	dir := filepath.Dir(path)
+	if err := os.MkdirAll(dir, 0755); err != nil {
+		writeJSON(w, http.StatusInternalServerError, UploadResponse{Error: fmt.Sprintf("mkdir: %v", err)})
+		return
+	}
+
+	f, err := os.OpenFile(path, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0755)
+	if err != nil {
+		writeJSON(w, http.StatusInternalServerError, UploadResponse{Error: fmt.Sprintf("create file: %v", err)})
+		return
+	}
+
+	n, err := io.Copy(f, r.Body)
+	f.Close()
+	if err != nil {
+		os.Remove(path)
+		writeJSON(w, http.StatusInternalServerError, UploadResponse{Error: fmt.Sprintf("write: %v", err)})
+		return
+	}
+
+	a.log.Printf("uploaded %d bytes to %s", n, path)
+	writeJSON(w, http.StatusOK, UploadResponse{OK: true, Size: n, Path: path})
+}
+
+// GET /artifacts — stream a directory as tar.gz.
+func (a *Agent) handleArtifacts(w http.ResponseWriter, r *http.Request) {
+	if r.Method != http.MethodGet {
+		http.Error(w, "method not allowed", http.StatusMethodNotAllowed)
+		return
+	}
+
+	dir := r.URL.Query().Get("dir")
+	if dir == "" {
+		writeJSON(w, http.StatusBadRequest, map[string]string{"error": "dir query parameter required"})
+		return
+	}
+
+	// Path safety: must be under UploadBasePath, no traversal.
+	if !isPathSafe(dir) {
+		writeJSON(w, http.StatusForbidden, map[string]string{"error": fmt.Sprintf("dir must be under %s with no '..'", UploadBasePath)})
+		return
+	}
+
+	info, err := os.Stat(dir)
+	if err != nil || !info.IsDir() {
+		writeJSON(w, http.StatusNotFound, map[string]string{"error": "directory not found"})
+		return
+	}
+
+	// Check if directory has any files.
+	entries, err := os.ReadDir(dir)
+	if err != nil || len(entries) == 0 {
+		writeJSON(w, http.StatusNotFound, map[string]string{"error": "directory empty"})
+		return
+	}
+
+	w.Header().Set("Content-Type", "application/gzip")
+	w.WriteHeader(http.StatusOK)
+
+	gw := gzip.NewWriter(w)
+	defer gw.Close()
+	tw := tar.NewWriter(gw)
+	defer tw.Close()
+
+	filepath.Walk(dir, func(path string, fi os.FileInfo, err error) error {
+		if err != nil {
+			return nil // skip unreadable files
+		}
+		if fi.IsDir() {
+			return nil
+		}
+
+		// Relative path inside the tar.
+		rel, err := filepath.Rel(dir, path)
+		if err != nil {
+			return nil
+		}
+		// Normalize to forward slashes for tar.
+		rel = filepath.ToSlash(rel)
+
+		hdr := &tar.Header{
+			Name:    rel,
+			Size:    fi.Size(),
+			Mode:    int64(fi.Mode()),
+			ModTime: fi.ModTime(),
+		}
+		if err := tw.WriteHeader(hdr); err != nil {
+			return err
+		}
+
+		f, err := os.Open(path)
+		if err != nil {
+			return nil
+		}
+		defer f.Close()
+		io.Copy(tw, f)
+		return nil
+	})
+}
+
+// POST /exec — ad-hoc command execution (disabled by default).
+func (a *Agent) handleExec(w http.ResponseWriter, r *http.Request) {
+	if r.Method != http.MethodPost {
+		http.Error(w, "method not allowed", http.StatusMethodNotAllowed)
+		return
+	}
+
+	if !a.AllowExec {
+		writeJSON(w, http.StatusForbidden, ExecResponse{Error: "exec endpoint disabled; start agent with --allow-exec"})
+		return
+	}
+
+	var req ExecRequest
+	if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
+		writeJSON(w, http.StatusBadRequest, ExecResponse{Error: fmt.Sprintf("decode: %v", err)})
+		return
+	}
+
+	ctx, cancel := context.WithTimeout(r.Context(), 60*time.Second)
+	defer cancel()
+
+	var stdout, stderr string
+	var exitCode int
+	var err error
+
+	if req.Root {
+		stdout, stderr, exitCode, err = a.localNode.RunRoot(ctx, req.Cmd)
+	} else {
+		stdout, stderr, exitCode, err = a.localNode.Run(ctx, req.Cmd)
+	}
+
+	if err != nil {
+		writeJSON(w, http.StatusOK, ExecResponse{
+			Stdout:   stdout,
+			Stderr:   stderr,
+			ExitCode: -1,
+			Error:    err.Error(),
+		})
+		return
+	}
+
+	writeJSON(w, http.StatusOK, ExecResponse{
+		Stdout:   stdout,
+		Stderr:   stderr,
+		ExitCode: exitCode,
+	})
+}
+
+// isPathSafe checks that a path is under UploadBasePath and has no traversal.
+func isPathSafe(p string) bool {
+	if strings.Contains(p, "..") {
+		return false
+	}
+	// Use forward-slash path cleaning (Unix paths, even on Windows coordinator).
+	cleaned := posixClean(p)
+	return strings.HasPrefix(cleaned, UploadBasePath)
+}
+
+// posixClean normalizes a path using forward slashes (Unix convention).
+func posixClean(p string) string {
+	// Replace backslashes, then clean.
+	p = strings.ReplaceAll(p, "\\", "/")
+	// Remove double slashes.
+	for strings.Contains(p, "//") {
+		p = strings.ReplaceAll(p, "//", "/")
+	}
+	// Remove trailing slash (unless root).
+	if len(p) > 1 && strings.HasSuffix(p, "/") {
+		p = p[:len(p)-1]
+	}
+	return p
+}
+
+func writeJSON(w http.ResponseWriter, status int, v interface{}) {
+	w.Header().Set("Content-Type", "application/json")
+	w.WriteHeader(status)
+	json.NewEncoder(w).Encode(v)
+}

weed/storage/blockvol/testrunner/agent_test.go

@@ -0,0 +1,426 @@
+package testrunner
+
+import (
+	"bytes"
+	"context"
+	"encoding/json"
+	"fmt"
+	"io"
+	"net/http"
+	"os"
+	"path/filepath"
+	"runtime"
+	"testing"
+	"time"
+)
+
+// startTestAgent starts an agent on a random port and returns its base URL.
+func startTestAgent(t *testing.T, cfg AgentConfig) (*Agent, string) {
+	t.Helper()
+	if cfg.Registry == nil {
+		cfg.Registry = NewRegistry()
+	}
+	agent := NewAgent(cfg)
+	ctx, cancel := context.WithCancel(context.Background())
+	t.Cleanup(cancel)
+
+	go agent.Start(ctx)
+	// Wait for listener.
+	for i := 0; i < 50; i++ {
+		if agent.ListenAddr() != "" {
+			break
+		}
+		time.Sleep(20 * time.Millisecond)
+	}
+	if agent.ListenAddr() == "" {
+		t.Fatal("agent didn't start")
+	}
+	baseURL := "http://" + agent.ListenAddr()
+	return agent, baseURL
+}
+
+func TestAgent_Health(t *testing.T) {
+	_, baseURL := startTestAgent(t, AgentConfig{Port: 0})
+
+	resp, err := http.Get(baseURL + "/health")
+	if err != nil {
+		t.Fatalf("GET /health: %v", err)
+	}
+	defer resp.Body.Close()
+
+	if resp.StatusCode != 200 {
+		t.Fatalf("status = %d", resp.StatusCode)
+	}
+
+	var hr HealthResponse
+	json.NewDecoder(resp.Body).Decode(&hr)
+	if !hr.OK {
+		t.Error("health not OK")
+	}
+	if hr.AgentID == "" {
+		t.Error("empty agent ID")
+	}
+	if hr.Hostname == "" {
+		t.Error("empty hostname")
+	}
+}
+
+func TestAgent_Health_NoAuth(t *testing.T) {
+	// /health should not require auth.
+	_, baseURL := startTestAgent(t, AgentConfig{Port: 0, Token: "secret"})
+
+	resp, err := http.Get(baseURL + "/health")
+	if err != nil {
+		t.Fatalf("GET /health: %v", err)
+	}
+	defer resp.Body.Close()
+
+	if resp.StatusCode != 200 {
+		t.Fatalf("/health without token should work, got %d", resp.StatusCode)
+	}
+}
+
+func TestAgent_Auth_Rejection(t *testing.T) {
+	_, baseURL := startTestAgent(t, AgentConfig{Port: 0, Token: "secret"})
+
+	// POST /phase without token → 401.
+	body := bytes.NewReader([]byte(`{}`))
+	resp, err := http.Post(baseURL+"/phase", "application/json", body)
+	if err != nil {
+		t.Fatalf("POST /phase: %v", err)
+	}
+	defer resp.Body.Close()
+
+	if resp.StatusCode != http.StatusUnauthorized {
+		t.Errorf("expected 401, got %d", resp.StatusCode)
+	}
+}
+
+func TestAgent_Auth_ValidToken(t *testing.T) {
+	_, baseURL := startTestAgent(t, AgentConfig{Port: 0, Token: "secret"})
+
+	req, _ := http.NewRequest("POST", baseURL+"/phase", bytes.NewReader([]byte(`{"phase_index":0, "actions":[], "global_vars":{}}`)))
+	req.Header.Set("Content-Type", "application/json")
+	req.Header.Set(AuthTokenHeader, "secret")
+
+	resp, err := http.DefaultClient.Do(req)
+	if err != nil {
+		t.Fatalf("POST /phase: %v", err)
+	}
+	defer resp.Body.Close()
+
+	if resp.StatusCode != 200 {
+		t.Errorf("expected 200 with valid token, got %d", resp.StatusCode)
+	}
+}
+
+func TestAgent_Phase_EchoAction(t *testing.T) {
+	registry := NewRegistry()
+	registry.RegisterFunc("echo_val", TierCore, func(ctx context.Context, actx *ActionContext, act Action) (map[string]string, error) {
+		return map[string]string{"value": act.Params["msg"]}, nil
+	})
+
+	_, baseURL := startTestAgent(t, AgentConfig{Port: 0, Registry: registry})
+
+	phaseReq := PhaseRequest{
+		PhaseIndex: 0,
+		PhaseName:  "test",
+		Actions: []Action{
+			{Action: "echo_val", SaveAs: "result", Params: map[string]string{"msg": "hello"}},
+		},
+		GlobalVars: map[string]string{},
+	}
+
+	body, _ := json.Marshal(phaseReq)
+	resp, err := http.Post(baseURL+"/phase", "application/json", bytes.NewReader(body))
+	if err != nil {
+		t.Fatalf("POST /phase: %v", err)
+	}
+	defer resp.Body.Close()
+
+	var phaseResp PhaseResponse
+	json.NewDecoder(resp.Body).Decode(&phaseResp)
+
+	if phaseResp.Error != "" {
+		t.Fatalf("phase error: %s", phaseResp.Error)
+	}
+	if len(phaseResp.Results) != 1 {
+		t.Fatalf("expected 1 result, got %d", len(phaseResp.Results))
+	}
+	if phaseResp.Results[0].Status != StatusPass {
+		t.Errorf("action status = %s", phaseResp.Results[0].Status)
+	}
+	if phaseResp.NewVars["result"] != "hello" {
+		t.Errorf("new_vars[result] = %q, want hello", phaseResp.NewVars["result"])
+	}
+}
+
+func TestAgent_Phase_FailStopsExecution(t *testing.T) {
+	registry := NewRegistry()
+	callOrder := []string{}
+	registry.RegisterFunc("a1", TierCore, func(ctx context.Context, actx *ActionContext, act Action) (map[string]string, error) {
+		callOrder = append(callOrder, "a1")
+		return nil, fmt.Errorf("fail")
+	})
+	registry.RegisterFunc("a2", TierCore, func(ctx context.Context, actx *ActionContext, act Action) (map[string]string, error) {
+		callOrder = append(callOrder, "a2")
+		return nil, nil
+	})
+
+	_, baseURL := startTestAgent(t, AgentConfig{Port: 0, Registry: registry})
+
+	phaseReq := PhaseRequest{
+		PhaseName: "test",
+		Actions: []Action{
+			{Action: "a1"},
+			{Action: "a2"}, // should not execute
+		},
+		GlobalVars: map[string]string{},
+	}
+
+	body, _ := json.Marshal(phaseReq)
+	resp, _ := http.Post(baseURL+"/phase", "application/json", bytes.NewReader(body))
+	defer resp.Body.Close()
+
+	var phaseResp PhaseResponse
+	json.NewDecoder(resp.Body).Decode(&phaseResp)
+
+	if phaseResp.Error == "" {
+		t.Error("expected error from failed action")
+	}
+	if len(callOrder) != 1 {
+		t.Errorf("expected only a1 to run, got %v", callOrder)
+	}
+}
+
+func TestAgent_Upload_PathSafety(t *testing.T) {
+	_, baseURL := startTestAgent(t, AgentConfig{Port: 0})
+
+	// Attempt path traversal → should be rejected.
+	req, _ := http.NewRequest("POST", baseURL+"/upload?path=/tmp/sw-test-runner/../etc/passwd", bytes.NewReader([]byte("evil")))
+	req.Header.Set("Content-Type", "application/octet-stream")
+	resp, err := http.DefaultClient.Do(req)
+	if err != nil {
+		t.Fatalf("upload: %v", err)
+	}
+	defer resp.Body.Close()
+
+	if resp.StatusCode != http.StatusForbidden {
+		t.Errorf("expected 403 for path traversal, got %d", resp.StatusCode)
+	}
+
+	// Attempt outside base path.
+	req2, _ := http.NewRequest("POST", baseURL+"/upload?path=/etc/evil", bytes.NewReader([]byte("evil")))
+	req2.Header.Set("Content-Type", "application/octet-stream")
+	resp2, _ := http.DefaultClient.Do(req2)
+	defer resp2.Body.Close()
+
+	if resp2.StatusCode != http.StatusForbidden {
+		t.Errorf("expected 403 for outside base path, got %d", resp2.StatusCode)
+	}
+}
+
+func TestAgent_Upload_ValidPath(t *testing.T) {
+	if runtime.GOOS == "windows" {
+		t.Skip("upload test requires /tmp on Unix")
+	}
+
+	_, baseURL := startTestAgent(t, AgentConfig{Port: 0})
+
+	// Create a temp subdir under /tmp/sw-test-runner/.
+	uploadDir := "/tmp/sw-test-runner/test-upload"
+	os.MkdirAll(uploadDir, 0755)
+	t.Cleanup(func() { os.RemoveAll(uploadDir) })
+
+	uploadPath := filepath.Join(uploadDir, "testfile.bin")
+	content := []byte("test binary content")
+
+	req, _ := http.NewRequest("POST", baseURL+"/upload?path="+uploadPath, bytes.NewReader(content))
+	req.Header.Set("Content-Type", "application/octet-stream")
+	resp, err := http.DefaultClient.Do(req)
+	if err != nil {
+		t.Fatalf("upload: %v", err)
+	}
+	defer resp.Body.Close()
+
+	var ur UploadResponse
+	json.NewDecoder(resp.Body).Decode(&ur)
+	if !ur.OK {
+		t.Fatalf("upload not OK: %s", ur.Error)
+	}
+	if ur.Size != int64(len(content)) {
+		t.Errorf("size = %d, want %d", ur.Size, len(content))
+	}
+
+	// Verify file contents.
+	got, _ := os.ReadFile(uploadPath)
+	if string(got) != string(content) {
+		t.Errorf("content mismatch")
+	}
+}
+
+func TestAgent_Exec_DisabledByDefault(t *testing.T) {
+	_, baseURL := startTestAgent(t, AgentConfig{Port: 0, AllowExec: false})
+
+	body, _ := json.Marshal(ExecRequest{Cmd: "echo hi"})
+	resp, err := http.Post(baseURL+"/exec", "application/json", bytes.NewReader(body))
+	if err != nil {
+		t.Fatalf("POST /exec: %v", err)
+	}
+	defer resp.Body.Close()
+
+	if resp.StatusCode != http.StatusForbidden {
+		t.Errorf("expected 403 when exec disabled, got %d", resp.StatusCode)
+	}
+}
+
+func TestAgent_Exec_Enabled(t *testing.T) {
+	if runtime.GOOS == "windows" {
+		t.Skip("exec test requires Unix shell")
+	}
+
+	_, baseURL := startTestAgent(t, AgentConfig{Port: 0, AllowExec: true})
+
+	body, _ := json.Marshal(ExecRequest{Cmd: "echo hello-exec"})
+	resp, err := http.Post(baseURL+"/exec", "application/json", bytes.NewReader(body))
+	if err != nil {
+		t.Fatalf("POST /exec: %v", err)
+	}
+	defer resp.Body.Close()
+
+	var er ExecResponse
+	json.NewDecoder(resp.Body).Decode(&er)
+	if er.ExitCode != 0 {
+		t.Errorf("exit code = %d, stderr: %s, error: %s", er.ExitCode, er.Stderr, er.Error)
+	}
+	if er.Stdout != "hello-exec\n" {
+		t.Errorf("stdout = %q", er.Stdout)
+	}
+}
+
+func TestAgent_Artifacts_PathSafety(t *testing.T) {
+	_, baseURL := startTestAgent(t, AgentConfig{Port: 0, Token: "secret"})
+
+	// Attempt with traversal.
+	req, _ := http.NewRequest("GET", baseURL+"/artifacts?dir=/tmp/sw-test-runner/../etc", nil)
+	req.Header.Set(AuthTokenHeader, "secret")
+	resp, err := http.DefaultClient.Do(req)
+	if err != nil {
+		t.Fatalf("GET /artifacts: %v", err)
+	}
+	defer resp.Body.Close()
+
+	if resp.StatusCode != http.StatusForbidden {
+		t.Errorf("expected 403 for traversal, got %d", resp.StatusCode)
+	}
+
+	// Attempt outside base path.
+	req2, _ := http.NewRequest("GET", baseURL+"/artifacts?dir=/etc", nil)
+	req2.Header.Set(AuthTokenHeader, "secret")
+	resp2, err := http.DefaultClient.Do(req2)
+	if err != nil {
+		t.Fatalf("GET /artifacts: %v", err)
+	}
+	defer resp2.Body.Close()
+
+	if resp2.StatusCode != http.StatusForbidden {
+		t.Errorf("expected 403 for outside base path, got %d", resp2.StatusCode)
+	}
+}
+
+func TestAgent_Artifacts_MissingDir(t *testing.T) {
+	_, baseURL := startTestAgent(t, AgentConfig{Port: 0})
+
+	req, _ := http.NewRequest("GET", baseURL+"/artifacts?dir=/tmp/sw-test-runner/nonexistent-"+fmt.Sprintf("%d", time.Now().UnixNano()), nil)
+	resp, err := http.DefaultClient.Do(req)
+	if err != nil {
+		t.Fatalf("GET /artifacts: %v", err)
+	}
+	defer resp.Body.Close()
+
+	if resp.StatusCode != http.StatusNotFound {
+		t.Errorf("expected 404 for missing dir, got %d", resp.StatusCode)
+	}
+}
+
+func TestAgent_Artifacts_NoAuth(t *testing.T) {
+	_, baseURL := startTestAgent(t, AgentConfig{Port: 0, Token: "secret"})
+
+	// No auth header should be rejected.
+	resp, err := http.Get(baseURL + "/artifacts?dir=/tmp/sw-test-runner/test")
+	if err != nil {
+		t.Fatalf("GET /artifacts: %v", err)
+	}
+	defer resp.Body.Close()
+
+	if resp.StatusCode != http.StatusUnauthorized {
+		t.Errorf("expected 401, got %d", resp.StatusCode)
+	}
+}
+
+func TestAgent_Artifacts_ValidDir(t *testing.T) {
+	if runtime.GOOS == "windows" {
+		t.Skip("artifacts test requires /tmp on Unix")
+	}
+
+	_, baseURL := startTestAgent(t, AgentConfig{Port: 0})
+
+	// Create test directory with files.
+	dir := fmt.Sprintf("/tmp/sw-test-runner/test-artifacts-%d", time.Now().UnixNano())
+	os.MkdirAll(dir, 0755)
+	t.Cleanup(func() { os.RemoveAll(dir) })
+
+	os.WriteFile(filepath.Join(dir, "log.txt"), []byte("test log content"), 0644)
+	os.WriteFile(filepath.Join(dir, "dmesg.txt"), []byte("kernel messages"), 0644)
+
+	req, _ := http.NewRequest("GET", baseURL+"/artifacts?dir="+dir, nil)
+	resp, err := http.DefaultClient.Do(req)
+	if err != nil {
+		t.Fatalf("GET /artifacts: %v", err)
+	}
+	defer resp.Body.Close()
+
+	if resp.StatusCode != http.StatusOK {
+		body, _ := io.ReadAll(resp.Body)
+		t.Fatalf("expected 200, got %d: %s", resp.StatusCode, string(body))
+	}
+
+	if ct := resp.Header.Get("Content-Type"); ct != "application/gzip" {
+		t.Errorf("Content-Type = %q, want application/gzip", ct)
+	}
+
+	// Verify it's valid gzip+tar.
+	body, _ := io.ReadAll(resp.Body)
+	if len(body) == 0 {
+		t.Fatal("empty response body")
+	}
+}
+
+func TestAgent_Phase_VarSubstitution(t *testing.T) {
+	registry := NewRegistry()
+	registry.RegisterFunc("concat", TierCore, func(ctx context.Context, actx *ActionContext, act Action) (map[string]string, error) {
+		return map[string]string{"value": act.Params["a"] + "-" + act.Params["b"]}, nil
+	})
+
+	_, baseURL := startTestAgent(t, AgentConfig{Port: 0, Registry: registry})
+
+	phaseReq := PhaseRequest{
+		PhaseName: "test",
+		Actions: []Action{
+			{Action: "concat", SaveAs: "out", Params: map[string]string{"a": "{{ x }}", "b": "{{ y }}"}},
+		},
+		GlobalVars: map[string]string{"x": "hello", "y": "world"},
+	}
+
+	body, _ := json.Marshal(phaseReq)
+	resp, _ := http.Post(baseURL+"/phase", "application/json", bytes.NewReader(body))
+	defer resp.Body.Close()
+
+	respBody, _ := io.ReadAll(resp.Body)
+	var phaseResp PhaseResponse
+	json.Unmarshal(respBody, &phaseResp)
+
+	if phaseResp.NewVars["out"] != "hello-world" {
+		t.Errorf("var substitution failed: out = %q", phaseResp.NewVars["out"])
+	}
+}

weed/storage/blockvol/testrunner/baseline.go

@@ -0,0 +1,102 @@
+package testrunner
+
+import (
+	"encoding/json"
+	"fmt"
+	"os"
+	"path/filepath"
+	"sort"
+	"time"
+)
+
+// Baseline is an immutable, versioned snapshot of SLO metrics from a test run.
+// Stored as JSON in testrunner/baselines/ — append-only, git-tracked.
+type Baseline struct {
+	Version       int                    `json:"version"`
+	GitSHA        string                 `json:"git_sha"`
+	Scenario      string                 `json:"scenario"`
+	ScenarioSHA   string                 `json:"scenario_sha256"`
+	Topology      BaselineTopology       `json:"topology"`
+	Config        BaselineConfig         `json:"config"`
+	Timestamp     time.Time              `json:"timestamp"`
+	Metrics       map[string]float64     `json:"metrics"`
+	DurationSec   float64                `json:"duration_sec"`
+	HardFailsPassed bool                 `json:"hard_fails_passed"`
+}
+
+// BaselineTopology describes the test cluster topology.
+type BaselineTopology struct {
+	Nodes   int      `json:"nodes"`
+	Servers []string `json:"servers"`
+}
+
+// BaselineConfig describes the volume configuration under test.
+type BaselineConfig struct {
+	ReplicaFactor int    `json:"rf"`
+	Durability    string `json:"durability"`
+	WALSize       string `json:"wal_size"`
+}
+
+// SaveBaseline writes a baseline to the baselines directory.
+// Filename: <date>-<git_sha>.json (date-first for chronological sort).
+func SaveBaseline(dir string, b *Baseline) (string, error) {
+	if err := os.MkdirAll(dir, 0755); err != nil {
+		return "", fmt.Errorf("baseline: mkdir %s: %w", dir, err)
+	}
+
+	sha := b.GitSHA
+	if len(sha) > 10 {
+		sha = sha[:10]
+	}
+	date := b.Timestamp.Format("20060102")
+	filename := fmt.Sprintf("%s-%s.json", date, sha)
+	path := filepath.Join(dir, filename)
+
+	// Append-only: fail if file already exists.
+	if _, err := os.Stat(path); err == nil {
+		return path, nil // idempotent: already saved
+	}
+
+	data, err := json.MarshalIndent(b, "", "  ")
+	if err != nil {
+		return "", fmt.Errorf("baseline: marshal: %w", err)
+	}
+
+	if err := os.WriteFile(path, data, 0644); err != nil {
+		return "", fmt.Errorf("baseline: write %s: %w", path, err)
+	}
+	return path, nil
+}
+
+// LoadLatestBaseline loads the most recent baseline from the directory.
+// Baselines are sorted by filename (which encodes date).
+func LoadLatestBaseline(dir string) (*Baseline, error) {
+	entries, err := os.ReadDir(dir)
+	if err != nil {
+		return nil, fmt.Errorf("baseline: read dir %s: %w", dir, err)
+	}
+
+	var jsonFiles []string
+	for _, e := range entries {
+		if !e.IsDir() && filepath.Ext(e.Name()) == ".json" {
+			jsonFiles = append(jsonFiles, e.Name())
+		}
+	}
+	if len(jsonFiles) == 0 {
+		return nil, fmt.Errorf("baseline: no baselines found in %s", dir)
+	}
+
+	sort.Strings(jsonFiles)
+	latest := jsonFiles[len(jsonFiles)-1]
+
+	data, err := os.ReadFile(filepath.Join(dir, latest))
+	if err != nil {
+		return nil, fmt.Errorf("baseline: read %s: %w", latest, err)
+	}
+
+	var b Baseline
+	if err := json.Unmarshal(data, &b); err != nil {
+		return nil, fmt.Errorf("baseline: unmarshal %s: %w", latest, err)
+	}
+	return &b, nil
+}

weed/storage/blockvol/testrunner/baseline_test.go

@@ -0,0 +1,109 @@
+package testrunner
+
+import (
+	"os"
+	"path/filepath"
+	"testing"
+	"time"
+)
+
+func TestSaveAndLoadBaseline(t *testing.T) {
+	dir := t.TempDir()
+
+	b := &Baseline{
+		Version:    1,
+		GitSHA:     "8b2b5f6f6abc123def",
+		Scenario:   "cp84-soak-4h.yaml",
+		ScenarioSHA: "abc123",
+		Topology:   BaselineTopology{Nodes: 2, Servers: []string{"m01", "M02"}},
+		Config:     BaselineConfig{ReplicaFactor: 2, Durability: "best_effort", WALSize: "64MB"},
+		Timestamp:  time.Date(2026, 3, 6, 12, 0, 0, 0, time.UTC),
+		Metrics: map[string]float64{
+			"p99_write_latency_seconds": 0.005,
+			"write_iops":               50000,
+		},
+		DurationSec:     14400,
+		HardFailsPassed: true,
+	}
+
+	path, err := SaveBaseline(dir, b)
+	if err != nil {
+		t.Fatalf("SaveBaseline: %v", err)
+	}
+	if filepath.Base(path) != "20260306-8b2b5f6f6a.json" {
+		t.Errorf("unexpected filename: %s", filepath.Base(path))
+	}
+
+	// Verify file exists.
+	if _, err := os.Stat(path); err != nil {
+		t.Fatalf("file not created: %v", err)
+	}
+
+	// Load it back.
+	loaded, err := LoadLatestBaseline(dir)
+	if err != nil {
+		t.Fatalf("LoadLatestBaseline: %v", err)
+	}
+	if loaded.GitSHA != b.GitSHA {
+		t.Errorf("GitSHA = %q, want %q", loaded.GitSHA, b.GitSHA)
+	}
+	if loaded.Metrics["write_iops"] != 50000 {
+		t.Errorf("write_iops = %f, want 50000", loaded.Metrics["write_iops"])
+	}
+}
+
+func TestSaveBaselineIdempotent(t *testing.T) {
+	dir := t.TempDir()
+	b := &Baseline{
+		Version:   1,
+		GitSHA:    "abc123",
+		Timestamp: time.Date(2026, 3, 6, 12, 0, 0, 0, time.UTC),
+		Metrics:   map[string]float64{},
+	}
+
+	p1, err := SaveBaseline(dir, b)
+	if err != nil {
+		t.Fatalf("first save: %v", err)
+	}
+	p2, err := SaveBaseline(dir, b)
+	if err != nil {
+		t.Fatalf("second save: %v", err)
+	}
+	if p1 != p2 {
+		t.Errorf("paths differ: %s vs %s", p1, p2)
+	}
+}
+
+func TestLoadLatestBaseline_Empty(t *testing.T) {
+	dir := t.TempDir()
+	_, err := LoadLatestBaseline(dir)
+	if err == nil {
+		t.Error("expected error for empty dir")
+	}
+}
+
+func TestLoadLatestBaseline_PicksLatest(t *testing.T) {
+	dir := t.TempDir()
+
+	old := &Baseline{
+		Version: 1, GitSHA: "old1234567",
+		Timestamp: time.Date(2026, 3, 1, 0, 0, 0, 0, time.UTC),
+		Metrics:   map[string]float64{"write_iops": 40000},
+	}
+	newer := &Baseline{
+		Version: 1, GitSHA: "new1234567",
+		Timestamp: time.Date(2026, 3, 6, 0, 0, 0, 0, time.UTC),
+		Metrics:   map[string]float64{"write_iops": 50000},
+	}
+
+	SaveBaseline(dir, old)
+	SaveBaseline(dir, newer)
+
+	loaded, err := LoadLatestBaseline(dir)
+	if err != nil {
+		t.Fatalf("LoadLatestBaseline: %v", err)
+	}
+	if loaded.Metrics["write_iops"] != 50000 {
+		t.Errorf("expected latest baseline (50000), got %f", loaded.Metrics["write_iops"])
+	}
+}

weed/storage/blockvol/testrunner/cmd/sw-test-runner/main.go

@@ -0,0 +1,521 @@
+package main
+
+import (
+	"context"
+	"flag"
+	"fmt"
+	"log"
+	"os"
+	"os/signal"
+	"sort"
+	"strings"
+	"time"
+
+	tr "github.com/seaweedfs/seaweedfs/weed/storage/blockvol/testrunner"
+	"github.com/seaweedfs/seaweedfs/weed/storage/blockvol/testrunner/actions"
+	"github.com/seaweedfs/seaweedfs/weed/storage/blockvol/testrunner/infra"
+)
+
+func main() {
+	if len(os.Args) < 2 {
+		usage()
+		os.Exit(1)
+	}
+
+	switch os.Args[1] {
+	case "run":
+		runCmd(os.Args[2:])
+	case "coordinator":
+		coordinatorCmd(os.Args[2:])
+	case "agent":
+		agentCmd(os.Args[2:])
+	case "console":
+		consoleCmd(os.Args[2:])
+	case "validate":
+		validateCmd(os.Args[2:])
+	case "list":
+		listCmd()
+	case "help", "-h", "--help":
+		usage()
+	default:
+		fmt.Fprintf(os.Stderr, "unknown command: %s\n", os.Args[1])
+		usage()
+		os.Exit(1)
+	}
+}
+
+func usage() {
+	fmt.Fprintf(os.Stderr, `sw-test-runner — YAML-driven test platform for SeaweedFS BlockVol
+
+Usage:
+  sw-test-runner run [flags] <scenario.yaml>           Run a test scenario (SSH mode)
+  sw-test-runner coordinator [flags] <scenario.yaml>   Run as coordinator (multi-node)
+  sw-test-runner agent [flags]                         Run as agent on test node
+  sw-test-runner console [flags]                       Start web console server
+  sw-test-runner validate <scenario.yaml>              Validate YAML without running
+  sw-test-runner list [flags]                          List registered actions
+  sw-test-runner help                                  Show this help
+
+Common flags:
+  -tiers <tiers>       Comma-separated enabled tiers: core,block,devops,chaos (default: all)
+
+Run flags:
+  -output <path>       Write JSON results to file
+  -junit <path>        Write JUnit XML to file
+  -html <path>         Write HTML report to file
+  -baseline <path>     Compare against baseline JSON
+  -artifacts <path>    Collect artifacts on failure to this directory
+
+Coordinator flags:
+  -port <port>         Listen port for agent registration (default: 9000)
+  -token <token>       Auth token for agent communication
+  -dry-run             Print execution plan without running
+  -output <path>       Write JSON results to file
+  -junit <path>        Write JUnit XML to file
+  -html <path>         Write HTML report to file
+  -artifacts <path>    Download artifacts from agents to this directory
+  -timeout <duration>  Agent registration timeout (default: 30s)
+
+Agent flags:
+  -port <port>         Listen port (default: 9100)
+  -coordinator <url>   Coordinator URL (e.g. http://192.168.1.100:9000)
+  -token <token>       Auth token for coordinator communication
+  -nodes <names>       Comma-separated node names this agent handles
+  -allow-exec          Enable /exec endpoint for ad-hoc commands
+  -persistent          Stay running, re-register with coordinator on each run
+
+Console flags:
+  -port <port>            Listen port (default: 9090)
+  -token <token>          Auth token for agents
+  -scenarios-dir <path>   Directory containing scenario YAML files
+`)
+}
+
+func runCmd(args []string) {
+	fs := flag.NewFlagSet("run", flag.ExitOnError)
+	outputPath := fs.String("output", "", "Write JSON results to file")
+	junitPath := fs.String("junit", "", "Write JUnit XML to file")
+	htmlPath := fs.String("html", "", "Write HTML report to file")
+	baselinePath := fs.String("baseline", "", "Compare against baseline JSON")
+	artifactsDir := fs.String("artifacts", "", "Collect artifacts on failure to this directory")
+	tiers := fs.String("tiers", "", "Comma-separated list of enabled tiers (core,block,devops,chaos)")
+	fs.Parse(args)
+
+	if fs.NArg() < 1 {
+		fmt.Fprintln(os.Stderr, "error: scenario file required")
+		os.Exit(1)
+	}
+	scenarioFile := fs.Arg(0)
+
+	logger := log.New(os.Stderr, "", log.LstdFlags)
+
+	scenario, err := tr.ParseFile(scenarioFile)
+	if err != nil {
+		logger.Fatalf("parse scenario: %v", err)
+	}
+
+	// Set up signal handling.
+	ctx, cancel := signal.NotifyContext(context.Background(), os.Interrupt)
+	defer cancel()
+
+	// Create registry with all actions.
+	registry := tr.NewRegistry()
+	actions.RegisterAll(registry)
+	if *tiers != "" {
+		registry.EnableTiers(parseTiers(*tiers))
+	}
+
+	logFunc := func(format string, args ...interface{}) {
+		logger.Printf(format, args...)
+	}
+
+	// Create engine.
+	engine := tr.NewEngine(registry, logFunc)
+
+	// Set up infrastructure.
+	actx, err := setupActionContext(scenario, logFunc)
+	if err != nil {
+		logger.Fatalf("setup: %v", err)
+	}
+	defer cleanupNodes(actx)
+
+	// Run scenario.
+	result := engine.Run(ctx, scenario, actx)
+
+	// Print summary.
+	tr.PrintSummary(os.Stdout, result)
+
+	// Write outputs.
+	if *outputPath != "" {
+		if err := tr.WriteJSON(result, *outputPath); err != nil {
+			logger.Printf("write JSON: %v", err)
+		} else {
+			logger.Printf("JSON results written to %s", *outputPath)
+		}
+	}
+
+	if *junitPath != "" {
+		if err := tr.WriteJUnitXML(result, *junitPath); err != nil {
+			logger.Printf("write JUnit: %v", err)
+		} else {
+			logger.Printf("JUnit XML written to %s", *junitPath)
+		}
+	}
+
+	if *htmlPath != "" {
+		if err := tr.WriteHTMLReport(result, *htmlPath); err != nil {
+			logger.Printf("write HTML: %v", err)
+		} else {
+			logger.Printf("HTML report written to %s", *htmlPath)
+		}
+	}
+
+	if *baselinePath != "" {
+		regressions, err := tr.BaselineCompare(result, *baselinePath)
+		if err != nil {
+			logger.Printf("baseline compare: %v", err)
+		} else if len(regressions) > 0 {
+			fmt.Fprintln(os.Stdout, "\nREGRESSIONS:")
+			for _, r := range regressions {
+				fmt.Fprintf(os.Stdout, "  - %s\n", r)
+			}
+		} else {
+			fmt.Fprintln(os.Stdout, "\nNo regressions detected.")
+		}
+	}
+
+	// Collect artifacts on failure.
+	if result.Status == tr.StatusFail && *artifactsDir != "" {
+		collectArtifacts(actx, *artifactsDir, logger)
+	}
+
+	if result.Status == tr.StatusFail {
+		os.Exit(1)
+	}
+}
+
+func collectArtifacts(actx *tr.ActionContext, dir string, logger *log.Logger) {
+	logger.Printf("collecting artifacts to %s ...", dir)
+	// Find any node for dmesg/lsblk collection.
+	var clientNode *infra.Node
+	for _, n := range actx.Nodes {
+		if nn, ok := n.(*infra.Node); ok {
+			clientNode = nn
+			break
+		}
+	}
+	if clientNode == nil {
+		logger.Printf("no nodes available for artifact collection")
+		return
+	}
+
+	collector := infra.NewArtifactCollector(dir, clientNode, logger)
+	for name, tgt := range actx.Targets {
+		if lc, ok := tgt.(infra.LogCollector); ok {
+			collector.CollectLabeled(lc, name)
+		}
+	}
+}
+
+func coordinatorCmd(args []string) {
+	fs := flag.NewFlagSet("coordinator", flag.ExitOnError)
+	port := fs.Int("port", 9000, "Listen port for agent registration")
+	token := fs.String("token", "", "Auth token for agent communication")
+	dryRun := fs.Bool("dry-run", false, "Print execution plan without running")
+	outputPath := fs.String("output", "", "Write JSON results to file")
+	junitPath := fs.String("junit", "", "Write JUnit XML to file")
+	htmlPath := fs.String("html", "", "Write HTML report to file")
+	artifactsDir := fs.String("artifacts", "", "Download artifacts from agents to this directory")
+	regTimeout := fs.String("timeout", "30s", "Agent registration timeout")
+	coordTiers := fs.String("tiers", "", "Comma-separated list of enabled tiers (core,block,devops,chaos)")
+	fs.Parse(args)
+
+	if fs.NArg() < 1 {
+		fmt.Fprintln(os.Stderr, "error: scenario file required")
+		os.Exit(1)
+	}
+	scenarioFile := fs.Arg(0)
+
+	logger := log.New(os.Stderr, "", log.LstdFlags)
+
+	scenario, err := tr.ParseFile(scenarioFile)
+	if err != nil {
+		logger.Fatalf("parse scenario: %v", err)
+	}
+
+	// Verify scenario has agents section.
+	if len(scenario.Topology.Agents) == 0 {
+		logger.Fatalf("scenario has no topology.agents section; use 'run' for SSH mode")
+	}
+
+	// Set up signal handling.
+	ctx, cancel := signal.NotifyContext(context.Background(), os.Interrupt)
+	defer cancel()
+
+	// Create registry.
+	registry := tr.NewRegistry()
+	actions.RegisterAll(registry)
+	if *coordTiers != "" {
+		registry.EnableTiers(parseTiers(*coordTiers))
+	}
+
+	// Create coordinator.
+	coord := tr.NewCoordinator(tr.CoordinatorConfig{
+		Port:     *port,
+		Token:    *token,
+		DryRun:   *dryRun,
+		Expected: scenario.Topology.Agents,
+		Logger:   log.New(os.Stderr, "[coord] ", log.LstdFlags),
+	})
+
+	if err := coord.Start(); err != nil {
+		logger.Fatalf("start coordinator: %v", err)
+	}
+	defer coord.Stop()
+
+	// Wait for agents.
+	timeout, _ := time.ParseDuration(*regTimeout)
+	if timeout == 0 {
+		timeout = 30 * time.Second
+	}
+	logger.Printf("waiting for %d agents (timeout=%s)...", len(scenario.Topology.Agents), timeout)
+	if err := coord.WaitForAgents(ctx, timeout); err != nil {
+		logger.Fatalf("%v", err)
+	}
+	logger.Printf("all %d agents registered", len(scenario.Topology.Agents))
+
+	// Run scenario.
+	result := coord.RunScenario(ctx, scenario, registry)
+
+	// Download artifacts from agents (on both pass and fail).
+	remoteArtifactsDir := scenario.Artifacts.Dir
+	if *artifactsDir != "" && remoteArtifactsDir != "" {
+		coord.DownloadAllArtifacts(ctx, remoteArtifactsDir, *artifactsDir, result)
+	}
+
+	// Print summary.
+	tr.PrintSummary(os.Stdout, result)
+
+	if *outputPath != "" {
+		if err := tr.WriteJSON(result, *outputPath); err != nil {
+			logger.Printf("write JSON: %v", err)
+		}
+	}
+	if *junitPath != "" {
+		if err := tr.WriteJUnitXML(result, *junitPath); err != nil {
+			logger.Printf("write JUnit: %v", err)
+		}
+	}
+	if *htmlPath != "" {
+		if err := tr.WriteHTMLReport(result, *htmlPath); err != nil {
+			logger.Printf("write HTML: %v", err)
+		} else {
+			logger.Printf("HTML report written to %s", *htmlPath)
+		}
+	}
+
+	if result.Status == tr.StatusFail {
+		os.Exit(1)
+	}
+}
+
+func agentCmd(args []string) {
+	fs := flag.NewFlagSet("agent", flag.ExitOnError)
+	port := fs.Int("port", 9100, "Listen port")
+	coordURL := fs.String("coordinator", "", "Coordinator URL (e.g. http://192.168.1.100:9000)")
+	token := fs.String("token", "", "Auth token")
+	nodes := fs.String("nodes", "", "Comma-separated node names this agent handles")
+	allowExec := fs.Bool("allow-exec", false, "Enable /exec endpoint")
+	persistent := fs.Bool("persistent", false, "Stay running, re-register with coordinator on each run")
+	fs.Parse(args)
+
+	logger := log.New(os.Stderr, "[agent] ", log.LstdFlags)
+
+	// Parse node names.
+	var nodeNames []string
+	if *nodes != "" {
+		for _, n := range strings.Split(*nodes, ",") {
+			n = strings.TrimSpace(n)
+			if n != "" {
+				nodeNames = append(nodeNames, n)
+			}
+		}
+	}
+
+	// Create registry.
+	registry := tr.NewRegistry()
+	actions.RegisterAll(registry)
+
+	// Create agent.
+	agent := tr.NewAgent(tr.AgentConfig{
+		Port:           *port,
+		CoordinatorURL: *coordURL,
+		Token:          *token,
+		AllowExec:      *allowExec,
+		Persistent:     *persistent,
+		Nodes:          nodeNames,
+		Registry:       registry,
+		Logger:         logger,
+	})
+
+	// Set up signal handling.
+	ctx, cancel := signal.NotifyContext(context.Background(), os.Interrupt)
+	defer cancel()
+
+	logger.Printf("starting agent (nodes=%v, exec=%v, persistent=%v)", nodeNames, *allowExec, *persistent)
+	if err := agent.Start(ctx); err != nil {
+		logger.Fatalf("agent error: %v", err)
+	}
+}
+
+func consoleCmd(args []string) {
+	fs := flag.NewFlagSet("console", flag.ExitOnError)
+	port := fs.Int("port", 9090, "Listen port for web UI")
+	token := fs.String("token", "", "Auth token for agents")
+	scenariosDir := fs.String("scenarios-dir", ".", "Directory containing scenario YAML files")
+	consoleTiers := fs.String("tiers", "", "Comma-separated list of enabled tiers")
+	fs.Parse(args)
+
+	logger := log.New(os.Stderr, "[console] ", log.LstdFlags)
+
+	registry := tr.NewRegistry()
+	actions.RegisterAll(registry)
+	if *consoleTiers != "" {
+		registry.EnableTiers(parseTiers(*consoleTiers))
+	}
+
+	console := tr.NewConsole(tr.ConsoleConfig{
+		Port:        *port,
+		Token:       *token,
+		ScenarioDir: *scenariosDir,
+		Registry:    registry,
+		Logger:      logger,
+	})
+
+	ctx, cancel := signal.NotifyContext(context.Background(), os.Interrupt)
+	defer cancel()
+
+	if err := console.Start(ctx); err != nil {
+		logger.Fatalf("console error: %v", err)
+	}
+}
+
+func validateCmd(args []string) {
+	if len(args) < 1 {
+		fmt.Fprintln(os.Stderr, "error: scenario file required")
+		os.Exit(1)
+	}
+
+	scenario, err := tr.ParseFile(args[0])
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "INVALID: %v\n", err)
+		os.Exit(1)
+	}
+
+	fmt.Printf("VALID: %s (%d phases, %d targets)\n",
+		scenario.Name, len(scenario.Phases), len(scenario.Targets))
+}
+
+func listCmd() {
+	// Parse --tiers flag from remaining args.
+	fs := flag.NewFlagSet("list", flag.ExitOnError)
+	listTiers := fs.String("tiers", "", "Comma-separated list of enabled tiers")
+	fs.Parse(os.Args[2:])
+
+	registry := tr.NewRegistry()
+	actions.RegisterAll(registry)
+	if *listTiers != "" {
+		registry.EnableTiers(parseTiers(*listTiers))
+	}
+
+	byTier := registry.ListByTier()
+	tierOrder := []string{tr.TierCore, tr.TierBlock, tr.TierDevOps, tr.TierChaos}
+
+	fmt.Println("Registered actions:")
+	for _, tier := range tierOrder {
+		names := byTier[tier]
+		if len(names) == 0 {
+			continue
+		}
+		// Skip tiers that are not enabled (if filtering).
+		if len(registry.EnabledTiers) > 0 && !registry.EnabledTiers[tier] {
+			continue
+		}
+		sort.Strings(names)
+		fmt.Printf("\n  [%s]\n", tier)
+		for _, name := range names {
+			fmt.Printf("    - %s\n", name)
+		}
+	}
+	fmt.Println()
+}
+
+// setupActionContext creates nodes, targets, and the action context from the scenario.
+func setupActionContext(s *tr.Scenario, logFunc func(string, ...interface{})) (*tr.ActionContext, error) {
+	actx := &tr.ActionContext{
+		Scenario: s,
+		Nodes:    make(map[string]tr.NodeRunner),
+		Targets:  make(map[string]tr.TargetRunner),
+		Vars:     make(map[string]string),
+		Log:      logFunc,
+	}
+
+	// Create and connect nodes.
+	for name, spec := range s.Topology.Nodes {
+		node := &infra.Node{
+			Host:    spec.Host,
+			User:    spec.User,
+			KeyFile: spec.KeyFile,
+			IsLocal: spec.IsLocal,
+		}
+		if err := node.Connect(); err != nil {
+			return nil, fmt.Errorf("connect node %s: %w", name, err)
+		}
+		actx.Nodes[name] = node
+	}
+
+	// Create targets.
+	for name, spec := range s.Targets {
+		nodeRunner, ok := actx.Nodes[spec.Node]
+		if !ok {
+			return nil, fmt.Errorf("target %s: node %s not found", name, spec.Node)
+		}
+		node, ok := nodeRunner.(*infra.Node)
+		if !ok {
+			return nil, fmt.Errorf("target %s: node %s is not infra.Node", name, spec.Node)
+		}
+		htSpec := infra.HATargetSpec{
+			VolSize:         spec.VolSize,
+			WALSize:         spec.WALSize,
+			IQN:             spec.IQN(),
+			ISCSIPort:       spec.ISCSIPort,
+			AdminPort:       spec.AdminPort,
+			ReplicaDataPort: spec.ReplicaDataPort,
+			ReplicaCtrlPort: spec.ReplicaCtrlPort,
+			RebuildPort:     spec.RebuildPort,
+			TPGID:           spec.TPGID,
+		}
+		ht := infra.NewHATargetFromSpec(node, name, htSpec)
+		actx.Targets[name] = ht
+	}
+
+	return actx, nil
+}
+
+func cleanupNodes(actx *tr.ActionContext) {
+	for _, n := range actx.Nodes {
+		n.Close()
+	}
+}
+
+// parseTiers splits a comma-separated tier string into a slice.
+func parseTiers(s string) []string {
+	var tiers []string
+	for _, t := range strings.Split(s, ",") {
+		t = strings.TrimSpace(t)
+		if t != "" {
+			tiers = append(tiers, t)
+		}
+	}
+	return tiers
+}

weed/storage/blockvol/testrunner/console.go

@@ -0,0 +1,590 @@
+package testrunner
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"html/template"
+	"log"
+	"net"
+	"net/http"
+	"os"
+	"path/filepath"
+	"sort"
+	"strings"
+	"sync"
+	"time"
+)
+
+// Console serves a web UI for interactive scenario execution.
+type Console struct {
+	Port        int
+	Token       string
+	ScenarioDir string
+	Registry    *Registry
+	Log         *log.Logger
+
+	coordinator *Coordinator
+	mu          sync.Mutex
+	running     bool
+	currentRun  *runState
+	results     map[string]*ScenarioResult
+	server      *http.Server
+	listener    net.Listener
+}
+
+type runState struct {
+	ScenarioName string    `json:"scenario"`
+	StartedAt    time.Time `json:"started_at"`
+	Status       string    `json:"status"` // "running", "done", "failed"
+	Result       *ScenarioResult
+}
+
+// ConsoleConfig holds configuration for creating a Console.
+type ConsoleConfig struct {
+	Port        int
+	Token       string
+	ScenarioDir string
+	Registry    *Registry
+	Logger      *log.Logger
+}
+
+// NewConsole creates a new Console server.
+func NewConsole(cfg ConsoleConfig) *Console {
+	logger := cfg.Logger
+	if logger == nil {
+		logger = log.New(os.Stderr, "[console] ", log.LstdFlags)
+	}
+
+	// Create an internal coordinator for agent management.
+	coord := NewCoordinator(CoordinatorConfig{
+		Port:     cfg.Port + 1, // agents register on port+1
+		Token:    cfg.Token,
+		Expected: make(map[string]string), // no expected agents by default
+		Logger:   log.New(os.Stderr, "[coord] ", log.LstdFlags),
+	})
+
+	return &Console{
+		Port:        cfg.Port,
+		Token:       cfg.Token,
+		ScenarioDir: cfg.ScenarioDir,
+		Registry:    cfg.Registry,
+		Log:         logger,
+		coordinator: coord,
+		results:     make(map[string]*ScenarioResult),
+	}
+}
+
+// Start begins serving the console web UI. Blocks until stopped.
+func (c *Console) Start(ctx context.Context) error {
+	mux := http.NewServeMux()
+	mux.HandleFunc("/", c.handleIndex)
+	mux.HandleFunc("/api/scenarios", c.handleScenarios)
+	mux.HandleFunc("/api/run", c.handleRun)
+	mux.HandleFunc("/api/status", c.handleStatus)
+	mux.HandleFunc("/api/result/", c.handleResult)
+	mux.HandleFunc("/api/report/", c.handleReport)
+	mux.HandleFunc("/api/agents", c.handleAgents)
+	mux.HandleFunc("/api/tiers", c.handleTiers)
+	mux.HandleFunc("/register", c.coordinator.handleRegister)
+
+	addr := fmt.Sprintf(":%d", c.Port)
+	ln, err := net.Listen("tcp", addr)
+	if err != nil {
+		return fmt.Errorf("listen %s: %w", addr, err)
+	}
+
+	c.mu.Lock()
+	c.listener = ln
+	c.server = &http.Server{Handler: mux}
+	c.mu.Unlock()
+
+	c.Log.Printf("console listening on http://localhost:%d", c.Port)
+	c.Log.Printf("scenarios dir: %s", c.ScenarioDir)
+
+	go func() {
+		<-ctx.Done()
+		c.Stop()
+	}()
+
+	if err := c.server.Serve(ln); err != nil && err != http.ErrServerClosed {
+		return err
+	}
+	return nil
+}
+
+// Stop gracefully shuts down the console server.
+func (c *Console) Stop() {
+	c.mu.Lock()
+	srv := c.server
+	c.mu.Unlock()
+
+	if srv != nil {
+		ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
+		defer cancel()
+		srv.Shutdown(ctx)
+	}
+}
+
+// ListenAddr returns the address the console is listening on.
+func (c *Console) ListenAddr() string {
+	c.mu.Lock()
+	defer c.mu.Unlock()
+	if c.listener != nil {
+		return c.listener.Addr().String()
+	}
+	return ""
+}
+
+// --- API Handlers ---
+
+type scenarioInfo struct {
+	Name   string `json:"name"`
+	File   string `json:"file"`
+	Phases int    `json:"phases"`
+}
+
+// GET /api/scenarios
+func (c *Console) handleScenarios(w http.ResponseWriter, r *http.Request) {
+	if r.Method != http.MethodGet {
+		http.Error(w, "method not allowed", http.StatusMethodNotAllowed)
+		return
+	}
+
+	files, err := filepath.Glob(filepath.Join(c.ScenarioDir, "*.yaml"))
+	if err != nil {
+		writeJSON(w, http.StatusInternalServerError, map[string]string{"error": err.Error()})
+		return
+	}
+
+	var scenarios []scenarioInfo
+	for _, f := range files {
+		s, err := ParseFile(f)
+		if err != nil {
+			continue // skip invalid files
+		}
+		scenarios = append(scenarios, scenarioInfo{
+			Name:   s.Name,
+			File:   filepath.Base(f),
+			Phases: len(s.Phases),
+		})
+	}
+
+	sort.Slice(scenarios, func(i, j int) bool { return scenarios[i].File < scenarios[j].File })
+	writeJSON(w, http.StatusOK, scenarios)
+}
+
+type runRequest struct {
+	Scenario string `json:"scenario"`
+}
+
+// POST /api/run
+func (c *Console) handleRun(w http.ResponseWriter, r *http.Request) {
+	if r.Method != http.MethodPost {
+		http.Error(w, "method not allowed", http.StatusMethodNotAllowed)
+		return
+	}
+
+	var req runRequest
+	if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
+		writeJSON(w, http.StatusBadRequest, map[string]string{"error": fmt.Sprintf("decode: %v", err)})
+		return
+	}
+
+	if req.Scenario == "" {
+		writeJSON(w, http.StatusBadRequest, map[string]string{"error": "scenario field required"})
+		return
+	}
+
+	c.mu.Lock()
+	if c.running {
+		c.mu.Unlock()
+		writeJSON(w, http.StatusConflict, map[string]string{"error": "a scenario is already running"})
+		return
+	}
+	c.running = true
+	c.currentRun = &runState{
+		ScenarioName: req.Scenario,
+		StartedAt:    time.Now(),
+		Status:       "running",
+	}
+	c.mu.Unlock()
+
+	// Parse and validate.
+	scenarioPath := filepath.Join(c.ScenarioDir, req.Scenario)
+	scenario, err := ParseFile(scenarioPath)
+	if err != nil {
+		c.mu.Lock()
+		c.running = false
+		c.currentRun = nil
+		c.mu.Unlock()
+		writeJSON(w, http.StatusBadRequest, map[string]string{"error": fmt.Sprintf("parse: %v", err)})
+		return
+	}
+
+	// Launch in background.
+	go c.executeScenario(scenario, req.Scenario)
+
+	writeJSON(w, http.StatusAccepted, map[string]string{"status": "started", "scenario": req.Scenario})
+}
+
+func (c *Console) executeScenario(scenario *Scenario, fileName string) {
+	ctx := context.Background()
+	if scenario.Timeout.Duration > 0 {
+		var cancel context.CancelFunc
+		ctx, cancel = context.WithTimeout(ctx, scenario.Timeout.Duration)
+		defer cancel()
+	}
+
+	logFunc := func(format string, args ...interface{}) {
+		c.Log.Printf(format, args...)
+	}
+
+	engine := NewEngine(c.Registry, logFunc)
+	actx := &ActionContext{
+		Scenario: scenario,
+		Nodes:    make(map[string]NodeRunner),
+		Targets:  make(map[string]TargetRunner),
+		Vars:     make(map[string]string),
+		Log:      logFunc,
+	}
+
+	result := engine.Run(ctx, scenario, actx)
+
+	name := strings.TrimSuffix(fileName, ".yaml")
+	c.mu.Lock()
+	c.results[name] = result
+	if c.currentRun != nil {
+		c.currentRun.Status = "done"
+		if result.Status == StatusFail {
+			c.currentRun.Status = "failed"
+		}
+		c.currentRun.Result = result
+	}
+	c.running = false
+	c.mu.Unlock()
+
+	c.Log.Printf("scenario %s completed: %s (%s)", fileName, result.Status, result.Duration)
+}
+
+type statusResponse struct {
+	Running  bool   `json:"running"`
+	Scenario string `json:"scenario,omitempty"`
+	Status   string `json:"status,omitempty"`
+	Elapsed  string `json:"elapsed,omitempty"`
+}
+
+// GET /api/status
+func (c *Console) handleStatus(w http.ResponseWriter, r *http.Request) {
+	if r.Method != http.MethodGet {
+		http.Error(w, "method not allowed", http.StatusMethodNotAllowed)
+		return
+	}
+
+	c.mu.Lock()
+	defer c.mu.Unlock()
+
+	resp := statusResponse{Running: c.running}
+	if c.currentRun != nil {
+		resp.Scenario = c.currentRun.ScenarioName
+		resp.Status = c.currentRun.Status
+		resp.Elapsed = time.Since(c.currentRun.StartedAt).Round(time.Second).String()
+	}
+
+	writeJSON(w, http.StatusOK, resp)
+}
+
+// GET /api/result/{name}
+func (c *Console) handleResult(w http.ResponseWriter, r *http.Request) {
+	if r.Method != http.MethodGet {
+		http.Error(w, "method not allowed", http.StatusMethodNotAllowed)
+		return
+	}
+
+	name := strings.TrimPrefix(r.URL.Path, "/api/result/")
+	if name == "" {
+		writeJSON(w, http.StatusBadRequest, map[string]string{"error": "name required"})
+		return
+	}
+
+	c.mu.Lock()
+	result, ok := c.results[name]
+	c.mu.Unlock()
+	if !ok {
+		writeJSON(w, http.StatusNotFound, map[string]string{"error": "no result for " + name})
+		return
+	}
+
+	writeJSON(w, http.StatusOK, result)
+}
+
+// GET /api/report/{name}
+func (c *Console) handleReport(w http.ResponseWriter, r *http.Request) {
+	if r.Method != http.MethodGet {
+		http.Error(w, "method not allowed", http.StatusMethodNotAllowed)
+		return
+	}
+
+	name := strings.TrimPrefix(r.URL.Path, "/api/report/")
+	if name == "" {
+		writeJSON(w, http.StatusBadRequest, map[string]string{"error": "name required"})
+		return
+	}
+
+	c.mu.Lock()
+	result, ok := c.results[name]
+	c.mu.Unlock()
+	if !ok {
+		writeJSON(w, http.StatusNotFound, map[string]string{"error": "no result for " + name})
+		return
+	}
+
+	// Render HTML report inline.
+	data := buildHTMLData(result)
+	tmpl, err := template.New("report").Parse(htmlTemplate)
+	if err != nil {
+		http.Error(w, "template error: "+err.Error(), http.StatusInternalServerError)
+		return
+	}
+
+	w.Header().Set("Content-Type", "text/html; charset=utf-8")
+	tmpl.Execute(w, data)
+}
+
+type agentInfo struct {
+	Name    string `json:"name"`
+	Addr    string `json:"addr"`
+	Healthy bool   `json:"healthy"`
+}
+
+// GET /api/agents
+func (c *Console) handleAgents(w http.ResponseWriter, r *http.Request) {
+	if r.Method != http.MethodGet {
+		http.Error(w, "method not allowed", http.StatusMethodNotAllowed)
+		return
+	}
+
+	c.coordinator.mu.Lock()
+	var agents []agentInfo
+	for name, info := range c.coordinator.agents {
+		agents = append(agents, agentInfo{
+			Name:    name,
+			Addr:    info.Addr,
+			Healthy: info.Healthy,
+		})
+	}
+	c.coordinator.mu.Unlock()
+
+	sort.Slice(agents, func(i, j int) bool { return agents[i].Name < agents[j].Name })
+	writeJSON(w, http.StatusOK, agents)
+}
+
+// GET /api/tiers
+func (c *Console) handleTiers(w http.ResponseWriter, r *http.Request) {
+	if r.Method != http.MethodGet {
+		http.Error(w, "method not allowed", http.StatusMethodNotAllowed)
+		return
+	}
+	writeJSON(w, http.StatusOK, c.Registry.ListByTier())
+}
+
+// GET /
+func (c *Console) handleIndex(w http.ResponseWriter, r *http.Request) {
+	if r.URL.Path != "/" {
+		http.NotFound(w, r)
+		return
+	}
+	w.Header().Set("Content-Type", "text/html; charset=utf-8")
+	w.Write([]byte(consoleSPA))
+}
+
+// consoleSPA is the embedded single-page application.
+var consoleSPA = strings.TrimSpace(`<!DOCTYPE html>
+<html lang="en">
+<head>
+<meta charset="utf-8">
+<title>sw-test-runner Console</title>
+<style>
+  * { margin: 0; padding: 0; box-sizing: border-box; }
+  body { font-family: -apple-system, BlinkMacSystemFont, "Segoe UI", Roboto, monospace; background: #1a1a2e; color: #e0e0e0; display: flex; flex-direction: column; height: 100vh; }
+  header { background: #16213e; padding: 12px 24px; display: flex; align-items: center; gap: 16px; border-bottom: 1px solid #2a2a4a; }
+  header h1 { font-size: 1.2em; color: #a0a0c0; }
+  .main { display: flex; flex: 1; overflow: hidden; }
+  .sidebar { width: 280px; border-right: 1px solid #2a2a4a; overflow-y: auto; padding: 12px; }
+  .content { flex: 1; display: flex; flex-direction: column; }
+  .scenario-item { padding: 8px 12px; margin: 4px 0; border-radius: 4px; cursor: pointer; border: 1px solid transparent; }
+  .scenario-item:hover { background: #2a2a4a; }
+  .scenario-item.selected { border-color: #4a9eff; background: #1a2a4a; }
+  .scenario-name { font-weight: bold; font-size: 0.9em; }
+  .scenario-meta { font-size: 0.75em; color: #888; margin-top: 2px; }
+  .toolbar { padding: 12px 24px; background: #16213e; border-bottom: 1px solid #2a2a4a; display: flex; align-items: center; gap: 12px; }
+  button { background: #2d6a4f; color: #b7e4c7; border: none; padding: 8px 16px; border-radius: 4px; cursor: pointer; font-size: 0.85em; font-weight: bold; }
+  button:hover { background: #3d7a5f; }
+  button:disabled { background: #333; color: #666; cursor: not-allowed; }
+  .status-text { font-size: 0.85em; color: #a0a0c0; }
+  .badge { display: inline-block; padding: 2px 8px; border-radius: 3px; font-size: 0.7em; font-weight: bold; }
+  .badge-core { background: #2d6a4f; color: #b7e4c7; }
+  .badge-block { background: #4a4a8a; color: #b0b0e0; }
+  .badge-devops { background: #7d6608; color: #f9e79f; }
+  .badge-chaos { background: #922b21; color: #f5b7b1; }
+  .report-frame { flex: 1; border: none; background: #1a1a2e; }
+  .placeholder { flex: 1; display: flex; align-items: center; justify-content: center; color: #555; font-size: 1.1em; }
+  footer { background: #16213e; padding: 8px 24px; border-top: 1px solid #2a2a4a; font-size: 0.75em; color: #666; display: flex; gap: 16px; }
+  .agent-badge { color: #2d6a4f; }
+  .agent-badge.offline { color: #922b21; }
+</style>
+</head>
+<body>
+<header>
+  <h1>sw-test-runner</h1>
+  <span style="color:#666;font-size:0.8em">Console</span>
+</header>
+<div class="main">
+  <div class="sidebar" id="sidebar">
+    <div style="color:#888;font-size:0.8em;margin-bottom:8px">SCENARIOS</div>
+    <div id="scenario-list">Loading...</div>
+  </div>
+  <div class="content">
+    <div class="toolbar">
+      <button id="run-btn" disabled onclick="runScenario()">Run</button>
+      <span class="status-text" id="status-text"></span>
+    </div>
+    <div id="report-container" class="placeholder">
+      Select a scenario and click Run
+    </div>
+  </div>
+</div>
+<footer>
+  <span id="agent-status">Agents: checking...</span>
+  <span id="tier-info"></span>
+</footer>
+<script>
+let selected = null;
+let polling = null;
+
+async function loadScenarios() {
+  try {
+    const resp = await fetch('/api/scenarios');
+    const data = await resp.json();
+    const list = document.getElementById('scenario-list');
+    if (!data || data.length === 0) {
+      list.innerHTML = '<div style="color:#666">No scenarios found</div>';
+      return;
+    }
+    list.innerHTML = '';
+    data.forEach(s => {
+      const div = document.createElement('div');
+      div.className = 'scenario-item';
+      div.innerHTML = '<div class="scenario-name">' + s.name + '</div>' +
+        '<div class="scenario-meta">' + s.file + ' &middot; ' + s.phases + ' phases</div>';
+      div.onclick = () => selectScenario(s, div);
+      list.appendChild(div);
+    });
+  } catch (e) {
+    document.getElementById('scenario-list').textContent = 'Error: ' + e.message;
+  }
+}
+
+function selectScenario(s, el) {
+  document.querySelectorAll('.scenario-item').forEach(e => e.classList.remove('selected'));
+  el.classList.add('selected');
+  selected = s;
+  document.getElementById('run-btn').disabled = false;
+}
+
+async function runScenario() {
+  if (!selected) return;
+  const btn = document.getElementById('run-btn');
+  btn.disabled = true;
+  document.getElementById('status-text').textContent = 'Starting ' + selected.file + '...';
+  document.getElementById('report-container').innerHTML = '<div class="placeholder">Running...</div>';
+
+  try {
+    const resp = await fetch('/api/run', {
+      method: 'POST',
+      headers: {'Content-Type': 'application/json'},
+      body: JSON.stringify({scenario: selected.file})
+    });
+    if (!resp.ok) {
+      const err = await resp.json();
+      document.getElementById('status-text').textContent = 'Error: ' + (err.error || resp.statusText);
+      btn.disabled = false;
+      return;
+    }
+    startPolling();
+  } catch (e) {
+    document.getElementById('status-text').textContent = 'Error: ' + e.message;
+    btn.disabled = false;
+  }
+}
+
+function startPolling() {
+  if (polling) clearInterval(polling);
+  polling = setInterval(async () => {
+    try {
+      const resp = await fetch('/api/status');
+      const data = await resp.json();
+      if (data.running) {
+        document.getElementById('status-text').textContent = data.scenario + ' — ' + data.status + ' (' + data.elapsed + ')';
+      } else {
+        clearInterval(polling);
+        polling = null;
+        document.getElementById('status-text').textContent = data.scenario + ' — ' + data.status + (data.elapsed ? ' (' + data.elapsed + ')' : '');
+        document.getElementById('run-btn').disabled = false;
+        if (data.scenario) {
+          const name = data.scenario.replace('.yaml', '');
+          document.getElementById('report-container').innerHTML = '<iframe class="report-frame" src="/api/report/' + name + '"></iframe>';
+        }
+      }
+    } catch (e) { /* ignore polling errors */ }
+  }, 2000);
+}
+
+async function loadAgents() {
+  try {
+    const resp = await fetch('/api/agents');
+    const data = await resp.json();
+    const el = document.getElementById('agent-status');
+    if (!data || data.length === 0) {
+      el.textContent = 'Agents: none connected';
+    } else {
+      el.innerHTML = 'Agents: ' + data.map(a =>
+        '<span class="agent-badge' + (a.healthy ? '' : ' offline') + '">' + a.name + '</span>'
+      ).join(' ');
+    }
+  } catch (e) {
+    document.getElementById('agent-status').textContent = 'Agents: error';
+  }
+}
+
+async function loadTiers() {
+  try {
+    const resp = await fetch('/api/tiers');
+    const data = await resp.json();
+    let count = 0;
+    for (const tier in data) count += data[tier].length;
+    document.getElementById('tier-info').textContent = count + ' actions registered';
+  } catch (e) { /* ignore */ }
+}
+
+// Check if a run was already in progress.
+async function checkStatus() {
+  try {
+    const resp = await fetch('/api/status');
+    const data = await resp.json();
+    if (data.running) {
+      document.getElementById('status-text').textContent = data.scenario + ' — ' + data.status + ' (' + data.elapsed + ')';
+      document.getElementById('run-btn').disabled = true;
+      startPolling();
+    }
+  } catch (e) { /* ignore */ }
+}
+
+loadScenarios();
+loadAgents();
+loadTiers();
+checkStatus();
+setInterval(loadAgents, 10000);
+</script>
+</body>
+</html>`)

weed/storage/blockvol/testrunner/console_test.go

@@ -0,0 +1,332 @@
+package testrunner
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"net/http"
+	"os"
+	"path/filepath"
+	"strings"
+	"testing"
+	"time"
+)
+
+func TestConsole_ScenariosEndpoint(t *testing.T) {
+	dir := t.TempDir()
+	writeTestScenario(t, dir, "smoke.yaml", "Smoke Test", 2)
+	writeTestScenario(t, dir, "ha.yaml", "HA Test", 4)
+
+	console, baseURL := startTestConsole(t, dir)
+	_ = console
+
+	resp, err := http.Get(baseURL + "/api/scenarios")
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer resp.Body.Close()
+
+	if resp.StatusCode != 200 {
+		t.Fatalf("status = %d", resp.StatusCode)
+	}
+
+	var scenarios []scenarioInfo
+	json.NewDecoder(resp.Body).Decode(&scenarios)
+
+	if len(scenarios) != 2 {
+		t.Fatalf("scenarios = %d, want 2", len(scenarios))
+	}
+}
+
+func TestConsole_StatusEndpoint_NoRun(t *testing.T) {
+	dir := t.TempDir()
+	_, baseURL := startTestConsole(t, dir)
+
+	resp, err := http.Get(baseURL + "/api/status")
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer resp.Body.Close()
+
+	var st statusResponse
+	json.NewDecoder(resp.Body).Decode(&st)
+
+	if st.Running {
+		t.Error("should not be running")
+	}
+}
+
+func TestConsole_RunEndpoint_MissingScenario(t *testing.T) {
+	dir := t.TempDir()
+	_, baseURL := startTestConsole(t, dir)
+
+	resp, err := http.Post(baseURL+"/api/run", "application/json",
+		strings.NewReader(`{"scenario":""}`))
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer resp.Body.Close()
+
+	if resp.StatusCode != 400 {
+		t.Errorf("status = %d, want 400", resp.StatusCode)
+	}
+}
+
+func TestConsole_RunEndpoint_InvalidScenario(t *testing.T) {
+	dir := t.TempDir()
+	_, baseURL := startTestConsole(t, dir)
+
+	resp, err := http.Post(baseURL+"/api/run", "application/json",
+		strings.NewReader(`{"scenario":"nonexistent.yaml"}`))
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer resp.Body.Close()
+
+	if resp.StatusCode != 400 {
+		t.Errorf("status = %d, want 400", resp.StatusCode)
+	}
+}
+
+func TestConsole_RunAndPollStatus(t *testing.T) {
+	dir := t.TempDir()
+	// Write a scenario with a registered action.
+	writeTestScenarioWithAction(t, dir, "test.yaml", "Test Run", "console_noop")
+
+	registry := NewRegistry()
+	registry.RegisterFunc("console_noop", TierCore, func(ctx context.Context, actx *ActionContext, act Action) (map[string]string, error) {
+		return map[string]string{"value": "done"}, nil
+	})
+
+	console, baseURL := startTestConsoleWithRegistry(t, dir, registry)
+	_ = console
+
+	// Run the scenario.
+	resp, err := http.Post(baseURL+"/api/run", "application/json",
+		strings.NewReader(`{"scenario":"test.yaml"}`))
+	if err != nil {
+		t.Fatal(err)
+	}
+	resp.Body.Close()
+
+	if resp.StatusCode != 202 {
+		t.Fatalf("status = %d, want 202", resp.StatusCode)
+	}
+
+	// Poll until done.
+	deadline := time.After(5 * time.Second)
+	for {
+		select {
+		case <-deadline:
+			t.Fatal("timeout waiting for scenario to complete")
+		case <-time.After(100 * time.Millisecond):
+		}
+
+		statusResp, err := http.Get(baseURL + "/api/status")
+		if err != nil {
+			continue
+		}
+		var st statusResponse
+		json.NewDecoder(statusResp.Body).Decode(&st)
+		statusResp.Body.Close()
+
+		if !st.Running && st.Status != "" {
+			if st.Status != "done" {
+				t.Errorf("status = %s, want done", st.Status)
+			}
+			break
+		}
+	}
+
+	// Check result endpoint.
+	resultResp, err := http.Get(baseURL + "/api/result/test")
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer resultResp.Body.Close()
+
+	if resultResp.StatusCode != 200 {
+		t.Fatalf("result status = %d", resultResp.StatusCode)
+	}
+
+	var result ScenarioResult
+	json.NewDecoder(resultResp.Body).Decode(&result)
+	if result.Status != StatusPass {
+		t.Errorf("result.Status = %s, want PASS", result.Status)
+	}
+}
+
+func TestConsole_ConflictOnDoubleRun(t *testing.T) {
+	dir := t.TempDir()
+	writeTestScenarioWithAction(t, dir, "slow.yaml", "Slow Test", "console_slow")
+
+	registry := NewRegistry()
+	registry.RegisterFunc("console_slow", TierCore, func(ctx context.Context, actx *ActionContext, act Action) (map[string]string, error) {
+		time.Sleep(2 * time.Second)
+		return nil, nil
+	})
+
+	_, baseURL := startTestConsoleWithRegistry(t, dir, registry)
+
+	// Start first run.
+	resp1, _ := http.Post(baseURL+"/api/run", "application/json",
+		strings.NewReader(`{"scenario":"slow.yaml"}`))
+	resp1.Body.Close()
+	if resp1.StatusCode != 202 {
+		t.Fatalf("first run status = %d", resp1.StatusCode)
+	}
+
+	// Wait a moment for it to start.
+	time.Sleep(100 * time.Millisecond)
+
+	// Try second run — should get 409.
+	resp2, _ := http.Post(baseURL+"/api/run", "application/json",
+		strings.NewReader(`{"scenario":"slow.yaml"}`))
+	resp2.Body.Close()
+	if resp2.StatusCode != 409 {
+		t.Errorf("second run status = %d, want 409", resp2.StatusCode)
+	}
+}
+
+func TestConsole_AgentsEndpoint(t *testing.T) {
+	dir := t.TempDir()
+	_, baseURL := startTestConsole(t, dir)
+
+	resp, err := http.Get(baseURL + "/api/agents")
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer resp.Body.Close()
+
+	var agents []agentInfo
+	json.NewDecoder(resp.Body).Decode(&agents)
+
+	// No agents connected by default.
+	if len(agents) != 0 {
+		t.Errorf("agents = %d, want 0", len(agents))
+	}
+}
+
+func TestConsole_TiersEndpoint(t *testing.T) {
+	dir := t.TempDir()
+	registry := NewRegistry()
+	registry.RegisterFunc("a", TierCore, func(ctx context.Context, actx *ActionContext, act Action) (map[string]string, error) {
+		return nil, nil
+	})
+	registry.RegisterFunc("b", TierBlock, func(ctx context.Context, actx *ActionContext, act Action) (map[string]string, error) {
+		return nil, nil
+	})
+
+	_, baseURL := startTestConsoleWithRegistry(t, dir, registry)
+
+	resp, err := http.Get(baseURL + "/api/tiers")
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer resp.Body.Close()
+
+	var tiers map[string][]string
+	json.NewDecoder(resp.Body).Decode(&tiers)
+
+	if len(tiers["core"]) != 1 || tiers["core"][0] != "a" {
+		t.Errorf("core tier = %v", tiers["core"])
+	}
+	if len(tiers["block"]) != 1 || tiers["block"][0] != "b" {
+		t.Errorf("block tier = %v", tiers["block"])
+	}
+}
+
+func TestConsole_ReportNotFound(t *testing.T) {
+	dir := t.TempDir()
+	_, baseURL := startTestConsole(t, dir)
+
+	resp, err := http.Get(baseURL + "/api/report/nonexistent")
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer resp.Body.Close()
+
+	if resp.StatusCode != 404 {
+		t.Errorf("status = %d, want 404", resp.StatusCode)
+	}
+}
+
+func TestConsole_IndexPage(t *testing.T) {
+	dir := t.TempDir()
+	_, baseURL := startTestConsole(t, dir)
+
+	resp, err := http.Get(baseURL + "/")
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer resp.Body.Close()
+
+	if resp.StatusCode != 200 {
+		t.Fatalf("status = %d", resp.StatusCode)
+	}
+
+	ct := resp.Header.Get("Content-Type")
+	if !strings.Contains(ct, "text/html") {
+		t.Errorf("content-type = %s, want text/html", ct)
+	}
+}
+
+// --- Helpers ---
+
+func startTestConsole(t *testing.T, scenariosDir string) (*Console, string) {
+	t.Helper()
+	return startTestConsoleWithRegistry(t, scenariosDir, NewRegistry())
+}
+
+func startTestConsoleWithRegistry(t *testing.T, scenariosDir string, registry *Registry) (*Console, string) {
+	t.Helper()
+
+	console := NewConsole(ConsoleConfig{
+		Port:        0,
+		ScenarioDir: scenariosDir,
+		Registry:    registry,
+	})
+
+	// Override to use ephemeral port — bind directly.
+	ctx, cancel := context.WithCancel(context.Background())
+	t.Cleanup(cancel)
+
+	go console.Start(ctx)
+
+	// Wait for listener.
+	for i := 0; i < 50; i++ {
+		if console.ListenAddr() != "" {
+			break
+		}
+		time.Sleep(20 * time.Millisecond)
+	}
+
+	addr := console.ListenAddr()
+	if addr == "" {
+		t.Fatal("console failed to start")
+	}
+
+	return console, fmt.Sprintf("http://%s", addr)
+}
+
+func writeTestScenario(t *testing.T, dir, filename, name string, phases int) {
+	t.Helper()
+	content := fmt.Sprintf("name: %s\ntimeout: 30s\nphases:\n", name)
+	for i := 0; i < phases; i++ {
+		content += fmt.Sprintf("  - name: phase%d\n    actions:\n      - action: noop\n", i)
+	}
+	os.WriteFile(filepath.Join(dir, filename), []byte(content), 0644)
+}
+
+func writeTestScenarioWithAction(t *testing.T, dir, filename, name, action string) {
+	t.Helper()
+	content := fmt.Sprintf(`name: %s
+timeout: 30s
+phases:
+  - name: main
+    actions:
+      - action: %s
+        save_as: result
+`, name, action)
+	os.WriteFile(filepath.Join(dir, filename), []byte(content), 0644)
+}

weed/storage/blockvol/testrunner/coordinator_test.go

@@ -0,0 +1,680 @@
+package testrunner
+
+import (
+	"bytes"
+	"context"
+	"encoding/json"
+	"fmt"
+	"net/http"
+	"testing"
+	"time"
+)
+
+// newTestCoordinator creates a coordinator on a random port for testing.
+func newTestCoordinator(t *testing.T, expected map[string]string) *Coordinator {
+	t.Helper()
+	c := NewCoordinator(CoordinatorConfig{
+		Port:     0, // random
+		Token:    "test-token",
+		Expected: expected,
+	})
+	if err := c.Start(); err != nil {
+		t.Fatalf("start coordinator: %v", err)
+	}
+	t.Cleanup(func() { c.Stop() })
+	return c
+}
+
+func TestCoordinator_RegisterAgent(t *testing.T) {
+	c := newTestCoordinator(t, map[string]string{"agent1": "127.0.0.1:9100"})
+
+	c.RegisterAgent("agent1", "127.0.0.1:9100")
+
+	if !c.IsAgentRegistered("agent1") {
+		t.Error("agent1 should be registered")
+	}
+	if c.IsAgentRegistered("agent2") {
+		t.Error("agent2 should not be registered")
+	}
+	if c.RegisteredAgentCount() != 1 {
+		t.Errorf("count = %d, want 1", c.RegisteredAgentCount())
+	}
+}
+
+func TestCoordinator_WaitForAgents_AlreadyRegistered(t *testing.T) {
+	c := newTestCoordinator(t, map[string]string{"a1": "host:9100"})
+	c.RegisterAgent("a1", "host:9100")
+
+	ctx := context.Background()
+	if err := c.WaitForAgents(ctx, 1*time.Second); err != nil {
+		t.Fatalf("WaitForAgents: %v", err)
+	}
+}
+
+func TestCoordinator_WaitForAgents_Timeout(t *testing.T) {
+	c := newTestCoordinator(t, map[string]string{"a1": "host:9100", "a2": "host:9200"})
+	c.RegisterAgent("a1", "host:9100")
+	// a2 never registers
+
+	ctx := context.Background()
+	err := c.WaitForAgents(ctx, 100*time.Millisecond)
+	if err == nil {
+		t.Fatal("expected timeout error")
+	}
+}
+
+func TestCoordinator_WaitForAgents_NoExpected(t *testing.T) {
+	c := newTestCoordinator(t, map[string]string{})
+	ctx := context.Background()
+	if err := c.WaitForAgents(ctx, 100*time.Millisecond); err != nil {
+		t.Fatalf("WaitForAgents with empty expected: %v", err)
+	}
+}
+
+func TestCoordinator_BuildNodeAgentMap(t *testing.T) {
+	c := newTestCoordinator(t, map[string]string{"tp01": "host:9100"})
+	c.RegisterAgent("tp01", "host:9100")
+
+	s := &Scenario{
+		Topology: Topology{
+			Agents: map[string]string{"tp01": "host:9100"},
+			Nodes: map[string]NodeSpec{
+				"node1": {Host: "host", Agent: "tp01"},
+				"node2": {Host: "host2"},
+			},
+		},
+	}
+
+	m := c.buildNodeAgentMap(s)
+	if m["node1"] != "tp01" {
+		t.Errorf("node1 → %q, want tp01", m["node1"])
+	}
+}
+
+func TestCoordinator_ResolveActionAgent(t *testing.T) {
+	c := newTestCoordinator(t, map[string]string{"tp01": "h:9100", "m01": "h:9200"})
+	c.RegisterAgent("tp01", "h:9100")
+	c.RegisterAgent("m01", "h:9200")
+
+	s := &Scenario{
+		Targets: map[string]TargetSpec{
+			"primary": {Node: "node_tp01"},
+		},
+		Topology: Topology{
+			Nodes: map[string]NodeSpec{
+				"node_tp01":  {Agent: "tp01"},
+				"node_m01":   {Agent: "m01"},
+			},
+		},
+	}
+	nodeToAgent := map[string]string{"node_tp01": "tp01", "node_m01": "m01"}
+
+	// By node.
+	agent := c.resolveActionAgent(s, Action{Node: "node_tp01"}, nodeToAgent)
+	if agent != "tp01" {
+		t.Errorf("by node: got %q, want tp01", agent)
+	}
+
+	// By target.
+	agent = c.resolveActionAgent(s, Action{Target: "primary"}, nodeToAgent)
+	if agent != "tp01" {
+		t.Errorf("by target: got %q, want tp01", agent)
+	}
+
+	// Fallback.
+	agent = c.resolveActionAgent(s, Action{}, nodeToAgent)
+	if agent == "" {
+		t.Error("fallback should return some agent")
+	}
+}
+
+// --- Integration: Coordinator + Agent ---
+
+func TestCoordinatorAgent_PhaseDispatch(t *testing.T) {
+	if testing.Short() {
+		t.Skip("integration test")
+	}
+
+	// Create a test action that returns a value.
+	registry := NewRegistry()
+	registry.RegisterFunc("echo_test", TierCore, func(ctx context.Context, actx *ActionContext, act Action) (map[string]string, error) {
+		msg := act.Params["msg"]
+		if msg == "" {
+			msg = "default"
+		}
+		return map[string]string{"value": msg}, nil
+	})
+	registry.RegisterFunc("fail_test", TierCore, func(ctx context.Context, actx *ActionContext, act Action) (map[string]string, error) {
+		return nil, fmt.Errorf("intentional failure")
+	})
+
+	// Start agent.
+	agent := NewAgent(AgentConfig{
+		Port:     0,
+		Token:    "tok",
+		Nodes:    []string{"test_node"},
+		Registry: registry,
+	})
+
+	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+	defer cancel()
+
+	go agent.Start(ctx)
+	// Wait for listener.
+	for i := 0; i < 50; i++ {
+		if agent.ListenAddr() != "" {
+			break
+		}
+		time.Sleep(20 * time.Millisecond)
+	}
+	if agent.ListenAddr() == "" {
+		t.Fatal("agent didn't start")
+	}
+
+	// Start coordinator.
+	coord := NewCoordinator(CoordinatorConfig{
+		Port:     0,
+		Token:    "tok",
+		Expected: map[string]string{"test_node": agent.ListenAddr()},
+	})
+	if err := coord.Start(); err != nil {
+		t.Fatalf("start coordinator: %v", err)
+	}
+	defer coord.Stop()
+
+	// Register agent.
+	coord.RegisterAgent("test_node", agent.ListenAddr())
+
+	// Define scenario.
+	s := &Scenario{
+		Name:    "test-dispatch",
+		Timeout: Duration{30 * time.Second},
+		Env:     map[string]string{"greeting": "hello"},
+		Topology: Topology{
+			Agents: map[string]string{"test_node": agent.ListenAddr()},
+			Nodes:  map[string]NodeSpec{"test_node": {Host: "127.0.0.1", Agent: "test_node"}},
+		},
+		Phases: []Phase{
+			{
+				Name: "echo_phase",
+				Actions: []Action{
+					{Action: "echo_test", Node: "test_node", SaveAs: "echo_result", Params: map[string]string{"msg": "world"}},
+				},
+			},
+		},
+	}
+
+	result := coord.RunScenario(ctx, s, registry)
+	if result.Status != StatusPass {
+		t.Fatalf("scenario failed: %s", result.Error)
+	}
+	if len(result.Phases) != 1 {
+		t.Fatalf("expected 1 phase, got %d", len(result.Phases))
+	}
+	if len(result.Phases[0].Actions) != 1 {
+		t.Fatalf("expected 1 action result, got %d", len(result.Phases[0].Actions))
+	}
+	if result.Phases[0].Actions[0].Status != StatusPass {
+		t.Errorf("action status = %s", result.Phases[0].Actions[0].Status)
+	}
+}
+
+func TestCoordinatorAgent_VarMerge(t *testing.T) {
+	if testing.Short() {
+		t.Skip("integration test")
+	}
+
+	// Two actions: first produces a var, second uses it.
+	registry := NewRegistry()
+	callCount := 0
+	registry.RegisterFunc("produce_var", TierCore, func(ctx context.Context, actx *ActionContext, act Action) (map[string]string, error) {
+		callCount++
+		return map[string]string{"value": fmt.Sprintf("produced-%d", callCount)}, nil
+	})
+	registry.RegisterFunc("check_var", TierCore, func(ctx context.Context, actx *ActionContext, act Action) (map[string]string, error) {
+		expected := act.Params["expected"]
+		got := act.Params["actual"]
+		if got != expected {
+			return nil, fmt.Errorf("var mismatch: got %q, want %q", got, expected)
+		}
+		return map[string]string{"value": "ok"}, nil
+	})
+
+	agent := NewAgent(AgentConfig{Port: 0, Token: "tok", Nodes: []string{"n1"}, Registry: registry})
+	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+	defer cancel()
+	go agent.Start(ctx)
+	for i := 0; i < 50; i++ {
+		if agent.ListenAddr() != "" {
+			break
+		}
+		time.Sleep(20 * time.Millisecond)
+	}
+
+	coord := NewCoordinator(CoordinatorConfig{Port: 0, Token: "tok", Expected: map[string]string{"n1": agent.ListenAddr()}})
+	coord.Start()
+	defer coord.Stop()
+	coord.RegisterAgent("n1", agent.ListenAddr())
+
+	s := &Scenario{
+		Name:    "var-merge",
+		Timeout: Duration{30 * time.Second},
+		Topology: Topology{
+			Agents: map[string]string{"n1": agent.ListenAddr()},
+			Nodes:  map[string]NodeSpec{"n1": {Host: "127.0.0.1", Agent: "n1"}},
+		},
+		Phases: []Phase{
+			{
+				Name: "produce",
+				Actions: []Action{
+					{Action: "produce_var", Node: "n1", SaveAs: "my_var"},
+				},
+			},
+			{
+				Name: "consume",
+				Actions: []Action{
+					{Action: "check_var", Node: "n1", Params: map[string]string{"expected": "produced-1", "actual": "{{ my_var }}"}},
+				},
+			},
+		},
+	}
+
+	result := coord.RunScenario(ctx, s, registry)
+	if result.Status != StatusPass {
+		t.Fatalf("scenario failed: %s", result.Error)
+	}
+}
+
+func TestCoordinatorAgent_FailureStopsNormalPhases(t *testing.T) {
+	if testing.Short() {
+		t.Skip("integration test")
+	}
+
+	registry := NewRegistry()
+	registry.RegisterFunc("pass_action", TierCore, func(ctx context.Context, actx *ActionContext, act Action) (map[string]string, error) {
+		return nil, nil
+	})
+	registry.RegisterFunc("fail_action", TierCore, func(ctx context.Context, actx *ActionContext, act Action) (map[string]string, error) {
+		return nil, fmt.Errorf("boom")
+	})
+
+	agent := NewAgent(AgentConfig{Port: 0, Token: "tok", Nodes: []string{"n1"}, Registry: registry})
+	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+	defer cancel()
+	go agent.Start(ctx)
+	for i := 0; i < 50; i++ {
+		if agent.ListenAddr() != "" {
+			break
+		}
+		time.Sleep(20 * time.Millisecond)
+	}
+
+	coord := NewCoordinator(CoordinatorConfig{Port: 0, Token: "tok", Expected: map[string]string{"n1": agent.ListenAddr()}})
+	coord.Start()
+	defer coord.Stop()
+	coord.RegisterAgent("n1", agent.ListenAddr())
+
+	s := &Scenario{
+		Name:    "fail-test",
+		Timeout: Duration{30 * time.Second},
+		Topology: Topology{
+			Agents: map[string]string{"n1": agent.ListenAddr()},
+			Nodes:  map[string]NodeSpec{"n1": {Host: "127.0.0.1", Agent: "n1"}},
+		},
+		Phases: []Phase{
+			{Name: "p1", Actions: []Action{{Action: "fail_action", Node: "n1"}}},
+			{Name: "p2", Actions: []Action{{Action: "pass_action", Node: "n1"}}}, // should be skipped
+			{Name: "cleanup", Always: true, Actions: []Action{{Action: "pass_action", Node: "n1"}}},
+		},
+	}
+
+	result := coord.RunScenario(ctx, s, registry)
+	if result.Status != StatusFail {
+		t.Fatal("expected FAIL")
+	}
+	// p1 (failed) + cleanup (always) = 2 phases, p2 skipped.
+	if len(result.Phases) != 2 {
+		t.Errorf("expected 2 phases (p1+cleanup), got %d", len(result.Phases))
+	}
+}
+
+func TestCoordinatorAgent_RetryOnFailure(t *testing.T) {
+	if testing.Short() {
+		t.Skip("integration test")
+	}
+
+	registry := NewRegistry()
+	callCount := 0
+	registry.RegisterFunc("flaky_action", TierCore, func(ctx context.Context, actx *ActionContext, act Action) (map[string]string, error) {
+		callCount++
+		if callCount < 3 {
+			return nil, fmt.Errorf("flaky error %d", callCount)
+		}
+		return map[string]string{"value": "success"}, nil
+	})
+
+	agent := NewAgent(AgentConfig{Port: 0, Token: "tok", Nodes: []string{"n1"}, Registry: registry})
+	ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
+	defer cancel()
+	go agent.Start(ctx)
+	for i := 0; i < 50; i++ {
+		if agent.ListenAddr() != "" {
+			break
+		}
+		time.Sleep(20 * time.Millisecond)
+	}
+
+	coord := NewCoordinator(CoordinatorConfig{Port: 0, Token: "tok", Expected: map[string]string{"n1": agent.ListenAddr()}})
+	coord.Start()
+	defer coord.Stop()
+	coord.RegisterAgent("n1", agent.ListenAddr())
+
+	s := &Scenario{
+		Name:    "retry-test",
+		Timeout: Duration{30 * time.Second},
+		Topology: Topology{
+			Agents: map[string]string{"n1": agent.ListenAddr()},
+			Nodes:  map[string]NodeSpec{"n1": {Host: "127.0.0.1", Agent: "n1"}},
+		},
+		Phases: []Phase{
+			{Name: "p1", Actions: []Action{{Action: "flaky_action", Node: "n1", Retry: 3}}},
+		},
+	}
+
+	result := coord.RunScenario(ctx, s, registry)
+	if result.Status != StatusPass {
+		t.Fatalf("expected PASS after retries, got %s: %s", result.Status, result.Error)
+	}
+}
+
+func TestCoordinatorAgent_VarsInResult(t *testing.T) {
+	if testing.Short() {
+		t.Skip("integration test")
+	}
+
+	registry := NewRegistry()
+	registry.RegisterFunc("produce", TierCore, func(ctx context.Context, actx *ActionContext, act Action) (map[string]string, error) {
+		return map[string]string{"value": "produced-value"}, nil
+	})
+
+	agent := NewAgent(AgentConfig{Port: 0, Token: "tok", Nodes: []string{"n1"}, Registry: registry})
+	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+	defer cancel()
+	go agent.Start(ctx)
+	for i := 0; i < 50; i++ {
+		if agent.ListenAddr() != "" {
+			break
+		}
+		time.Sleep(20 * time.Millisecond)
+	}
+
+	coord := NewCoordinator(CoordinatorConfig{Port: 0, Token: "tok", Expected: map[string]string{"n1": agent.ListenAddr()}})
+	coord.Start()
+	defer coord.Stop()
+	coord.RegisterAgent("n1", agent.ListenAddr())
+
+	s := &Scenario{
+		Name:    "vars-result",
+		Timeout: Duration{30 * time.Second},
+		Env:     map[string]string{"env_key": "env_val"},
+		Topology: Topology{
+			Agents: map[string]string{"n1": agent.ListenAddr()},
+			Nodes:  map[string]NodeSpec{"n1": {Host: "127.0.0.1", Agent: "n1"}},
+		},
+		Phases: []Phase{
+			{Name: "p1", Actions: []Action{{Action: "produce", Node: "n1", SaveAs: "out_var"}}},
+		},
+	}
+
+	result := coord.RunScenario(ctx, s, registry)
+	if result.Status != StatusPass {
+		t.Fatalf("expected PASS, got %s: %s", result.Status, result.Error)
+	}
+
+	// Verify vars are in the result.
+	if result.Vars == nil {
+		t.Fatal("result.Vars is nil")
+	}
+	if result.Vars["out_var"] != "produced-value" {
+		t.Errorf("result.Vars[out_var] = %q, want produced-value", result.Vars["out_var"])
+	}
+	if result.Vars["env_key"] != "env_val" {
+		t.Errorf("result.Vars[env_key] = %q, want env_val", result.Vars["env_key"])
+	}
+}
+
+func TestCoordinatorAgent_DryRun(t *testing.T) {
+	coord := NewCoordinator(CoordinatorConfig{
+		Port:     0,
+		DryRun:   true,
+		Expected: map[string]string{"n1": "h:9100"},
+	})
+	coord.Start()
+	defer coord.Stop()
+	coord.RegisterAgent("n1", "h:9100")
+
+	s := &Scenario{
+		Name: "dry-run-test",
+		Topology: Topology{
+			Agents: map[string]string{"n1": "h:9100"},
+			Nodes:  map[string]NodeSpec{"n1": {Host: "h", Agent: "n1"}},
+		},
+		Phases: []Phase{
+			{Name: "p1", Actions: []Action{{Action: "echo", Node: "n1"}}},
+		},
+	}
+
+	result := coord.RunScenario(context.Background(), s, NewRegistry())
+	// Dry run should always pass (no execution).
+	if result.Status != StatusPass {
+		t.Errorf("dry run status = %s", result.Status)
+	}
+}
+
+func TestCoordinator_RegisterTokenValidation(t *testing.T) {
+	// Coordinator with token should reject unauthenticated /register.
+	c := NewCoordinator(CoordinatorConfig{
+		Port:     0,
+		Token:    "secret",
+		Expected: map[string]string{"n1": "h:9100"},
+	})
+	if err := c.Start(); err != nil {
+		t.Fatalf("start: %v", err)
+	}
+	defer c.Stop()
+
+	regReq := RegisterRequest{
+		AgentID:  "a1",
+		Hostname: "h",
+		IP:       "127.0.0.1",
+		Port:     9100,
+		Nodes:    []string{"n1"},
+	}
+	body, _ := json.Marshal(regReq)
+
+	// No token → 401.
+	resp, err := http.Post("http://"+c.ListenAddr()+"/register", "application/json", bytes.NewReader(body))
+	if err != nil {
+		t.Fatalf("POST /register: %v", err)
+	}
+	resp.Body.Close()
+	if resp.StatusCode != http.StatusUnauthorized {
+		t.Errorf("no token: expected 401, got %d", resp.StatusCode)
+	}
+
+	// Wrong token → 401.
+	req, _ := http.NewRequest("POST", "http://"+c.ListenAddr()+"/register", bytes.NewReader(body))
+	req.Header.Set("Content-Type", "application/json")
+	req.Header.Set(AuthTokenHeader, "wrong")
+	resp2, _ := http.DefaultClient.Do(req)
+	resp2.Body.Close()
+	if resp2.StatusCode != http.StatusUnauthorized {
+		t.Errorf("wrong token: expected 401, got %d", resp2.StatusCode)
+	}
+
+	// Correct token → 200.
+	req3, _ := http.NewRequest("POST", "http://"+c.ListenAddr()+"/register", bytes.NewReader(body))
+	req3.Header.Set("Content-Type", "application/json")
+	req3.Header.Set(AuthTokenHeader, "secret")
+	resp3, _ := http.DefaultClient.Do(req3)
+	defer resp3.Body.Close()
+	if resp3.StatusCode != http.StatusOK {
+		t.Errorf("correct token: expected 200, got %d", resp3.StatusCode)
+	}
+	var rr RegisterResponse
+	json.NewDecoder(resp3.Body).Decode(&rr)
+	if !rr.OK {
+		t.Errorf("registration should succeed with correct token, error: %s", rr.Error)
+	}
+}
+
+func TestCoordinator_RegisterNoToken(t *testing.T) {
+	// Coordinator without token should accept any /register.
+	c := NewCoordinator(CoordinatorConfig{
+		Port:     0,
+		Token:    "", // no token
+		Expected: map[string]string{"n1": "h:9100"},
+	})
+	if err := c.Start(); err != nil {
+		t.Fatalf("start: %v", err)
+	}
+	defer c.Stop()
+
+	regReq := RegisterRequest{
+		AgentID:  "a1",
+		Hostname: "h",
+		IP:       "127.0.0.1",
+		Port:     9100,
+		Nodes:    []string{"n1"},
+	}
+	body, _ := json.Marshal(regReq)
+
+	resp, err := http.Post("http://"+c.ListenAddr()+"/register", "application/json", bytes.NewReader(body))
+	if err != nil {
+		t.Fatalf("POST /register: %v", err)
+	}
+	defer resp.Body.Close()
+	if resp.StatusCode != http.StatusOK {
+		t.Errorf("expected 200 without token requirement, got %d", resp.StatusCode)
+	}
+}
+
+func TestAgent_PersistentReRegistration(t *testing.T) {
+	if testing.Short() {
+		t.Skip("integration test")
+	}
+
+	registry := NewRegistry()
+	registry.RegisterFunc("noop", TierCore, func(ctx context.Context, actx *ActionContext, act Action) (map[string]string, error) {
+		return nil, nil
+	})
+
+	// Start persistent agent.
+	agent := NewAgent(AgentConfig{
+		Port:       0,
+		Token:      "tok",
+		Persistent: true,
+		Nodes:      []string{"n1"},
+		Registry:   registry,
+	})
+
+	ctx, cancel := context.WithTimeout(context.Background(), 15*time.Second)
+	defer cancel()
+
+	go agent.Start(ctx)
+	for i := 0; i < 50; i++ {
+		if agent.ListenAddr() != "" {
+			break
+		}
+		time.Sleep(20 * time.Millisecond)
+	}
+	if agent.ListenAddr() == "" {
+		t.Fatal("agent didn't start")
+	}
+
+	// Start first coordinator — agent should register.
+	coord1 := NewCoordinator(CoordinatorConfig{
+		Port:     0,
+		Token:    "tok",
+		Expected: map[string]string{"n1": agent.ListenAddr()},
+	})
+	// Update agent's coordinator URL to point at coord1.
+	agent.CoordinatorURL = "http://" + coord1.ListenAddr()
+	// Since agent is already running with persistent loop pointing to original URL,
+	// we need a different approach: start coordinator first, then the agent connects.
+
+	// Actually, let's restart: stop agent, start coord, then start agent pointing at it.
+	cancel()
+	time.Sleep(100 * time.Millisecond)
+
+	// Start coordinator 1.
+	coord1 = NewCoordinator(CoordinatorConfig{
+		Port:     0,
+		Token:    "tok",
+		Expected: map[string]string{"n1": "will-be-set"},
+	})
+	if err := coord1.Start(); err != nil {
+		t.Fatalf("start coord1: %v", err)
+	}
+
+	// Start persistent agent pointing at coord1.
+	ctx2, cancel2 := context.WithTimeout(context.Background(), 15*time.Second)
+	defer cancel2()
+
+	agent2 := NewAgent(AgentConfig{
+		Port:           0,
+		CoordinatorURL: "http://" + coord1.ListenAddr(),
+		Token:          "tok",
+		Persistent:     true,
+		Nodes:          []string{"n1"},
+		Registry:       registry,
+	})
+	go agent2.Start(ctx2)
+	for i := 0; i < 50; i++ {
+		if agent2.ListenAddr() != "" {
+			break
+		}
+		time.Sleep(20 * time.Millisecond)
+	}
+
+	// Wait for registration.
+	if err := coord1.WaitForAgents(ctx2, 10*time.Second); err != nil {
+		t.Fatalf("agent didn't register with coord1: %v", err)
+	}
+	if !coord1.IsAgentRegistered("n1") {
+		t.Fatal("n1 not registered on coord1")
+	}
+
+	// Stop coord1, start coord2 on the SAME port.
+	addr := coord1.ListenAddr()
+	coord1.Stop()
+	time.Sleep(200 * time.Millisecond)
+
+	// Extract port from addr.
+	var port int
+	fmt.Sscanf(addr, "[::]:%d", &port)
+	if port == 0 {
+		fmt.Sscanf(addr, "0.0.0.0:%d", &port)
+	}
+	if port == 0 {
+		fmt.Sscanf(addr, "127.0.0.1:%d", &port)
+	}
+
+	coord2 := NewCoordinator(CoordinatorConfig{
+		Port:     port,
+		Token:    "tok",
+		Expected: map[string]string{"n1": agent2.ListenAddr()},
+	})
+	if err := coord2.Start(); err != nil {
+		t.Fatalf("start coord2: %v", err)
+	}
+	defer coord2.Stop()
+
+	// Agent should re-register with coord2 within ~10s.
+	if err := coord2.WaitForAgents(ctx2, 12*time.Second); err != nil {
+		t.Fatalf("agent didn't re-register with coord2: %v", err)
+	}
+	if !coord2.IsAgentRegistered("n1") {
+		t.Fatal("n1 not registered on coord2 after re-registration")
+	}
+}

weed/storage/blockvol/testrunner/engine.go

@@ -0,0 +1,299 @@
+package testrunner
+
+import (
+	"context"
+	"fmt"
+	"regexp"
+	"strings"
+	"sync"
+	"time"
+
+	"gopkg.in/yaml.v3"
+)
+
+var varPattern = regexp.MustCompile(`\{\{\s*(\w+)\s*\}\}`)
+
+// Engine executes a Scenario using the given Registry.
+type Engine struct {
+	registry *Registry
+	log      func(format string, args ...interface{})
+}
+
+// NewEngine creates an engine with the given registry and logger.
+func NewEngine(registry *Registry, log func(format string, args ...interface{})) *Engine {
+	if log == nil {
+		log = func(string, ...interface{}) {}
+	}
+	return &Engine{registry: registry, log: log}
+}
+
+// Run executes the scenario end-to-end and returns the result.
+func (e *Engine) Run(ctx context.Context, s *Scenario, actx *ActionContext) *ScenarioResult {
+	start := time.Now()
+	result := &ScenarioResult{
+		Name:   s.Name,
+		Status: StatusPass,
+	}
+
+	// Apply scenario timeout.
+	if s.Timeout.Duration > 0 {
+		var cancel context.CancelFunc
+		ctx, cancel = context.WithTimeout(ctx, s.Timeout.Duration)
+		defer cancel()
+	}
+
+	// Seed vars from env.
+	if actx.Vars == nil {
+		actx.Vars = make(map[string]string)
+	}
+	for k, v := range s.Env {
+		actx.Vars[k] = v
+	}
+
+	// Separate always-phases for deferred cleanup.
+	var normalPhases, alwaysPhases []Phase
+	for _, p := range s.Phases {
+		if p.Always {
+			alwaysPhases = append(alwaysPhases, p)
+		} else {
+			normalPhases = append(normalPhases, p)
+		}
+	}
+
+	// Execute normal phases sequentially.
+	failed := false
+	for _, phase := range normalPhases {
+		pr := e.runPhase(ctx, actx, phase)
+		result.Phases = append(result.Phases, pr)
+		if pr.Status == StatusFail {
+			failed = true
+			result.Status = StatusFail
+			result.Error = fmt.Sprintf("phase %q failed: %s", phase.Name, pr.Error)
+			break
+		}
+	}
+
+	// Always-phases run regardless of failure.
+	for _, phase := range alwaysPhases {
+		pr := e.runPhase(ctx, actx, phase)
+		result.Phases = append(result.Phases, pr)
+	}
+
+	result.Duration = time.Since(start)
+	if !failed {
+		result.Status = StatusPass
+	}
+
+	// Preserve all final vars in the result for downstream reporting.
+	if len(actx.Vars) > 0 {
+		result.Vars = make(map[string]string, len(actx.Vars))
+		for k, v := range actx.Vars {
+			result.Vars[k] = v
+		}
+	}
+
+	return result
+}
+
+// runPhase executes a single phase (sequential or parallel).
+func (e *Engine) runPhase(ctx context.Context, actx *ActionContext, phase Phase) PhaseResult {
+	start := time.Now()
+	pr := PhaseResult{
+		Name:   phase.Name,
+		Status: StatusPass,
+	}
+
+	e.log("[phase] %s", phase.Name)
+
+	if phase.Parallel {
+		pr = e.runPhaseParallel(ctx, actx, phase)
+	} else {
+		pr = e.runPhaseSequential(ctx, actx, phase)
+	}
+
+	pr.Duration = time.Since(start)
+	return pr
+}
+
+func (e *Engine) runPhaseSequential(ctx context.Context, actx *ActionContext, phase Phase) PhaseResult {
+	pr := PhaseResult{
+		Name:   phase.Name,
+		Status: StatusPass,
+	}
+
+	for i, act := range phase.Actions {
+		ar := e.runAction(ctx, actx, act)
+		pr.Actions = append(pr.Actions, ar)
+		if ar.Status == StatusFail && !act.IgnoreError {
+			pr.Status = StatusFail
+			pr.Error = fmt.Sprintf("action %d (%s) failed: %s", i, act.Action, ar.Error)
+			return pr
+		}
+	}
+	return pr
+}
+
+func (e *Engine) runPhaseParallel(ctx context.Context, actx *ActionContext, phase Phase) PhaseResult {
+	pr := PhaseResult{
+		Name:   phase.Name,
+		Status: StatusPass,
+	}
+
+	results := make([]ActionResult, len(phase.Actions))
+	var wg sync.WaitGroup
+	for i, act := range phase.Actions {
+		wg.Add(1)
+		go func(idx int, a Action) {
+			defer wg.Done()
+			results[idx] = e.runAction(ctx, actx, a)
+		}(i, act)
+	}
+	wg.Wait()
+
+	for i, ar := range results {
+		pr.Actions = append(pr.Actions, ar)
+		if ar.Status == StatusFail && !phase.Actions[i].IgnoreError {
+			pr.Status = StatusFail
+			if pr.Error == "" {
+				pr.Error = fmt.Sprintf("action %d (%s) failed: %s", i, phase.Actions[i].Action, ar.Error)
+			}
+		}
+	}
+	return pr
+}
+
+// runAction resolves variables and executes a single action.
+func (e *Engine) runAction(ctx context.Context, actx *ActionContext, act Action) ActionResult {
+	start := time.Now()
+
+	// Resolve variables in the action.
+	resolved := resolveAction(act, actx.Vars)
+
+	// Serialize resolved action to YAML for report display.
+	yamlDef := marshalActionYAML(resolved)
+
+	handler, err := e.registry.Get(resolved.Action)
+	if err != nil {
+		return ActionResult{
+			Action:   resolved.Action,
+			Status:   StatusFail,
+			Duration: time.Since(start),
+			Error:    err.Error(),
+		}
+	}
+
+	// Handle delay param.
+	if d, ok := resolved.Params["delay"]; ok {
+		dur, err := time.ParseDuration(d)
+		if err == nil {
+			e.log("  [delay] %s", d)
+			select {
+			case <-time.After(dur):
+			case <-ctx.Done():
+				return ActionResult{
+					Action:   resolved.Action,
+					Status:   StatusFail,
+					Duration: time.Since(start),
+					Error:    ctx.Err().Error(),
+				}
+			}
+		}
+	}
+
+	e.log("  [action] %s", resolved.Action)
+
+	output, err := handler.Execute(ctx, actx, resolved)
+
+	ar := ActionResult{
+		Action:   resolved.Action,
+		Duration: time.Since(start),
+		YAML:     yamlDef,
+	}
+
+	if err != nil {
+		ar.Status = StatusFail
+		ar.Error = err.Error()
+		if act.IgnoreError {
+			ar.Status = StatusPass
+			e.log("  [action] %s failed (ignored): %v", resolved.Action, err)
+		}
+	} else {
+		ar.Status = StatusPass
+	}
+
+	// Store output as var if save_as is set.
+	if resolved.SaveAs != "" && output != nil {
+		if v, ok := output["value"]; ok {
+			actx.Vars[resolved.SaveAs] = v
+			e.log("  [var] %s = %s", resolved.SaveAs, truncate(v, 60))
+		}
+	}
+
+	// Store all output keys with double-underscore prefix for cleanup vars.
+	if output != nil {
+		for k, v := range output {
+			if strings.HasPrefix(k, "__") {
+				actx.Vars[k] = v
+			}
+		}
+	}
+
+	if output != nil {
+		if v, ok := output["value"]; ok {
+			ar.Output = truncate(v, 4096)
+		}
+	}
+
+	return ar
+}
+
+// resolveAction substitutes {{ var }} references in the action's fields.
+func resolveAction(act Action, vars map[string]string) Action {
+	resolved := Action{
+		Action:      act.Action,
+		Target:      act.Target,
+		Replica:     act.Replica,
+		Node:        act.Node,
+		SaveAs:      act.SaveAs,
+		IgnoreError: act.IgnoreError,
+		Params:      make(map[string]string),
+	}
+
+	// Copy and resolve params.
+	for k, v := range act.Params {
+		resolved.Params[k] = resolveVars(v, vars)
+	}
+
+	return resolved
+}
+
+// resolveVars replaces {{ name }} with the value from vars.
+func resolveVars(s string, vars map[string]string) string {
+	return varPattern.ReplaceAllStringFunc(s, func(match string) string {
+		sub := varPattern.FindStringSubmatch(match)
+		if len(sub) < 2 {
+			return match
+		}
+		name := sub[1]
+		if v, ok := vars[name]; ok {
+			return v
+		}
+		return match // leave unresolved
+	})
+}
+
+func truncate(s string, max int) string {
+	if len(s) <= max {
+		return s
+	}
+	return s[:max] + "..."
+}
+
+// marshalActionYAML serializes a resolved action to YAML for report display.
+func marshalActionYAML(act Action) string {
+	data, err := yaml.Marshal(act)
+	if err != nil {
+		return ""
+	}
+	return string(data)
+}

weed/storage/blockvol/testrunner/engine_test.go

@@ -0,0 +1,515 @@
+package testrunner
+
+import (
+	"context"
+	"fmt"
+	"testing"
+	"time"
+)
+
+// mockHandler records calls and returns configured outputs.
+type mockHandler struct {
+	calls   []Action
+	outputs map[string]string
+	err     error
+}
+
+func (m *mockHandler) Execute(ctx context.Context, actx *ActionContext, act Action) (map[string]string, error) {
+	m.calls = append(m.calls, act)
+	if m.err != nil {
+		return nil, m.err
+	}
+	return m.outputs, nil
+}
+
+func TestEngine_BasicFlow(t *testing.T) {
+	registry := NewRegistry()
+
+	step1 := &mockHandler{outputs: map[string]string{"value": "hello"}}
+	step2 := &mockHandler{outputs: map[string]string{"value": "world"}}
+
+	registry.Register("step1", TierCore, step1)
+	registry.Register("step2", TierCore, step2)
+
+	scenario := &Scenario{
+		Name:    "basic-test",
+		Timeout: Duration{5 * time.Second},
+		Phases: []Phase{
+			{
+				Name: "phase1",
+				Actions: []Action{
+					{Action: "step1", SaveAs: "v1"},
+					{Action: "step2", SaveAs: "v2"},
+				},
+			},
+		},
+	}
+
+	engine := NewEngine(registry, nil)
+	actx := &ActionContext{
+		Scenario: scenario,
+		Vars:     make(map[string]string),
+		Log:      func(string, ...interface{}) {},
+	}
+	result := engine.Run(context.Background(), scenario, actx)
+
+	if result.Status != StatusPass {
+		t.Errorf("status = %s, want PASS", result.Status)
+	}
+	if len(result.Phases) != 1 {
+		t.Fatalf("phases = %d, want 1", len(result.Phases))
+	}
+	if len(result.Phases[0].Actions) != 2 {
+		t.Fatalf("actions = %d, want 2", len(result.Phases[0].Actions))
+	}
+	if actx.Vars["v1"] != "hello" {
+		t.Errorf("v1 = %q, want %q", actx.Vars["v1"], "hello")
+	}
+	if actx.Vars["v2"] != "world" {
+		t.Errorf("v2 = %q, want %q", actx.Vars["v2"], "world")
+	}
+}
+
+func TestEngine_FailureStopsPhase(t *testing.T) {
+	registry := NewRegistry()
+
+	step1 := &mockHandler{err: fmt.Errorf("boom")}
+	step2 := &mockHandler{outputs: map[string]string{"value": "ok"}}
+
+	registry.Register("step1", TierCore, step1)
+	registry.Register("step2", TierCore, step2)
+
+	scenario := &Scenario{
+		Name:    "fail-test",
+		Timeout: Duration{5 * time.Second},
+		Phases: []Phase{
+			{
+				Name: "phase1",
+				Actions: []Action{
+					{Action: "step1"},
+					{Action: "step2"}, // should not run
+				},
+			},
+		},
+	}
+
+	engine := NewEngine(registry, nil)
+	actx := &ActionContext{
+		Scenario: scenario,
+		Vars:     make(map[string]string),
+		Log:      func(string, ...interface{}) {},
+	}
+	result := engine.Run(context.Background(), scenario, actx)
+
+	if result.Status != StatusFail {
+		t.Errorf("status = %s, want FAIL", result.Status)
+	}
+	if len(result.Phases[0].Actions) != 1 {
+		t.Errorf("actions = %d, want 1 (should stop after failure)", len(result.Phases[0].Actions))
+	}
+	if len(step2.calls) != 0 {
+		t.Error("step2 should not have been called")
+	}
+}
+
+func TestEngine_IgnoreError(t *testing.T) {
+	registry := NewRegistry()
+
+	step1 := &mockHandler{err: fmt.Errorf("expected error")}
+	step2 := &mockHandler{outputs: map[string]string{"value": "ok"}}
+
+	registry.Register("step1", TierCore, step1)
+	registry.Register("step2", TierCore, step2)
+
+	scenario := &Scenario{
+		Name:    "ignore-test",
+		Timeout: Duration{5 * time.Second},
+		Phases: []Phase{
+			{
+				Name: "phase1",
+				Actions: []Action{
+					{Action: "step1", IgnoreError: true},
+					{Action: "step2"},
+				},
+			},
+		},
+	}
+
+	engine := NewEngine(registry, nil)
+	actx := &ActionContext{
+		Scenario: scenario,
+		Vars:     make(map[string]string),
+		Log:      func(string, ...interface{}) {},
+	}
+	result := engine.Run(context.Background(), scenario, actx)
+
+	if result.Status != StatusPass {
+		t.Errorf("status = %s, want PASS (error was ignored)", result.Status)
+	}
+	if len(step2.calls) != 1 {
+		t.Error("step2 should have been called despite step1 error")
+	}
+}
+
+func TestEngine_AlwaysPhaseRunsAfterFailure(t *testing.T) {
+	registry := NewRegistry()
+
+	failStep := &mockHandler{err: fmt.Errorf("fail")}
+	cleanStep := &mockHandler{}
+
+	registry.Register("fail_step", TierCore, failStep)
+	registry.Register("clean_step", TierCore, cleanStep)
+
+	scenario := &Scenario{
+		Name:    "always-test",
+		Timeout: Duration{5 * time.Second},
+		Phases: []Phase{
+			{
+				Name: "main",
+				Actions: []Action{
+					{Action: "fail_step"},
+				},
+			},
+			{
+				Name:   "cleanup",
+				Always: true,
+				Actions: []Action{
+					{Action: "clean_step"},
+				},
+			},
+		},
+	}
+
+	engine := NewEngine(registry, nil)
+	actx := &ActionContext{
+		Scenario: scenario,
+		Vars:     make(map[string]string),
+		Log:      func(string, ...interface{}) {},
+	}
+	result := engine.Run(context.Background(), scenario, actx)
+
+	if result.Status != StatusFail {
+		t.Errorf("status = %s, want FAIL", result.Status)
+	}
+	if len(result.Phases) != 2 {
+		t.Fatalf("phases = %d, want 2 (cleanup should still run)", len(result.Phases))
+	}
+	if len(cleanStep.calls) != 1 {
+		t.Error("cleanup step should have been called")
+	}
+}
+
+func TestEngine_VarSubstitution(t *testing.T) {
+	registry := NewRegistry()
+
+	step1 := &mockHandler{outputs: map[string]string{"value": "/dev/sda"}}
+	step2 := &mockHandler{}
+
+	registry.Register("step1", TierCore, step1)
+	registry.Register("step2", TierCore, step2)
+
+	scenario := &Scenario{
+		Name:    "var-test",
+		Timeout: Duration{5 * time.Second},
+		Phases: []Phase{
+			{
+				Name: "phase1",
+				Actions: []Action{
+					{Action: "step1", SaveAs: "device"},
+					{Action: "step2", Params: map[string]string{"device": "{{ device }}"}},
+				},
+			},
+		},
+	}
+
+	engine := NewEngine(registry, nil)
+	actx := &ActionContext{
+		Scenario: scenario,
+		Vars:     make(map[string]string),
+		Log:      func(string, ...interface{}) {},
+	}
+	engine.Run(context.Background(), scenario, actx)
+
+	if len(step2.calls) != 1 {
+		t.Fatalf("step2 not called")
+	}
+	if step2.calls[0].Params["device"] != "/dev/sda" {
+		t.Errorf("device = %q, want /dev/sda", step2.calls[0].Params["device"])
+	}
+}
+
+func TestEngine_EnvVars(t *testing.T) {
+	registry := NewRegistry()
+
+	step := &mockHandler{}
+	registry.Register("step", TierCore, step)
+
+	scenario := &Scenario{
+		Name:    "env-test",
+		Timeout: Duration{5 * time.Second},
+		Env:     map[string]string{"repo_dir": "/tmp/repo"},
+		Phases: []Phase{
+			{
+				Name: "phase1",
+				Actions: []Action{
+					{Action: "step", Params: map[string]string{"dir": "{{ repo_dir }}"}},
+				},
+			},
+		},
+	}
+
+	engine := NewEngine(registry, nil)
+	actx := &ActionContext{
+		Scenario: scenario,
+		Vars:     make(map[string]string),
+		Log:      func(string, ...interface{}) {},
+	}
+	engine.Run(context.Background(), scenario, actx)
+
+	if len(step.calls) != 1 {
+		t.Fatalf("step not called")
+	}
+	if step.calls[0].Params["dir"] != "/tmp/repo" {
+		t.Errorf("dir = %q, want /tmp/repo", step.calls[0].Params["dir"])
+	}
+}
+
+func TestEngine_Timeout(t *testing.T) {
+	registry := NewRegistry()
+
+	slowStep := ActionHandlerFunc(func(ctx context.Context, actx *ActionContext, act Action) (map[string]string, error) {
+		select {
+		case <-ctx.Done():
+			return nil, ctx.Err()
+		case <-time.After(10 * time.Second):
+			return nil, nil
+		}
+	})
+	registry.Register("slow", TierCore, slowStep)
+
+	scenario := &Scenario{
+		Name:    "timeout-test",
+		Timeout: Duration{100 * time.Millisecond},
+		Phases: []Phase{
+			{
+				Name: "phase1",
+				Actions: []Action{
+					{Action: "slow"},
+				},
+			},
+		},
+	}
+
+	engine := NewEngine(registry, nil)
+	actx := &ActionContext{
+		Scenario: scenario,
+		Vars:     make(map[string]string),
+		Log:      func(string, ...interface{}) {},
+	}
+
+	start := time.Now()
+	result := engine.Run(context.Background(), scenario, actx)
+	elapsed := time.Since(start)
+
+	if result.Status != StatusFail {
+		t.Errorf("status = %s, want FAIL", result.Status)
+	}
+	if elapsed > 2*time.Second {
+		t.Errorf("took %v, expected timeout at ~100ms", elapsed)
+	}
+}
+
+func TestEngine_UnknownAction(t *testing.T) {
+	registry := NewRegistry()
+	scenario := &Scenario{
+		Name:    "unknown-test",
+		Timeout: Duration{5 * time.Second},
+		Phases: []Phase{
+			{
+				Name: "phase1",
+				Actions: []Action{
+					{Action: "nonexistent"},
+				},
+			},
+		},
+	}
+
+	engine := NewEngine(registry, nil)
+	actx := &ActionContext{
+		Scenario: scenario,
+		Vars:     make(map[string]string),
+		Log:      func(string, ...interface{}) {},
+	}
+	result := engine.Run(context.Background(), scenario, actx)
+
+	if result.Status != StatusFail {
+		t.Errorf("status = %s, want FAIL for unknown action", result.Status)
+	}
+}
+
+func TestEngine_ParallelPhase(t *testing.T) {
+	registry := NewRegistry()
+
+	callCount := 0
+	step := ActionHandlerFunc(func(ctx context.Context, actx *ActionContext, act Action) (map[string]string, error) {
+		callCount++
+		time.Sleep(50 * time.Millisecond)
+		return nil, nil
+	})
+	registry.Register("step", TierCore, step)
+
+	scenario := &Scenario{
+		Name:    "parallel-test",
+		Timeout: Duration{5 * time.Second},
+		Phases: []Phase{
+			{
+				Name:     "phase1",
+				Parallel: true,
+				Actions: []Action{
+					{Action: "step"},
+					{Action: "step"},
+					{Action: "step"},
+				},
+			},
+		},
+	}
+
+	engine := NewEngine(registry, nil)
+	actx := &ActionContext{
+		Scenario: scenario,
+		Vars:     make(map[string]string),
+		Log:      func(string, ...interface{}) {},
+	}
+
+	start := time.Now()
+	result := engine.Run(context.Background(), scenario, actx)
+	elapsed := time.Since(start)
+
+	if result.Status != StatusPass {
+		t.Errorf("status = %s, want PASS", result.Status)
+	}
+	// Parallel: should take ~50ms, not 150ms.
+	if elapsed > 200*time.Millisecond {
+		t.Errorf("took %v, parallel should be faster", elapsed)
+	}
+}
+
+func TestResolveVars(t *testing.T) {
+	vars := map[string]string{
+		"device": "/dev/sda",
+		"md5":    "abc123",
+	}
+
+	tests := []struct {
+		input string
+		want  string
+	}{
+		{"{{ device }}", "/dev/sda"},
+		{"{{md5}}", "abc123"},
+		{"dev={{ device }}, hash={{ md5 }}", "dev=/dev/sda, hash=abc123"},
+		{"no vars", "no vars"},
+		{"{{ unknown }}", "{{ unknown }}"}, // unresolved stays
+	}
+
+	for _, tt := range tests {
+		got := resolveVars(tt.input, vars)
+		if got != tt.want {
+			t.Errorf("resolveVars(%q) = %q, want %q", tt.input, got, tt.want)
+		}
+	}
+}
+
+func TestEngine_VarsInResult(t *testing.T) {
+	registry := NewRegistry()
+	step := &mockHandler{outputs: map[string]string{"value": "saved-data"}}
+	registry.Register("step", TierCore, step)
+
+	scenario := &Scenario{
+		Name:    "vars-result-test",
+		Timeout: Duration{5 * time.Second},
+		Env:     map[string]string{"env_key": "env_val"},
+		Phases: []Phase{
+			{
+				Name: "phase1",
+				Actions: []Action{
+					{Action: "step", SaveAs: "my_var"},
+				},
+			},
+		},
+	}
+
+	engine := NewEngine(registry, nil)
+	actx := &ActionContext{
+		Scenario: scenario,
+		Vars:     make(map[string]string),
+		Log:      func(string, ...interface{}) {},
+	}
+	result := engine.Run(context.Background(), scenario, actx)
+
+	if result.Status != StatusPass {
+		t.Fatalf("status = %s: %s", result.Status, result.Error)
+	}
+
+	// Verify vars are copied into result.
+	if result.Vars == nil {
+		t.Fatal("result.Vars is nil")
+	}
+	if result.Vars["my_var"] != "saved-data" {
+		t.Errorf("result.Vars[my_var] = %q, want saved-data", result.Vars["my_var"])
+	}
+	if result.Vars["env_key"] != "env_val" {
+		t.Errorf("result.Vars[env_key] = %q, want env_val", result.Vars["env_key"])
+	}
+}
+
+func TestEngine_CleanupVars(t *testing.T) {
+	registry := NewRegistry()
+
+	inject := &mockHandler{outputs: map[string]string{
+		"__cleanup_netem": "tc qdisc del dev eth0 root",
+	}}
+	clear := ActionHandlerFunc(func(ctx context.Context, actx *ActionContext, act Action) (map[string]string, error) {
+		cmd := actx.Vars["__cleanup_netem"]
+		if cmd == "" {
+			return nil, fmt.Errorf("cleanup var not found")
+		}
+		return map[string]string{"value": cmd}, nil
+	})
+
+	registry.Register("inject", TierCore, inject)
+	registry.Register("clear", TierCore, clear)
+
+	scenario := &Scenario{
+		Name:    "cleanup-var-test",
+		Timeout: Duration{5 * time.Second},
+		Phases: []Phase{
+			{
+				Name: "inject",
+				Actions: []Action{
+					{Action: "inject"},
+				},
+			},
+			{
+				Name: "clear",
+				Actions: []Action{
+					{Action: "clear", SaveAs: "result"},
+				},
+			},
+		},
+	}
+
+	engine := NewEngine(registry, nil)
+	actx := &ActionContext{
+		Scenario: scenario,
+		Vars:     make(map[string]string),
+		Log:      func(string, ...interface{}) {},
+	}
+	result := engine.Run(context.Background(), scenario, actx)
+
+	if result.Status != StatusPass {
+		t.Errorf("status = %s, want PASS: %s", result.Status, result.Error)
+	}
+	if actx.Vars["result"] != "tc qdisc del dev eth0 root" {
+		t.Errorf("result = %q", actx.Vars["result"])
+	}
+}

weed/storage/blockvol/testrunner/infra/artifacts.go

@@ -0,0 +1,94 @@
+package infra
+
+import (
+	"context"
+	"fmt"
+	"log"
+	"os"
+	"path/filepath"
+	"time"
+)
+
+// LogCollector is implemented by any target that can provide a log file.
+type LogCollector interface {
+	CollectLog() (string, error)
+}
+
+// ArtifactCollector gathers diagnostic info on failure.
+type ArtifactCollector struct {
+	Dir  string // base artifacts directory
+	Node *Node  // initiator node for dmesg/lsblk
+	Log  *log.Logger
+}
+
+// NewArtifactCollector creates a collector rooted at the given directory.
+func NewArtifactCollector(dir string, node *Node, logger *log.Logger) *ArtifactCollector {
+	if logger == nil {
+		logger = log.Default()
+	}
+	return &ArtifactCollector{
+		Dir:  dir,
+		Node: node,
+		Log:  logger,
+	}
+}
+
+// Collect gathers diagnostics when failed is true.
+func (a *ArtifactCollector) Collect(failed bool, tgt LogCollector, label string) {
+	if !failed {
+		return
+	}
+	a.CollectLabeled(tgt, label)
+}
+
+// CollectLabeled gathers diagnostics unconditionally.
+func (a *ArtifactCollector) CollectLabeled(tgt LogCollector, label string) {
+	ts := time.Now().Format("20060102-150405")
+	testDir := filepath.Join(a.Dir, ts)
+	if err := os.MkdirAll(testDir, 0755); err != nil {
+		a.Log.Printf("artifacts: mkdir failed: %v", err)
+		return
+	}
+
+	ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
+	defer cancel()
+
+	if tgt != nil {
+		if logContent, err := tgt.CollectLog(); err == nil && logContent != "" {
+			a.writeArtifact(filepath.Join(testDir, label+".log"), logContent)
+		}
+	}
+
+	if stdout, _, _, err := a.Node.RunRoot(ctx, "iscsiadm -m session 2>&1"); err == nil {
+		a.writeArtifact(filepath.Join(testDir, "iscsi-session.txt"), stdout)
+	}
+
+	if stdout, _, _, err := a.Node.RunRoot(ctx, "dmesg | tail -200"); err == nil {
+		a.writeArtifact(filepath.Join(testDir, "dmesg.txt"), stdout)
+	}
+
+	if stdout, _, _, err := a.Node.Run(ctx, "lsblk 2>&1"); err == nil {
+		a.writeArtifact(filepath.Join(testDir, "lsblk.txt"), stdout)
+	}
+
+	a.Log.Printf("artifacts saved to %s", testDir)
+}
+
+func (a *ArtifactCollector) writeArtifact(path, content string) {
+	if err := os.WriteFile(path, []byte(content), 0644); err != nil {
+		a.Log.Printf("artifacts: write %s: %v", path, err)
+	} else {
+		a.Log.Printf("artifacts: wrote %s (%d bytes)", filepath.Base(path), len(content))
+	}
+}
+
+// CollectPerf saves performance results to a timestamped JSON file.
+func (a *ArtifactCollector) CollectPerf(name string, data string) {
+	ts := time.Now().Format("20060102-150405")
+	path := filepath.Join(a.Dir, fmt.Sprintf("perf-%s-%s.json", name, ts))
+	if err := os.MkdirAll(filepath.Dir(path), 0755); err != nil {
+		a.Log.Printf("artifacts: mkdir failed: %v", err)
+		return
+	}
+	a.writeArtifact(path, data)
+}

weed/storage/blockvol/testrunner/infra/fault.go

@@ -0,0 +1,132 @@
+package infra
+
+import (
+	"context"
+	"fmt"
+	"strings"
+	"time"
+)
+
+// InjectNetem adds a netem delay on the node's outbound traffic to targetIP.
+// Returns the cleanup command string (for storage in vars).
+func InjectNetem(ctx context.Context, node *Node, targetIP string, delayMs int) (cleanupCmd string, err error) {
+	iface, _, code, err := node.RunRoot(ctx, fmt.Sprintf(
+		"ip route get %s | head -1 | awk '{for(i=1;i<=NF;i++) if($i==\"dev\") print $(i+1)}'", targetIP))
+	iface = strings.TrimSpace(iface)
+	if err != nil || code != 0 || iface == "" {
+		return "", fmt.Errorf("find interface for %s: iface=%q code=%d err=%v", targetIP, iface, code, err)
+	}
+
+	_, stderr, code, err := node.RunRoot(ctx, fmt.Sprintf(
+		"tc qdisc add dev %s root netem delay %dms", iface, delayMs))
+	if err != nil || code != 0 {
+		return "", fmt.Errorf("tc qdisc add: code=%d stderr=%s err=%v", code, stderr, err)
+	}
+
+	cleanupCmd = fmt.Sprintf("tc qdisc del dev %s root 2>/dev/null", iface)
+	return cleanupCmd, nil
+}
+
+// InjectIptablesDrop blocks outbound TCP traffic from node to targetIP on the given ports.
+// Returns the cleanup command string.
+func InjectIptablesDrop(ctx context.Context, node *Node, targetIP string, ports []int) (cleanupCmd string, err error) {
+	for i, port := range ports {
+		_, stderr, code, err := node.RunRoot(ctx, fmt.Sprintf(
+			"iptables -A OUTPUT -d %s -p tcp --dport %d -j DROP", targetIP, port))
+		if err != nil || code != 0 {
+			// Rollback already-added rules
+			for j := 0; j < i; j++ {
+				node.RunRoot(ctx, fmt.Sprintf(
+					"iptables -D OUTPUT -d %s -p tcp --dport %d -j DROP 2>/dev/null", targetIP, ports[j]))
+			}
+			return "", fmt.Errorf("iptables add port %d: code=%d stderr=%s err=%v", port, code, stderr, err)
+		}
+	}
+
+	// Build cleanup command that removes all rules.
+	var cmds []string
+	for _, port := range ports {
+		cmds = append(cmds, fmt.Sprintf(
+			"iptables -D OUTPUT -d %s -p tcp --dport %d -j DROP 2>/dev/null", targetIP, port))
+	}
+	cleanupCmd = strings.Join(cmds, " && ")
+	return cleanupCmd, nil
+}
+
+// FillDisk fills the filesystem at dir, leaving ~4MB free.
+// Returns the cleanup command string.
+func FillDisk(ctx context.Context, node *Node, dir string) (cleanupCmd string, err error) {
+	stdout, _, code, err := node.RunRoot(ctx, fmt.Sprintf(
+		"df -BM --output=avail %s | tail -1 | tr -d ' M'", dir))
+	if err != nil || code != 0 {
+		return "", fmt.Errorf("df: code=%d err=%v", code, err)
+	}
+	availMB := 0
+	fmt.Sscanf(strings.TrimSpace(stdout), "%d", &availMB)
+	if availMB < 8 {
+		return "", fmt.Errorf("not enough space to fill: %dMB available", availMB)
+	}
+	fillMB := availMB - 4
+
+	// Use fallocate (instant) instead of dd (linear time).
+	_, stderr, code, err := node.RunRoot(ctx, fmt.Sprintf(
+		"fallocate -l %dM %s/fillfile", fillMB, dir))
+	if err != nil || code != 0 {
+		// Fallback to dd if fallocate not available.
+		_, stderr, code, err = node.RunRoot(ctx, fmt.Sprintf(
+			"dd if=/dev/zero of=%s/fillfile bs=1M count=%d 2>/dev/null", dir, fillMB))
+		if err != nil || code != 0 {
+			stdout2, _, _, _ := node.RunRoot(ctx, fmt.Sprintf("test -f %s/fillfile && echo ok", dir))
+			if !strings.Contains(stdout2, "ok") {
+				return "", fmt.Errorf("fillDisk: code=%d stderr=%s err=%v", code, stderr, err)
+			}
+		}
+	}
+
+	cleanupCmd = fmt.Sprintf("rm -f %s/fillfile", dir)
+	return cleanupCmd, nil
+}
+
+// CorruptWALRegion overwrites nBytes within the WAL section of the volume file.
+func CorruptWALRegion(ctx context.Context, node *Node, volPath string, nBytes int) error {
+	const walOffset = 4096 // SuperblockSize
+
+	stdout, _, code, err := node.RunRoot(ctx, fmt.Sprintf("stat -c %%s %s", volPath))
+	if err != nil || code != 0 {
+		return fmt.Errorf("stat %s: code=%d err=%v", volPath, code, err)
+	}
+	fileSize := 0
+	fmt.Sscanf(strings.TrimSpace(stdout), "%d", &fileSize)
+
+	walEnd := walOffset + 64*1024*1024
+	if walEnd > fileSize {
+		walEnd = fileSize
+	}
+	walUsable := walEnd - walOffset
+	if walUsable < nBytes*2 {
+		return fmt.Errorf("WAL region too small: %d", walUsable)
+	}
+	seekPos := walOffset + walUsable/3
+
+	_, stderr, code, err := node.RunRoot(ctx, fmt.Sprintf(
+		"python3 -c \"import sys; sys.stdout.buffer.write(b'\\xff'*%d)\" | dd of=%s bs=1 seek=%d conv=notrunc 2>/dev/null",
+		nBytes, volPath, seekPos))
+	if err != nil || code != 0 {
+		return fmt.Errorf("corrupt WAL region: code=%d stderr=%s err=%v", code, stderr, err)
+	}
+	return nil
+}
+
+// ClearFault executes a cleanup command stored in vars.
+func ClearFault(ctx context.Context, node *Node, cleanupCmd string) error {
+	if cleanupCmd == "" {
+		return nil
+	}
+	cctx, cancel := context.WithTimeout(ctx, 10*time.Second)
+	defer cancel()
+	_, stderr, code, err := node.RunRoot(cctx, cleanupCmd)
+	if err != nil || code != 0 {
+		return fmt.Errorf("clear fault: code=%d stderr=%s err=%v", code, stderr, err)
+	}
+	return nil
+}

weed/storage/blockvol/testrunner/infra/ha_target.go

@@ -0,0 +1,412 @@
+package infra
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"net/http"
+	"strings"
+	"time"
+)
+
+// HATarget extends Target with HA-specific admin HTTP endpoints.
+type HATarget struct {
+	*Target
+	AdminPort   int
+	ReplicaData int // replica receiver data port
+	ReplicaCtrl int // replica receiver ctrl port
+	RebuildPort int
+	TPGID       int // ALUA target port group ID (0 = omit flag)
+}
+
+// StatusResp matches the JSON returned by GET /status.
+type StatusResp struct {
+	Path          string `json:"path"`
+	Epoch         uint64 `json:"epoch"`
+	Role          string `json:"role"`
+	WALHeadLSN    uint64 `json:"wal_head_lsn"`
+	CheckpointLSN uint64 `json:"checkpoint_lsn"`
+	HasLease      bool   `json:"has_lease"`
+	Healthy       bool   `json:"healthy"`
+	VolumeSize    uint64 `json:"volume_size"`
+}
+
+// SnapshotEntry matches the JSON returned by POST /snapshot {action:"list"}.
+type SnapshotEntry struct {
+	ID        uint32 `json:"id"`
+	BaseLSN   uint64 `json:"base_lsn"`
+	CreatedAt string `json:"created_at"`
+	CoWBlocks uint64 `json:"cow_blocks"`
+}
+
+// NewHATarget creates an HATarget with the given ports.
+func NewHATarget(node *Node, cfg TargetConfig, adminPort, replicaData, replicaCtrl, rebuildPort int) *HATarget {
+	return &HATarget{
+		Target:      NewTarget(node, cfg),
+		AdminPort:   adminPort,
+		ReplicaData: replicaData,
+		ReplicaCtrl: replicaCtrl,
+		RebuildPort: rebuildPort,
+	}
+}
+
+// HATargetSpec holds the parameters needed to create an HATarget from YAML config.
+type HATargetSpec struct {
+	VolSize         string
+	WALSize         string
+	IQN             string
+	ISCSIPort       int
+	AdminPort       int
+	ReplicaDataPort int
+	ReplicaCtrlPort int
+	RebuildPort     int
+	TPGID           int
+}
+
+// NewHATargetFromSpec creates an HATarget from an HATargetSpec and Node.
+func NewHATargetFromSpec(node *Node, name string, spec HATargetSpec) *HATarget {
+	volSize := spec.VolSize
+	if volSize == "" {
+		volSize = "100M"
+	}
+	walSize := spec.WALSize
+	if walSize == "" {
+		walSize = "64M"
+	}
+
+	cfg := TargetConfig{
+		VolSize: volSize,
+		WALSize: walSize,
+		IQN:     spec.IQN,
+		Port:    spec.ISCSIPort,
+	}
+
+	ht := NewHATarget(node, cfg, spec.AdminPort, spec.ReplicaDataPort, spec.ReplicaCtrlPort, spec.RebuildPort)
+	ht.TPGID = spec.TPGID
+
+	// Use unique file paths per target name.
+	ht.BinPath = "/tmp/iscsi-target-test"
+	ht.VolFile = fmt.Sprintf("/tmp/blockvol-%s.blk", name)
+	ht.LogFile = fmt.Sprintf("/tmp/iscsi-target-%s.log", name)
+	return ht
+}
+
+// Start overrides Target.Start to add HA-specific flags.
+func (h *HATarget) Start(ctx context.Context, create bool) error {
+	// Remove old log
+	h.Node.Run(ctx, fmt.Sprintf("rm -f %s", h.LogFile))
+
+	args := fmt.Sprintf("-vol %s -addr :%d -iqn %s",
+		h.VolFile, h.Config.Port, h.Config.IQN)
+
+	if create {
+		h.Node.Run(ctx, fmt.Sprintf("rm -f %s %s.wal", h.VolFile, h.VolFile))
+		args += fmt.Sprintf(" -create -size %s", h.Config.VolSize)
+	}
+
+	if h.AdminPort > 0 {
+		args += fmt.Sprintf(" -admin 0.0.0.0:%d", h.AdminPort)
+	}
+	if h.ReplicaData > 0 && h.ReplicaCtrl > 0 {
+		args += fmt.Sprintf(" -replica-data :%d -replica-ctrl :%d", h.ReplicaData, h.ReplicaCtrl)
+	}
+	if h.RebuildPort > 0 {
+		args += fmt.Sprintf(" -rebuild-listen :%d", h.RebuildPort)
+	}
+	if h.TPGID > 0 {
+		args += fmt.Sprintf(" -tpg-id %d", h.TPGID)
+	}
+
+	cmd := fmt.Sprintf("setsid -f %s %s >%s 2>&1", h.BinPath, args, h.LogFile)
+	_, stderr, code, err := h.Node.Run(ctx, cmd)
+	if err != nil || code != 0 {
+		return fmt.Errorf("start ha target: code=%d stderr=%s err=%v", code, stderr, err)
+	}
+
+	if err := h.WaitForPort(ctx); err != nil {
+		return err
+	}
+
+	if h.AdminPort > 0 {
+		if err := h.waitForAdminPort(ctx); err != nil {
+			return err
+		}
+	}
+
+	// Discover PID by matching the unique volume file path.
+	stdout, _, _, _ := h.Node.Run(ctx, fmt.Sprintf("ps -eo pid,args | grep '%s' | grep -v grep | awk '{print $1}'", h.VolFile))
+	pidStr := strings.TrimSpace(stdout)
+	if idx := strings.IndexByte(pidStr, '\n'); idx > 0 {
+		pidStr = pidStr[:idx]
+	}
+	pid := 0
+	fmt.Sscanf(pidStr, "%d", &pid)
+	if pid == 0 {
+		return fmt.Errorf("find ha target PID: %q", pidStr)
+	}
+	h.Pid = pid
+	return nil
+}
+
+func (h *HATarget) waitForAdminPort(ctx context.Context) error {
+	for {
+		select {
+		case <-ctx.Done():
+			return fmt.Errorf("wait for admin port %d: %w", h.AdminPort, ctx.Err())
+		default:
+		}
+		stdout, _, code, _ := h.Node.Run(ctx, fmt.Sprintf("ss -tln | grep :%d", h.AdminPort))
+		if code == 0 && strings.Contains(stdout, fmt.Sprintf(":%d", h.AdminPort)) {
+			return nil
+		}
+		time.Sleep(200 * time.Millisecond)
+	}
+}
+
+// curlPost executes a POST via curl on the node.
+func (h *HATarget) curlPost(ctx context.Context, path string, body interface{}) (int, string, error) {
+	data, err := json.Marshal(body)
+	if err != nil {
+		return 0, "", err
+	}
+	cmd := fmt.Sprintf("curl -s -w '\\n%%{http_code}' -X POST -H 'Content-Type: application/json' -d '%s' http://127.0.0.1:%d%s 2>&1",
+		string(data), h.AdminPort, path)
+	stdout, _, code, err := h.Node.Run(ctx, cmd)
+	if err != nil || code != 0 {
+		return 0, "", fmt.Errorf("curl POST %s: code=%d err=%v stdout=%s", path, code, err, stdout)
+	}
+	return parseCurlOutput(stdout)
+}
+
+// curlGet executes a GET via curl on the node.
+func (h *HATarget) curlGet(ctx context.Context, path string) (int, string, error) {
+	cmd := fmt.Sprintf("curl -s -w '\\n%%{http_code}' http://127.0.0.1:%d%s 2>&1", h.AdminPort, path)
+	stdout, _, code, err := h.Node.Run(ctx, cmd)
+	if err != nil || code != 0 {
+		return 0, "", fmt.Errorf("curl GET %s: code=%d err=%v stdout=%s", path, code, err, stdout)
+	}
+	return parseCurlOutput(stdout)
+}
+
+// parseCurlOutput splits curl -w '\n%{http_code}' output into body and status.
+func parseCurlOutput(output string) (int, string, error) {
+	output = strings.TrimSpace(output)
+	idx := strings.LastIndex(output, "\n")
+	if idx < 0 {
+		var code int
+		if _, err := fmt.Sscanf(output, "%d", &code); err != nil {
+			return 0, "", fmt.Errorf("parse curl status: %q", output)
+		}
+		return code, "", nil
+	}
+	body := output[:idx]
+	var httpCode int
+	if _, err := fmt.Sscanf(strings.TrimSpace(output[idx+1:]), "%d", &httpCode); err != nil {
+		return 0, "", fmt.Errorf("parse curl status from %q", output[idx+1:])
+	}
+	return httpCode, body, nil
+}
+
+// Assign sends POST /assign to inject a role/epoch assignment.
+func (h *HATarget) Assign(ctx context.Context, epoch uint64, role uint32, leaseTTLMs uint32) error {
+	code, body, err := h.curlPost(ctx, "/assign", map[string]interface{}{
+		"epoch":        epoch,
+		"role":         role,
+		"lease_ttl_ms": leaseTTLMs,
+	})
+	if err != nil {
+		return fmt.Errorf("assign request: %w", err)
+	}
+	if code != http.StatusOK {
+		return fmt.Errorf("assign failed (HTTP %d): %s", code, body)
+	}
+	return nil
+}
+
+// AssignRaw sends POST /assign and returns HTTP status code and body.
+func (h *HATarget) AssignRaw(ctx context.Context, body interface{}) (int, string, error) {
+	return h.curlPost(ctx, "/assign", body)
+}
+
+// Status sends GET /status and returns the parsed response.
+func (h *HATarget) Status(ctx context.Context) (*StatusResp, error) {
+	code, body, err := h.curlGet(ctx, "/status")
+	if err != nil {
+		return nil, fmt.Errorf("status request: %w", err)
+	}
+	if code != http.StatusOK {
+		return nil, fmt.Errorf("status failed (HTTP %d): %s", code, body)
+	}
+	var st StatusResp
+	if err := json.NewDecoder(strings.NewReader(body)).Decode(&st); err != nil {
+		return nil, fmt.Errorf("decode status: %w", err)
+	}
+	return &st, nil
+}
+
+// SetReplica sends POST /replica to configure WAL shipping target.
+func (h *HATarget) SetReplica(ctx context.Context, dataAddr, ctrlAddr string) error {
+	code, body, err := h.curlPost(ctx, "/replica", map[string]string{
+		"data_addr": dataAddr,
+		"ctrl_addr": ctrlAddr,
+	})
+	if err != nil {
+		return fmt.Errorf("replica request: %w", err)
+	}
+	if code != http.StatusOK {
+		return fmt.Errorf("replica failed (HTTP %d): %s", code, body)
+	}
+	return nil
+}
+
+// SetReplicaRaw sends POST /replica and returns HTTP status + body.
+func (h *HATarget) SetReplicaRaw(ctx context.Context, body interface{}) (int, string, error) {
+	return h.curlPost(ctx, "/replica", body)
+}
+
+// StartRebuildEndpoint sends POST /rebuild {action:"start"}.
+func (h *HATarget) StartRebuildEndpoint(ctx context.Context, listenAddr string) error {
+	code, body, err := h.curlPost(ctx, "/rebuild", map[string]string{
+		"action":      "start",
+		"listen_addr": listenAddr,
+	})
+	if err != nil {
+		return fmt.Errorf("rebuild start: %w", err)
+	}
+	if code != http.StatusOK {
+		return fmt.Errorf("rebuild start failed (HTTP %d): %s", code, body)
+	}
+	return nil
+}
+
+// StartRebuildClient sends POST /rebuild {action:"connect"}.
+func (h *HATarget) StartRebuildClient(ctx context.Context, rebuildAddr string, epoch uint64) error {
+	code, body, err := h.curlPost(ctx, "/rebuild", map[string]interface{}{
+		"action":       "connect",
+		"rebuild_addr": rebuildAddr,
+		"epoch":        epoch,
+	})
+	if err != nil {
+		return fmt.Errorf("rebuild connect: %w", err)
+	}
+	if code != http.StatusOK {
+		return fmt.Errorf("rebuild connect failed (HTTP %d): %s", code, body)
+	}
+	return nil
+}
+
+// StopRebuildEndpoint sends POST /rebuild {action:"stop"}.
+func (h *HATarget) StopRebuildEndpoint(ctx context.Context) error {
+	code, body, err := h.curlPost(ctx, "/rebuild", map[string]string{"action": "stop"})
+	if err != nil {
+		return fmt.Errorf("rebuild stop: %w", err)
+	}
+	if code != http.StatusOK {
+		return fmt.Errorf("rebuild stop failed (HTTP %d): %s", code, body)
+	}
+	return nil
+}
+
+// CreateSnapshot sends POST /snapshot {action:"create", id:N}.
+func (h *HATarget) CreateSnapshot(ctx context.Context, id uint32) error {
+	code, body, err := h.curlPost(ctx, "/snapshot", map[string]interface{}{
+		"action": "create",
+		"id":     id,
+	})
+	if err != nil {
+		return fmt.Errorf("create snapshot %d: %w", id, err)
+	}
+	if code != http.StatusOK {
+		return fmt.Errorf("create snapshot %d failed (HTTP %d): %s", id, code, body)
+	}
+	return nil
+}
+
+// DeleteSnapshot sends POST /snapshot {action:"delete", id:N}.
+func (h *HATarget) DeleteSnapshot(ctx context.Context, id uint32) error {
+	code, body, err := h.curlPost(ctx, "/snapshot", map[string]interface{}{
+		"action": "delete",
+		"id":     id,
+	})
+	if err != nil {
+		return fmt.Errorf("delete snapshot %d: %w", id, err)
+	}
+	if code != http.StatusOK {
+		return fmt.Errorf("delete snapshot %d failed (HTTP %d): %s", id, code, body)
+	}
+	return nil
+}
+
+// ListSnapshots sends POST /snapshot {action:"list"} and returns the entries.
+func (h *HATarget) ListSnapshots(ctx context.Context) ([]SnapshotEntry, error) {
+	code, body, err := h.curlPost(ctx, "/snapshot", map[string]string{
+		"action": "list",
+	})
+	if err != nil {
+		return nil, fmt.Errorf("list snapshots: %w", err)
+	}
+	if code != http.StatusOK {
+		return nil, fmt.Errorf("list snapshots failed (HTTP %d): %s", code, body)
+	}
+	var resp struct {
+		Snapshots []SnapshotEntry `json:"snapshots"`
+	}
+	if err := json.NewDecoder(strings.NewReader(body)).Decode(&resp); err != nil {
+		return nil, fmt.Errorf("decode snapshots: %w", err)
+	}
+	return resp.Snapshots, nil
+}
+
+// Resize sends POST /resize {new_size_bytes:N}.
+func (h *HATarget) Resize(ctx context.Context, newSizeBytes uint64) error {
+	code, body, err := h.curlPost(ctx, "/resize", map[string]interface{}{
+		"new_size_bytes": newSizeBytes,
+	})
+	if err != nil {
+		return fmt.Errorf("resize to %d: %w", newSizeBytes, err)
+	}
+	if code != http.StatusOK {
+		return fmt.Errorf("resize to %d failed (HTTP %d): %s", newSizeBytes, code, body)
+	}
+	return nil
+}
+
+// WaitForRole polls GET /status until the target reports the expected role.
+func (h *HATarget) WaitForRole(ctx context.Context, expectedRole string) error {
+	for {
+		select {
+		case <-ctx.Done():
+			return fmt.Errorf("wait for role %s: %w", expectedRole, ctx.Err())
+		default:
+		}
+		st, err := h.Status(ctx)
+		if err == nil && st.Role == expectedRole {
+			return nil
+		}
+		time.Sleep(500 * time.Millisecond)
+	}
+}
+
+// WaitForLSN polls GET /status until wal_head_lsn >= minLSN.
+func (h *HATarget) WaitForLSN(ctx context.Context, minLSN uint64) error {
+	for {
+		select {
+		case <-ctx.Done():
+			return fmt.Errorf("wait for LSN >= %d: %w", minLSN, ctx.Err())
+		default:
+		}
+		st, err := h.Status(ctx)
+		if err == nil && st.WALHeadLSN >= minLSN {
+			return nil
+		}
+		time.Sleep(500 * time.Millisecond)
+	}
+}
+
+// HostAddr returns the target's node host (for building replica addresses).
+func (h *HATarget) HostAddr() string {
+	if h.Node.IsLocal {
+		return "127.0.0.1"
+	}
+	return h.Node.Host
+}

weed/storage/blockvol/testrunner/infra/iscsi_client.go

@@ -0,0 +1,203 @@
+package infra
+
+import (
+	"context"
+	"fmt"
+	"strings"
+	"time"
+)
+
+// ISCSIClient wraps iscsiadm commands on a node.
+type ISCSIClient struct {
+	Node       *Node
+	TargetHost string
+	TargetPort int
+}
+
+// NewISCSIClient creates an iSCSI client bound to a node.
+func NewISCSIClient(node *Node) *ISCSIClient {
+	return &ISCSIClient{Node: node}
+}
+
+// Discover runs iSCSI SendTargets discovery and returns discovered IQNs.
+func (c *ISCSIClient) Discover(ctx context.Context, host string, port int) ([]string, error) {
+	c.TargetHost = host
+	c.TargetPort = port
+
+	cmd := fmt.Sprintf("iscsiadm -m discovery -t sendtargets -p %s:%d", host, port)
+	stdout, stderr, code, err := c.Node.RunRoot(ctx, cmd)
+	if err != nil {
+		return nil, fmt.Errorf("discovery error: %w", err)
+	}
+	if code != 0 {
+		return nil, fmt.Errorf("discovery failed (code %d): %s", code, stderr)
+	}
+
+	var iqns []string
+	for _, line := range strings.Split(stdout, "\n") {
+		line = strings.TrimSpace(line)
+		if line == "" {
+			continue
+		}
+		parts := strings.Fields(line)
+		if len(parts) >= 2 {
+			iqns = append(iqns, parts[1])
+		}
+	}
+
+	for _, iqn := range iqns {
+		c.fixNodePortal(ctx, iqn, host, port)
+	}
+
+	return iqns, nil
+}
+
+func (c *ISCSIClient) fixNodePortal(ctx context.Context, iqn, host string, port int) {
+	stdout, _, _, _ := c.Node.RunRoot(ctx,
+		fmt.Sprintf("iscsiadm -m node -T %s 2>/dev/null", iqn))
+
+	hasWildcard := false
+	for _, line := range strings.Split(stdout, "\n") {
+		if strings.Contains(line, "node.conn[0].address") {
+			if strings.Contains(line, "::") || strings.Contains(line, "0.0.0.0") {
+				hasWildcard = true
+			}
+		}
+	}
+	if !hasWildcard {
+		return
+	}
+
+	c.Node.RunRoot(ctx, fmt.Sprintf("iscsiadm -m node -T %s -o delete 2>/dev/null", iqn))
+	portal := fmt.Sprintf("%s:%d", host, port)
+	c.Node.RunRoot(ctx, fmt.Sprintf("iscsiadm -m node -T %s -p %s -o new 2>/dev/null", iqn, portal))
+}
+
+// Login connects to the target and returns the device path (e.g. /dev/sda).
+func (c *ISCSIClient) Login(ctx context.Context, iqn string) (string, error) {
+	var cmd string
+	if c.TargetHost != "" && c.TargetHost != "127.0.0.1" && c.TargetHost != "localhost" {
+		portal := fmt.Sprintf("%s:%d", c.TargetHost, c.TargetPort)
+		cmd = fmt.Sprintf("iscsiadm -m node -T %s -p %s --login", iqn, portal)
+	} else {
+		cmd = fmt.Sprintf("iscsiadm -m node -T %s --login", iqn)
+	}
+	_, stderr, code, err := c.Node.RunRoot(ctx, cmd)
+	if err != nil {
+		return "", fmt.Errorf("login error: %w", err)
+	}
+	if code != 0 {
+		return "", fmt.Errorf("login failed (code %d): %s", code, stderr)
+	}
+
+	return c.waitForDevice(ctx, iqn)
+}
+
+// Logout disconnects from the target.
+func (c *ISCSIClient) Logout(ctx context.Context, iqn string) error {
+	cmd := fmt.Sprintf("iscsiadm -m node -T %s --logout", iqn)
+	_, stderr, code, err := c.Node.RunRoot(ctx, cmd)
+	if err != nil {
+		return fmt.Errorf("logout error: %w", err)
+	}
+	if code != 0 {
+		return fmt.Errorf("logout failed (code %d): %s", code, stderr)
+	}
+	return nil
+}
+
+// GetDevice returns the device path for an active session.
+func (c *ISCSIClient) GetDevice(ctx context.Context, iqn string) (string, error) {
+	return c.waitForDevice(ctx, iqn)
+}
+
+func (c *ISCSIClient) waitForDevice(ctx context.Context, iqn string) (string, error) {
+	deadline := time.Now().Add(30 * time.Second)
+	rescanned := false
+	for time.Now().Before(deadline) {
+		select {
+		case <-ctx.Done():
+			return "", ctx.Err()
+		default:
+		}
+
+		stdout, _, code, _ := c.Node.RunRoot(ctx, "iscsiadm -m session -P3")
+		if code == 0 {
+			dev := ParseDeviceFromSession(stdout, iqn)
+			if dev != "" {
+				return dev, nil
+			}
+		}
+
+		if !rescanned && time.Until(deadline) < 25*time.Second {
+			c.Node.RunRoot(ctx, "iscsiadm -m session -R")
+			rescanned = true
+		}
+		time.Sleep(500 * time.Millisecond)
+	}
+	return "", fmt.Errorf("device for %s did not appear within 30s", iqn)
+}
+
+// ParseDeviceFromSession extracts /dev/sdX from iscsiadm -m session -P3 output.
+func ParseDeviceFromSession(output, iqn string) string {
+	lines := strings.Split(output, "\n")
+	inTarget := false
+	for _, line := range lines {
+		if strings.Contains(line, "Target: "+iqn) {
+			inTarget = true
+			continue
+		}
+		if inTarget && strings.Contains(line, "Target: ") {
+			break
+		}
+		if inTarget && strings.Contains(line, "Attached scsi disk") {
+			fields := strings.Fields(line)
+			for i, f := range fields {
+				if f == "disk" && i+1 < len(fields) {
+					return "/dev/" + fields[i+1]
+				}
+			}
+		}
+	}
+	return ""
+}
+
+// WaitForSession polls until a session for the given IQN is in LOGGED_IN state.
+func (c *ISCSIClient) WaitForSession(ctx context.Context, iqn string) error {
+	for {
+		select {
+		case <-ctx.Done():
+			return fmt.Errorf("session %s did not recover: %w", iqn, ctx.Err())
+		default:
+		}
+
+		stdout, _, code, _ := c.Node.RunRoot(ctx, "iscsiadm -m session")
+		if code == 0 && strings.Contains(stdout, iqn) {
+			return nil
+		}
+		time.Sleep(500 * time.Millisecond)
+	}
+}
+
+// CleanupAll force-logouts sessions matching the IQN prefix only.
+func (c *ISCSIClient) CleanupAll(ctx context.Context, iqnPrefix string) error {
+	stdout, _, _, _ := c.Node.RunRoot(ctx, "iscsiadm -m session 2>&1")
+	if stdout == "" || strings.Contains(stdout, "No active sessions") {
+		return nil
+	}
+
+	for _, line := range strings.Split(stdout, "\n") {
+		line = strings.TrimSpace(line)
+		if !strings.Contains(line, iqnPrefix) {
+			continue
+		}
+		fields := strings.Fields(line)
+		for _, f := range fields {
+			if strings.HasPrefix(f, iqnPrefix) {
+				c.Node.RunRoot(ctx, fmt.Sprintf("iscsiadm -m node -T %s --logout 2>/dev/null", f))
+				c.Node.RunRoot(ctx, fmt.Sprintf("iscsiadm -m node -T %s -o delete 2>/dev/null", f))
+			}
+		}
+	}
+	return nil
+}

weed/storage/blockvol/testrunner/infra/node.go

@@ -0,0 +1,362 @@
+package infra
+
+import (
+	"bytes"
+	"context"
+	"fmt"
+	"io"
+	"net"
+	"os"
+	"os/exec"
+	"strings"
+	"sync"
+	"time"
+
+	"golang.org/x/crypto/ssh"
+)
+
+// Node represents an SSH-accessible (or local WSL2/native) machine.
+type Node struct {
+	Host     string
+	User     string
+	KeyFile  string
+	IsLocal  bool // WSL2 mode: use exec.CommandContext via wsl instead of SSH
+	IsNative bool // Native local mode: use bash -c directly (for Linux agents)
+
+	mu     sync.Mutex
+	client *ssh.Client
+}
+
+// Connect establishes the SSH connection (no-op for local/native mode).
+func (n *Node) Connect() error {
+	if n.IsLocal || n.IsNative {
+		return nil
+	}
+
+	key, err := os.ReadFile(n.KeyFile)
+	if err != nil {
+		return fmt.Errorf("read SSH key %s: %w", n.KeyFile, err)
+	}
+	signer, err := ssh.ParsePrivateKey(key)
+	if err != nil {
+		return fmt.Errorf("parse SSH key: %w", err)
+	}
+
+	config := &ssh.ClientConfig{
+		User:            n.User,
+		Auth:            []ssh.AuthMethod{ssh.PublicKeys(signer)},
+		HostKeyCallback: ssh.InsecureIgnoreHostKey(),
+		Timeout:         10 * time.Second,
+	}
+
+	addr := n.Host
+	if !strings.Contains(addr, ":") {
+		addr += ":22"
+	}
+
+	n.mu.Lock()
+	defer n.mu.Unlock()
+	n.client, err = ssh.Dial("tcp", addr, config)
+	if err != nil {
+		return fmt.Errorf("SSH dial %s: %w", addr, err)
+	}
+	return nil
+}
+
+// Run executes a command and returns stdout, stderr, exit code.
+func (n *Node) Run(ctx context.Context, cmd string) (stdout, stderr string, exitCode int, err error) {
+	if n.IsNative {
+		return n.runNative(ctx, cmd)
+	}
+	if n.IsLocal {
+		return n.runLocal(ctx, cmd)
+	}
+	return n.runSSH(ctx, cmd)
+}
+
+func (n *Node) runNative(ctx context.Context, cmd string) (string, string, int, error) {
+	c := exec.CommandContext(ctx, "bash", "-c", cmd)
+	var outBuf, errBuf bytes.Buffer
+	c.Stdout = &outBuf
+	c.Stderr = &errBuf
+
+	err := c.Run()
+	if ctx.Err() != nil {
+		return outBuf.String(), errBuf.String(), -1, fmt.Errorf("command timed out: %w", ctx.Err())
+	}
+	if err != nil {
+		if exitErr, ok := err.(*exec.ExitError); ok {
+			return outBuf.String(), errBuf.String(), exitErr.ExitCode(), nil
+		}
+		return outBuf.String(), errBuf.String(), -1, err
+	}
+	return outBuf.String(), errBuf.String(), 0, nil
+}
+
+func (n *Node) runLocal(ctx context.Context, cmd string) (string, string, int, error) {
+	c := exec.CommandContext(ctx, "wsl", "-e", "bash", "-c", cmd)
+	var outBuf, errBuf bytes.Buffer
+	c.Stdout = &outBuf
+	c.Stderr = &errBuf
+
+	err := c.Run()
+	if ctx.Err() != nil {
+		return outBuf.String(), errBuf.String(), -1, fmt.Errorf("command timed out: %w", ctx.Err())
+	}
+	if err != nil {
+		if exitErr, ok := err.(*exec.ExitError); ok {
+			return outBuf.String(), errBuf.String(), exitErr.ExitCode(), nil
+		}
+		return outBuf.String(), errBuf.String(), -1, err
+	}
+	return outBuf.String(), errBuf.String(), 0, nil
+}
+
+func (n *Node) runSSH(ctx context.Context, cmd string) (string, string, int, error) {
+	n.mu.Lock()
+	if n.client == nil {
+		n.mu.Unlock()
+		return "", "", -1, fmt.Errorf("SSH not connected")
+	}
+	session, err := n.client.NewSession()
+	n.mu.Unlock()
+	if err != nil {
+		return "", "", -1, fmt.Errorf("new SSH session: %w", err)
+	}
+	defer session.Close()
+
+	var outBuf, errBuf bytes.Buffer
+	session.Stdout = &outBuf
+	session.Stderr = &errBuf
+
+	done := make(chan error, 1)
+	go func() { done <- session.Run(cmd) }()
+
+	select {
+	case <-ctx.Done():
+		_ = session.Signal(ssh.SIGKILL)
+		return outBuf.String(), errBuf.String(), -1, fmt.Errorf("command timed out: %w", ctx.Err())
+	case err := <-done:
+		if err != nil {
+			if exitErr, ok := err.(*ssh.ExitError); ok {
+				return outBuf.String(), errBuf.String(), exitErr.ExitStatus(), nil
+			}
+			return outBuf.String(), errBuf.String(), -1, err
+		}
+		return outBuf.String(), errBuf.String(), 0, nil
+	}
+}
+
+// RunRoot executes a command with sudo -n (non-interactive).
+func (n *Node) RunRoot(ctx context.Context, cmd string) (string, string, int, error) {
+	return n.Run(ctx, "sudo -n "+cmd)
+}
+
+// Upload copies a local file to the remote node via SCP.
+func (n *Node) Upload(local, remote string) error {
+	if n.IsNative {
+		ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second)
+		defer cancel()
+		_, stderr, code, err := n.Run(ctx, fmt.Sprintf("cp %s %s && chmod +x %s", local, remote, remote))
+		if err != nil || code != 0 {
+			return fmt.Errorf("native upload: code=%d stderr=%s err=%v", code, stderr, err)
+		}
+		return nil
+	}
+	if n.IsLocal {
+		ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second)
+		defer cancel()
+		wslLocal := ToWSLPath(local)
+		_, stderr, code, err := n.Run(ctx, fmt.Sprintf("cp %s %s && chmod +x %s", wslLocal, remote, remote))
+		if err != nil || code != 0 {
+			return fmt.Errorf("local upload: code=%d stderr=%s err=%v", code, stderr, err)
+		}
+		return nil
+	}
+	return n.scpUpload(local, remote)
+}
+
+func (n *Node) scpUpload(local, remote string) error {
+	data, err := os.ReadFile(local)
+	if err != nil {
+		return fmt.Errorf("read local file %s: %w", local, err)
+	}
+
+	n.mu.Lock()
+	if n.client == nil {
+		n.mu.Unlock()
+		return fmt.Errorf("SSH not connected")
+	}
+	session, err := n.client.NewSession()
+	n.mu.Unlock()
+	if err != nil {
+		return fmt.Errorf("new SSH session: %w", err)
+	}
+	defer session.Close()
+
+	// Use cat+chmod over stdin — simpler and more reliable than SCP protocol.
+	w, err := session.StdinPipe()
+	if err != nil {
+		return fmt.Errorf("stdin pipe: %w", err)
+	}
+
+	go func() {
+		w.Write(data)
+		w.Close()
+	}()
+
+	cmd := fmt.Sprintf("cat > %s && chmod +x %s", remote, remote)
+	if err := session.Run(cmd); err != nil {
+		return fmt.Errorf("upload run: %w", err)
+	}
+	return nil
+}
+
+// Download copies a remote file to local via SCP.
+func (n *Node) Download(remote, local string) error {
+	if n.IsNative {
+		ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second)
+		defer cancel()
+		_, stderr, code, err := n.Run(ctx, fmt.Sprintf("cp %s %s", remote, local))
+		if err != nil || code != 0 {
+			return fmt.Errorf("native download: code=%d stderr=%s err=%v", code, stderr, err)
+		}
+		return nil
+	}
+	if n.IsLocal {
+		ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second)
+		defer cancel()
+		wslLocal := ToWSLPath(local)
+		_, stderr, code, err := n.Run(ctx, fmt.Sprintf("cp %s %s", remote, wslLocal))
+		if err != nil || code != 0 {
+			return fmt.Errorf("local download: code=%d stderr=%s err=%v", code, stderr, err)
+		}
+		return nil
+	}
+	return n.scpDownload(remote, local)
+}
+
+func (n *Node) scpDownload(remote, local string) error {
+	n.mu.Lock()
+	if n.client == nil {
+		n.mu.Unlock()
+		return fmt.Errorf("SSH not connected")
+	}
+	session, err := n.client.NewSession()
+	n.mu.Unlock()
+	if err != nil {
+		return fmt.Errorf("new SSH session: %w", err)
+	}
+	defer session.Close()
+
+	var buf bytes.Buffer
+	session.Stdout = &buf
+	if err := session.Run(fmt.Sprintf("cat %s", remote)); err != nil {
+		return fmt.Errorf("read remote %s: %w", remote, err)
+	}
+	return os.WriteFile(local, buf.Bytes(), 0644)
+}
+
+// Kill sends SIGKILL to a process by PID.
+func (n *Node) Kill(pid int) error {
+	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+	defer cancel()
+	_, _, _, err := n.RunRoot(ctx, fmt.Sprintf("kill -9 %d", pid))
+	return err
+}
+
+// HasCommand checks if a command is available on the node.
+func (n *Node) HasCommand(cmd string) bool {
+	ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
+	defer cancel()
+	_, _, code, err := n.Run(ctx, fmt.Sprintf("which %s", cmd))
+	return err == nil && code == 0
+}
+
+// Close closes the SSH connection.
+func (n *Node) Close() {
+	n.mu.Lock()
+	defer n.mu.Unlock()
+	if n.client != nil {
+		n.client.Close()
+		n.client = nil
+	}
+}
+
+// DialTCP opens a direct TCP connection through the SSH tunnel.
+func (n *Node) DialTCP(addr string) (net.Conn, error) {
+	if n.IsLocal || n.IsNative {
+		return net.DialTimeout("tcp", addr, 5*time.Second)
+	}
+	n.mu.Lock()
+	defer n.mu.Unlock()
+	if n.client == nil {
+		return nil, fmt.Errorf("SSH not connected")
+	}
+	return n.client.Dial("tcp", addr)
+}
+
+// StreamRun executes a command and streams stdout to the writer.
+func (n *Node) StreamRun(ctx context.Context, cmd string, w io.Writer) error {
+	if n.IsNative {
+		c := exec.CommandContext(ctx, "bash", "-c", cmd)
+		c.Stdout = w
+		c.Stderr = w
+		return c.Run()
+	}
+	if n.IsLocal {
+		c := exec.CommandContext(ctx, "wsl", "-e", "bash", "-c", cmd)
+		c.Stdout = w
+		c.Stderr = w
+		return c.Run()
+	}
+
+	n.mu.Lock()
+	if n.client == nil {
+		n.mu.Unlock()
+		return fmt.Errorf("SSH not connected")
+	}
+	session, err := n.client.NewSession()
+	n.mu.Unlock()
+	if err != nil {
+		return err
+	}
+	defer session.Close()
+
+	session.Stdout = w
+	session.Stderr = w
+
+	done := make(chan error, 1)
+	go func() { done <- session.Run(cmd) }()
+
+	select {
+	case <-ctx.Done():
+		_ = session.Signal(ssh.SIGKILL)
+		return ctx.Err()
+	case err := <-done:
+		return err
+	}
+}
+
+// ToWSLPath converts a Windows path to a WSL path.
+func ToWSLPath(winPath string) string {
+	p := strings.ReplaceAll(winPath, "\\", "/")
+	if len(p) >= 2 && p[1] == ':' {
+		drive := strings.ToLower(string(p[0]))
+		p = "/mnt/" + drive + p[2:]
+	}
+	return p
+}
+
+func remoteName(path string) string {
+	parts := strings.Split(path, "/")
+	return parts[len(parts)-1]
+}
+
+func remoteDir(path string) string {
+	idx := strings.LastIndex(path, "/")
+	if idx < 0 {
+		return "."
+	}
+	return path[:idx]
+}

weed/storage/blockvol/testrunner/infra/target.go

@@ -0,0 +1,196 @@
+package infra
+
+import (
+	"context"
+	"fmt"
+	"os"
+	"os/exec"
+	"strconv"
+	"strings"
+	"time"
+)
+
+// TargetConfig configures an iSCSI target instance.
+type TargetConfig struct {
+	VolSize string // e.g. "100M"
+	WALSize string // e.g. "64M" (default), "4M" for WAL pressure tests
+	IQN     string
+	Port    int
+}
+
+// DefaultTargetConfig returns a default target config for integration tests.
+func DefaultTargetConfig() TargetConfig {
+	return TargetConfig{
+		VolSize: "100M",
+		WALSize: "64M",
+		IQN:     "iqn.2024.com.seaweedfs:test1",
+		Port:    3260,
+	}
+}
+
+// Target manages the lifecycle of an iscsi-target process on a remote node.
+type Target struct {
+	Node    *Node
+	Config  TargetConfig
+	BinPath string // remote path to iscsi-target binary
+	Pid     int
+	LogFile string // remote path to target's stderr log
+	VolFile string // remote path to volume file
+}
+
+// NewTarget creates a Target bound to a node.
+func NewTarget(node *Node, config TargetConfig) *Target {
+	return &Target{
+		Node:    node,
+		Config:  config,
+		BinPath: "/tmp/iscsi-target-test",
+		VolFile: "/tmp/blockvol-test.blk",
+		LogFile: "/tmp/iscsi-target-test.log",
+	}
+}
+
+// SetBinPath overrides the remote binary path.
+func (t *Target) SetBinPath(p string) { t.BinPath = p }
+
+// SetVolFile overrides the remote volume file path.
+func (t *Target) SetVolFile(p string) { t.VolFile = p }
+
+// SetLogFile overrides the remote log file path.
+func (t *Target) SetLogFile(p string) { t.LogFile = p }
+
+// Build cross-compiles the iscsi-target binary for linux/amd64.
+func (t *Target) Build(ctx context.Context, repoDir string) error {
+	binDir := repoDir + "/weed/storage/blockvol/iscsi/cmd/iscsi-target"
+	outPath := repoDir + "/iscsi-target-linux"
+
+	cmd := exec.CommandContext(ctx, "go", "build", "-o", outPath, ".")
+	cmd.Dir = binDir
+	cmd.Env = append(os.Environ(), "GOOS=linux", "GOARCH=amd64", "CGO_ENABLED=0")
+	out, err := cmd.CombinedOutput()
+	if err != nil {
+		return fmt.Errorf("build failed: %s\n%w", out, err)
+	}
+	return nil
+}
+
+// Deploy uploads the pre-built binary to the target node.
+func (t *Target) Deploy(localBin string) error {
+	return t.Node.Upload(localBin, t.BinPath)
+}
+
+// Start launches the target process. If create is true, a new volume is created.
+func (t *Target) Start(ctx context.Context, create bool) error {
+	// Remove old log
+	t.Node.Run(ctx, fmt.Sprintf("rm -f %s", t.LogFile))
+
+	args := fmt.Sprintf("-vol %s -addr :%d -iqn %s",
+		t.VolFile, t.Config.Port, t.Config.IQN)
+
+	if create {
+		t.Node.Run(ctx, fmt.Sprintf("rm -f %s %s.wal", t.VolFile, t.VolFile))
+		args += fmt.Sprintf(" -create -size %s", t.Config.VolSize)
+	}
+
+	cmd := fmt.Sprintf("setsid -f %s %s >%s 2>&1", t.BinPath, args, t.LogFile)
+	_, stderr, code, err := t.Node.Run(ctx, cmd)
+	if err != nil || code != 0 {
+		return fmt.Errorf("start target: code=%d stderr=%s err=%v", code, stderr, err)
+	}
+
+	if err := t.WaitForPort(ctx); err != nil {
+		return err
+	}
+
+	// Discover PID by matching the binary name
+	stdout, _, _, _ := t.Node.Run(ctx, fmt.Sprintf("ps -eo pid,args | grep '%s' | grep -v grep | awk '{print $1}'", t.BinPath))
+	pidStr := strings.TrimSpace(stdout)
+	if idx := strings.IndexByte(pidStr, '\n'); idx > 0 {
+		pidStr = pidStr[:idx]
+	}
+	pid, err := strconv.Atoi(pidStr)
+	if err != nil {
+		return fmt.Errorf("find target PID: %q: %w", pidStr, err)
+	}
+	t.Pid = pid
+	return nil
+}
+
+// Stop sends SIGTERM, waits up to 10s, then Kill9.
+func (t *Target) Stop(ctx context.Context) error {
+	if t.Pid == 0 {
+		return nil
+	}
+
+	t.Node.Run(ctx, fmt.Sprintf("kill %d", t.Pid))
+
+	deadline := time.Now().Add(10 * time.Second)
+	for time.Now().Before(deadline) {
+		_, _, code, _ := t.Node.Run(ctx, fmt.Sprintf("kill -0 %d 2>/dev/null", t.Pid))
+		if code != 0 {
+			t.Pid = 0
+			return nil
+		}
+		time.Sleep(500 * time.Millisecond)
+	}
+
+	return t.Kill9()
+}
+
+// Kill9 sends SIGKILL immediately.
+func (t *Target) Kill9() error {
+	if t.Pid == 0 {
+		return nil
+	}
+	ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
+	defer cancel()
+	t.Node.Run(ctx, fmt.Sprintf("kill -9 %d", t.Pid))
+	t.Pid = 0
+	return nil
+}
+
+// Restart stops the target and starts it again (preserving the volume).
+func (t *Target) Restart(ctx context.Context) error {
+	if err := t.Stop(ctx); err != nil {
+		return fmt.Errorf("restart stop: %w", err)
+	}
+	return t.Start(ctx, false)
+}
+
+// WaitForPort polls until the target port is listening.
+func (t *Target) WaitForPort(ctx context.Context) error {
+	for {
+		select {
+		case <-ctx.Done():
+			return fmt.Errorf("wait for port %d: %w", t.Config.Port, ctx.Err())
+		default:
+		}
+
+		stdout, _, code, _ := t.Node.Run(ctx, fmt.Sprintf("ss -tln | grep :%d", t.Config.Port))
+		if code == 0 && strings.Contains(stdout, fmt.Sprintf(":%d", t.Config.Port)) {
+			return nil
+		}
+		time.Sleep(200 * time.Millisecond)
+	}
+}
+
+// CollectLog downloads the target's log file contents.
+func (t *Target) CollectLog() (string, error) {
+	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+	defer cancel()
+	stdout, _, _, err := t.Node.Run(ctx, fmt.Sprintf("cat %s 2>/dev/null", t.LogFile))
+	if err != nil {
+		return "", err
+	}
+	return stdout, nil
+}
+
+// Cleanup removes the volume file, WAL, and log from the target node.
+func (t *Target) Cleanup(ctx context.Context) {
+	t.Node.Run(ctx, fmt.Sprintf("rm -f %s %s.wal %s", t.VolFile, t.VolFile, t.LogFile))
+}
+
+// PID returns the current target process ID.
+func (t *Target) PID() int { return t.Pid }
+
+// VolFilePath returns the remote volume file path.
+func (t *Target) VolFilePath() string { return t.VolFile }

weed/storage/blockvol/testrunner/local_node.go

@@ -0,0 +1,104 @@
+package testrunner
+
+import (
+	"bytes"
+	"context"
+	"fmt"
+	"io"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"strings"
+)
+
+// LocalNode implements NodeRunner by executing commands locally via os/exec.
+// Used by agents that run on the same machine as the targets they manage.
+type LocalNode struct {
+	hostname string
+	isRoot   bool
+}
+
+// NewLocalNode creates a LocalNode, detecting root status.
+func NewLocalNode(hostname string) *LocalNode {
+	n := &LocalNode{hostname: hostname}
+	n.isRoot = n.detectRoot()
+	return n
+}
+
+// Hostname returns the node's hostname.
+func (n *LocalNode) Hostname() string { return n.hostname }
+
+// IsRoot returns whether the process is running as root.
+func (n *LocalNode) IsRoot() bool { return n.isRoot }
+
+func (n *LocalNode) detectRoot() bool {
+	// Try "id -u" to check if running as root (uid 0).
+	ctx, cancel := context.WithTimeout(context.Background(), 5*secondDuration)
+	defer cancel()
+	out, _, code, err := n.Run(ctx, "id -u")
+	if err != nil || code != 0 {
+		return false
+	}
+	return strings.TrimSpace(out) == "0"
+}
+
+// secondDuration avoids importing time in the constant.
+const secondDuration = 1_000_000_000 // time.Second
+
+// Run executes a command locally via bash -c.
+func (n *LocalNode) Run(ctx context.Context, cmd string) (stdout, stderr string, exitCode int, err error) {
+	c := exec.CommandContext(ctx, "bash", "-c", cmd)
+	var outBuf, errBuf bytes.Buffer
+	c.Stdout = &outBuf
+	c.Stderr = &errBuf
+
+	runErr := c.Run()
+	if ctx.Err() != nil {
+		return outBuf.String(), errBuf.String(), -1, fmt.Errorf("command timed out: %w", ctx.Err())
+	}
+	if runErr != nil {
+		if exitErr, ok := runErr.(*exec.ExitError); ok {
+			return outBuf.String(), errBuf.String(), exitErr.ExitCode(), nil
+		}
+		return outBuf.String(), errBuf.String(), -1, runErr
+	}
+	return outBuf.String(), errBuf.String(), 0, nil
+}
+
+// RunRoot executes a command as root. If already root, runs directly.
+// Otherwise uses sudo -n (non-interactive, fails if password required).
+func (n *LocalNode) RunRoot(ctx context.Context, cmd string) (stdout, stderr string, exitCode int, err error) {
+	if n.isRoot {
+		return n.Run(ctx, cmd)
+	}
+	return n.Run(ctx, "sudo -n "+cmd)
+}
+
+// Upload copies a file from local source to a local destination path.
+func (n *LocalNode) Upload(local, remote string) error {
+	// Ensure destination directory exists.
+	dir := filepath.Dir(remote)
+	if err := os.MkdirAll(dir, 0755); err != nil {
+		return fmt.Errorf("mkdir %s: %w", dir, err)
+	}
+
+	src, err := os.Open(local)
+	if err != nil {
+		return fmt.Errorf("open source %s: %w", local, err)
+	}
+	defer src.Close()
+
+	dst, err := os.OpenFile(remote, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0755)
+	if err != nil {
+		return fmt.Errorf("create dest %s: %w", remote, err)
+	}
+	defer dst.Close()
+
+	if _, err := io.Copy(dst, src); err != nil {
+		return fmt.Errorf("copy %s → %s: %w", local, remote, err)
+	}
+	return nil
+}
+
+// Close is a no-op for LocalNode (no connection to close).
+func (n *LocalNode) Close() {}

weed/storage/blockvol/testrunner/local_node_test.go

@@ -0,0 +1,122 @@
+package testrunner
+
+import (
+	"context"
+	"os"
+	"path/filepath"
+	"runtime"
+	"testing"
+	"time"
+)
+
+func skipIfWindows(t *testing.T) {
+	if runtime.GOOS == "windows" {
+		t.Skip("LocalNode tests require Unix shell")
+	}
+}
+
+func TestLocalNode_Run_Echo(t *testing.T) {
+	skipIfWindows(t)
+	n := NewLocalNode("test")
+	ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
+	defer cancel()
+
+	stdout, stderr, code, err := n.Run(ctx, "echo hello")
+	if err != nil {
+		t.Fatalf("Run: %v", err)
+	}
+	if code != 0 {
+		t.Fatalf("exit code %d, stderr: %s", code, stderr)
+	}
+	if got := stdout; got != "hello\n" {
+		t.Errorf("stdout = %q, want %q", got, "hello\n")
+	}
+}
+
+func TestLocalNode_Run_ExitCode(t *testing.T) {
+	skipIfWindows(t)
+	n := NewLocalNode("test")
+	ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
+	defer cancel()
+
+	_, _, code, err := n.Run(ctx, "exit 42")
+	if err != nil {
+		t.Fatalf("Run: %v", err)
+	}
+	if code != 42 {
+		t.Errorf("exit code = %d, want 42", code)
+	}
+}
+
+func TestLocalNode_Run_Timeout(t *testing.T) {
+	skipIfWindows(t)
+	n := NewLocalNode("test")
+	ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond)
+	defer cancel()
+
+	_, _, _, err := n.Run(ctx, "sleep 10")
+	if err == nil {
+		t.Fatal("expected timeout error")
+	}
+}
+
+func TestLocalNode_RunRoot_NonRoot(t *testing.T) {
+	skipIfWindows(t)
+	n := NewLocalNode("test")
+	// Force non-root to test sudo path.
+	n.isRoot = false
+	ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
+	defer cancel()
+
+	// sudo -n will likely fail without sudoers config, but should not panic.
+	_, _, _, err := n.RunRoot(ctx, "echo test")
+	// We only care that it doesn't panic — error is expected on most CI.
+	_ = err
+}
+
+func TestLocalNode_Upload(t *testing.T) {
+	skipIfWindows(t)
+	n := NewLocalNode("test")
+
+	dir := t.TempDir()
+	src := filepath.Join(dir, "src.bin")
+	dst := filepath.Join(dir, "subdir", "dst.bin")
+
+	// Create source file.
+	content := []byte("test binary content 12345")
+	if err := os.WriteFile(src, content, 0644); err != nil {
+		t.Fatalf("write src: %v", err)
+	}
+
+	// Upload (should create subdir).
+	if err := n.Upload(src, dst); err != nil {
+		t.Fatalf("Upload: %v", err)
+	}
+
+	// Verify.
+	got, err := os.ReadFile(dst)
+	if err != nil {
+		t.Fatalf("read dst: %v", err)
+	}
+	if string(got) != string(content) {
+		t.Errorf("content mismatch: got %q", got)
+	}
+
+	// Verify executable permission.
+	info, _ := os.Stat(dst)
+	if info.Mode()&0100 == 0 {
+		t.Error("file is not executable")
+	}
+}
+
+func TestLocalNode_Close(t *testing.T) {
+	n := NewLocalNode("test")
+	n.Close() // should be no-op, no panic
+}
+
+func TestLocalNode_DetectRoot(t *testing.T) {
+	skipIfWindows(t)
+	n := NewLocalNode("test")
+	// We just verify it doesn't panic and returns a bool.
+	_ = n.IsRoot()
+}

weed/storage/blockvol/testrunner/metrics.go

@@ -0,0 +1,156 @@
+package testrunner
+
+import (
+	"bufio"
+	"fmt"
+	"math"
+	"sort"
+	"strconv"
+	"strings"
+)
+
+// MetricsSample holds a snapshot of Prometheus-format metrics from a target.
+type MetricsSample struct {
+	Target  string             `json:"target"`
+	Metrics map[string]float64 `json:"metrics"`
+}
+
+// ParsePrometheusText parses the text exposition format from /metrics into
+// a flat map of metric_name → value. Histograms/summaries produce multiple
+// entries (e.g. "metric_bucket{le=\"0.01\"}" → value).
+func ParsePrometheusText(body string) map[string]float64 {
+	metrics := make(map[string]float64)
+	scanner := bufio.NewScanner(strings.NewReader(body))
+	for scanner.Scan() {
+		line := strings.TrimSpace(scanner.Text())
+		if line == "" || strings.HasPrefix(line, "#") {
+			continue
+		}
+		name, value, ok := parseMetricLine(line)
+		if ok {
+			metrics[name] = value
+		}
+	}
+	return metrics
+}
+
+// parseMetricLine parses "metric_name{labels} value" or "metric_name value".
+func parseMetricLine(line string) (string, float64, bool) {
+	// Find the last space that separates the key from the value.
+	// Handle possible timestamp after value.
+	parts := strings.Fields(line)
+	if len(parts) < 2 {
+		return "", 0, false
+	}
+	name := parts[0]
+	val, err := strconv.ParseFloat(parts[1], 64)
+	if err != nil {
+		return "", 0, false
+	}
+	return name, val, true
+}
+
+// PerfStats summarizes latency or throughput samples.
+type PerfStats struct {
+	Count  int     `json:"count"`
+	Min    float64 `json:"min"`
+	Max    float64 `json:"max"`
+	Mean   float64 `json:"mean"`
+	StdDev float64 `json:"stddev"`
+	P50    float64 `json:"p50"`
+	P90    float64 `json:"p90"`
+	P99    float64 `json:"p99"`
+}
+
+// ComputeStats computes percentile and summary statistics from a slice of values.
+func ComputeStats(values []float64) PerfStats {
+	if len(values) == 0 {
+		return PerfStats{}
+	}
+
+	sorted := make([]float64, len(values))
+	copy(sorted, values)
+	sort.Float64s(sorted)
+
+	n := len(sorted)
+	sum := 0.0
+	for _, v := range sorted {
+		sum += v
+	}
+	mean := sum / float64(n)
+
+	variance := 0.0
+	for _, v := range sorted {
+		d := v - mean
+		variance += d * d
+	}
+	variance /= float64(n)
+
+	return PerfStats{
+		Count:  n,
+		Min:    sorted[0],
+		Max:    sorted[n-1],
+		Mean:   mean,
+		StdDev: math.Sqrt(variance),
+		P50:    percentile(sorted, 0.50),
+		P90:    percentile(sorted, 0.90),
+		P99:    percentile(sorted, 0.99),
+	}
+}
+
+func percentile(sorted []float64, p float64) float64 {
+	if len(sorted) == 0 {
+		return 0
+	}
+	if len(sorted) == 1 {
+		return sorted[0]
+	}
+	idx := p * float64(len(sorted)-1)
+	lower := int(idx)
+	upper := lower + 1
+	if upper >= len(sorted) {
+		return sorted[len(sorted)-1]
+	}
+	frac := idx - float64(lower)
+	return sorted[lower]*(1-frac) + sorted[upper]*frac
+}
+
+// FormatStats returns a one-line summary of PerfStats.
+func FormatStats(name string, s PerfStats) string {
+	return fmt.Sprintf("%s: n=%d mean=%.2f stddev=%.2f p50=%.2f p90=%.2f p99=%.2f min=%.2f max=%.2f",
+		name, s.Count, s.Mean, s.StdDev, s.P50, s.P90, s.P99, s.Min, s.Max)
+}
+
+// ParsePerfLogLines extracts numeric values from PERF[5s] log lines.
+// Expected format: "PERF[5s] iops=1234 lat_us=567 ..."
+// Returns map of field_name → []values across all matching lines.
+func ParsePerfLogLines(logContent string) map[string][]float64 {
+	result := make(map[string][]float64)
+	scanner := bufio.NewScanner(strings.NewReader(logContent))
+	for scanner.Scan() {
+		line := scanner.Text()
+		idx := strings.Index(line, "PERF[")
+		if idx < 0 {
+			continue
+		}
+		// Extract everything after "PERF[Xs] "
+		rest := line[idx:]
+		closeBracket := strings.Index(rest, "]")
+		if closeBracket < 0 {
+			continue
+		}
+		fields := strings.Fields(rest[closeBracket+1:])
+		for _, f := range fields {
+			parts := strings.SplitN(f, "=", 2)
+			if len(parts) != 2 {
+				continue
+			}
+			val, err := strconv.ParseFloat(parts[1], 64)
+			if err != nil {
+				continue
+			}
+			result[parts[0]] = append(result[parts[0]], val)
+		}
+	}
+	return result
+}

weed/storage/blockvol/testrunner/metrics_test.go

@@ -0,0 +1,130 @@
+package testrunner
+
+import (
+	"math"
+	"testing"
+)
+
+func TestParsePrometheusText(t *testing.T) {
+	body := `# HELP blockvol_write_ops_total Total write operations.
+# TYPE blockvol_write_ops_total counter
+blockvol_write_ops_total 1234
+blockvol_read_ops_total 5678
+blockvol_write_latency_seconds_bucket{le="0.001"} 100
+blockvol_write_latency_seconds_bucket{le="0.01"} 900
+blockvol_write_latency_seconds_bucket{le="+Inf"} 1234
+blockvol_write_latency_seconds_sum 12.34
+blockvol_write_latency_seconds_count 1234
+`
+	m := ParsePrometheusText(body)
+
+	if m["blockvol_write_ops_total"] != 1234 {
+		t.Errorf("write_ops_total = %v, want 1234", m["blockvol_write_ops_total"])
+	}
+	if m["blockvol_read_ops_total"] != 5678 {
+		t.Errorf("read_ops_total = %v, want 5678", m["blockvol_read_ops_total"])
+	}
+	if m[`blockvol_write_latency_seconds_bucket{le="0.001"}`] != 100 {
+		t.Errorf("bucket le=0.001 = %v, want 100", m[`blockvol_write_latency_seconds_bucket{le="0.001"}`])
+	}
+}
+
+func TestParsePrometheusText_Empty(t *testing.T) {
+	m := ParsePrometheusText("")
+	if len(m) != 0 {
+		t.Errorf("expected empty map, got %d entries", len(m))
+	}
+}
+
+func TestComputeStats(t *testing.T) {
+	values := []float64{1, 2, 3, 4, 5, 6, 7, 8, 9, 10}
+	s := ComputeStats(values)
+
+	if s.Count != 10 {
+		t.Errorf("count = %d, want 10", s.Count)
+	}
+	if s.Min != 1 {
+		t.Errorf("min = %v, want 1", s.Min)
+	}
+	if s.Max != 10 {
+		t.Errorf("max = %v, want 10", s.Max)
+	}
+	if math.Abs(s.Mean-5.5) > 0.01 {
+		t.Errorf("mean = %v, want 5.5", s.Mean)
+	}
+	if math.Abs(s.P50-5.5) > 0.01 {
+		t.Errorf("p50 = %v, want ~5.5", s.P50)
+	}
+	if s.P99 < 9.5 {
+		t.Errorf("p99 = %v, want >= 9.5", s.P99)
+	}
+}
+
+func TestComputeStats_Empty(t *testing.T) {
+	s := ComputeStats(nil)
+	if s.Count != 0 {
+		t.Errorf("expected count=0 for empty input, got %d", s.Count)
+	}
+}
+
+func TestComputeStats_Single(t *testing.T) {
+	s := ComputeStats([]float64{42.0})
+	if s.Count != 1 || s.Min != 42 || s.Max != 42 || s.P50 != 42 || s.P99 != 42 {
+		t.Errorf("single value stats wrong: %+v", s)
+	}
+}
+
+func TestParsePerfLogLines(t *testing.T) {
+	log := `2024-01-01 12:00:00 INFO starting target
+2024-01-01 12:00:05 PERF[5s] iops=1234 lat_us=567 bw_mb=48.5
+2024-01-01 12:00:10 PERF[5s] iops=1300 lat_us=520 bw_mb=50.2
+2024-01-01 12:00:15 PERF[5s] iops=1250 lat_us=540 bw_mb=49.0
+2024-01-01 12:00:20 INFO done
+`
+	result := ParsePerfLogLines(log)
+
+	if len(result["iops"]) != 3 {
+		t.Fatalf("expected 3 iops samples, got %d", len(result["iops"]))
+	}
+	if result["iops"][0] != 1234 {
+		t.Errorf("first iops = %v, want 1234", result["iops"][0])
+	}
+	if len(result["lat_us"]) != 3 {
+		t.Fatalf("expected 3 lat_us samples, got %d", len(result["lat_us"]))
+	}
+	if len(result["bw_mb"]) != 3 {
+		t.Fatalf("expected 3 bw_mb samples, got %d", len(result["bw_mb"]))
+	}
+}
+
+func TestParsePerfLogLines_Empty(t *testing.T) {
+	result := ParsePerfLogLines("no perf lines here\njust regular log output\n")
+	if len(result) != 0 {
+		t.Errorf("expected empty result, got %d entries", len(result))
+	}
+}
+
+func TestFormatStats(t *testing.T) {
+	s := PerfStats{Count: 100, Mean: 5.5, StdDev: 2.87, P50: 5.0, P90: 9.0, P99: 10.0, Min: 1, Max: 10}
+	str := FormatStats("iops", s)
+	if str == "" {
+		t.Error("FormatStats returned empty string")
+	}
+	// Just verify it contains the name and key stats.
+	if !contains(str, "iops") || !contains(str, "n=100") || !contains(str, "p99=10.00") {
+		t.Errorf("unexpected format: %s", str)
+	}
+}
+
+func contains(s, sub string) bool {
+	return len(s) >= len(sub) && findSubstring(s, sub)
+}
+
+func findSubstring(s, sub string) bool {
+	for i := 0; i <= len(s)-len(sub); i++ {
+		if s[i:i+len(sub)] == sub {
+			return true
+		}
+	}
+	return false
+}

weed/storage/blockvol/testrunner/parser.go

@@ -0,0 +1,198 @@
+package testrunner
+
+import (
+	"fmt"
+	"os"
+	"strings"
+
+	"gopkg.in/yaml.v3"
+)
+
+// ParseFile reads and parses a YAML scenario file.
+func ParseFile(path string) (*Scenario, error) {
+	data, err := os.ReadFile(path)
+	if err != nil {
+		return nil, fmt.Errorf("read scenario %s: %w", path, err)
+	}
+	return Parse(data)
+}
+
+// Parse parses YAML bytes into a Scenario and validates it.
+func Parse(data []byte) (*Scenario, error) {
+	var s Scenario
+	if err := yaml.Unmarshal(data, &s); err != nil {
+		return nil, fmt.Errorf("parse YAML: %w", err)
+	}
+	if err := validate(&s); err != nil {
+		return nil, fmt.Errorf("validate: %w", err)
+	}
+	return &s, nil
+}
+
+// validate checks referential integrity and required fields.
+func validate(s *Scenario) error {
+	if s.Name == "" {
+		return fmt.Errorf("scenario name is required")
+	}
+
+	// Check that every target references a valid node.
+	for tName, tSpec := range s.Targets {
+		if tSpec.Node == "" {
+			return fmt.Errorf("target %q: node is required", tName)
+		}
+		if _, ok := s.Topology.Nodes[tSpec.Node]; !ok {
+			return fmt.Errorf("target %q: node %q not found in topology", tName, tSpec.Node)
+		}
+		if tSpec.IQNSuffix == "" {
+			return fmt.Errorf("target %q: iqn_suffix is required", tName)
+		}
+	}
+
+	// Check port conflicts among targets on the same node.
+	type nodePort struct {
+		node string
+		port int
+	}
+	used := make(map[nodePort]string) // nodePort -> target name
+	for tName, tSpec := range s.Targets {
+		ports := []int{tSpec.ISCSIPort, tSpec.AdminPort, tSpec.ReplicaDataPort, tSpec.ReplicaCtrlPort, tSpec.RebuildPort}
+		for _, p := range ports {
+			if p == 0 {
+				continue
+			}
+			np := nodePort{tSpec.Node, p}
+			if other, ok := used[np]; ok {
+				return fmt.Errorf("port conflict: targets %q and %q both use port %d on node %q",
+					other, tName, p, tSpec.Node)
+			}
+			used[np] = tName
+		}
+	}
+
+	// Validate agents section (coordinator mode).
+	if len(s.Topology.Agents) > 0 {
+		for nodeName, nodeSpec := range s.Topology.Nodes {
+			if nodeSpec.Agent != "" {
+				if _, ok := s.Topology.Agents[nodeSpec.Agent]; !ok {
+					return fmt.Errorf("node %q: agent %q not found in topology.agents", nodeName, nodeSpec.Agent)
+				}
+			}
+		}
+	}
+
+	// Check phases and actions.
+	if len(s.Phases) == 0 {
+		return fmt.Errorf("at least one phase is required")
+	}
+	for _, phase := range s.Phases {
+		if phase.Name == "" {
+			return fmt.Errorf("phase name is required")
+		}
+
+		// Validate save_as uniqueness within parallel phases.
+		if phase.Parallel {
+			saveAsSet := make(map[string]int)
+			for i, act := range phase.Actions {
+				if act.SaveAs != "" {
+					if prev, ok := saveAsSet[act.SaveAs]; ok {
+						return fmt.Errorf("phase %q (parallel): save_as %q used by both action %d and %d",
+							phase.Name, act.SaveAs, prev, i)
+					}
+					saveAsSet[act.SaveAs] = i
+				}
+			}
+		}
+
+		for i, act := range phase.Actions {
+			if act.Action == "" {
+				return fmt.Errorf("phase %q, action %d: action type is required", phase.Name, i)
+			}
+			// Validate target references.
+			if act.Target != "" {
+				if _, ok := s.Targets[act.Target]; !ok {
+					return fmt.Errorf("phase %q, action %d (%s): target %q not found",
+						phase.Name, i, act.Action, act.Target)
+				}
+			}
+			if act.Replica != "" {
+				if _, ok := s.Targets[act.Replica]; !ok {
+					return fmt.Errorf("phase %q, action %d (%s): replica %q not found",
+						phase.Name, i, act.Action, act.Replica)
+				}
+			}
+			// Validate node references in actions.
+			if act.Node != "" {
+				if _, ok := s.Topology.Nodes[act.Node]; !ok {
+					return fmt.Errorf("phase %q, action %d (%s): node %q not found",
+						phase.Name, i, act.Action, act.Node)
+				}
+			}
+		}
+	}
+
+	// Validate variable references ({{ var }}) don't reference undefined save_as.
+	defined := make(map[string]bool)
+	// Add env vars.
+	for k := range s.Env {
+		defined[k] = true
+	}
+	for _, phase := range s.Phases {
+		if phase.Always {
+			continue // cleanup phases may use vars from any prior phase
+		}
+		for _, act := range phase.Actions {
+			// Check var references in all string fields.
+			refs := extractVarRefs(act)
+			for _, ref := range refs {
+				if !defined[ref] && !strings.HasPrefix(ref, "__") {
+					// Allow forward refs (they'll be resolved at runtime); just warn-level
+				}
+			}
+			if act.SaveAs != "" {
+				defined[act.SaveAs] = true
+			}
+		}
+	}
+
+	return nil
+}
+
+// extractVarRefs finds all {{ var }} references in action fields.
+func extractVarRefs(act Action) []string {
+	var refs []string
+	fields := collectStringFields(act)
+	for _, f := range fields {
+		refs = append(refs, extractVarsFromString(f)...)
+	}
+	return refs
+}
+
+// collectStringFields returns all string values from an action's params and known fields.
+func collectStringFields(act Action) []string {
+	var fields []string
+	for _, v := range act.Params {
+		fields = append(fields, v)
+	}
+	return fields
+}
+
+// extractVarsFromString finds all {{ name }} patterns in a string.
+func extractVarsFromString(s string) []string {
+	var vars []string
+	for {
+		start := strings.Index(s, "{{")
+		if start < 0 {
+			break
+		}
+		end := strings.Index(s[start:], "}}")
+		if end < 0 {
+			break
+		}
+		name := strings.TrimSpace(s[start+2 : start+end])
+		if name != "" {
+			vars = append(vars, name)
+		}
+		s = s[start+end+2:]
+	}
+	return vars
+}

weed/storage/blockvol/testrunner/parser_test.go

@@ -0,0 +1,385 @@
+package testrunner
+
+import (
+	"testing"
+	"time"
+)
+
+func TestParse_ValidScenario(t *testing.T) {
+	yaml := `
+name: test-scenario
+timeout: 5m
+env:
+  repo_dir: "/tmp/repo"
+
+topology:
+  nodes:
+    node1:
+      host: "192.168.1.1"
+      user: testdev
+      key: "/tmp/key"
+
+targets:
+  primary:
+    node: node1
+    vol_size: 100M
+    iscsi_port: 3260
+    admin_port: 8080
+    iqn_suffix: test-primary
+  replica:
+    node: node1
+    vol_size: 100M
+    iscsi_port: 3261
+    admin_port: 8081
+    replica_data_port: 9011
+    replica_ctrl_port: 9012
+    rebuild_port: 9013
+    iqn_suffix: test-replica
+
+phases:
+  - name: setup
+    actions:
+      - action: build_deploy
+      - action: start_target
+        target: primary
+        create: "true"
+  - name: cleanup
+    always: true
+    actions:
+      - action: stop_all_targets
+        ignore_error: true
+`
+	s, err := Parse([]byte(yaml))
+	if err != nil {
+		t.Fatalf("Parse failed: %v", err)
+	}
+
+	if s.Name != "test-scenario" {
+		t.Errorf("name = %q, want %q", s.Name, "test-scenario")
+	}
+	if s.Timeout.Duration != 5*time.Minute {
+		t.Errorf("timeout = %v, want 5m", s.Timeout.Duration)
+	}
+	if len(s.Topology.Nodes) != 1 {
+		t.Errorf("nodes = %d, want 1", len(s.Topology.Nodes))
+	}
+	if len(s.Targets) != 2 {
+		t.Errorf("targets = %d, want 2", len(s.Targets))
+	}
+	if len(s.Phases) != 2 {
+		t.Errorf("phases = %d, want 2", len(s.Phases))
+	}
+
+	// Check target IQN generation.
+	if iqn := s.Targets["primary"].IQN(); iqn != "iqn.2024.com.seaweedfs:test-primary" {
+		t.Errorf("primary IQN = %q", iqn)
+	}
+
+	// Check always flag.
+	if !s.Phases[1].Always {
+		t.Error("cleanup phase should have always=true")
+	}
+}
+
+func TestParse_MissingName(t *testing.T) {
+	yaml := `
+phases:
+  - name: test
+    actions:
+      - action: exec
+        cmd: "echo hi"
+`
+	_, err := Parse([]byte(yaml))
+	if err == nil {
+		t.Fatal("expected error for missing name")
+	}
+}
+
+func TestParse_InvalidNodeRef(t *testing.T) {
+	yaml := `
+name: bad-ref
+topology:
+  nodes:
+    node1:
+      host: "1.2.3.4"
+      user: test
+      key: "/tmp/key"
+targets:
+  tgt:
+    node: nonexistent
+    iscsi_port: 3260
+    iqn_suffix: test
+phases:
+  - name: test
+    actions:
+      - action: exec
+        cmd: "echo"
+`
+	_, err := Parse([]byte(yaml))
+	if err == nil {
+		t.Fatal("expected error for invalid node ref")
+	}
+}
+
+func TestParse_PortConflict(t *testing.T) {
+	yaml := `
+name: port-conflict
+topology:
+  nodes:
+    node1:
+      host: "1.2.3.4"
+      user: test
+      key: "/tmp/key"
+targets:
+  tgt1:
+    node: node1
+    iscsi_port: 3260
+    iqn_suffix: t1
+  tgt2:
+    node: node1
+    iscsi_port: 3260
+    iqn_suffix: t2
+phases:
+  - name: test
+    actions:
+      - action: exec
+        cmd: "echo"
+`
+	_, err := Parse([]byte(yaml))
+	if err == nil {
+		t.Fatal("expected error for port conflict")
+	}
+}
+
+func TestParse_InvalidTargetRef(t *testing.T) {
+	yaml := `
+name: bad-target-ref
+topology:
+  nodes:
+    node1:
+      host: "1.2.3.4"
+      user: test
+      key: "/tmp/key"
+targets:
+  primary:
+    node: node1
+    iscsi_port: 3260
+    iqn_suffix: test
+phases:
+  - name: test
+    actions:
+      - action: start_target
+        target: nonexistent
+`
+	_, err := Parse([]byte(yaml))
+	if err == nil {
+		t.Fatal("expected error for invalid target ref in action")
+	}
+}
+
+func TestParse_MissingIQNSuffix(t *testing.T) {
+	yaml := `
+name: missing-iqn
+topology:
+  nodes:
+    node1:
+      host: "1.2.3.4"
+      user: test
+      key: "/tmp/key"
+targets:
+  tgt:
+    node: node1
+    iscsi_port: 3260
+phases:
+  - name: test
+    actions:
+      - action: exec
+        cmd: "echo"
+`
+	_, err := Parse([]byte(yaml))
+	if err == nil {
+		t.Fatal("expected error for missing iqn_suffix")
+	}
+}
+
+func TestParse_NoPhases(t *testing.T) {
+	yaml := `
+name: no-phases
+`
+	_, err := Parse([]byte(yaml))
+	if err == nil {
+		t.Fatal("expected error for no phases")
+	}
+}
+
+func TestParse_AgentTopology_Valid(t *testing.T) {
+	yaml := `
+name: agent-topo
+topology:
+  agents:
+    tp01: "192.168.1.188:9100"
+    m01: "192.168.1.181:9100"
+  nodes:
+    tp01_node:
+      host: "192.168.1.188"
+      agent: tp01
+    m01_node:
+      host: "192.168.1.181"
+      agent: m01
+targets:
+  primary:
+    node: tp01_node
+    iscsi_port: 3260
+    iqn_suffix: primary
+phases:
+  - name: test
+    actions:
+      - action: exec
+        node: tp01_node
+        cmd: "echo"
+`
+	s, err := Parse([]byte(yaml))
+	if err != nil {
+		t.Fatalf("Parse failed: %v", err)
+	}
+	if len(s.Topology.Agents) != 2 {
+		t.Errorf("agents = %d, want 2", len(s.Topology.Agents))
+	}
+	if s.Topology.Nodes["tp01_node"].Agent != "tp01" {
+		t.Error("tp01_node should have agent=tp01")
+	}
+}
+
+func TestParse_AgentTopology_InvalidAgentRef(t *testing.T) {
+	yaml := `
+name: bad-agent-ref
+topology:
+  agents:
+    tp01: "192.168.1.188:9100"
+  nodes:
+    node1:
+      host: "192.168.1.181"
+      agent: nonexistent
+phases:
+  - name: test
+    actions:
+      - action: exec
+        cmd: "echo"
+`
+	_, err := Parse([]byte(yaml))
+	if err == nil {
+		t.Fatal("expected error for invalid agent reference")
+	}
+}
+
+func TestParse_ParallelPhase_SaveAsConflict(t *testing.T) {
+	yaml := `
+name: save-as-conflict
+topology:
+  nodes:
+    node1:
+      host: "1.2.3.4"
+      user: test
+      key: "/tmp/key"
+phases:
+  - name: parallel_phase
+    parallel: true
+    actions:
+      - action: exec
+        node: node1
+        save_as: my_var
+        cmd: "echo a"
+      - action: exec
+        node: node1
+        save_as: my_var
+        cmd: "echo b"
+`
+	_, err := Parse([]byte(yaml))
+	if err == nil {
+		t.Fatal("expected error for save_as conflict in parallel phase")
+	}
+}
+
+func TestParse_SequentialPhase_SaveAsDuplicate_Allowed(t *testing.T) {
+	yaml := `
+name: save-as-sequential
+topology:
+  nodes:
+    node1:
+      host: "1.2.3.4"
+      user: test
+      key: "/tmp/key"
+phases:
+  - name: seq_phase
+    actions:
+      - action: exec
+        node: node1
+        save_as: device
+        cmd: "echo a"
+      - action: exec
+        node: node1
+        save_as: device
+        cmd: "echo b"
+`
+	_, err := Parse([]byte(yaml))
+	if err != nil {
+		t.Fatalf("sequential save_as duplicate should be allowed, got: %v", err)
+	}
+}
+
+func TestParse_ActionRetryAndTimeout(t *testing.T) {
+	yaml := `
+name: retry-test
+topology:
+  nodes:
+    n1:
+      host: "1.2.3.4"
+      user: test
+      key: "/tmp/key"
+phases:
+  - name: test
+    actions:
+      - action: start_target
+        node: n1
+        retry: 2
+        timeout: 30s
+        cmd: "echo"
+`
+	s, err := Parse([]byte(yaml))
+	if err != nil {
+		t.Fatalf("Parse failed: %v", err)
+	}
+	act := s.Phases[0].Actions[0]
+	if act.Retry != 2 {
+		t.Errorf("retry = %d, want 2", act.Retry)
+	}
+	if act.Timeout != "30s" {
+		t.Errorf("timeout = %q, want 30s", act.Timeout)
+	}
+}
+
+func TestExtractVarsFromString(t *testing.T) {
+	tests := []struct {
+		input string
+		want  []string
+	}{
+		{"{{ device }}", []string{"device"}},
+		{"{{written_md5}}", []string{"written_md5"}},
+		{"prefix {{ a }} middle {{ b }} suffix", []string{"a", "b"}},
+		{"no vars here", nil},
+		{"{{ }}", nil},
+	}
+
+	for _, tt := range tests {
+		got := extractVarsFromString(tt.input)
+		if len(got) != len(tt.want) {
+			t.Errorf("extractVarsFromString(%q) = %v, want %v", tt.input, got, tt.want)
+			continue
+		}
+		for i := range got {
+			if got[i] != tt.want[i] {
+				t.Errorf("extractVarsFromString(%q)[%d] = %q, want %q", tt.input, i, got[i], tt.want[i])
+			}
+		}
+	}
+}

weed/storage/blockvol/testrunner/protocol.go

@@ -0,0 +1,73 @@
+package testrunner
+
+// AuthTokenHeader is the HTTP header used for coordinator↔agent authentication.
+const AuthTokenHeader = "X-Auth-Token"
+
+// UploadBasePath is the required prefix for all upload paths.
+const UploadBasePath = "/tmp/sw-test-runner/"
+
+// RegisterRequest is sent by an agent to the coordinator at POST /register.
+type RegisterRequest struct {
+	AgentID      string   `json:"agent_id"`
+	Hostname     string   `json:"hostname"`
+	IP           string   `json:"ip"`
+	Port         int      `json:"port"`
+	Nodes        []string `json:"nodes"`
+	Capabilities []string `json:"capabilities"`
+}
+
+// RegisterResponse is returned by the coordinator after successful registration.
+type RegisterResponse struct {
+	OK          bool   `json:"ok"`
+	AgentIndex  int    `json:"agent_index"`
+	TotalAgents int    `json:"total_agents"`
+	Error       string `json:"error,omitempty"`
+}
+
+// PhaseRequest is sent by the coordinator to an agent at POST /phase.
+type PhaseRequest struct {
+	PhaseIndex int               `json:"phase_index"`
+	PhaseName  string            `json:"phase_name"`
+	Actions    []Action          `json:"actions"`
+	GlobalVars map[string]string `json:"global_vars"`
+	Scenario   *Scenario         `json:"scenario,omitempty"`
+}
+
+// PhaseResponse is returned by the agent after executing a phase.
+type PhaseResponse struct {
+	PhaseIndex int               `json:"phase_index"`
+	Results    []ActionResult    `json:"results"`
+	NewVars    map[string]string `json:"new_vars"`
+	Error      string            `json:"error,omitempty"`
+}
+
+// ExecRequest is sent by the coordinator to an agent at POST /exec.
+type ExecRequest struct {
+	Cmd  string `json:"cmd"`
+	Root bool   `json:"root"`
+}
+
+// ExecResponse is returned by the agent after executing a command.
+type ExecResponse struct {
+	Stdout   string `json:"stdout"`
+	Stderr   string `json:"stderr"`
+	ExitCode int    `json:"exit_code"`
+	Error    string `json:"error,omitempty"`
+}
+
+// UploadResponse is returned by the agent after receiving a file upload.
+type UploadResponse struct {
+	OK    bool   `json:"ok"`
+	Size  int64  `json:"size"`
+	Path  string `json:"path"`
+	Error string `json:"error,omitempty"`
+}
+
+// HealthResponse is returned by the agent at GET /health.
+type HealthResponse struct {
+	OK       bool   `json:"ok"`
+	AgentID  string `json:"agent_id"`
+	UptimeS  int64  `json:"uptime_s"`
+	HasRoot  bool   `json:"has_root"`
+	Hostname string `json:"hostname"`
+}

weed/storage/blockvol/testrunner/protocol_test.go

@@ -0,0 +1,36 @@
+package testrunner
+
+import (
+	"testing"
+)
+
+func TestIsPathSafe(t *testing.T) {
+	tests := []struct {
+		path string
+		want bool
+	}{
+		{"/tmp/sw-test-runner/binary", true},
+		{"/tmp/sw-test-runner/subdir/file", true},
+		{"/tmp/sw-test-runner/nested/deep/file", true},
+		{"/tmp/other/file", false},
+		{"/etc/passwd", false},
+		{"/tmp/sw-test-runner/../etc/passwd", false},
+		{"../etc/passwd", false},
+		{"/tmp/sw-test-runner/../../root", false},
+		{"", false},
+		{"/tmp/sw-test-runner-evil/file", false},
+	}
+
+	for _, tt := range tests {
+		got := isPathSafe(tt.path)
+		if got != tt.want {
+			t.Errorf("isPathSafe(%q) = %v, want %v", tt.path, got, tt.want)
+		}
+	}
+}
+
+func TestAuthTokenHeaderConstant(t *testing.T) {
+	if AuthTokenHeader != "X-Auth-Token" {
+		t.Errorf("AuthTokenHeader = %q", AuthTokenHeader)
+	}
+}

weed/storage/blockvol/testrunner/registry.go

@@ -0,0 +1,166 @@
+package testrunner
+
+import (
+	"context"
+	"fmt"
+)
+
+// ActionContext provides action handlers with access to the scenario's
+// infrastructure (nodes, targets) and variable store.
+type ActionContext struct {
+	Scenario    *Scenario
+	Nodes       map[string]NodeRunner
+	Targets     map[string]TargetRunner
+	Vars        map[string]string
+	Log         func(format string, args ...interface{})
+	Coordinator *Coordinator // non-nil when running in coordinator mode
+}
+
+// NodeRunner abstracts remote command execution (implemented by infra.Node).
+type NodeRunner interface {
+	Run(ctx context.Context, cmd string) (stdout, stderr string, exitCode int, err error)
+	RunRoot(ctx context.Context, cmd string) (stdout, stderr string, exitCode int, err error)
+	Upload(local, remote string) error
+	Close()
+}
+
+// TargetRunner abstracts iSCSI target lifecycle (implemented by infra.HATarget).
+type TargetRunner interface {
+	Start(ctx context.Context, create bool) error
+	Stop(ctx context.Context) error
+	Kill9() error
+	CollectLog() (string, error)
+	Cleanup(ctx context.Context)
+	PID() int
+	VolFilePath() string
+}
+
+// HATargetRunner extends TargetRunner with HA admin operations.
+type HATargetRunner interface {
+	TargetRunner
+	Assign(ctx context.Context, epoch uint64, role uint32, leaseTTLMs uint32) error
+	Status(ctx context.Context) (*StatusResult, error)
+	SetReplica(ctx context.Context, dataAddr, ctrlAddr string) error
+	WaitForRole(ctx context.Context, expectedRole string) error
+	WaitForLSN(ctx context.Context, minLSN uint64) error
+	StartRebuildEndpoint(ctx context.Context, listenAddr string) error
+	StartRebuildClient(ctx context.Context, rebuildAddr string, epoch uint64) error
+	StopRebuildEndpoint(ctx context.Context) error
+}
+
+// StatusResult mirrors the JSON from GET /status.
+type StatusResult struct {
+	Epoch      uint64 `json:"epoch"`
+	Role       string `json:"role"`
+	WALHeadLSN uint64 `json:"wal_head_lsn"`
+	HasLease   bool   `json:"has_lease"`
+	Healthy    bool   `json:"healthy"`
+}
+
+// ActionHandler executes a single action type.
+type ActionHandler interface {
+	Execute(ctx context.Context, actx *ActionContext, act Action) (map[string]string, error)
+}
+
+// ActionHandlerFunc adapts a function to the ActionHandler interface.
+type ActionHandlerFunc func(ctx context.Context, actx *ActionContext, act Action) (map[string]string, error)
+
+func (f ActionHandlerFunc) Execute(ctx context.Context, actx *ActionContext, act Action) (map[string]string, error) {
+	return f(ctx, actx, act)
+}
+
+// Tier constants for action classification.
+const (
+	TierCore   = "core"   // exec, sleep, assert_*, print
+	TierBlock  = "block"  // start_target, iscsi_*, dd_*, fio_*, assign, metrics...
+	TierDevOps = "devops" // start_weed_master, start_weed_volume, cluster_status...
+	TierChaos  = "chaos"  // inject_netem, inject_partition, fill_disk, corrupt_wal
+)
+
+type actionEntry struct {
+	handler ActionHandler
+	tier    string
+}
+
+// Registry maps action names to handlers with tier-based gating.
+type Registry struct {
+	handlers     map[string]actionEntry
+	EnabledTiers map[string]bool // nil or empty = all tiers allowed
+}
+
+// NewRegistry creates an empty registry.
+func NewRegistry() *Registry {
+	return &Registry{handlers: make(map[string]actionEntry)}
+}
+
+// Register adds a handler for an action name with a tier.
+func (r *Registry) Register(name, tier string, h ActionHandler) {
+	r.handlers[name] = actionEntry{handler: h, tier: tier}
+}
+
+// RegisterFunc adds a function handler for an action name with a tier.
+func (r *Registry) RegisterFunc(name, tier string, f ActionHandlerFunc) {
+	r.handlers[name] = actionEntry{handler: f, tier: tier}
+}
+
+// EnableTiers sets which tiers are allowed. Pass nil or empty to allow all.
+func (r *Registry) EnableTiers(tiers []string) {
+	if len(tiers) == 0 {
+		r.EnabledTiers = nil
+		return
+	}
+	r.EnabledTiers = make(map[string]bool, len(tiers))
+	for _, t := range tiers {
+		r.EnabledTiers[t] = true
+	}
+}
+
+// Get returns the handler for an action name, or an error.
+// Returns an error if the action's tier is not enabled.
+func (r *Registry) Get(name string) (ActionHandler, error) {
+	entry, ok := r.handlers[name]
+	if !ok {
+		return nil, fmt.Errorf("unknown action: %q", name)
+	}
+	if len(r.EnabledTiers) > 0 && !r.EnabledTiers[entry.tier] {
+		return nil, fmt.Errorf("action %q requires tier %q (enabled: %v)", name, entry.tier, r.tierList())
+	}
+	return entry.handler, nil
+}
+
+// List returns all registered action names (respecting tier gating).
+func (r *Registry) List() []string {
+	names := make([]string, 0, len(r.handlers))
+	for name, entry := range r.handlers {
+		if len(r.EnabledTiers) > 0 && !r.EnabledTiers[entry.tier] {
+			continue
+		}
+		names = append(names, name)
+	}
+	return names
+}
+
+// ListByTier returns action names grouped by tier.
+func (r *Registry) ListByTier() map[string][]string {
+	result := make(map[string][]string)
+	for name, entry := range r.handlers {
+		result[entry.tier] = append(result[entry.tier], name)
+	}
+	return result
+}
+
+// ActionTier returns the tier for a registered action, or "" if not found.
+func (r *Registry) ActionTier(name string) string {
+	if entry, ok := r.handlers[name]; ok {
+		return entry.tier
+	}
+	return ""
+}
+
+func (r *Registry) tierList() []string {
+	list := make([]string, 0, len(r.EnabledTiers))
+	for t := range r.EnabledTiers {
+		list = append(list, t)
+	}
+	return list
+}

weed/storage/blockvol/testrunner/registry_test.go

@@ -0,0 +1,104 @@
+package testrunner
+
+import (
+	"context"
+	"sort"
+	"testing"
+)
+
+func TestRegistry_TierGating(t *testing.T) {
+	r := NewRegistry()
+	noop := ActionHandlerFunc(func(ctx context.Context, actx *ActionContext, act Action) (map[string]string, error) {
+		return nil, nil
+	})
+
+	r.RegisterFunc("exec", TierCore, noop)
+	r.RegisterFunc("dd_write", TierBlock, noop)
+	r.RegisterFunc("inject_netem", TierChaos, noop)
+	r.RegisterFunc("start_weed_master", TierDevOps, noop)
+
+	// No gating — all accessible.
+	if _, err := r.Get("exec"); err != nil {
+		t.Errorf("exec ungated: %v", err)
+	}
+	if _, err := r.Get("dd_write"); err != nil {
+		t.Errorf("dd_write ungated: %v", err)
+	}
+
+	// Enable only core.
+	r.EnableTiers([]string{TierCore})
+	if _, err := r.Get("exec"); err != nil {
+		t.Errorf("exec (core enabled): %v", err)
+	}
+	if _, err := r.Get("dd_write"); err == nil {
+		t.Error("dd_write should be blocked when only core is enabled")
+	}
+	if _, err := r.Get("inject_netem"); err == nil {
+		t.Error("inject_netem should be blocked")
+	}
+
+	// List should only return core actions.
+	names := r.List()
+	if len(names) != 1 || names[0] != "exec" {
+		t.Errorf("List() = %v, want [exec]", names)
+	}
+
+	// Reset gating.
+	r.EnableTiers(nil)
+	names = r.List()
+	if len(names) != 4 {
+		t.Errorf("List() after reset = %d, want 4", len(names))
+	}
+}
+
+func TestRegistry_ListByTier(t *testing.T) {
+	r := NewRegistry()
+	noop := ActionHandlerFunc(func(ctx context.Context, actx *ActionContext, act Action) (map[string]string, error) {
+		return nil, nil
+	})
+
+	r.RegisterFunc("a", TierCore, noop)
+	r.RegisterFunc("b", TierCore, noop)
+	r.RegisterFunc("c", TierBlock, noop)
+
+	byTier := r.ListByTier()
+	coreActions := byTier[TierCore]
+	sort.Strings(coreActions)
+	if len(coreActions) != 2 || coreActions[0] != "a" || coreActions[1] != "b" {
+		t.Errorf("core = %v, want [a b]", coreActions)
+	}
+	if len(byTier[TierBlock]) != 1 || byTier[TierBlock][0] != "c" {
+		t.Errorf("block = %v, want [c]", byTier[TierBlock])
+	}
+}
+
+func TestRegistry_ActionTier(t *testing.T) {
+	r := NewRegistry()
+	noop := ActionHandlerFunc(func(ctx context.Context, actx *ActionContext, act Action) (map[string]string, error) {
+		return nil, nil
+	})
+
+	r.RegisterFunc("foo", TierChaos, noop)
+
+	if tier := r.ActionTier("foo"); tier != TierChaos {
+		t.Errorf("tier = %q, want chaos", tier)
+	}
+	if tier := r.ActionTier("nonexistent"); tier != "" {
+		t.Errorf("nonexistent tier = %q, want empty", tier)
+	}
+}
+
+func TestRegistry_EmptyTiersAllowsAll(t *testing.T) {
+	r := NewRegistry()
+	noop := ActionHandlerFunc(func(ctx context.Context, actx *ActionContext, act Action) (map[string]string, error) {
+		return nil, nil
+	})
+
+	r.RegisterFunc("x", TierDevOps, noop)
+
+	// Empty slice should not restrict.
+	r.EnableTiers([]string{})
+	if _, err := r.Get("x"); err != nil {
+		t.Errorf("empty tiers should allow all: %v", err)
+	}
+}

weed/storage/blockvol/testrunner/regression.go

@@ -0,0 +1,200 @@
+package testrunner
+
+import (
+	"fmt"
+	"math"
+	"strings"
+)
+
+// RegressionResult holds the comparison of one metric against baseline.
+type RegressionResult struct {
+	Metric       string
+	BaselineVal  float64
+	CurrentVal   float64
+	ThresholdPct float64 // positive = allowed increase, negative = allowed decrease
+	Pass         bool
+	Reason       string
+}
+
+// RegressionReport is the full regression comparison output.
+type RegressionReport struct {
+	Results       []RegressionResult
+	HardFails     []HardFailResult
+	OverallPass   bool
+	BaselineGitSHA string
+	CurrentGitSHA  string
+}
+
+// HardFailResult is the result of a hard fail condition check.
+type HardFailResult struct {
+	Condition string
+	Pass      bool
+	Detail    string
+}
+
+// RegressionThresholds defines the allowed deviation per metric.
+var RegressionThresholds = map[string]float64{
+	"p99_write_latency_seconds": 0.10,  // fail if >10% increase
+	"p99_read_latency_seconds":  0.10,  // fail if >10% increase
+	"write_iops":                -0.05, // fail if >5% decrease
+	"read_iops":                 -0.05, // fail if >5% decrease
+}
+
+// CompareBaseline runs regression checks against a baseline.
+func CompareBaseline(baseline, current map[string]float64) []RegressionResult {
+	var results []RegressionResult
+	for metric, threshold := range RegressionThresholds {
+		bv, bOk := baseline[metric]
+		cv, cOk := current[metric]
+		if !bOk || !cOk {
+			continue
+		}
+		if bv == 0 {
+			results = append(results, RegressionResult{
+				Metric: metric, BaselineVal: bv, CurrentVal: cv,
+				ThresholdPct: threshold, Pass: true, Reason: "baseline is 0, skipped",
+			})
+			continue
+		}
+
+		r := RegressionResult{
+			Metric: metric, BaselineVal: bv, CurrentVal: cv,
+			ThresholdPct: threshold,
+		}
+
+		pctChange := (cv - bv) / math.Abs(bv)
+		if threshold >= 0 {
+			// Positive threshold: fail if increase exceeds threshold.
+			r.Pass = pctChange <= threshold
+			r.Reason = fmt.Sprintf("change=%.1f%% (limit=+%.1f%%)", pctChange*100, threshold*100)
+		} else {
+			// Negative threshold: fail if decrease exceeds |threshold|.
+			r.Pass = pctChange >= threshold
+			r.Reason = fmt.Sprintf("change=%.1f%% (limit=%.1f%%)", pctChange*100, threshold*100)
+		}
+		results = append(results, r)
+	}
+	return results
+}
+
+// HardFailChecks are immediate test failure conditions (no comparison needed).
+// Each check takes the current metrics map and returns pass/fail.
+var HardFailChecks = []struct {
+	Name  string
+	Check func(m map[string]float64) (bool, string)
+}{
+	{
+		Name: "data_mismatch",
+		Check: func(m map[string]float64) (bool, string) {
+			if v, ok := m["data_mismatch_count"]; ok && v > 0 {
+				return false, fmt.Sprintf("%.0f data mismatches detected", v)
+			}
+			return true, ""
+		},
+	},
+	{
+		Name: "promotion_panic",
+		Check: func(m map[string]float64) (bool, string) {
+			if v, ok := m["promotion_panic_count"]; ok && v > 0 {
+				return false, fmt.Sprintf("%.0f promotion panics", v)
+			}
+			return true, ""
+		},
+	},
+	{
+		Name: "scrub_false_positive_burst",
+		Check: func(m map[string]float64) (bool, string) {
+			if v, ok := m["scrub_false_positives_per_pass"]; ok && v > 3 {
+				return false, fmt.Sprintf("%.0f false positives in one scrub pass (limit=3)", v)
+			}
+			return true, ""
+		},
+	},
+	{
+		Name: "barrier_lag_lsn_unbounded",
+		Check: func(m map[string]float64) (bool, string) {
+			if v, ok := m["barrier_lag_lsn_max"]; ok && v > 1000 {
+				return false, fmt.Sprintf("barrier_lag_lsn=%.0f (limit=1000)", v)
+			}
+			return true, ""
+		},
+	},
+	{
+		Name: "barrier_error_rate",
+		Check: func(m map[string]float64) (bool, string) {
+			total, tOk := m["barrier_requests_total"]
+			failed, fOk := m["barrier_failures_total"]
+			if tOk && fOk && total > 0 && (failed/total) > 0.05 {
+				return false, fmt.Sprintf("barrier error rate=%.1f%% (limit=5%%)", (failed/total)*100)
+			}
+			return true, ""
+		},
+	},
+	{
+		Name: "health_zero_without_fault",
+		Check: func(m map[string]float64) (bool, string) {
+			health, hOk := m["health_score"]
+			faultActive, fOk := m["fault_active"]
+			if hOk && health == 0.0 && (!fOk || faultActive == 0) {
+				return false, "health score dropped to 0.0 without injected fault"
+			}
+			return true, ""
+		},
+	},
+	{
+		Name: "wal_full_stall",
+		Check: func(m map[string]float64) (bool, string) {
+			if v, ok := m["wal_full_duration_seconds"]; ok && v > 10 {
+				return false, fmt.Sprintf("WAL full for %.1fs (limit=10s)", v)
+			}
+			return true, ""
+		},
+	},
+}
+
+// RunHardFailChecks evaluates all hard fail conditions against current metrics.
+func RunHardFailChecks(metrics map[string]float64) []HardFailResult {
+	results := make([]HardFailResult, 0, len(HardFailChecks))
+	for _, hf := range HardFailChecks {
+		pass, detail := hf.Check(metrics)
+		results = append(results, HardFailResult{
+			Condition: hf.Name,
+			Pass:      pass,
+			Detail:    detail,
+		})
+	}
+	return results
+}
+
+// FormatRegressionReport produces a human-readable report string.
+func FormatRegressionReport(r *RegressionReport) string {
+	var sb strings.Builder
+	sb.WriteString("=== SLO Regression Report ===\n")
+	sb.WriteString(fmt.Sprintf("Baseline: %s  Current: %s\n\n", r.BaselineGitSHA, r.CurrentGitSHA))
+
+	sb.WriteString("--- Metric Comparisons ---\n")
+	for _, res := range r.Results {
+		status := "PASS"
+		if !res.Pass {
+			status = "FAIL"
+		}
+		sb.WriteString(fmt.Sprintf("  [%s] %s: baseline=%.4f current=%.4f %s\n",
+			status, res.Metric, res.BaselineVal, res.CurrentVal, res.Reason))
+	}
+
+	sb.WriteString("\n--- Hard Fail Conditions ---\n")
+	for _, hf := range r.HardFails {
+		status := "PASS"
+		if !hf.Pass {
+			status = "FAIL"
+		}
+		detail := hf.Detail
+		if detail == "" {
+			detail = "ok"
+		}
+		sb.WriteString(fmt.Sprintf("  [%s] %s: %s\n", status, hf.Condition, detail))
+	}
+
+	sb.WriteString(fmt.Sprintf("\nOverall: %s\n", map[bool]string{true: "PASS", false: "FAIL"}[r.OverallPass]))
+	return sb.String()
+}

weed/storage/blockvol/testrunner/regression_test.go

@@ -0,0 +1,163 @@
+package testrunner
+
+import (
+	"testing"
+)
+
+func TestCompareBaseline_P99Increase(t *testing.T) {
+	baseline := map[string]float64{"p99_write_latency_seconds": 0.005}
+	current := map[string]float64{"p99_write_latency_seconds": 0.006} // +20%
+
+	results := CompareBaseline(baseline, current)
+	if len(results) == 0 {
+		t.Fatal("expected results")
+	}
+	for _, r := range results {
+		if r.Metric == "p99_write_latency_seconds" && r.Pass {
+			t.Error("expected FAIL for 20% latency increase (limit 10%)")
+		}
+	}
+}
+
+func TestCompareBaseline_P99WithinLimit(t *testing.T) {
+	baseline := map[string]float64{"p99_write_latency_seconds": 0.005}
+	current := map[string]float64{"p99_write_latency_seconds": 0.0054} // +8%
+
+	results := CompareBaseline(baseline, current)
+	for _, r := range results {
+		if r.Metric == "p99_write_latency_seconds" && !r.Pass {
+			t.Errorf("expected PASS for 8%% increase (limit 10%%): %s", r.Reason)
+		}
+	}
+}
+
+func TestCompareBaseline_IOPSDecrease(t *testing.T) {
+	baseline := map[string]float64{"write_iops": 50000}
+	current := map[string]float64{"write_iops": 46000} // -8%
+
+	results := CompareBaseline(baseline, current)
+	for _, r := range results {
+		if r.Metric == "write_iops" && r.Pass {
+			t.Error("expected FAIL for 8% IOPS decrease (limit 5%)")
+		}
+	}
+}
+
+func TestCompareBaseline_IOPSWithinLimit(t *testing.T) {
+	baseline := map[string]float64{"write_iops": 50000}
+	current := map[string]float64{"write_iops": 48000} // -4%
+
+	results := CompareBaseline(baseline, current)
+	for _, r := range results {
+		if r.Metric == "write_iops" && !r.Pass {
+			t.Errorf("expected PASS for 4%% decrease (limit 5%%): %s", r.Reason)
+		}
+	}
+}
+
+func TestHardFail_DataMismatch(t *testing.T) {
+	metrics := map[string]float64{"data_mismatch_count": 1}
+	results := RunHardFailChecks(metrics)
+	for _, r := range results {
+		if r.Condition == "data_mismatch" && r.Pass {
+			t.Error("expected FAIL for data mismatch")
+		}
+	}
+}
+
+func TestHardFail_BarrierLagUnbounded(t *testing.T) {
+	metrics := map[string]float64{"barrier_lag_lsn_max": 1500}
+	results := RunHardFailChecks(metrics)
+	for _, r := range results {
+		if r.Condition == "barrier_lag_lsn_unbounded" && r.Pass {
+			t.Error("expected FAIL for barrier_lag_lsn=1500")
+		}
+	}
+}
+
+func TestHardFail_BarrierLagOK(t *testing.T) {
+	metrics := map[string]float64{"barrier_lag_lsn_max": 50}
+	results := RunHardFailChecks(metrics)
+	for _, r := range results {
+		if r.Condition == "barrier_lag_lsn_unbounded" && !r.Pass {
+			t.Error("expected PASS for barrier_lag_lsn=50")
+		}
+	}
+}
+
+func TestHardFail_BarrierErrorRate(t *testing.T) {
+	metrics := map[string]float64{
+		"barrier_requests_total": 100,
+		"barrier_failures_total": 10, // 10% error rate
+	}
+	results := RunHardFailChecks(metrics)
+	for _, r := range results {
+		if r.Condition == "barrier_error_rate" && r.Pass {
+			t.Error("expected FAIL for 10% barrier error rate")
+		}
+	}
+}
+
+func TestHardFail_HealthZero(t *testing.T) {
+	metrics := map[string]float64{"health_score": 0.0, "fault_active": 0}
+	results := RunHardFailChecks(metrics)
+	for _, r := range results {
+		if r.Condition == "health_zero_without_fault" && r.Pass {
+			t.Error("expected FAIL for health=0 without fault")
+		}
+	}
+}
+
+func TestHardFail_HealthZeroDuringFault_OK(t *testing.T) {
+	metrics := map[string]float64{"health_score": 0.0, "fault_active": 1}
+	results := RunHardFailChecks(metrics)
+	for _, r := range results {
+		if r.Condition == "health_zero_without_fault" && !r.Pass {
+			t.Error("expected PASS for health=0 during active fault")
+		}
+	}
+}
+
+func TestHardFail_WALFullStall(t *testing.T) {
+	metrics := map[string]float64{"wal_full_duration_seconds": 15}
+	results := RunHardFailChecks(metrics)
+	for _, r := range results {
+		if r.Condition == "wal_full_stall" && r.Pass {
+			t.Error("expected FAIL for WAL full 15s")
+		}
+	}
+}
+
+func TestHardFail_AllPass(t *testing.T) {
+	metrics := map[string]float64{
+		"health_score":      1.0,
+		"barrier_lag_lsn_max": 10,
+	}
+	results := RunHardFailChecks(metrics)
+	for _, r := range results {
+		if !r.Pass {
+			t.Errorf("expected all PASS, got FAIL for %s: %s", r.Condition, r.Detail)
+		}
+	}
+}
+
+func TestFormatRegressionReport(t *testing.T) {
+	report := &RegressionReport{
+		Results: []RegressionResult{
+			{Metric: "p99_write_latency_seconds", BaselineVal: 0.005, CurrentVal: 0.006, Pass: false, Reason: "change=+20%"},
+		},
+		HardFails: []HardFailResult{
+			{Condition: "data_mismatch", Pass: true},
+		},
+		OverallPass:    false,
+		BaselineGitSHA: "abc123",
+		CurrentGitSHA:  "def456",
+	}
+	s := FormatRegressionReport(report)
+	if s == "" {
+		t.Error("expected non-empty report")
+	}
+	if len(s) < 50 {
+		t.Errorf("report too short: %s", s)
+	}
+}

weed/storage/blockvol/testrunner/reporter.go

@@ -0,0 +1,324 @@
+package testrunner
+
+import (
+	"encoding/json"
+	"encoding/xml"
+	"fmt"
+	"io"
+	"os"
+	"sort"
+	"strings"
+	"time"
+)
+
+// WriteJSON writes the scenario result as JSON to the given path.
+func WriteJSON(result *ScenarioResult, path string) error {
+	data, err := json.MarshalIndent(result, "", "  ")
+	if err != nil {
+		return fmt.Errorf("marshal JSON: %w", err)
+	}
+	return os.WriteFile(path, data, 0644)
+}
+
+// PrintSummary writes a human-readable summary table to w.
+func PrintSummary(w io.Writer, result *ScenarioResult) {
+	fmt.Fprintf(w, "\n=== %s === %s (%s)\n\n",
+		result.Name, result.Status, result.Duration.Round(time.Millisecond))
+
+	passCount, failCount, totalActions := 0, 0, 0
+	for _, pr := range result.Phases {
+		status := string(pr.Status)
+		fmt.Fprintf(w, "  %-20s %s  (%s)\n",
+			pr.Name, status, pr.Duration.Round(time.Millisecond))
+		for _, ar := range pr.Actions {
+			totalActions++
+			if ar.Status == StatusPass {
+				passCount++
+			} else {
+				failCount++
+			}
+			marker := " "
+			if ar.Status == StatusFail {
+				marker = "X"
+			}
+			fmt.Fprintf(w, "    %s %-25s %s\n", marker, ar.Action, ar.Duration.Round(time.Millisecond))
+			if ar.Error != "" {
+				fmt.Fprintf(w, "      ERROR: %s\n", ar.Error)
+			}
+		}
+	}
+
+	fmt.Fprintf(w, "\n  %d actions: %d passed, %d failed\n", totalActions, passCount, failCount)
+
+	// Render perf stats and metrics tables from result.Vars.
+	if len(result.Vars) > 0 {
+		printPerfTable(w, result.Vars)
+		printMetricsTable(w, result.Vars)
+	}
+
+	fmt.Fprintln(w)
+
+	if result.Error != "" {
+		fmt.Fprintf(w, "  ERROR: %s\n\n", result.Error)
+	}
+}
+
+// printPerfTable scans vars for PerfStats-format values (FormatStats output)
+// and renders a formatted table. Matches lines like "name: n=100 mean=1.23 ...".
+func printPerfTable(w io.Writer, vars map[string]string) {
+	type perfEntry struct {
+		name  string
+		stats PerfStats
+	}
+	var entries []perfEntry
+
+	for key, val := range vars {
+		// Try parsing as PerfStats JSON first.
+		var ps PerfStats
+		if err := json.Unmarshal([]byte(val), &ps); err == nil && ps.Count > 0 {
+			entries = append(entries, perfEntry{name: key, stats: ps})
+			continue
+		}
+		// Try parsing FormatStats text: "name: n=100 mean=1.23 ..."
+		if ps, ok := parsePerfStatsLine(val); ok {
+			entries = append(entries, perfEntry{name: key, stats: ps})
+		}
+	}
+
+	if len(entries) == 0 {
+		return
+	}
+
+	sort.Slice(entries, func(i, j int) bool { return entries[i].name < entries[j].name })
+
+	fmt.Fprintf(w, "\n  === Performance ===\n")
+	fmt.Fprintf(w, "  %-16s %6s %12s %12s %12s %12s %12s %12s\n",
+		"Metric", "Count", "Mean", "P50", "P90", "P99", "Min", "Max")
+	for _, e := range entries {
+		fmt.Fprintf(w, "  %-16s %6d %12.1f %12.1f %12.1f %12.1f %12.1f %12.1f\n",
+			e.name, e.stats.Count, e.stats.Mean, e.stats.P50, e.stats.P90, e.stats.P99, e.stats.Min, e.stats.Max)
+	}
+}
+
+// parsePerfStatsLine parses a FormatStats-style line: "name: n=100 mean=1.23 stddev=0.5 p50=1.2 p90=1.5 p99=1.8 min=0.5 max=2.0"
+func parsePerfStatsLine(s string) (PerfStats, bool) {
+	if !strings.Contains(s, "n=") || !strings.Contains(s, "p99=") {
+		return PerfStats{}, false
+	}
+
+	var ps PerfStats
+	fields := strings.Fields(s)
+	parsed := 0
+	for _, f := range fields {
+		parts := strings.SplitN(f, "=", 2)
+		if len(parts) != 2 {
+			continue
+		}
+		var val float64
+		if _, err := fmt.Sscanf(parts[1], "%f", &val); err != nil {
+			continue
+		}
+		switch parts[0] {
+		case "n":
+			ps.Count = int(val)
+			parsed++
+		case "mean":
+			ps.Mean = val
+			parsed++
+		case "stddev":
+			ps.StdDev = val
+			parsed++
+		case "p50":
+			ps.P50 = val
+			parsed++
+		case "p90":
+			ps.P90 = val
+			parsed++
+		case "p99":
+			ps.P99 = val
+			parsed++
+		case "min":
+			ps.Min = val
+			parsed++
+		case "max":
+			ps.Max = val
+			parsed++
+		}
+	}
+	return ps, parsed >= 3 && ps.Count > 0
+}
+
+// printMetricsTable scans vars for MetricsSample JSON values and renders a table.
+func printMetricsTable(w io.Writer, vars map[string]string) {
+	type metricsGroup struct {
+		name    string
+		sample  MetricsSample
+	}
+	var groups []metricsGroup
+
+	for key, val := range vars {
+		var ms MetricsSample
+		if err := json.Unmarshal([]byte(val), &ms); err != nil {
+			continue
+		}
+		if len(ms.Metrics) == 0 {
+			continue
+		}
+		label := key
+		if ms.Target != "" {
+			label = ms.Target
+		}
+		groups = append(groups, metricsGroup{name: label, sample: ms})
+	}
+
+	if len(groups) == 0 {
+		return
+	}
+
+	sort.Slice(groups, func(i, j int) bool { return groups[i].name < groups[j].name })
+
+	for _, g := range groups {
+		fmt.Fprintf(w, "\n  === Metrics (%s) ===\n", g.name)
+		// Sort metric names.
+		names := make([]string, 0, len(g.sample.Metrics))
+		for name := range g.sample.Metrics {
+			names = append(names, name)
+		}
+		sort.Strings(names)
+		for _, name := range names {
+			fmt.Fprintf(w, "  %-40s %s\n", name, formatMetricValue(g.sample.Metrics[name]))
+		}
+	}
+}
+
+// formatMetricValue formats a float with commas for readability.
+func formatMetricValue(v float64) string {
+	if v == float64(int64(v)) {
+		return formatInt(int64(v))
+	}
+	return fmt.Sprintf("%.2f", v)
+}
+
+// formatInt formats an integer with comma separators.
+func formatInt(n int64) string {
+	s := fmt.Sprintf("%d", n)
+	if n < 0 {
+		return s
+	}
+	// Insert commas.
+	var result strings.Builder
+	for i, c := range s {
+		if i > 0 && (len(s)-i)%3 == 0 {
+			result.WriteByte(',')
+		}
+		result.WriteRune(c)
+	}
+	return result.String()
+}
+
+// --- JUnit XML output ---
+
+// JUnitTestSuites is the root of JUnit XML.
+type JUnitTestSuites struct {
+	XMLName xml.Name         `xml:"testsuites"`
+	Suites  []JUnitTestSuite `xml:"testsuite"`
+}
+
+// JUnitTestSuite corresponds to one scenario.
+type JUnitTestSuite struct {
+	XMLName  xml.Name        `xml:"testsuite"`
+	Name     string          `xml:"name,attr"`
+	Tests    int             `xml:"tests,attr"`
+	Failures int             `xml:"failures,attr"`
+	Time     float64         `xml:"time,attr"`
+	Cases    []JUnitTestCase `xml:"testcase"`
+}
+
+// JUnitTestCase corresponds to one action.
+type JUnitTestCase struct {
+	XMLName   xml.Name      `xml:"testcase"`
+	Name      string        `xml:"name,attr"`
+	ClassName string        `xml:"classname,attr"`
+	Time      float64       `xml:"time,attr"`
+	Failure   *JUnitFailure `xml:"failure,omitempty"`
+}
+
+// JUnitFailure describes a test failure.
+type JUnitFailure struct {
+	Message string `xml:"message,attr"`
+	Content string `xml:",chardata"`
+}
+
+// WriteJUnitXML writes JUnit XML output to the given path.
+func WriteJUnitXML(result *ScenarioResult, path string) error {
+	suite := JUnitTestSuite{
+		Name: result.Name,
+		Time: result.Duration.Seconds(),
+	}
+
+	for _, pr := range result.Phases {
+		for _, ar := range pr.Actions {
+			suite.Tests++
+			tc := JUnitTestCase{
+				Name:      fmt.Sprintf("%s/%s", pr.Name, ar.Action),
+				ClassName: result.Name,
+				Time:      ar.Duration.Seconds(),
+			}
+			if ar.Status == StatusFail {
+				suite.Failures++
+				tc.Failure = &JUnitFailure{
+					Message: ar.Error,
+					Content: ar.Output,
+				}
+			}
+			suite.Cases = append(suite.Cases, tc)
+		}
+	}
+
+	suites := JUnitTestSuites{Suites: []JUnitTestSuite{suite}}
+	data, err := xml.MarshalIndent(suites, "", "  ")
+	if err != nil {
+		return fmt.Errorf("marshal JUnit XML: %w", err)
+	}
+
+	header := []byte(xml.Header)
+	data = append(header, data...)
+	return os.WriteFile(path, data, 0644)
+}
+
+// --- Baseline comparison ---
+
+// BaselineCompare compares a result against a baseline JSON file.
+// Returns a list of regressions (actions that passed in baseline but fail now).
+func BaselineCompare(result *ScenarioResult, baselinePath string) ([]string, error) {
+	data, err := os.ReadFile(baselinePath)
+	if err != nil {
+		return nil, fmt.Errorf("read baseline: %w", err)
+	}
+	var baseline ScenarioResult
+	if err := json.Unmarshal(data, &baseline); err != nil {
+		return nil, fmt.Errorf("parse baseline: %w", err)
+	}
+
+	// Build map of baseline action statuses.
+	baseActions := make(map[string]ResultStatus)
+	for _, pr := range baseline.Phases {
+		for _, ar := range pr.Actions {
+			key := fmt.Sprintf("%s/%s", pr.Name, ar.Action)
+			baseActions[key] = ar.Status
+		}
+	}
+
+	var regressions []string
+	for _, pr := range result.Phases {
+		for _, ar := range pr.Actions {
+			key := fmt.Sprintf("%s/%s", pr.Name, ar.Action)
+			if baseStatus, ok := baseActions[key]; ok {
+				if baseStatus == StatusPass && ar.Status == StatusFail {
+					regressions = append(regressions, fmt.Sprintf("%s: was PASS, now FAIL: %s", key, ar.Error))
+				}
+			}
+		}
+	}
+	return regressions, nil
+}

weed/storage/blockvol/testrunner/reporter_html.go

@@ -0,0 +1,301 @@
+package testrunner
+
+import (
+	"encoding/json"
+	"fmt"
+	"html/template"
+	"os"
+	"sort"
+	"strings"
+	"time"
+)
+
+// WriteHTMLReport generates a single-file HTML report from a ScenarioResult.
+func WriteHTMLReport(result *ScenarioResult, path string) error {
+	data := buildHTMLData(result)
+
+	f, err := os.Create(path)
+	if err != nil {
+		return fmt.Errorf("create %s: %w", path, err)
+	}
+	defer f.Close()
+
+	tmpl, err := template.New("report").Parse(htmlTemplate)
+	if err != nil {
+		return fmt.Errorf("parse template: %w", err)
+	}
+
+	return tmpl.Execute(f, data)
+}
+
+// htmlData is the template context.
+type htmlData struct {
+	Name      string
+	Status    string
+	StatusCSS string
+	Duration  string
+	Timestamp string
+	Phases    []htmlPhase
+	Perf      []htmlPerfEntry
+	Metrics   []htmlMetricsGroup
+	Artifacts []ArtifactEntry
+	Error     string
+	HasPerf   bool
+	HasMetrics bool
+	HasArtifacts bool
+}
+
+type htmlPhase struct {
+	Name     string
+	Status   string
+	StatusCSS string
+	Duration string
+	Actions  []htmlAction
+}
+
+type htmlAction struct {
+	Name     string
+	Status   string
+	StatusCSS string
+	Duration string
+	Output   string
+	Error    string
+	YAML     string
+}
+
+type htmlPerfEntry struct {
+	Name  string
+	Count int
+	Mean  string
+	P50   string
+	P90   string
+	P99   string
+	Min   string
+	Max   string
+}
+
+type htmlMetricsGroup struct {
+	Name    string
+	Entries []htmlMetricEntry
+}
+
+type htmlMetricEntry struct {
+	Name  string
+	Value string
+}
+
+func buildHTMLData(result *ScenarioResult) htmlData {
+	d := htmlData{
+		Name:      result.Name,
+		Status:    string(result.Status),
+		StatusCSS: statusCSS(result.Status),
+		Duration:  result.Duration.Round(time.Millisecond).String(),
+		Timestamp: time.Now().Format(time.RFC3339),
+		Error:     result.Error,
+		Artifacts: result.Artifacts,
+		HasArtifacts: len(result.Artifacts) > 0,
+	}
+
+	for _, pr := range result.Phases {
+		hp := htmlPhase{
+			Name:      pr.Name,
+			Status:    string(pr.Status),
+			StatusCSS: statusCSS(pr.Status),
+			Duration:  pr.Duration.Round(time.Millisecond).String(),
+		}
+		for _, ar := range pr.Actions {
+			hp.Actions = append(hp.Actions, htmlAction{
+				Name:      ar.Action,
+				Status:    string(ar.Status),
+				StatusCSS: statusCSS(ar.Status),
+				Duration:  ar.Duration.Round(time.Millisecond).String(),
+				Output:    ar.Output,
+				Error:     ar.Error,
+				YAML:      ar.YAML,
+			})
+		}
+		d.Phases = append(d.Phases, hp)
+	}
+
+	// Extract perf stats from vars.
+	if len(result.Vars) > 0 {
+		d.Perf = extractHTMLPerf(result.Vars)
+		d.HasPerf = len(d.Perf) > 0
+		d.Metrics = extractHTMLMetrics(result.Vars)
+		d.HasMetrics = len(d.Metrics) > 0
+	}
+
+	return d
+}
+
+func statusCSS(s ResultStatus) string {
+	switch s {
+	case StatusPass:
+		return "pass"
+	case StatusFail:
+		return "fail"
+	case StatusSkip:
+		return "skip"
+	default:
+		return ""
+	}
+}
+
+func extractHTMLPerf(vars map[string]string) []htmlPerfEntry {
+	var entries []htmlPerfEntry
+	for key, val := range vars {
+		var ps PerfStats
+		if err := json.Unmarshal([]byte(val), &ps); err == nil && ps.Count > 0 {
+			entries = append(entries, perfToHTML(key, ps))
+			continue
+		}
+		if ps, ok := parsePerfStatsLine(val); ok {
+			entries = append(entries, perfToHTML(key, ps))
+		}
+	}
+	sort.Slice(entries, func(i, j int) bool { return entries[i].Name < entries[j].Name })
+	return entries
+}
+
+func perfToHTML(name string, ps PerfStats) htmlPerfEntry {
+	return htmlPerfEntry{
+		Name:  name,
+		Count: ps.Count,
+		Mean:  fmt.Sprintf("%.1f", ps.Mean),
+		P50:   fmt.Sprintf("%.1f", ps.P50),
+		P90:   fmt.Sprintf("%.1f", ps.P90),
+		P99:   fmt.Sprintf("%.1f", ps.P99),
+		Min:   fmt.Sprintf("%.1f", ps.Min),
+		Max:   fmt.Sprintf("%.1f", ps.Max),
+	}
+}
+
+func extractHTMLMetrics(vars map[string]string) []htmlMetricsGroup {
+	var groups []htmlMetricsGroup
+	for key, val := range vars {
+		var ms MetricsSample
+		if err := json.Unmarshal([]byte(val), &ms); err != nil || len(ms.Metrics) == 0 {
+			continue
+		}
+		label := key
+		if ms.Target != "" {
+			label = ms.Target
+		}
+		names := make([]string, 0, len(ms.Metrics))
+		for n := range ms.Metrics {
+			names = append(names, n)
+		}
+		sort.Strings(names)
+		g := htmlMetricsGroup{Name: label}
+		for _, n := range names {
+			g.Entries = append(g.Entries, htmlMetricEntry{
+				Name:  n,
+				Value: formatMetricValue(ms.Metrics[n]),
+			})
+		}
+		groups = append(groups, g)
+	}
+	sort.Slice(groups, func(i, j int) bool { return groups[i].Name < groups[j].Name })
+	return groups
+}
+
+var htmlTemplate = strings.TrimSpace(`
+<!DOCTYPE html>
+<html lang="en">
+<head>
+<meta charset="utf-8">
+<title>{{.Name}} — Test Report</title>
+<style>
+  * { margin: 0; padding: 0; box-sizing: border-box; }
+  body { font-family: -apple-system, BlinkMacSystemFont, "Segoe UI", Roboto, monospace; background: #1a1a2e; color: #e0e0e0; padding: 24px; }
+  h1 { font-size: 1.4em; margin-bottom: 8px; }
+  h2 { font-size: 1.1em; margin: 24px 0 8px; color: #a0a0c0; }
+  .header { display: flex; align-items: center; gap: 12px; margin-bottom: 16px; }
+  .badge { padding: 4px 12px; border-radius: 4px; font-weight: bold; font-size: 0.85em; }
+  .badge.pass { background: #2d6a4f; color: #b7e4c7; }
+  .badge.fail { background: #922b21; color: #f5b7b1; }
+  .badge.skip { background: #7d6608; color: #f9e79f; }
+  .meta { color: #888; font-size: 0.85em; }
+  table { border-collapse: collapse; width: 100%; margin-top: 8px; }
+  th, td { text-align: left; padding: 6px 12px; border-bottom: 1px solid #2a2a4a; }
+  th { color: #a0a0c0; font-size: 0.8em; text-transform: uppercase; }
+  tr:hover { background: #2a2a4a; }
+  .phase-row { background: #16213e; font-weight: bold; }
+  .action-row td:first-child { padding-left: 32px; }
+  .error { color: #e74c3c; font-size: 0.85em; padding: 4px 12px 4px 32px; }
+  .output { color: #888; font-size: 0.8em; padding: 2px 12px 2px 32px; white-space: pre-wrap; word-break: break-all; max-height: 200px; overflow-y: auto; }
+  .perf-table th, .perf-table td { text-align: right; }
+  .perf-table th:first-child, .perf-table td:first-child { text-align: left; }
+  .artifact-list { list-style: none; }
+  .artifact-list li { padding: 4px 0; font-size: 0.85em; color: #a0a0c0; }
+  .artifact-list .size { color: #666; margin-left: 8px; }
+</style>
+</head>
+<body>
+  <div class="header">
+    <h1>{{.Name}}</h1>
+    <span class="badge {{.StatusCSS}}">{{.Status}}</span>
+    <span class="meta">{{.Duration}}</span>
+  </div>
+  <div class="meta">Generated: {{.Timestamp}}</div>
+
+  <h2>Phases &amp; Actions</h2>
+  <table>
+    <tr><th>Name</th><th>Status</th><th>Duration</th></tr>
+    {{range .Phases}}
+    <tr class="phase-row">
+      <td>{{.Name}}</td>
+      <td><span class="badge {{.StatusCSS}}">{{.Status}}</span></td>
+      <td>{{.Duration}}</td>
+    </tr>
+    {{range .Actions}}
+    <tr class="action-row">
+      <td>{{.Name}}</td>
+      <td><span class="badge {{.StatusCSS}}">{{.Status}}</span></td>
+      <td>{{.Duration}}</td>
+    </tr>
+    {{if .Error}}<tr><td colspan="3" class="error">ERROR: {{.Error}}</td></tr>{{end}}
+    {{if .Output}}<tr><td colspan="3" class="output">{{.Output}}</td></tr>{{end}}
+    {{if .YAML}}<tr><td colspan="3" style="padding-left:32px"><details><summary style="cursor:pointer;color:#a0a0c0;font-size:0.8em">YAML definition</summary><pre class="output">{{.YAML}}</pre></details></td></tr>{{end}}
+    {{end}}
+    {{end}}
+  </table>
+
+  {{if .HasPerf}}
+  <h2>Performance</h2>
+  <table class="perf-table">
+    <tr><th>Metric</th><th>Count</th><th>Mean</th><th>P50</th><th>P90</th><th>P99</th><th>Min</th><th>Max</th></tr>
+    {{range .Perf}}
+    <tr><td>{{.Name}}</td><td>{{.Count}}</td><td>{{.Mean}}</td><td>{{.P50}}</td><td>{{.P90}}</td><td>{{.P99}}</td><td>{{.Min}}</td><td>{{.Max}}</td></tr>
+    {{end}}
+  </table>
+  {{end}}
+
+  {{if .HasMetrics}}
+  {{range .Metrics}}
+  <h2>Metrics ({{.Name}})</h2>
+  <table>
+    <tr><th>Metric</th><th>Value</th></tr>
+    {{range .Entries}}
+    <tr><td>{{.Name}}</td><td>{{.Value}}</td></tr>
+    {{end}}
+  </table>
+  {{end}}
+  {{end}}
+
+  {{if .HasArtifacts}}
+  <h2>Artifacts</h2>
+  <ul class="artifact-list">
+    {{range .Artifacts}}
+    <li>{{.Agent}}: {{.Path}} <span class="size">({{.Size}} bytes)</span></li>
+    {{end}}
+  </ul>
+  {{end}}
+
+  {{if .Error}}
+  <h2>Error</h2>
+  <pre class="error" style="padding:12px">{{.Error}}</pre>
+  {{end}}
+</body>
+</html>
+`)

weed/storage/blockvol/testrunner/reporter_html_test.go

@@ -0,0 +1,169 @@
+package testrunner
+
+import (
+	"os"
+	"path/filepath"
+	"strings"
+	"testing"
+	"time"
+)
+
+func TestWriteHTMLReport_Basic(t *testing.T) {
+	result := &ScenarioResult{
+		Name:     "html-test",
+		Status:   StatusPass,
+		Duration: 2500 * time.Millisecond,
+		Phases: []PhaseResult{
+			{
+				Name:     "setup",
+				Status:   StatusPass,
+				Duration: 1000 * time.Millisecond,
+				Actions: []ActionResult{
+					{Action: "build_deploy", Status: StatusPass, Duration: 800 * time.Millisecond},
+					{Action: "start_target", Status: StatusPass, Duration: 200 * time.Millisecond},
+				},
+			},
+			{
+				Name:     "verify",
+				Status:   StatusPass,
+				Duration: 500 * time.Millisecond,
+				Actions: []ActionResult{
+					{Action: "assert_equal", Status: StatusPass, Duration: 5 * time.Millisecond},
+				},
+			},
+		},
+	}
+
+	dir := t.TempDir()
+	path := filepath.Join(dir, "report.html")
+
+	if err := WriteHTMLReport(result, path); err != nil {
+		t.Fatalf("WriteHTMLReport: %v", err)
+	}
+
+	data, err := os.ReadFile(path)
+	if err != nil {
+		t.Fatalf("read: %v", err)
+	}
+
+	html := string(data)
+	if !strings.Contains(html, "html-test") {
+		t.Error("HTML should contain scenario name")
+	}
+	if !strings.Contains(html, "PASS") {
+		t.Error("HTML should contain PASS status")
+	}
+	if !strings.Contains(html, "build_deploy") {
+		t.Error("HTML should contain action name")
+	}
+	if !strings.Contains(html, "<!DOCTYPE html>") {
+		t.Error("HTML should be valid HTML document")
+	}
+}
+
+func TestWriteHTMLReport_WithFailure(t *testing.T) {
+	result := &ScenarioResult{
+		Name:     "fail-html",
+		Status:   StatusFail,
+		Duration: 300 * time.Millisecond,
+		Error:    "test <broke> & failed",
+		Phases: []PhaseResult{
+			{
+				Name:   "main",
+				Status: StatusFail,
+				Actions: []ActionResult{
+					{Action: "bad_step", Status: StatusFail, Error: "something <bad> happened"},
+				},
+			},
+		},
+	}
+
+	dir := t.TempDir()
+	path := filepath.Join(dir, "report.html")
+
+	if err := WriteHTMLReport(result, path); err != nil {
+		t.Fatalf("WriteHTMLReport: %v", err)
+	}
+
+	data, _ := os.ReadFile(path)
+	html := string(data)
+
+	if !strings.Contains(html, "FAIL") {
+		t.Error("HTML should contain FAIL")
+	}
+	// Verify HTML escaping (review item #5).
+	if strings.Contains(html, "<broke>") {
+		t.Error("HTML should escape angle brackets in error text")
+	}
+	if !strings.Contains(html, "&lt;broke&gt;") {
+		t.Error("HTML should contain escaped angle brackets")
+	}
+}
+
+func TestWriteHTMLReport_WithPerfAndMetrics(t *testing.T) {
+	result := &ScenarioResult{
+		Name:     "perf-html",
+		Status:   StatusPass,
+		Duration: 1 * time.Second,
+		Phases:   []PhaseResult{{Name: "bench", Status: StatusPass}},
+		Vars: map[string]string{
+			"iops":    `{"count":100,"min":9000,"max":15000,"mean":12345,"stddev":500,"p50":12200,"p90":13500,"p99":14000}`,
+			"metrics": `{"target":"primary","metrics":{"blockvol_write_bytes_total":1234567,"blockvol_read_bytes_total":987654}}`,
+		},
+	}
+
+	dir := t.TempDir()
+	path := filepath.Join(dir, "report.html")
+
+	if err := WriteHTMLReport(result, path); err != nil {
+		t.Fatalf("WriteHTMLReport: %v", err)
+	}
+
+	data, _ := os.ReadFile(path)
+	html := string(data)
+
+	if !strings.Contains(html, "Performance") {
+		t.Error("HTML should contain Performance section")
+	}
+	if !strings.Contains(html, "iops") {
+		t.Error("HTML should contain perf metric name")
+	}
+	if !strings.Contains(html, "Metrics") {
+		t.Error("HTML should contain Metrics section")
+	}
+	if !strings.Contains(html, "blockvol_write_bytes_total") {
+		t.Error("HTML should contain metric name")
+	}
+}
+
+func TestWriteHTMLReport_WithArtifacts(t *testing.T) {
+	result := &ScenarioResult{
+		Name:     "artifacts-html",
+		Status:   StatusFail,
+		Duration: 500 * time.Millisecond,
+		Artifacts: []ArtifactEntry{
+			{Agent: "m01", Path: "/tmp/artifacts/m01/target.log", Size: 4096},
+			{Agent: "m02", Path: "/tmp/artifacts/m02/dmesg.log", Size: 1024},
+		},
+	}
+
+	dir := t.TempDir()
+	path := filepath.Join(dir, "report.html")
+
+	if err := WriteHTMLReport(result, path); err != nil {
+		t.Fatalf("WriteHTMLReport: %v", err)
+	}
+
+	data, _ := os.ReadFile(path)
+	html := string(data)
+
+	if !strings.Contains(html, "Artifacts") {
+		t.Error("HTML should contain Artifacts section")
+	}
+	if !strings.Contains(html, "target.log") {
+		t.Error("HTML should contain artifact path")
+	}
+	if !strings.Contains(html, "4096 bytes") {
+		t.Error("HTML should contain artifact size")
+	}
+}

weed/storage/blockvol/testrunner/reporter_test.go

@@ -0,0 +1,368 @@
+package testrunner
+
+import (
+	"bytes"
+	"encoding/json"
+	"encoding/xml"
+	"os"
+	"path/filepath"
+	"strings"
+	"testing"
+	"time"
+)
+
+func TestPrintSummary(t *testing.T) {
+	result := &ScenarioResult{
+		Name:     "test-scenario",
+		Status:   StatusPass,
+		Duration: 1500 * time.Millisecond,
+		Phases: []PhaseResult{
+			{
+				Name:     "setup",
+				Status:   StatusPass,
+				Duration: 800 * time.Millisecond,
+				Actions: []ActionResult{
+					{Action: "build_deploy", Status: StatusPass, Duration: 500 * time.Millisecond},
+					{Action: "start_target", Status: StatusPass, Duration: 300 * time.Millisecond},
+				},
+			},
+			{
+				Name:     "verify",
+				Status:   StatusPass,
+				Duration: 700 * time.Millisecond,
+				Actions: []ActionResult{
+					{Action: "assert_equal", Status: StatusPass, Duration: 1 * time.Millisecond},
+				},
+			},
+		},
+	}
+
+	var buf bytes.Buffer
+	PrintSummary(&buf, result)
+	output := buf.String()
+
+	if !strings.Contains(output, "test-scenario") {
+		t.Error("summary should contain scenario name")
+	}
+	if !strings.Contains(output, "PASS") {
+		t.Error("summary should contain PASS")
+	}
+	if !strings.Contains(output, "3 actions") {
+		t.Error("summary should contain action count")
+	}
+}
+
+func TestPrintSummary_WithFailure(t *testing.T) {
+	result := &ScenarioResult{
+		Name:     "fail-scenario",
+		Status:   StatusFail,
+		Duration: 500 * time.Millisecond,
+		Error:    "something broke",
+		Phases: []PhaseResult{
+			{
+				Name:   "main",
+				Status: StatusFail,
+				Actions: []ActionResult{
+					{Action: "step1", Status: StatusFail, Error: "boom"},
+				},
+			},
+		},
+	}
+
+	var buf bytes.Buffer
+	PrintSummary(&buf, result)
+	output := buf.String()
+
+	if !strings.Contains(output, "FAIL") {
+		t.Error("summary should contain FAIL")
+	}
+	if !strings.Contains(output, "boom") {
+		t.Error("summary should contain error message")
+	}
+}
+
+func TestWriteJSON(t *testing.T) {
+	result := &ScenarioResult{
+		Name:     "json-test",
+		Status:   StatusPass,
+		Duration: 100 * time.Millisecond,
+	}
+
+	dir := t.TempDir()
+	path := filepath.Join(dir, "result.json")
+
+	if err := WriteJSON(result, path); err != nil {
+		t.Fatalf("WriteJSON: %v", err)
+	}
+
+	data, err := os.ReadFile(path)
+	if err != nil {
+		t.Fatalf("read: %v", err)
+	}
+
+	var loaded ScenarioResult
+	if err := json.Unmarshal(data, &loaded); err != nil {
+		t.Fatalf("unmarshal: %v", err)
+	}
+
+	if loaded.Name != "json-test" {
+		t.Errorf("name = %q", loaded.Name)
+	}
+	if loaded.Status != StatusPass {
+		t.Errorf("status = %s", loaded.Status)
+	}
+}
+
+func TestWriteJUnitXML(t *testing.T) {
+	result := &ScenarioResult{
+		Name:     "junit-test",
+		Status:   StatusFail,
+		Duration: 200 * time.Millisecond,
+		Phases: []PhaseResult{
+			{
+				Name:   "main",
+				Status: StatusFail,
+				Actions: []ActionResult{
+					{Action: "ok_step", Status: StatusPass, Duration: 50 * time.Millisecond},
+					{Action: "fail_step", Status: StatusFail, Duration: 100 * time.Millisecond, Error: "oops"},
+				},
+			},
+		},
+	}
+
+	dir := t.TempDir()
+	path := filepath.Join(dir, "result.xml")
+
+	if err := WriteJUnitXML(result, path); err != nil {
+		t.Fatalf("WriteJUnitXML: %v", err)
+	}
+
+	data, err := os.ReadFile(path)
+	if err != nil {
+		t.Fatalf("read: %v", err)
+	}
+
+	var suites JUnitTestSuites
+	if err := xml.Unmarshal(data, &suites); err != nil {
+		t.Fatalf("unmarshal XML: %v", err)
+	}
+
+	if len(suites.Suites) != 1 {
+		t.Fatalf("suites = %d, want 1", len(suites.Suites))
+	}
+	suite := suites.Suites[0]
+	if suite.Tests != 2 {
+		t.Errorf("tests = %d, want 2", suite.Tests)
+	}
+	if suite.Failures != 1 {
+		t.Errorf("failures = %d, want 1", suite.Failures)
+	}
+}
+
+func TestPrintSummary_WithPerfTable(t *testing.T) {
+	result := &ScenarioResult{
+		Name:     "perf-scenario",
+		Status:   StatusPass,
+		Duration: 2 * time.Second,
+		Phases: []PhaseResult{
+			{
+				Name:   "bench",
+				Status: StatusPass,
+				Actions: []ActionResult{
+					{Action: "dd_write", Status: StatusPass, Duration: 1 * time.Second},
+				},
+			},
+		},
+		Vars: map[string]string{
+			"iops": `{"count":100,"min":9000,"max":15000,"mean":12345,"stddev":500,"p50":12200,"p90":13500,"p99":14000}`,
+		},
+	}
+
+	var buf bytes.Buffer
+	PrintSummary(&buf, result)
+	output := buf.String()
+
+	if !strings.Contains(output, "Performance") {
+		t.Error("summary should contain Performance table header")
+	}
+	if !strings.Contains(output, "iops") {
+		t.Error("summary should contain metric name 'iops'")
+	}
+	if !strings.Contains(output, "12345") {
+		t.Error("summary should contain mean value")
+	}
+}
+
+func TestPrintSummary_WithMetricsTable(t *testing.T) {
+	result := &ScenarioResult{
+		Name:     "metrics-scenario",
+		Status:   StatusPass,
+		Duration: 1 * time.Second,
+		Phases:   []PhaseResult{{Name: "m", Status: StatusPass}},
+		Vars: map[string]string{
+			"primary_metrics": `{"target":"primary","metrics":{"blockvol_write_bytes_total":1234567,"blockvol_read_bytes_total":987654}}`,
+		},
+	}
+
+	var buf bytes.Buffer
+	PrintSummary(&buf, result)
+	output := buf.String()
+
+	if !strings.Contains(output, "Metrics (primary)") {
+		t.Error("summary should contain Metrics section with target name")
+	}
+	if !strings.Contains(output, "blockvol_write_bytes_total") {
+		t.Error("summary should contain metric name")
+	}
+}
+
+func TestPrintSummary_WithPerfStatsLine(t *testing.T) {
+	result := &ScenarioResult{
+		Name:     "perf-line-scenario",
+		Status:   StatusPass,
+		Duration: 1 * time.Second,
+		Phases:   []PhaseResult{{Name: "p", Status: StatusPass}},
+		Vars: map[string]string{
+			"lat_stats": "lat_us: n=50 mean=420.5 stddev=30.1 p50=415.0 p90=460.0 p99=500.0 min=350.0 max=550.0",
+		},
+	}
+
+	var buf bytes.Buffer
+	PrintSummary(&buf, result)
+	output := buf.String()
+
+	if !strings.Contains(output, "Performance") {
+		t.Error("summary should render perf table from FormatStats-style line")
+	}
+	if !strings.Contains(output, "lat_stats") {
+		t.Error("summary should contain metric name from var key")
+	}
+}
+
+func TestParsePerfStatsLine(t *testing.T) {
+	line := "iops: n=100 mean=12345.00 stddev=500.00 p50=12200.00 p90=13500.00 p99=14000.00 min=9000.00 max=15000.00"
+	ps, ok := parsePerfStatsLine(line)
+	if !ok {
+		t.Fatal("should parse FormatStats-style line")
+	}
+	if ps.Count != 100 {
+		t.Errorf("count = %d, want 100", ps.Count)
+	}
+	if ps.Mean != 12345 {
+		t.Errorf("mean = %f, want 12345", ps.Mean)
+	}
+	if ps.P99 != 14000 {
+		t.Errorf("p99 = %f, want 14000", ps.P99)
+	}
+}
+
+func TestParsePerfStatsLine_Invalid(t *testing.T) {
+	_, ok := parsePerfStatsLine("not a perf line")
+	if ok {
+		t.Error("should not parse non-perf text")
+	}
+}
+
+func TestFormatInt(t *testing.T) {
+	tests := []struct {
+		n    int64
+		want string
+	}{
+		{0, "0"},
+		{999, "999"},
+		{1000, "1,000"},
+		{1234567, "1,234,567"},
+	}
+	for _, tt := range tests {
+		got := formatInt(tt.n)
+		if got != tt.want {
+			t.Errorf("formatInt(%d) = %q, want %q", tt.n, got, tt.want)
+		}
+	}
+}
+
+func TestWriteJSON_WithVars(t *testing.T) {
+	result := &ScenarioResult{
+		Name:   "json-vars-test",
+		Status: StatusPass,
+		Vars: map[string]string{
+			"key1": "value1",
+			"key2": `{"nested":"json"}`,
+		},
+		Artifacts: []ArtifactEntry{
+			{Agent: "m01", Path: "/tmp/a/log.txt", Size: 512},
+		},
+	}
+
+	dir := t.TempDir()
+	path := filepath.Join(dir, "result.json")
+
+	if err := WriteJSON(result, path); err != nil {
+		t.Fatalf("WriteJSON: %v", err)
+	}
+
+	data, _ := os.ReadFile(path)
+	var loaded ScenarioResult
+	json.Unmarshal(data, &loaded)
+
+	if loaded.Vars["key1"] != "value1" {
+		t.Errorf("vars[key1] = %q", loaded.Vars["key1"])
+	}
+	if loaded.Vars["key2"] != `{"nested":"json"}` {
+		t.Errorf("vars[key2] = %q", loaded.Vars["key2"])
+	}
+	if len(loaded.Artifacts) != 1 {
+		t.Fatalf("artifacts = %d, want 1", len(loaded.Artifacts))
+	}
+	if loaded.Artifacts[0].Agent != "m01" {
+		t.Errorf("artifact agent = %q", loaded.Artifacts[0].Agent)
+	}
+}
+
+func TestBaselineCompare(t *testing.T) {
+	baseline := &ScenarioResult{
+		Name: "baseline",
+		Phases: []PhaseResult{
+			{
+				Name: "main",
+				Actions: []ActionResult{
+					{Action: "step1", Status: StatusPass},
+					{Action: "step2", Status: StatusPass},
+					{Action: "step3", Status: StatusFail},
+				},
+			},
+		},
+	}
+
+	dir := t.TempDir()
+	baselinePath := filepath.Join(dir, "baseline.json")
+	if err := WriteJSON(baseline, baselinePath); err != nil {
+		t.Fatalf("write baseline: %v", err)
+	}
+
+	current := &ScenarioResult{
+		Name: "current",
+		Phases: []PhaseResult{
+			{
+				Name: "main",
+				Actions: []ActionResult{
+					{Action: "step1", Status: StatusPass}, // still pass
+					{Action: "step2", Status: StatusFail, Error: "regression"}, // regression!
+					{Action: "step3", Status: StatusFail}, // was fail, still fail (not regression)
+				},
+			},
+		},
+	}
+
+	regressions, err := BaselineCompare(current, baselinePath)
+	if err != nil {
+		t.Fatalf("compare: %v", err)
+	}
+
+	if len(regressions) != 1 {
+		t.Fatalf("regressions = %d, want 1", len(regressions))
+	}
+	if !strings.Contains(regressions[0], "step2") {
+		t.Errorf("regression should mention step2: %s", regressions[0])
+	}
+}

weed/storage/blockvol/testrunner/scenarios/consistency-epoch.yaml

@@ -0,0 +1,80 @@
+name: consistency-epoch
+timeout: 5m
+env:
+  repo_dir: "C:/work/seaweedfs"
+
+topology:
+  nodes:
+    target_node:
+      host: "192.168.1.184"
+      user: testdev
+      key: "C:/work/dev_server/testdev_key"
+    client_node:
+      host: "192.168.1.181"
+      user: testdev
+      key: "C:/work/dev_server/testdev_key"
+
+targets:
+  primary:
+    node: target_node
+    vol_size: 100M
+    iscsi_port: 3260
+    admin_port: 8080
+    iqn_suffix: epoch-primary
+
+phases:
+  - name: setup
+    actions:
+      - action: build_deploy
+      - action: start_target
+        target: primary
+        create: "true"
+
+  - name: epoch_monotonicity
+    actions:
+      - action: assign
+        target: primary
+        epoch: "1"
+        role: primary
+        lease_ttl: 30s
+      - action: assert_status
+        target: primary
+        role: primary
+      - action: assign
+        target: primary
+        epoch: "2"
+        role: primary
+        lease_ttl: 30s
+      - action: status
+        target: primary
+        save_as: status_e2
+      - action: print
+        msg: "status after epoch 2: {{ status_e2 }}"
+
+  - name: stale_epoch_reject
+    actions:
+      - action: exec
+        node: target_node
+        cmd: "curl -s -w '\\n%{http_code}' -X POST -H 'Content-Type: application/json' -d '{\"epoch\":1,\"role\":1,\"lease_ttl_ms\":30000}' http://127.0.0.1:8080/assign"
+        save_as: stale_result
+      - action: print
+        msg: "stale epoch result: {{ stale_result }}"
+
+  - name: epoch_persist
+    actions:
+      - action: stop_target
+        target: primary
+      - action: start_target
+        target: primary
+        create: "false"
+      - action: status
+        target: primary
+        save_as: post_restart_status
+      - action: print
+        msg: "status after restart: {{ post_restart_status }}"
+
+  - name: cleanup
+    always: true
+    actions:
+      - action: stop_all_targets
+        ignore_error: true

weed/storage/blockvol/testrunner/scenarios/consistency-lease.yaml

@@ -0,0 +1,80 @@
+name: consistency-lease
+timeout: 5m
+env:
+  repo_dir: "C:/work/seaweedfs"
+
+topology:
+  nodes:
+    target_node:
+      host: "192.168.1.184"
+      user: testdev
+      key: "C:/work/dev_server/testdev_key"
+    client_node:
+      host: "192.168.1.181"
+      user: testdev
+      key: "C:/work/dev_server/testdev_key"
+
+targets:
+  primary:
+    node: target_node
+    vol_size: 100M
+    iscsi_port: 3260
+    admin_port: 8080
+    iqn_suffix: lease-primary
+  replica:
+    node: target_node
+    vol_size: 100M
+    iscsi_port: 3261
+    admin_port: 8081
+    replica_data_port: 9011
+    replica_ctrl_port: 9012
+    iqn_suffix: lease-replica
+
+phases:
+  - name: setup
+    actions:
+      - action: build_deploy
+      - action: start_target
+        target: primary
+        create: "true"
+      - action: start_target
+        target: replica
+        create: "true"
+
+  - name: lease_expiry
+    actions:
+      - action: assign
+        target: primary
+        epoch: "1"
+        role: primary
+        lease_ttl: 5s
+      - action: assert_status
+        target: primary
+        role: primary
+      - action: sleep
+        duration: 7s
+      - action: assert_status
+        target: primary
+        has_lease: "false"
+
+  - name: split_brain_prevention
+    actions:
+      - action: assign
+        target: primary
+        epoch: "2"
+        role: primary
+        lease_ttl: 30s
+      - action: assign
+        target: replica
+        epoch: "2"
+        role: replica
+        lease_ttl: 30s
+      - action: set_replica
+        target: primary
+        replica: replica
+
+  - name: cleanup
+    always: true
+    actions:
+      - action: stop_all_targets
+        ignore_error: true

weed/storage/blockvol/testrunner/scenarios/coord-dev-cycle.yaml

@@ -0,0 +1,139 @@
+name: coord-dev-cycle
+timeout: 5m
+env:
+  repo_dir: "/c/work/seaweedfs"
+
+topology:
+  agents:
+    target_agent: "192.168.1.184:9100"
+    client_agent: "192.168.1.181:9100"
+
+  nodes:
+    target_node:
+      host: "192.168.1.184"
+      agent: target_agent
+    client_node:
+      host: "192.168.1.181"
+      agent: client_agent
+
+targets:
+  primary:
+    node: target_node
+    vol_size: 100M
+    iscsi_port: 3260
+    admin_port: 8080
+    iqn_suffix: dev-primary
+  replica:
+    node: target_node
+    vol_size: 100M
+    iscsi_port: 3261
+    admin_port: 8081
+    replica_data_port: 9011
+    replica_ctrl_port: 9012
+    rebuild_port: 9013
+    iqn_suffix: dev-replica
+
+phases:
+  # Phase 0: Kill stale processes from previous runs
+  - name: pre_cleanup
+    actions:
+      - action: kill_stale
+        node: target_node
+        process: iscsi-target-test
+        ignore_error: true
+      - action: kill_stale
+        node: client_node
+        iscsi_cleanup: "true"
+        ignore_error: true
+
+  # Phase 1: Build and deploy iscsi-target binary
+  - name: build_deploy
+    actions:
+      - action: build_deploy
+
+  # Phase 2: Start targets, set up HA replication
+  - name: setup
+    actions:
+      - action: start_target
+        target: primary
+        create: "true"
+      - action: start_target
+        target: replica
+        create: "true"
+      - action: assign
+        target: replica
+        epoch: "1"
+        role: replica
+        lease_ttl: 30s
+      - action: assign
+        target: primary
+        epoch: "1"
+        role: primary
+        lease_ttl: 30s
+      - action: set_replica
+        target: primary
+        replica: replica
+      - action: iscsi_login
+        target: primary
+        node: client_node
+        save_as: device
+
+  # Phase 3: Write data, verify replication
+  - name: write_and_replicate
+    actions:
+      - action: dd_write
+        node: client_node
+        device: "{{ device }}"
+        bs: 1M
+        count: "1"
+        save_as: written_md5
+      - action: wait_lsn
+        target: replica
+        min_lsn: "1"
+        timeout: 10s
+
+  # Phase 4: Kill primary, promote replica
+  - name: failover
+    actions:
+      - action: kill_target
+        target: primary
+      - action: iscsi_cleanup
+        node: client_node
+        ignore_error: true
+      - action: assign
+        target: replica
+        epoch: "2"
+        role: primary
+        lease_ttl: 30s
+      - action: wait_role
+        target: replica
+        role: primary
+        timeout: 5s
+
+  # Phase 5: Verify data survived failover
+  - name: verify
+    actions:
+      - action: iscsi_login
+        target: replica
+        node: client_node
+        save_as: device2
+      - action: dd_read_md5
+        node: client_node
+        device: "{{ device2 }}"
+        bs: 1M
+        count: "1"
+        save_as: read_md5
+      - action: assert_equal
+        actual: "{{ read_md5 }}"
+        expected: "{{ written_md5 }}"
+
+  # Phase 6: Cleanup (always runs, even on failure)
+  - name: cleanup
+    always: true
+    actions:
+      - action: iscsi_cleanup
+        node: client_node
+        ignore_error: true
+      - action: stop_all_targets
+        aggressive: "true"
+        ignore_error: true

weed/storage/blockvol/testrunner/scenarios/coord-ha-failover.yaml

@@ -0,0 +1,116 @@
+name: coord-ha-failover
+timeout: 5m
+env:
+  repo_dir: "/opt/work/seaweedfs"
+
+topology:
+  agents:
+    target_agent: "192.168.1.184:9100"
+    client_agent: "192.168.1.181:9100"
+
+  nodes:
+    target_node:
+      host: "192.168.1.184"
+      agent: target_agent
+    client_node:
+      host: "192.168.1.181"
+      agent: client_agent
+
+targets:
+  primary:
+    node: target_node
+    vol_size: 100M
+    iscsi_port: 3260
+    admin_port: 8080
+    iqn_suffix: ha-primary
+  replica:
+    node: target_node
+    vol_size: 100M
+    iscsi_port: 3261
+    admin_port: 8081
+    replica_data_port: 9011
+    replica_ctrl_port: 9012
+    rebuild_port: 9013
+    iqn_suffix: ha-replica
+
+phases:
+  - name: setup
+    actions:
+      - action: start_target
+        target: primary
+        create: "true"
+      - action: start_target
+        target: replica
+        create: "true"
+      - action: assign
+        target: replica
+        epoch: "1"
+        role: replica
+        lease_ttl: 30s
+      - action: assign
+        target: primary
+        epoch: "1"
+        role: primary
+        lease_ttl: 30s
+      - action: set_replica
+        target: primary
+        replica: replica
+      - action: iscsi_login
+        target: primary
+        node: client_node
+        save_as: device
+
+  - name: write_and_replicate
+    actions:
+      - action: dd_write
+        node: client_node
+        device: "{{ device }}"
+        bs: 1M
+        count: "1"
+        save_as: written_md5
+      - action: wait_lsn
+        target: replica
+        min_lsn: "1"
+        timeout: 10s
+
+  - name: failover
+    actions:
+      - action: kill_target
+        target: primary
+      - action: iscsi_cleanup
+        node: client_node
+        ignore_error: true
+      - action: assign
+        target: replica
+        epoch: "2"
+        role: primary
+        lease_ttl: 30s
+      - action: wait_role
+        target: replica
+        role: primary
+        timeout: 5s
+
+  - name: verify
+    actions:
+      - action: iscsi_login
+        target: replica
+        node: client_node
+        save_as: device2
+      - action: dd_read_md5
+        node: client_node
+        device: "{{ device2 }}"
+        bs: 1M
+        count: "1"
+        save_as: read_md5
+      - action: assert_equal
+        actual: "{{ read_md5 }}"
+        expected: "{{ written_md5 }}"
+
+  - name: cleanup
+    always: true
+    actions:
+      - action: iscsi_cleanup
+        node: client_node
+        ignore_error: true
+      - action: stop_all_targets
+        ignore_error: true

weed/storage/blockvol/testrunner/scenarios/coord-smoke-iscsi.yaml

@@ -0,0 +1,66 @@
+name: coord-smoke-iscsi
+timeout: 5m
+env:
+  repo_dir: "/opt/work/seaweedfs"
+
+topology:
+  agents:
+    target_agent: "192.168.1.184:9100"
+    client_agent: "192.168.1.181:9100"
+
+  nodes:
+    target_node:
+      host: "192.168.1.184"
+      agent: target_agent
+    client_node:
+      host: "192.168.1.181"
+      agent: client_agent
+
+targets:
+  primary:
+    node: target_node
+    vol_size: 100M
+    iscsi_port: 3260
+    admin_port: 8080
+    iqn_suffix: coord-smoke-primary
+
+phases:
+  - name: setup
+    actions:
+      - action: start_target
+        target: primary
+        create: "true"
+
+  - name: iscsi_connect
+    actions:
+      - action: iscsi_login
+        target: primary
+        node: client_node
+        save_as: device
+
+  - name: write_verify
+    actions:
+      - action: dd_write
+        node: client_node
+        device: "{{ device }}"
+        bs: 1M
+        count: "1"
+        save_as: written_md5
+      - action: dd_read_md5
+        node: client_node
+        device: "{{ device }}"
+        bs: 1M
+        count: "1"
+        save_as: read_md5
+      - action: assert_equal
+        actual: "{{ written_md5 }}"
+        expected: "{{ read_md5 }}"
+
+  - name: cleanup
+    always: true
+    actions:
+      - action: iscsi_cleanup
+        node: client_node
+        ignore_error: true
+      - action: stop_all_targets
+        ignore_error: true

weed/storage/blockvol/testrunner/scenarios/cp83-snapshot-expand.yaml

@@ -0,0 +1,199 @@
+name: cp83-snapshot-expand
+timeout: 5m
+env:
+  repo_dir: "C:/work/seaweedfs"
+
+topology:
+  nodes:
+    target_node:
+      host: "192.168.1.184"
+      user: testdev
+      key: "C:/work/dev_server/testdev_key"
+    client_node:
+      host: "192.168.1.181"
+      user: testdev
+      key: "C:/work/dev_server/testdev_key"
+
+targets:
+  primary:
+    node: target_node
+    vol_size: 50M
+    iscsi_port: 3262
+    admin_port: 8082
+    iqn_suffix: cp83-snap
+
+phases:
+  # Phase 1: Clean slate + start target
+  - name: setup
+    actions:
+      - action: kill_stale
+        node: target_node
+      - action: kill_stale
+        node: client_node
+        iscsi_cleanup: "true"
+      - action: exec
+        node: target_node
+        cmd: "rm -f /tmp/blockvol-primary.blk.snap.*"
+      - action: build_deploy
+      - action: start_target
+        target: primary
+        create: "true"
+      - action: assign
+        target: primary
+        epoch: "1"
+        role: primary
+        lease_ttl: 120s
+
+  # Phase 2: Connect iSCSI, record original size
+  - name: iscsi_connect
+    actions:
+      - action: iscsi_login
+        target: primary
+        node: client_node
+        save_as: device
+      - action: get_block_size
+        node: client_node
+        device: "{{ device }}"
+        save_as: original_size
+
+  # Phase 3: Write initial data at two offsets
+  - name: write_initial_data
+    actions:
+      # 10 MB at offset 0
+      - action: dd_write
+        node: client_node
+        device: "{{ device }}"
+        bs: 1M
+        count: "10"
+        save_as: md5_at_0
+      # 5 MB at offset 20M (seek=20 with bs=1M)
+      - action: dd_write
+        node: client_node
+        device: "{{ device }}"
+        bs: 1M
+        count: "5"
+        seek: "20"
+        save_as: md5_at_20M
+
+  # Phase 4: Expand volume 50M -> 100M while iSCSI session active
+  - name: expand
+    actions:
+      - action: resize
+        target: primary
+        new_size: 100M
+      - action: iscsi_rescan
+        node: client_node
+      - action: get_block_size
+        node: client_node
+        device: "{{ device }}"
+        save_as: expanded_size
+      - action: assert_greater
+        actual: "{{ expanded_size }}"
+        expected: "{{ original_size }}"
+
+  # Phase 5: Verify original data intact after expand
+  - name: verify_data_after_expand
+    actions:
+      # Read 10 MB at offset 0
+      - action: dd_read_md5
+        node: client_node
+        device: "{{ device }}"
+        bs: 1M
+        count: "10"
+        save_as: verify_md5_at_0
+      - action: assert_equal
+        actual: "{{ verify_md5_at_0 }}"
+        expected: "{{ md5_at_0 }}"
+      # Read 5 MB at offset 20M
+      - action: dd_read_md5
+        node: client_node
+        device: "{{ device }}"
+        bs: 1M
+        count: "5"
+        skip: "20"
+        save_as: verify_md5_at_20M
+      - action: assert_equal
+        actual: "{{ verify_md5_at_20M }}"
+        expected: "{{ md5_at_20M }}"
+
+  # Phase 6: Write to expanded area (beyond original 50M)
+  - name: write_expanded_area
+    actions:
+      # 5 MB at offset 60M (in expanded region)
+      - action: dd_write
+        node: client_node
+        device: "{{ device }}"
+        bs: 1M
+        count: "5"
+        seek: "60"
+        save_as: md5_at_60M
+      - action: dd_read_md5
+        node: client_node
+        device: "{{ device }}"
+        bs: 1M
+        count: "5"
+        skip: "60"
+        save_as: verify_md5_at_60M
+      - action: assert_equal
+        actual: "{{ verify_md5_at_60M }}"
+        expected: "{{ md5_at_60M }}"
+
+  # Phase 7: Create snapshots on expanded volume
+  - name: snapshot_on_expanded
+    actions:
+      - action: snapshot_create
+        target: primary
+        id: "1"
+      - action: snapshot_list
+        target: primary
+        save_as: snap_count_1
+      - action: assert_equal
+        actual: "{{ snap_count_1 }}"
+        expected: "1"
+      - action: snapshot_create
+        target: primary
+        id: "2"
+      - action: snapshot_list
+        target: primary
+        save_as: snap_count_2
+      - action: assert_equal
+        actual: "{{ snap_count_2 }}"
+        expected: "2"
+
+  # Phase 8: Delete snapshots, then expand again (100M -> 150M)
+  - name: delete_snap_and_expand_again
+    actions:
+      - action: snapshot_delete
+        target: primary
+        id: "1"
+      - action: snapshot_delete
+        target: primary
+        id: "2"
+      - action: snapshot_list
+        target: primary
+        save_as: snap_count_0
+      - action: assert_equal
+        actual: "{{ snap_count_0 }}"
+        expected: "0"
+      - action: resize
+        target: primary
+        new_size: 150M
+      - action: iscsi_rescan
+        node: client_node
+      - action: get_block_size
+        node: client_node
+        device: "{{ device }}"
+        save_as: final_size
+      - action: assert_greater
+        actual: "{{ final_size }}"
+        expected: "{{ expanded_size }}"
+
+  # Phase 9: Cleanup (always runs)
+  - name: cleanup
+    always: true
+    actions:
+      - action: iscsi_cleanup
+        node: client_node
+        ignore_error: true
+      - action: stop_all_targets
+        ignore_error: true

weed/storage/blockvol/testrunner/scenarios/cp84-soak-4h.yaml

@@ -0,0 +1,189 @@
+name: cp84-soak-4h
+timeout: 5h
+env:
+  repo_dir: "C:/work/seaweedfs"
+
+topology:
+  nodes:
+    target_node:
+      host: "192.168.1.184"
+      user: testdev
+      key: "C:/work/dev_server/testdev_key"
+    client_node:
+      host: "192.168.1.181"
+      user: testdev
+      key: "C:/work/dev_server/testdev_key"
+
+targets:
+  primary:
+    node: target_node
+    vol_size: 200M
+    iscsi_port: 3260
+    admin_port: 8080
+    iqn_suffix: soak-4h-primary
+  replica:
+    node: target_node
+    vol_size: 200M
+    iscsi_port: 3261
+    admin_port: 8081
+    replica_data_port: 9011
+    replica_ctrl_port: 9012
+    iqn_suffix: soak-4h-replica
+
+phases:
+  # Phase 1: Setup — build, deploy, start targets, wire replication.
+  - name: setup
+    actions:
+      - action: kill_stale
+        node: target_node
+        ignore_error: true
+      - action: build_deploy
+      - action: start_target
+        target: primary
+        create: "true"
+      - action: start_target
+        target: replica
+        create: "true"
+      - action: assign
+        target: replica
+        epoch: "1"
+        role: replica
+        lease_ttl: 30s
+      - action: assign
+        target: primary
+        epoch: "1"
+        role: primary
+        lease_ttl: 30s
+      - action: set_replica
+        target: primary
+        replica: replica
+      - action: iscsi_login
+        target: primary
+        node: client_node
+        save_as: device
+
+  # Phase 2: Baseline metrics scrape (pre-load).
+  - name: baseline_scrape
+    actions:
+      - action: scrape_metrics
+        target: primary
+        save_as: metrics_baseline
+
+  # Phase 3: Steady-state load (2 hours).
+  # Mixed read/write with periodic metrics scrape every 30s.
+  - name: steady_state
+    actions:
+      - action: dd_write
+        node: client_node
+        device: "{{ device }}"
+        bs: 4k
+        count: "256"
+        save_as: ss_write_md5
+      - action: dd_read_md5
+        node: client_node
+        device: "{{ device }}"
+        bs: 4k
+        count: "256"
+        save_as: ss_read_md5
+      - action: assert_equal
+        actual: "{{ ss_read_md5 }}"
+        expected: "{{ ss_write_md5 }}"
+      - action: scrape_metrics
+        target: primary
+        save_as: metrics_steady
+
+  # Phase 4: Inject 200ms replica network delay (fault window = 10 min).
+  - name: fault_inject
+    actions:
+      - action: inject_netem
+        node: target_node
+        target_ip: "127.0.0.1"
+        delay_ms: "200"
+      - action: sleep
+        duration: 5s
+      # Write under fault to verify primary still serves.
+      - action: dd_write
+        node: client_node
+        device: "{{ device }}"
+        bs: 4k
+        count: "64"
+        seek: "512"
+        save_as: fault_write_md5
+      - action: dd_read_md5
+        node: client_node
+        device: "{{ device }}"
+        bs: 4k
+        count: "64"
+        skip: "512"
+        save_as: fault_read_md5
+      - action: assert_equal
+        actual: "{{ fault_read_md5 }}"
+        expected: "{{ fault_write_md5 }}"
+      - action: scrape_metrics
+        target: primary
+        save_as: metrics_fault
+
+  # Phase 5: Clear fault — restore normal network.
+  - name: fault_clear
+    actions:
+      - action: clear_fault
+        type: netem
+        node: target_node
+      - action: sleep
+        duration: 5s
+
+  # Phase 6: Post-fault steady-state — verify recovery.
+  - name: post_fault_verify
+    actions:
+      # Re-read original data to verify no corruption.
+      - action: dd_read_md5
+        node: client_node
+        device: "{{ device }}"
+        bs: 4k
+        count: "256"
+        save_as: pf_read_md5
+      - action: assert_equal
+        actual: "{{ pf_read_md5 }}"
+        expected: "{{ ss_write_md5 }}"
+      # Write new data post-fault.
+      - action: dd_write
+        node: client_node
+        device: "{{ device }}"
+        bs: 4k
+        count: "128"
+        seek: "1024"
+        save_as: pf_write_md5
+      - action: dd_read_md5
+        node: client_node
+        device: "{{ device }}"
+        bs: 4k
+        count: "128"
+        skip: "1024"
+        save_as: pf_verify_md5
+      - action: assert_equal
+        actual: "{{ pf_verify_md5 }}"
+        expected: "{{ pf_write_md5 }}"
+
+  # Phase 7: Final metrics scrape + perf summary.
+  - name: final_metrics
+    actions:
+      - action: scrape_metrics
+        target: primary
+        save_as: metrics_final
+      - action: perf_summary
+        target: primary
+        save_as: perf_stats
+
+  # Phase 8: Cleanup (always runs).
+  - name: cleanup
+    always: true
+    actions:
+      - action: clear_fault
+        type: netem
+        node: target_node
+        ignore_error: true
+      - action: iscsi_cleanup
+        node: client_node
+        ignore_error: true
+      - action: stop_all_targets
+        ignore_error: true

weed/storage/blockvol/testrunner/scenarios/crash-recovery.yaml

@@ -0,0 +1,87 @@
+name: crash-recovery
+timeout: 5m
+env:
+  repo_dir: "C:/work/seaweedfs"
+
+topology:
+  nodes:
+    target_node:
+      host: "192.168.1.184"
+      user: testdev
+      key: "C:/work/dev_server/testdev_key"
+    client_node:
+      host: "192.168.1.181"
+      user: testdev
+      key: "C:/work/dev_server/testdev_key"
+
+targets:
+  primary:
+    node: target_node
+    vol_size: 100M
+    iscsi_port: 3260
+    admin_port: 8080
+    iqn_suffix: crash-primary
+
+phases:
+  - name: setup
+    actions:
+      - action: build_deploy
+      - action: start_target
+        target: primary
+        create: "true"
+      - action: iscsi_login
+        target: primary
+        node: client_node
+        save_as: device
+
+  - name: write_data
+    actions:
+      - action: dd_write
+        node: client_node
+        device: "{{ device }}"
+        bs: 1M
+        count: "1"
+        save_as: written_md5
+
+  - name: crash
+    actions:
+      # Brief pause to ensure fsync fully completes on target.
+      - action: sleep
+        duration: 1s
+      # Kill while session is still active (like the existing Kill9Fsync test).
+      - action: kill_target
+        target: primary
+      # Clean up stale iSCSI kernel state after kill.
+      - action: iscsi_cleanup
+        node: client_node
+        ignore_error: true
+
+  - name: restart_and_verify
+    actions:
+      - action: start_target
+        target: primary
+        create: "false"
+      - action: sleep
+        duration: 2s
+      - action: iscsi_login
+        target: primary
+        node: client_node
+        save_as: device2
+      - action: dd_read_md5
+        node: client_node
+        device: "{{ device2 }}"
+        bs: 1M
+        count: "1"
+        save_as: read_md5
+      - action: assert_equal
+        actual: "{{ read_md5 }}"
+        expected: "{{ written_md5 }}"
+
+  - name: cleanup
+    always: true
+    actions:
+      - action: iscsi_cleanup
+        node: client_node
+        ignore_error: true
+      - action: stop_all_targets
+        ignore_error: true

weed/storage/blockvol/testrunner/scenarios/fault-disk-full.yaml

@@ -0,0 +1,76 @@
+name: fault-disk-full
+timeout: 5m
+env:
+  repo_dir: "C:/work/seaweedfs"
+
+topology:
+  nodes:
+    target_node:
+      host: "192.168.1.184"
+      user: testdev
+      key: "C:/work/dev_server/testdev_key"
+    client_node:
+      host: "192.168.1.181"
+      user: testdev
+      key: "C:/work/dev_server/testdev_key"
+
+targets:
+  primary:
+    node: target_node
+    vol_size: 50M
+    iscsi_port: 3260
+    admin_port: 8080
+    iqn_suffix: fault-diskfull-primary
+
+phases:
+  - name: setup
+    actions:
+      - action: build_deploy
+      - action: start_target
+        target: primary
+        create: "true"
+      - action: iscsi_login
+        target: primary
+        node: client_node
+        save_as: device
+
+  - name: fill_disk
+    actions:
+      - action: fill_disk
+        node: target_node
+        dir: /tmp
+
+  - name: write_fails
+    actions:
+      - action: dd_write
+        node: client_node
+        device: "{{ device }}"
+        bs: 1M
+        count: "1"
+        ignore_error: true
+        save_as: write_result
+
+  - name: clear_and_recover
+    actions:
+      - action: clear_fault
+        type: fill_disk
+        node: target_node
+      - action: dd_write
+        node: client_node
+        device: "{{ device }}"
+        bs: 4k
+        count: "1"
+        save_as: recovery_md5
+
+  - name: cleanup
+    always: true
+    actions:
+      - action: clear_fault
+        type: fill_disk
+        node: target_node
+        ignore_error: true
+      - action: iscsi_cleanup
+        node: client_node
+        ignore_error: true
+      - action: stop_all_targets
+        ignore_error: true

weed/storage/blockvol/testrunner/scenarios/fault-netem.yaml

@@ -0,0 +1,88 @@
+name: fault-netem
+timeout: 5m
+env:
+  repo_dir: "C:/work/seaweedfs"
+
+topology:
+  nodes:
+    target_node:
+      host: "192.168.1.184"
+      user: testdev
+      key: "C:/work/dev_server/testdev_key"
+    client_node:
+      host: "192.168.1.181"
+      user: testdev
+      key: "C:/work/dev_server/testdev_key"
+
+targets:
+  primary:
+    node: target_node
+    vol_size: 100M
+    iscsi_port: 3260
+    admin_port: 8080
+    iqn_suffix: fault-netem-primary
+  replica:
+    node: target_node
+    vol_size: 100M
+    iscsi_port: 3261
+    admin_port: 8081
+    replica_data_port: 9011
+    replica_ctrl_port: 9012
+    iqn_suffix: fault-netem-replica
+
+phases:
+  - name: setup
+    actions:
+      - action: build_deploy
+      - action: start_target
+        target: primary
+        create: "true"
+      - action: start_target
+        target: replica
+        create: "true"
+      - action: assign
+        target: replica
+        epoch: "1"
+        role: replica
+        lease_ttl: 30s
+      - action: assign
+        target: primary
+        epoch: "1"
+        role: primary
+        lease_ttl: 30s
+      - action: set_replica
+        target: primary
+        replica: replica
+      - action: iscsi_login
+        target: primary
+        node: client_node
+        save_as: device
+
+  - name: inject_delay
+    actions:
+      - action: inject_netem
+        node: target_node
+        target_ip: "127.0.0.1"
+        delay_ms: "200"
+
+  - name: write_under_delay
+    actions:
+      - action: dd_write
+        node: client_node
+        device: "{{ device }}"
+        bs: 4k
+        count: "10"
+        save_as: written_md5
+
+  - name: cleanup
+    always: true
+    actions:
+      - action: clear_fault
+        type: netem
+        node: target_node
+        ignore_error: true
+      - action: iscsi_cleanup
+        node: client_node
+        ignore_error: true
+      - action: stop_all_targets
+        ignore_error: true

weed/storage/blockvol/testrunner/scenarios/fault-partition.yaml

@@ -0,0 +1,96 @@
+name: fault-partition
+timeout: 5m
+env:
+  repo_dir: "C:/work/seaweedfs"
+
+topology:
+  nodes:
+    target_node:
+      host: "192.168.1.184"
+      user: testdev
+      key: "C:/work/dev_server/testdev_key"
+    client_node:
+      host: "192.168.1.181"
+      user: testdev
+      key: "C:/work/dev_server/testdev_key"
+
+targets:
+  primary:
+    node: target_node
+    vol_size: 100M
+    iscsi_port: 3260
+    admin_port: 8080
+    iqn_suffix: fault-part-primary
+  replica:
+    node: target_node
+    vol_size: 100M
+    iscsi_port: 3261
+    admin_port: 8081
+    replica_data_port: 9011
+    replica_ctrl_port: 9012
+    rebuild_port: 9013
+    iqn_suffix: fault-part-replica
+
+phases:
+  - name: setup
+    actions:
+      - action: build_deploy
+      - action: start_target
+        target: primary
+        create: "true"
+      - action: start_target
+        target: replica
+        create: "true"
+      - action: assign
+        target: replica
+        epoch: "1"
+        role: replica
+        lease_ttl: 10s
+      - action: assign
+        target: primary
+        epoch: "1"
+        role: primary
+        lease_ttl: 10s
+      - action: set_replica
+        target: primary
+        replica: replica
+
+  - name: inject_partition
+    actions:
+      - action: inject_partition
+        node: target_node
+        target_ip: "127.0.0.1"
+        ports: "9011,9012"
+
+  - name: wait_for_lease_expiry
+    actions:
+      - action: sleep
+        duration: 15s
+      - action: assert_status
+        target: primary
+        has_lease: "false"
+
+  - name: promote_replica
+    actions:
+      - action: assign
+        target: replica
+        epoch: "2"
+        role: primary
+        lease_ttl: 30s
+      - action: wait_role
+        target: replica
+        role: primary
+        timeout: 5s
+
+  - name: cleanup
+    always: true
+    actions:
+      - action: clear_fault
+        type: partition
+        node: target_node
+        ignore_error: true
+      - action: iscsi_cleanup
+        node: client_node
+        ignore_error: true
+      - action: stop_all_targets
+        ignore_error: true

weed/storage/blockvol/testrunner/scenarios/ha-failover.yaml

@@ -0,0 +1,115 @@
+name: ha-failover
+timeout: 5m
+env:
+  repo_dir: "C:/work/seaweedfs"
+
+topology:
+  nodes:
+    target_node:
+      host: "192.168.1.184"
+      user: testdev
+      key: "C:/work/dev_server/testdev_key"
+    client_node:
+      host: "192.168.1.181"
+      user: testdev
+      key: "C:/work/dev_server/testdev_key"
+
+targets:
+  primary:
+    node: target_node
+    vol_size: 100M
+    iscsi_port: 3260
+    admin_port: 8080
+    iqn_suffix: ha-primary
+  replica:
+    node: target_node
+    vol_size: 100M
+    iscsi_port: 3261
+    admin_port: 8081
+    replica_data_port: 9011
+    replica_ctrl_port: 9012
+    rebuild_port: 9013
+    iqn_suffix: ha-replica
+
+phases:
+  - name: setup
+    actions:
+      - action: build_deploy
+      - action: start_target
+        target: primary
+        create: "true"
+      - action: start_target
+        target: replica
+        create: "true"
+      - action: assign
+        target: replica
+        epoch: "1"
+        role: replica
+        lease_ttl: 30s
+      - action: assign
+        target: primary
+        epoch: "1"
+        role: primary
+        lease_ttl: 30s
+      - action: set_replica
+        target: primary
+        replica: replica
+      - action: iscsi_login
+        target: primary
+        node: client_node
+        save_as: device
+
+  - name: write_and_replicate
+    actions:
+      - action: dd_write
+        node: client_node
+        device: "{{ device }}"
+        bs: 1M
+        count: "1"
+        save_as: written_md5
+      - action: wait_lsn
+        target: replica
+        min_lsn: "1"
+        timeout: 10s
+
+  - name: failover
+    actions:
+      - action: kill_target
+        target: primary
+      - action: iscsi_cleanup
+        node: client_node
+        ignore_error: true
+      - action: assign
+        target: replica
+        epoch: "2"
+        role: primary
+        lease_ttl: 30s
+      - action: wait_role
+        target: replica
+        role: primary
+        timeout: 5s
+
+  - name: verify
+    actions:
+      - action: iscsi_login
+        target: replica
+        node: client_node
+        save_as: device2
+      - action: dd_read_md5
+        node: client_node
+        device: "{{ device2 }}"
+        bs: 1M
+        count: "1"
+        save_as: read_md5
+      - action: assert_equal
+        actual: "{{ read_md5 }}"
+        expected: "{{ written_md5 }}"
+
+  - name: cleanup
+    always: true
+    actions:
+      - action: iscsi_cleanup
+        node: client_node
+        ignore_error: true
+      - action: stop_all_targets
+        ignore_error: true

weed/storage/blockvol/testrunner/scenarios/ha-full-lifecycle.yaml

@@ -0,0 +1,166 @@
+name: ha-full-lifecycle
+timeout: 10m
+env:
+  repo_dir: "C:/work/seaweedfs"
+
+topology:
+  nodes:
+    target_node:
+      host: "192.168.1.184"
+      user: testdev
+      key: "C:/work/dev_server/testdev_key"
+    client_node:
+      host: "192.168.1.181"
+      user: testdev
+      key: "C:/work/dev_server/testdev_key"
+
+targets:
+  primary:
+    node: target_node
+    vol_size: 100M
+    iscsi_port: 3260
+    admin_port: 8080
+    rebuild_port: 9020
+    iqn_suffix: lifecycle-primary
+  replica:
+    node: target_node
+    vol_size: 100M
+    iscsi_port: 3261
+    admin_port: 8081
+    replica_data_port: 9011
+    replica_ctrl_port: 9012
+    rebuild_port: 9013
+    iqn_suffix: lifecycle-replica
+
+phases:
+  - name: setup
+    actions:
+      - action: build_deploy
+      - action: start_target
+        target: primary
+        create: "true"
+      - action: start_target
+        target: replica
+        create: "true"
+      - action: assign
+        target: replica
+        epoch: "1"
+        role: replica
+        lease_ttl: 60s
+      - action: assign
+        target: primary
+        epoch: "1"
+        role: primary
+        lease_ttl: 60s
+      - action: set_replica
+        target: primary
+        replica: replica
+
+  - name: initial_write
+    actions:
+      - action: iscsi_login
+        target: primary
+        node: client_node
+        save_as: device
+      - action: dd_write
+        node: client_node
+        device: "{{ device }}"
+        bs: 1M
+        count: "2"
+        save_as: md5_epoch1
+      - action: wait_lsn
+        target: replica
+        min_lsn: "1"
+        timeout: 10s
+
+  - name: failover_1
+    actions:
+      - action: kill_target
+        target: primary
+      - action: iscsi_cleanup
+        node: client_node
+        ignore_error: true
+      - action: assign
+        target: replica
+        epoch: "2"
+        role: primary
+        lease_ttl: 60s
+      - action: wait_role
+        target: replica
+        role: primary
+        timeout: 5s
+
+  - name: write_after_failover_1
+    actions:
+      - action: iscsi_login
+        target: replica
+        node: client_node
+        save_as: device2
+      - action: dd_write
+        node: client_node
+        device: "{{ device2 }}"
+        bs: 1M
+        count: "1"
+        save_as: md5_epoch2
+
+  - name: rebuild_primary
+    actions:
+      - action: iscsi_logout
+        target: replica
+        node: client_node
+      - action: start_target
+        target: primary
+        create: "true"
+      - action: assign
+        target: primary
+        epoch: "2"
+        role: rebuilding
+        lease_ttl: 60s
+      - action: start_rebuild_client
+        target: primary
+        primary: replica
+        epoch: "2"
+      - action: wait_role
+        target: primary
+        role: replica
+        timeout: 30s
+
+  - name: failover_2
+    actions:
+      - action: kill_target
+        target: replica
+      - action: assign
+        target: primary
+        epoch: "3"
+        role: primary
+        lease_ttl: 60s
+      - action: wait_role
+        target: primary
+        role: primary
+        timeout: 5s
+
+  - name: verify_data
+    actions:
+      - action: iscsi_login
+        target: primary
+        node: client_node
+        save_as: device3
+      # Verify the epoch2 write (1MB at offset 0) survived double failover + rebuild.
+      - action: dd_read_md5
+        node: client_node
+        device: "{{ device3 }}"
+        bs: 1M
+        count: "1"
+        save_as: read_epoch2
+      - action: assert_equal
+        actual: "{{ read_epoch2 }}"
+        expected: "{{ md5_epoch2 }}"
+
+  - name: cleanup
+    always: true
+    actions:
+      - action: iscsi_cleanup
+        node: client_node
+        ignore_error: true
+      - action: stop_all_targets
+        ignore_error: true

weed/storage/blockvol/testrunner/scenarios/ha-io-continuity.yaml

@@ -0,0 +1,115 @@
+name: ha-io-continuity
+timeout: 5m
+env:
+  repo_dir: "C:/work/seaweedfs"
+
+topology:
+  nodes:
+    target_node:
+      host: "192.168.1.184"
+      user: testdev
+      key: "C:/work/dev_server/testdev_key"
+    client_node:
+      host: "192.168.1.181"
+      user: testdev
+      key: "C:/work/dev_server/testdev_key"
+
+targets:
+  primary:
+    node: target_node
+    vol_size: 100M
+    iscsi_port: 3260
+    admin_port: 8080
+    iqn_suffix: iocont-primary
+  replica:
+    node: target_node
+    vol_size: 100M
+    iscsi_port: 3261
+    admin_port: 8081
+    replica_data_port: 9011
+    replica_ctrl_port: 9012
+    rebuild_port: 9013
+    iqn_suffix: iocont-replica
+
+phases:
+  - name: setup
+    actions:
+      - action: build_deploy
+      - action: start_target
+        target: primary
+        create: "true"
+      - action: start_target
+        target: replica
+        create: "true"
+      - action: assign
+        target: replica
+        epoch: "1"
+        role: replica
+        lease_ttl: 30s
+      - action: assign
+        target: primary
+        epoch: "1"
+        role: primary
+        lease_ttl: 30s
+      - action: set_replica
+        target: primary
+        replica: replica
+      - action: iscsi_login
+        target: primary
+        node: client_node
+        save_as: device
+
+  - name: write_A
+    actions:
+      - action: dd_write
+        node: client_node
+        device: "{{ device }}"
+        bs: 512k
+        count: "1"
+        save_as: md5_A
+      - action: wait_lsn
+        target: replica
+        min_lsn: "1"
+        timeout: 10s
+
+  - name: failover
+    actions:
+      - action: kill_target
+        target: primary
+      - action: iscsi_cleanup
+        node: client_node
+        ignore_error: true
+      - action: assign
+        target: replica
+        epoch: "2"
+        role: primary
+        lease_ttl: 30s
+      - action: wait_role
+        target: replica
+        role: primary
+        timeout: 5s
+
+  - name: write_B_and_verify
+    actions:
+      - action: iscsi_login
+        target: replica
+        node: client_node
+        save_as: device2
+      - action: dd_read_md5
+        node: client_node
+        device: "{{ device2 }}"
+        bs: 512k
+        count: "1"
+        save_as: read_A
+      - action: assert_equal
+        actual: "{{ read_A }}"
+        expected: "{{ md5_A }}"
+
+  - name: cleanup
+    always: true
+    actions:
+      - action: iscsi_cleanup
+        node: client_node
+        ignore_error: true
+      - action: stop_all_targets
+        ignore_error: true

weed/storage/blockvol/testrunner/scenarios/ha-rebuild.yaml

@@ -0,0 +1,138 @@
+name: ha-rebuild
+timeout: 5m
+env:
+  repo_dir: "C:/work/seaweedfs"
+
+topology:
+  nodes:
+    target_node:
+      host: "192.168.1.184"
+      user: testdev
+      key: "C:/work/dev_server/testdev_key"
+    client_node:
+      host: "192.168.1.181"
+      user: testdev
+      key: "C:/work/dev_server/testdev_key"
+
+targets:
+  primary:
+    node: target_node
+    vol_size: 100M
+    iscsi_port: 3260
+    admin_port: 8080
+    rebuild_port: 9020
+    iqn_suffix: rebuild-primary
+  replica:
+    node: target_node
+    vol_size: 100M
+    iscsi_port: 3261
+    admin_port: 8081
+    replica_data_port: 9011
+    replica_ctrl_port: 9012
+    rebuild_port: 9013
+    iqn_suffix: rebuild-replica
+
+phases:
+  - name: setup
+    actions:
+      - action: build_deploy
+      - action: start_target
+        target: primary
+        create: "true"
+      - action: start_target
+        target: replica
+        create: "true"
+      - action: assign
+        target: replica
+        epoch: "1"
+        role: replica
+        lease_ttl: 30s
+      - action: assign
+        target: primary
+        epoch: "1"
+        role: primary
+        lease_ttl: 30s
+      - action: set_replica
+        target: primary
+        replica: replica
+      - action: iscsi_login
+        target: primary
+        node: client_node
+        save_as: device
+
+  - name: write_data
+    actions:
+      - action: dd_write
+        node: client_node
+        device: "{{ device }}"
+        bs: 1M
+        count: "1"
+        save_as: written_md5
+      - action: wait_lsn
+        target: replica
+        min_lsn: "1"
+        timeout: 10s
+
+  - name: kill_replica
+    actions:
+      - action: kill_target
+        target: replica
+
+  - name: rebuild
+    actions:
+      - action: start_target
+        target: replica
+        create: "true"
+      - action: assign
+        target: replica
+        epoch: "1"
+        role: rebuilding
+        lease_ttl: 30s
+      # Rebuild server auto-starts via rebuild_port in target spec.
+      - action: start_rebuild_client
+        target: replica
+        primary: primary
+        epoch: "1"
+      - action: wait_role
+        target: replica
+        role: replica
+        timeout: 30s
+
+  - name: verify_rebuild
+    actions:
+      - action: kill_target
+        target: primary
+      - action: iscsi_cleanup
+        node: client_node
+        ignore_error: true
+      - action: assign
+        target: replica
+        epoch: "2"
+        role: primary
+        lease_ttl: 30s
+      - action: wait_role
+        target: replica
+        role: primary
+        timeout: 5s
+      - action: iscsi_login
+        target: replica
+        node: client_node
+        save_as: device2
+      - action: dd_read_md5
+        node: client_node
+        device: "{{ device2 }}"
+        bs: 1M
+        count: "1"
+        save_as: read_md5
+      - action: assert_equal
+        actual: "{{ read_md5 }}"
+        expected: "{{ written_md5 }}"
+
+  - name: cleanup
+    always: true
+    actions:
+      - action: iscsi_cleanup
+        node: client_node
+        ignore_error: true
+      - action: stop_all_targets
+        ignore_error: true

weed/storage/blockvol/testrunner/scenarios/smoke-iscsi.yaml

@@ -0,0 +1,65 @@
+name: smoke-iscsi
+timeout: 5m
+env:
+  repo_dir: "C:/work/seaweedfs"
+
+topology:
+  nodes:
+    target_node:
+      host: "192.168.1.184"
+      user: testdev
+      key: "C:/work/dev_server/testdev_key"
+    client_node:
+      host: "192.168.1.181"
+      user: testdev
+      key: "C:/work/dev_server/testdev_key"
+
+targets:
+  primary:
+    node: target_node
+    vol_size: 100M
+    iscsi_port: 3260
+    admin_port: 8080
+    iqn_suffix: smoke-primary
+
+phases:
+  - name: setup
+    actions:
+      - action: build_deploy
+      - action: start_target
+        target: primary
+        create: "true"
+
+  - name: iscsi_connect
+    actions:
+      - action: iscsi_login
+        target: primary
+        node: client_node
+        save_as: device
+
+  - name: write_verify
+    actions:
+      - action: dd_write
+        node: client_node
+        device: "{{ device }}"
+        bs: 1M
+        count: "1"
+        save_as: written_md5
+      - action: dd_read_md5
+        node: client_node
+        device: "{{ device }}"
+        bs: 1M
+        count: "1"
+        save_as: read_md5
+      - action: assert_equal
+        actual: "{{ written_md5 }}"
+        expected: "{{ read_md5 }}"
+
+  - name: cleanup
+    always: true
+    actions:
+      - action: iscsi_cleanup
+        node: client_node
+        ignore_error: true
+      - action: stop_all_targets
+        ignore_error: true

weed/storage/blockvol/testrunner/scripts/unblock-windows.ps1

@@ -0,0 +1,23 @@
+# unblock-windows.ps1 — Unblock downloaded binaries and add Defender exclusion.
+# Run as Administrator: powershell -ExecutionPolicy Bypass -File .\unblock-windows.ps1
+#
+# This is needed because Windows SmartScreen blocks unsigned binaries downloaded
+# from the internet. Code signing is future work (requires certificate).
+
+$ErrorActionPreference = "Stop"
+$dir = (Get-Location).Path
+
+Write-Host "Unblocking binaries in $dir ..."
+
+$binaries = @("sw-test-runner.exe", "iscsi-target.exe", "weed.exe")
+foreach ($bin in $binaries) {
+    $path = Join-Path $dir $bin
+    if (Test-Path $path) {
+        Unblock-File $path
+        Write-Host "  Unblocked: $bin"
+    }
+}
+
+Write-Host "Adding Defender exclusion for $dir ..."
+Add-MpPreference -ExclusionPath $dir
+Write-Host "Done. Binaries should no longer trigger SmartScreen."

weed/storage/blockvol/testrunner/types.go

@@ -0,0 +1,146 @@
+package testrunner
+
+import "time"
+
+// Scenario is the top-level YAML structure for a test scenario.
+type Scenario struct {
+	Name     string            `yaml:"name"`
+	Timeout  Duration          `yaml:"timeout"`
+	Env      map[string]string `yaml:"env"`
+	Topology Topology          `yaml:"topology"`
+	Targets  map[string]TargetSpec `yaml:"targets"`
+	Phases   []Phase           `yaml:"phases"`
+	Artifacts ArtifactSpec     `yaml:"artifacts"`
+}
+
+// Duration wraps time.Duration for YAML unmarshaling (e.g. "5m", "30s").
+type Duration struct {
+	time.Duration
+}
+
+func (d *Duration) UnmarshalYAML(unmarshal func(interface{}) error) error {
+	var s string
+	if err := unmarshal(&s); err != nil {
+		return err
+	}
+	dur, err := time.ParseDuration(s)
+	if err != nil {
+		return err
+	}
+	d.Duration = dur
+	return nil
+}
+
+func (d Duration) MarshalYAML() (interface{}, error) {
+	return d.Duration.String(), nil
+}
+
+// Topology defines the set of nodes available.
+type Topology struct {
+	Agents map[string]string   `yaml:"agents"` // agent_name → "host:port" (coordinator mode)
+	Nodes  map[string]NodeSpec `yaml:"nodes"`
+}
+
+// NodeSpec defines a remote (or local) machine.
+type NodeSpec struct {
+	Host    string `yaml:"host"`
+	User    string `yaml:"user"`
+	KeyFile string `yaml:"key"`
+	IsLocal bool   `yaml:"is_local"`
+	Agent   string `yaml:"agent"` // maps node to an agent (coordinator mode)
+}
+
+// TargetSpec defines an iSCSI target instance.
+type TargetSpec struct {
+	Node            string `yaml:"node"`
+	VolSize         string `yaml:"vol_size"`
+	WALSize         string `yaml:"wal_size"`
+	ISCSIPort       int    `yaml:"iscsi_port"`
+	AdminPort       int    `yaml:"admin_port"`
+	ReplicaDataPort int    `yaml:"replica_data_port"`
+	ReplicaCtrlPort int    `yaml:"replica_ctrl_port"`
+	RebuildPort     int    `yaml:"rebuild_port"`
+	IQNSuffix       string `yaml:"iqn_suffix"`
+	TPGID           int    `yaml:"tpg_id"`
+}
+
+// IQN returns the full IQN from the suffix.
+func (ts TargetSpec) IQN() string {
+	return "iqn.2024.com.seaweedfs:" + ts.IQNSuffix
+}
+
+// Phase is a sequential group of actions.
+type Phase struct {
+	Name     string   `yaml:"name"`
+	Always   bool     `yaml:"always"`
+	Parallel bool     `yaml:"parallel"`
+	Actions  []Action `yaml:"actions"`
+}
+
+// Action is a single step within a phase.
+type Action struct {
+	Action      string            `yaml:"action" json:"action"`
+	Target      string            `yaml:"target" json:"target,omitempty"`
+	Replica     string            `yaml:"replica" json:"replica,omitempty"`
+	Node        string            `yaml:"node" json:"node,omitempty"`
+	SaveAs      string            `yaml:"save_as" json:"save_as,omitempty"`
+	IgnoreError bool              `yaml:"ignore_error" json:"ignore_error,omitempty"`
+	Retry       int               `yaml:"retry" json:"retry,omitempty"`
+	Timeout     string            `yaml:"timeout" json:"timeout,omitempty"`
+	Params      map[string]string `yaml:"params,inline" json:"params,omitempty"`
+}
+
+// ArtifactSpec configures what to collect on failure.
+type ArtifactSpec struct {
+	OnFailure []string `yaml:"on_failure"`
+	Dir       string   `yaml:"dir"`
+}
+
+// --- Result types ---
+
+// ScenarioResult is the final output of a scenario run.
+type ScenarioResult struct {
+	Name      string            `json:"name"`
+	Status    ResultStatus      `json:"status"`
+	Duration  time.Duration     `json:"duration_ms"`
+	Phases    []PhaseResult     `json:"phases"`
+	Error     string            `json:"error,omitempty"`
+	Vars      map[string]string `json:"vars,omitempty"`
+	Artifacts []ArtifactEntry   `json:"artifacts,omitempty"`
+}
+
+// ArtifactEntry records a collected artifact file.
+type ArtifactEntry struct {
+	Agent string `json:"agent"`
+	Path  string `json:"path"`
+	Size  int64  `json:"size"`
+}
+
+// PhaseResult captures the outcome of one phase.
+type PhaseResult struct {
+	Name     string         `json:"name"`
+	Status   ResultStatus   `json:"status"`
+	Duration time.Duration  `json:"duration_ms"`
+	Actions  []ActionResult `json:"actions"`
+	Error    string         `json:"error,omitempty"`
+}
+
+// ActionResult captures the outcome of one action.
+type ActionResult struct {
+	Action   string        `json:"action"`
+	Status   ResultStatus  `json:"status"`
+	Duration time.Duration `json:"duration_ms"`
+	Output   string        `json:"output,omitempty"`
+	Error    string        `json:"error,omitempty"`
+	YAML     string        `json:"yaml,omitempty"`
+}
+
+// ResultStatus is the status of a result.
+type ResultStatus string
+
+const (
+	StatusPass    ResultStatus = "PASS"
+	StatusFail    ResultStatus = "FAIL"
+	StatusSkip    ResultStatus = "SKIP"
+	StatusRunning ResultStatus = "RUNNING"
+)