feat: CP11B-3 safe ops — promotion hardening, preflight, manual promote

Six-task checkpoint hardening the promotion and failover paths:

T1: 4-gate candidate evaluation (heartbeat freshness, WAL lag, role,
    server liveness) with structured rejection reasons.
T2: Orphaned-primary re-evaluation on replica reconnect (B-06/B-08).
T3: Deferred timer safety — epoch validation prevents stale timers
    from firing on recreated/changed volumes (B-07).
T4: Rebuild addr cleanup on promotion (B-11), NVMe publication
    refresh on heartbeat, and preflight endpoint wiring.
T5: Manual promote API — POST /block/volume/{name}/promote with
    force flag, target server selection, and structured rejection
    response. Shared applyPromotionLocked/finalizePromotion helpers
    eliminate duplication between auto and manual paths.
T6: Read-only preflight endpoint (GET /block/volume/{name}/preflight)
    and blockapi client wrappers (Preflight, Promote).

BUG-T5-1: PromotionsTotal counter moved to finalizePromotion (shared
    by both auto and manual paths) to prevent metrics divergence.

24 files changed, ~6500 lines added. 42 new QA adversarial tests.

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
Ping QiuandClaude Opus 4.6 committed 2026-03-13 17:21:17 -07:00
1 parent ed11a09a61
commit 075ff52219
24 files changed
+6540 -114

No files matched your search

+55
View File
@@ -136,6 +136,61 @@ func (c *Client) ExpandVolume(ctx context.Context, name string, newSizeBytes uin
return out.CapacityBytes, nil
}
// PromoteVolume triggers a manual promotion for a block volume.
func (c *Client) PromoteVolume(ctx context.Context, name string, req PromoteVolumeRequest) (*PromoteVolumeResponse, error) {
body, err := json.Marshal(req)
if err != nil {
return nil, fmt.Errorf("marshal request: %w", err)
}
resp, err := c.doRequest(ctx, http.MethodPost, "/block/volume/"+name+"/promote", bytes.NewReader(body))
if err != nil {
return nil, err
}
defer resp.Body.Close()
if err := checkStatus(resp, http.StatusOK); err != nil {
return nil, err
}
var out PromoteVolumeResponse
if err := json.NewDecoder(resp.Body).Decode(&out); err != nil {
return nil, fmt.Errorf("decode response: %w", err)
}
return &out, nil
}
// BlockStatus fetches the block registry status metrics.
func (c *Client) BlockStatus(ctx context.Context) (*BlockStatusResponse, error) {
resp, err := c.doRequest(ctx, http.MethodGet, "/block/status", nil)
if err != nil {
return nil, err
}
defer resp.Body.Close()
if err := checkStatus(resp, http.StatusOK); err != nil {
return nil, err
}
var out BlockStatusResponse
if err := json.NewDecoder(resp.Body).Decode(&out); err != nil {
return nil, fmt.Errorf("decode response: %w", err)
}
return &out, nil
}
// Preflight returns the promotion preflight evaluation for a block volume.
func (c *Client) Preflight(ctx context.Context, name string) (*PreflightResponse, error) {
resp, err := c.doRequest(ctx, http.MethodGet, "/block/volume/"+name+"/preflight", nil)
if err != nil {
return nil, err
}
defer resp.Body.Close()
if err := checkStatus(resp, http.StatusOK); err != nil {
return nil, err
}
var out PreflightResponse
if err := json.NewDecoder(resp.Body).Decode(&out); err != nil {
return nil, fmt.Errorf("decode response: %w", err)
}
return &out, nil
}
// ListServers lists all block-capable volume servers.
func (c *Client) ListServers(ctx context.Context) ([]ServerInfo, error) {
resp, err := c.doRequest(ctx, http.MethodGet, "/block/servers", nil)
+48
View File
@@ -38,6 +38,8 @@ type VolumeInfo struct {
HealthScore float64 `json:"health_score"`
ReplicaDegraded bool `json:"replica_degraded,omitempty"`
DurabilityMode string `json:"durability_mode"` // CP8-3-1
NvmeAddr string `json:"nvme_addr,omitempty"`
NQN string `json:"nqn,omitempty"`
}
// ReplicaDetail describes one replica in the API response.
@@ -74,6 +76,52 @@ type ExpandVolumeResponse struct {
CapacityBytes uint64 `json:"capacity_bytes"`
}
// PromoteVolumeRequest is the request body for POST /block/volume/{name}/promote.
type PromoteVolumeRequest struct {
TargetServer string `json:"target_server,omitempty"` // specific replica, or empty for auto
Force bool `json:"force,omitempty"` // bypass soft safety checks
Reason string `json:"reason,omitempty"` // audit note
}
// PromoteVolumeResponse is the response for POST /block/volume/{name}/promote.
type PromoteVolumeResponse struct {
NewPrimary string `json:"new_primary"`
Epoch uint64 `json:"epoch"`
Reason string `json:"reason,omitempty"` // rejection reason if failed
Rejections []PreflightRejection `json:"rejections,omitempty"` // per-replica rejection details
}
// BlockStatusResponse is the response for GET /block/status.
type BlockStatusResponse struct {
VolumeCount int `json:"volume_count"`
ServerCount int `json:"server_count"`
PromotionLSNTolerance uint64 `json:"promotion_lsn_tolerance"`
BarrierLagLSN uint64 `json:"barrier_lag_lsn"`
PromotionsTotal int64 `json:"promotions_total"`
FailoversTotal int64 `json:"failovers_total"`
RebuildsTotal int64 `json:"rebuilds_total"`
AssignmentQueueDepth int `json:"assignment_queue_depth"`
}
// PreflightRejection describes why a specific replica was rejected for promotion.
type PreflightRejection struct {
Server string `json:"server"`
Reason string `json:"reason"` // "stale_heartbeat", "wal_lag", "wrong_role", "server_dead", "no_heartbeat"
}
// PreflightResponse is the response for GET /block/volume/{name}/preflight.
type PreflightResponse struct {
VolumeName string `json:"volume_name"`
Promotable bool `json:"promotable"`
Reason string `json:"reason,omitempty"`
CandidateServer string `json:"candidate_server,omitempty"`
CandidateHealth float64 `json:"candidate_health,omitempty"`
CandidateWALLSN uint64 `json:"candidate_wal_lsn,omitempty"`
Rejections []PreflightRejection `json:"rejections,omitempty"`
PrimaryServer string `json:"primary_server"`
PrimaryAlive bool `json:"primary_alive"`
}
// RoleFromString converts a role string to its uint32 wire value.
// Returns 0 (RoleNone) for unrecognized strings.
func RoleFromString(s string) uint32 {