mirror of
https://github.com/seaweedfs/seaweedfs.git
synced 2026-11-16 08:45:22 +01:00
feat: CP11B-3 safe ops — promotion hardening, preflight, manual promote
Six-task checkpoint hardening the promotion and failover paths:
T1: 4-gate candidate evaluation (heartbeat freshness, WAL lag, role,
server liveness) with structured rejection reasons.
T2: Orphaned-primary re-evaluation on replica reconnect (B-06/B-08).
T3: Deferred timer safety — epoch validation prevents stale timers
from firing on recreated/changed volumes (B-07).
T4: Rebuild addr cleanup on promotion (B-11), NVMe publication
refresh on heartbeat, and preflight endpoint wiring.
T5: Manual promote API — POST /block/volume/{name}/promote with
force flag, target server selection, and structured rejection
response. Shared applyPromotionLocked/finalizePromotion helpers
eliminate duplication between auto and manual paths.
T6: Read-only preflight endpoint (GET /block/volume/{name}/preflight)
and blockapi client wrappers (Preflight, Promote).
BUG-T5-1: PromotionsTotal counter moved to finalizePromotion (shared
by both auto and manual paths) to prevent metrics divergence.
24 files changed, ~6500 lines added. 42 new QA adversarial tests.
Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
1 parent
ed11a09a61
commit
075ff52219
24 files changed
+6540
-114
No files matched your search
@@ -136,6 +136,61 @@ func (c *Client) ExpandVolume(ctx context.Context, name string, newSizeBytes uin
|
||||
return out.CapacityBytes, nil
|
||||
}
|
||||
|
||||
// PromoteVolume triggers a manual promotion for a block volume.
|
||||
func (c *Client) PromoteVolume(ctx context.Context, name string, req PromoteVolumeRequest) (*PromoteVolumeResponse, error) {
|
||||
body, err := json.Marshal(req)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("marshal request: %w", err)
|
||||
}
|
||||
resp, err := c.doRequest(ctx, http.MethodPost, "/block/volume/"+name+"/promote", bytes.NewReader(body))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
if err := checkStatus(resp, http.StatusOK); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var out PromoteVolumeResponse
|
||||
if err := json.NewDecoder(resp.Body).Decode(&out); err != nil {
|
||||
return nil, fmt.Errorf("decode response: %w", err)
|
||||
}
|
||||
return &out, nil
|
||||
}
|
||||
|
||||
// BlockStatus fetches the block registry status metrics.
|
||||
func (c *Client) BlockStatus(ctx context.Context) (*BlockStatusResponse, error) {
|
||||
resp, err := c.doRequest(ctx, http.MethodGet, "/block/status", nil)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
if err := checkStatus(resp, http.StatusOK); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var out BlockStatusResponse
|
||||
if err := json.NewDecoder(resp.Body).Decode(&out); err != nil {
|
||||
return nil, fmt.Errorf("decode response: %w", err)
|
||||
}
|
||||
return &out, nil
|
||||
}
|
||||
|
||||
// Preflight returns the promotion preflight evaluation for a block volume.
|
||||
func (c *Client) Preflight(ctx context.Context, name string) (*PreflightResponse, error) {
|
||||
resp, err := c.doRequest(ctx, http.MethodGet, "/block/volume/"+name+"/preflight", nil)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
if err := checkStatus(resp, http.StatusOK); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var out PreflightResponse
|
||||
if err := json.NewDecoder(resp.Body).Decode(&out); err != nil {
|
||||
return nil, fmt.Errorf("decode response: %w", err)
|
||||
}
|
||||
return &out, nil
|
||||
}
|
||||
|
||||
// ListServers lists all block-capable volume servers.
|
||||
func (c *Client) ListServers(ctx context.Context) ([]ServerInfo, error) {
|
||||
resp, err := c.doRequest(ctx, http.MethodGet, "/block/servers", nil)
|
||||
|
||||
@@ -38,6 +38,8 @@ type VolumeInfo struct {
|
||||
HealthScore float64 `json:"health_score"`
|
||||
ReplicaDegraded bool `json:"replica_degraded,omitempty"`
|
||||
DurabilityMode string `json:"durability_mode"` // CP8-3-1
|
||||
NvmeAddr string `json:"nvme_addr,omitempty"`
|
||||
NQN string `json:"nqn,omitempty"`
|
||||
}
|
||||
|
||||
// ReplicaDetail describes one replica in the API response.
|
||||
@@ -74,6 +76,52 @@ type ExpandVolumeResponse struct {
|
||||
CapacityBytes uint64 `json:"capacity_bytes"`
|
||||
}
|
||||
|
||||
// PromoteVolumeRequest is the request body for POST /block/volume/{name}/promote.
|
||||
type PromoteVolumeRequest struct {
|
||||
TargetServer string `json:"target_server,omitempty"` // specific replica, or empty for auto
|
||||
Force bool `json:"force,omitempty"` // bypass soft safety checks
|
||||
Reason string `json:"reason,omitempty"` // audit note
|
||||
}
|
||||
|
||||
// PromoteVolumeResponse is the response for POST /block/volume/{name}/promote.
|
||||
type PromoteVolumeResponse struct {
|
||||
NewPrimary string `json:"new_primary"`
|
||||
Epoch uint64 `json:"epoch"`
|
||||
Reason string `json:"reason,omitempty"` // rejection reason if failed
|
||||
Rejections []PreflightRejection `json:"rejections,omitempty"` // per-replica rejection details
|
||||
}
|
||||
|
||||
// BlockStatusResponse is the response for GET /block/status.
|
||||
type BlockStatusResponse struct {
|
||||
VolumeCount int `json:"volume_count"`
|
||||
ServerCount int `json:"server_count"`
|
||||
PromotionLSNTolerance uint64 `json:"promotion_lsn_tolerance"`
|
||||
BarrierLagLSN uint64 `json:"barrier_lag_lsn"`
|
||||
PromotionsTotal int64 `json:"promotions_total"`
|
||||
FailoversTotal int64 `json:"failovers_total"`
|
||||
RebuildsTotal int64 `json:"rebuilds_total"`
|
||||
AssignmentQueueDepth int `json:"assignment_queue_depth"`
|
||||
}
|
||||
|
||||
// PreflightRejection describes why a specific replica was rejected for promotion.
|
||||
type PreflightRejection struct {
|
||||
Server string `json:"server"`
|
||||
Reason string `json:"reason"` // "stale_heartbeat", "wal_lag", "wrong_role", "server_dead", "no_heartbeat"
|
||||
}
|
||||
|
||||
// PreflightResponse is the response for GET /block/volume/{name}/preflight.
|
||||
type PreflightResponse struct {
|
||||
VolumeName string `json:"volume_name"`
|
||||
Promotable bool `json:"promotable"`
|
||||
Reason string `json:"reason,omitempty"`
|
||||
CandidateServer string `json:"candidate_server,omitempty"`
|
||||
CandidateHealth float64 `json:"candidate_health,omitempty"`
|
||||
CandidateWALLSN uint64 `json:"candidate_wal_lsn,omitempty"`
|
||||
Rejections []PreflightRejection `json:"rejections,omitempty"`
|
||||
PrimaryServer string `json:"primary_server"`
|
||||
PrimaryAlive bool `json:"primary_alive"`
|
||||
}
|
||||
|
||||
// RoleFromString converts a role string to its uint32 wire value.
|
||||
// Returns 0 (RoleNone) for unrecognized strings.
|
||||
func RoleFromString(s string) uint32 {
|
||||
|
||||
Reference in new issue
Block a user