Files
seaweedfs/weed/server/master_block_assignment_queue.go
T
Ping QiuandClaude Opus 4.6 8b2b5f6f66 feat: Phase 6 CP6-3 -- failover + rebuild in Kubernetes, 126 tests
Wire low-level fencing primitives to master/VS control plane and CSI:

- Proto: replica/rebuild address fields on assignment/info/response messages
- Assignment queue: retain-until-confirmed (Peek+Confirm), stale epoch pruning
- VS assignment receiver: processes assignments from HeartbeatResponse
- BlockService replication: ProcessAssignments, deterministic ports (FNV hash)
- Registry replica tracking: SetReplica/ClearReplica/SwapPrimaryReplica
- CreateBlockVolume: primary + replica, enqueues assignments, single-copy mode
- Failover: lease-aware promotion, deferred timers with cancellation on reconnect
- ControllerPublish: returns fresh primary iSCSI address after failover
- Recovery: recoverBlockVolumes drains pendingRebuilds, enqueues Rebuilding
- Real integration tests on M02: failover address switch, rebuild data
  consistency, full lifecycle failover+rebuild (3 tests, all PASS)

Review fixes (12 findings, 5 High, 5 Medium, 2 Low):
- R1-1: AllocateBlockVolume returns replication ports
- R1-2: setupPrimaryReplication starts rebuild server
- R1-3: VS sends periodic block heartbeat for assignment confirmation
- R2-F1: LastLeaseGrant set before Register (no stale-lease race)
- R2-F2: Deferred promotion timers cancelled on VS reconnect
- R2-F3: SwapPrimaryReplica uses RoleToWire instead of uint32(1)
- R2-F4: DeleteBlockVolume deletes replica (best-effort)
- R2-F5: SwapPrimaryReplica computes epoch atomically under lock
- QA: SetReplica removes old replica from byServer index (BUG-QA-CP63-1)

126 CP6-3 tests (67 dev + 48 QA + 8 integration + 3 real).
Cumulative Phase 6: 352 tests. All PASS.

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
2026-03-05 00:52:05 -08:00

126 lines
3.2 KiB
Go

package weed_server
import (
"sync"
"github.com/seaweedfs/seaweedfs/weed/storage/blockvol"
)
// BlockAssignmentQueue holds pending assignments per volume server.
// Assignments are retained until confirmed by a matching heartbeat (F1).
type BlockAssignmentQueue struct {
mu sync.Mutex
queues map[string][]blockvol.BlockVolumeAssignment // server -> pending
}
// NewBlockAssignmentQueue creates an empty queue.
func NewBlockAssignmentQueue() *BlockAssignmentQueue {
return &BlockAssignmentQueue{
queues: make(map[string][]blockvol.BlockVolumeAssignment),
}
}
// Enqueue adds a single assignment to the server's queue.
func (q *BlockAssignmentQueue) Enqueue(server string, a blockvol.BlockVolumeAssignment) {
q.mu.Lock()
defer q.mu.Unlock()
q.queues[server] = append(q.queues[server], a)
}
// EnqueueBatch adds multiple assignments to the server's queue.
func (q *BlockAssignmentQueue) EnqueueBatch(server string, as []blockvol.BlockVolumeAssignment) {
if len(as) == 0 {
return
}
q.mu.Lock()
defer q.mu.Unlock()
q.queues[server] = append(q.queues[server], as...)
}
// Peek returns a copy of pending assignments for the server without removing them.
// Stale assignments (superseded by a newer epoch for the same path) are pruned.
func (q *BlockAssignmentQueue) Peek(server string) []blockvol.BlockVolumeAssignment {
q.mu.Lock()
defer q.mu.Unlock()
pending := q.queues[server]
if len(pending) == 0 {
return nil
}
// Prune stale: keep only the latest epoch per path.
latest := make(map[string]uint64, len(pending))
for _, a := range pending {
if a.Epoch > latest[a.Path] {
latest[a.Path] = a.Epoch
}
}
pruned := pending[:0]
for _, a := range pending {
if a.Epoch >= latest[a.Path] {
pruned = append(pruned, a)
}
}
q.queues[server] = pruned
// Return a copy.
out := make([]blockvol.BlockVolumeAssignment, len(pruned))
copy(out, pruned)
return out
}
// Confirm removes a matching assignment (same path and epoch) from the server's queue.
func (q *BlockAssignmentQueue) Confirm(server string, path string, epoch uint64) {
q.mu.Lock()
defer q.mu.Unlock()
pending := q.queues[server]
for i, a := range pending {
if a.Path == path && a.Epoch == epoch {
q.queues[server] = append(pending[:i], pending[i+1:]...)
return
}
}
}
// ConfirmFromHeartbeat batch-confirms assignments that match reported heartbeat info.
// An assignment is confirmed if the VS reports (path, epoch) that matches.
func (q *BlockAssignmentQueue) ConfirmFromHeartbeat(server string, infos []blockvol.BlockVolumeInfoMessage) {
if len(infos) == 0 {
return
}
q.mu.Lock()
defer q.mu.Unlock()
pending := q.queues[server]
if len(pending) == 0 {
return
}
// Build a set of reported (path, epoch) pairs.
type key struct {
path string
epoch uint64
}
reported := make(map[key]bool, len(infos))
for _, info := range infos {
reported[key{info.Path, info.Epoch}] = true
}
// Keep only assignments not confirmed.
kept := pending[:0]
for _, a := range pending {
if !reported[key{a.Path, a.Epoch}] {
kept = append(kept, a)
}
}
q.queues[server] = kept
}
// Pending returns the number of pending assignments for the server.
func (q *BlockAssignmentQueue) Pending(server string) int {
q.mu.Lock()
defer q.mu.Unlock()
return len(q.queues[server])
}