mirror of
https://github.com/seaweedfs/seaweedfs.git
synced 2026-10-13 01:28:37 +02:00
Ping QiuandClaude Opus 4.6
a7b1b4cb22
ReplicaInfo now carries NvmeAddr/NQN. Fields are populated during replica allocation (tryCreateOneReplica), updated from replica heartbeats, and copied in PromoteBestReplica. This ensures master lookup returns correct NVMe endpoints immediately after failover, without waiting for the first post-promotion heartbeat. Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
1012 lines
31 KiB
Go
1012 lines
31 KiB
Go
package weed_server
|
||
|
||
import (
|
||
"fmt"
|
||
"path/filepath"
|
||
"sort"
|
||
"strings"
|
||
"sync"
|
||
"sync/atomic"
|
||
"time"
|
||
|
||
"github.com/seaweedfs/seaweedfs/weed/glog"
|
||
"github.com/seaweedfs/seaweedfs/weed/pb/master_pb"
|
||
"github.com/seaweedfs/seaweedfs/weed/storage/blockvol"
|
||
)
|
||
|
||
// VolumeStatus tracks the lifecycle of a block volume entry.
|
||
type VolumeStatus int
|
||
|
||
const (
|
||
StatusPending VolumeStatus = iota // Created via RPC, not yet confirmed by heartbeat
|
||
StatusActive // Confirmed by heartbeat from volume server
|
||
)
|
||
|
||
// ReplicaInfo tracks one replica of a block volume (CP8-2).
|
||
type ReplicaInfo struct {
|
||
Server string // replica VS address
|
||
Path string // file path on replica VS
|
||
ISCSIAddr string // iSCSI target address
|
||
IQN string // iSCSI qualified name
|
||
NvmeAddr string // NVMe/TCP target address (ip:port), empty if NVMe disabled
|
||
NQN string // NVMe subsystem NQN, empty if NVMe disabled
|
||
DataAddr string // WAL receiver data listen addr
|
||
CtrlAddr string // WAL receiver ctrl listen addr
|
||
HealthScore float64 // from heartbeat (0.0-1.0)
|
||
WALHeadLSN uint64 // from heartbeat
|
||
WALLag uint64 // computed: primary.WALHeadLSN - replica.WALHeadLSN
|
||
LastHeartbeat time.Time // last heartbeat received from this replica
|
||
Role uint32 // replica role (RoleReplica, RoleRebuilding, etc.)
|
||
}
|
||
|
||
const (
|
||
// DefaultPromotionLSNTolerance is the max WAL LSN lag allowed for promotion eligibility.
|
||
// Configurable per-registry via SetPromotionLSNTolerance.
|
||
DefaultPromotionLSNTolerance uint64 = 100
|
||
)
|
||
|
||
// BlockVolumeEntry tracks one block volume across the cluster.
|
||
type BlockVolumeEntry struct {
|
||
Name string
|
||
VolumeServer string // volume server address (ip:port or grpc addr)
|
||
Path string // file path on volume server
|
||
IQN string
|
||
ISCSIAddr string
|
||
NvmeAddr string // NVMe/TCP target address (ip:port), empty if NVMe disabled
|
||
NQN string // NVMe subsystem NQN, empty if NVMe disabled
|
||
SizeBytes uint64
|
||
ReplicaPlacement string // SeaweedFS placement string: "000", "001", "010", "100"
|
||
Epoch uint64
|
||
Role uint32
|
||
Status VolumeStatus
|
||
|
||
// Deprecated scalar replica fields (CP6-3). Use Replicas[] for new code.
|
||
ReplicaServer string
|
||
ReplicaPath string
|
||
ReplicaISCSIAddr string
|
||
ReplicaIQN string
|
||
ReplicaDataAddr string
|
||
ReplicaCtrlAddr string
|
||
RebuildListenAddr string // rebuild server listen addr on primary
|
||
|
||
// CP8-2: Multi-replica support.
|
||
ReplicaFactor int // 2 or 3 (default 2)
|
||
Replicas []ReplicaInfo // one per replica (RF-1 entries)
|
||
HealthScore float64 // primary health score from heartbeat
|
||
ReplicaDegraded bool // primary reports degraded replicas
|
||
WALHeadLSN uint64 // primary WAL head LSN from heartbeat
|
||
|
||
// CP8-3-1: Durability mode.
|
||
DurabilityMode string // "best_effort", "sync_all", "sync_quorum"
|
||
|
||
// Lease tracking for failover (CP6-3 F2).
|
||
LastLeaseGrant time.Time
|
||
LeaseTTL time.Duration
|
||
}
|
||
|
||
// HasReplica returns true if this volume has any replica (checks both new and deprecated fields).
|
||
func (e *BlockVolumeEntry) HasReplica() bool {
|
||
return len(e.Replicas) > 0 || e.ReplicaServer != ""
|
||
}
|
||
|
||
// FirstReplica returns the first replica info, or nil if none.
|
||
func (e *BlockVolumeEntry) FirstReplica() *ReplicaInfo {
|
||
if len(e.Replicas) > 0 {
|
||
return &e.Replicas[0]
|
||
}
|
||
return nil
|
||
}
|
||
|
||
// ReplicaByServer returns the replica hosted on the given server, or nil.
|
||
func (e *BlockVolumeEntry) ReplicaByServer(server string) *ReplicaInfo {
|
||
for i := range e.Replicas {
|
||
if e.Replicas[i].Server == server {
|
||
return &e.Replicas[i]
|
||
}
|
||
}
|
||
return nil
|
||
}
|
||
|
||
// BestReplicaForPromotion returns the best replica for promotion, or nil if none eligible.
|
||
// Criteria: highest HealthScore, tie-break by highest WALHeadLSN, then first in list.
|
||
func (e *BlockVolumeEntry) BestReplicaForPromotion() *ReplicaInfo {
|
||
if len(e.Replicas) == 0 {
|
||
return nil
|
||
}
|
||
best := 0
|
||
for i := 1; i < len(e.Replicas); i++ {
|
||
if e.Replicas[i].HealthScore > e.Replicas[best].HealthScore {
|
||
best = i
|
||
} else if e.Replicas[i].HealthScore == e.Replicas[best].HealthScore &&
|
||
e.Replicas[i].WALHeadLSN > e.Replicas[best].WALHeadLSN {
|
||
best = i
|
||
}
|
||
}
|
||
return &e.Replicas[best]
|
||
}
|
||
|
||
// BlockVolumeRegistry is the in-memory registry of block volumes.
|
||
// Rebuilt from heartbeats on master restart (no persistence).
|
||
type BlockVolumeRegistry struct {
|
||
mu sync.RWMutex
|
||
volumes map[string]*BlockVolumeEntry // keyed by name
|
||
byServer map[string]map[string]bool // server -> set of volume names
|
||
blockServers map[string]bool // servers known to support block volumes
|
||
|
||
// Promotion eligibility: max WAL LSN lag for replica to be promotable.
|
||
promotionLSNTolerance uint64
|
||
|
||
// inflight guards concurrent CreateBlockVolume for the same name.
|
||
inflight sync.Map // name -> *inflightEntry
|
||
|
||
// Metrics (CP8-4).
|
||
PromotionsTotal atomic.Uint64
|
||
FailoversTotal atomic.Uint64
|
||
RebuildsTotal atomic.Uint64
|
||
}
|
||
|
||
type inflightEntry struct{}
|
||
|
||
// NewBlockVolumeRegistry creates an empty registry.
|
||
func NewBlockVolumeRegistry() *BlockVolumeRegistry {
|
||
return &BlockVolumeRegistry{
|
||
volumes: make(map[string]*BlockVolumeEntry),
|
||
byServer: make(map[string]map[string]bool),
|
||
blockServers: make(map[string]bool),
|
||
promotionLSNTolerance: DefaultPromotionLSNTolerance,
|
||
}
|
||
}
|
||
|
||
// Register adds an entry to the registry.
|
||
// Returns error if a volume with the same name already exists.
|
||
func (r *BlockVolumeRegistry) Register(entry *BlockVolumeEntry) error {
|
||
r.mu.Lock()
|
||
defer r.mu.Unlock()
|
||
if _, ok := r.volumes[entry.Name]; ok {
|
||
return fmt.Errorf("block volume %q already registered", entry.Name)
|
||
}
|
||
r.volumes[entry.Name] = entry
|
||
r.addToServer(entry.VolumeServer, entry.Name)
|
||
// Also index replica servers so ListByServer finds them.
|
||
for _, ri := range entry.Replicas {
|
||
r.addToServer(ri.Server, entry.Name)
|
||
}
|
||
return nil
|
||
}
|
||
|
||
// Unregister removes and returns the entry. Returns nil if not found.
|
||
func (r *BlockVolumeRegistry) Unregister(name string) *BlockVolumeEntry {
|
||
r.mu.Lock()
|
||
defer r.mu.Unlock()
|
||
entry, ok := r.volumes[name]
|
||
if !ok {
|
||
return nil
|
||
}
|
||
delete(r.volumes, name)
|
||
r.removeFromServer(entry.VolumeServer, name)
|
||
for _, ri := range entry.Replicas {
|
||
r.removeFromServer(ri.Server, name)
|
||
}
|
||
return entry
|
||
}
|
||
|
||
// UpdateSize updates the size of a registered volume.
|
||
// Called only after a successful VS expand to keep registry in sync.
|
||
func (r *BlockVolumeRegistry) UpdateSize(name string, newSizeBytes uint64) error {
|
||
r.mu.Lock()
|
||
defer r.mu.Unlock()
|
||
entry, ok := r.volumes[name]
|
||
if !ok {
|
||
return fmt.Errorf("block volume %q not found in registry", name)
|
||
}
|
||
entry.SizeBytes = newSizeBytes
|
||
return nil
|
||
}
|
||
|
||
// Lookup returns the entry for the given name.
|
||
func (r *BlockVolumeRegistry) Lookup(name string) (*BlockVolumeEntry, bool) {
|
||
r.mu.RLock()
|
||
defer r.mu.RUnlock()
|
||
e, ok := r.volumes[name]
|
||
return e, ok
|
||
}
|
||
|
||
// ListByServer returns all entries hosted on the given server.
|
||
func (r *BlockVolumeRegistry) ListByServer(server string) []*BlockVolumeEntry {
|
||
r.mu.RLock()
|
||
defer r.mu.RUnlock()
|
||
names, ok := r.byServer[server]
|
||
if !ok {
|
||
return nil
|
||
}
|
||
entries := make([]*BlockVolumeEntry, 0, len(names))
|
||
for name := range names {
|
||
if e, ok := r.volumes[name]; ok {
|
||
entries = append(entries, e)
|
||
}
|
||
}
|
||
return entries
|
||
}
|
||
|
||
// UpdateFullHeartbeat reconciles the registry from a full heartbeat.
|
||
// Called on the first heartbeat from a volume server.
|
||
// Marks reported volumes as Active, removes entries for this server
|
||
// that are not reported (stale).
|
||
func (r *BlockVolumeRegistry) UpdateFullHeartbeat(server string, infos []*master_pb.BlockVolumeInfoMessage) {
|
||
r.mu.Lock()
|
||
defer r.mu.Unlock()
|
||
|
||
// Mark server as block-capable since it sent block volume info.
|
||
r.blockServers[server] = true
|
||
|
||
// Build set of reported paths.
|
||
reported := make(map[string]*master_pb.BlockVolumeInfoMessage, len(infos))
|
||
for _, info := range infos {
|
||
reported[info.Path] = info
|
||
}
|
||
|
||
// Find entries for this server that are NOT reported -> reconcile.
|
||
if names, ok := r.byServer[server]; ok {
|
||
for name := range names {
|
||
entry := r.volumes[name]
|
||
if entry == nil {
|
||
continue
|
||
}
|
||
if entry.VolumeServer == server {
|
||
// Server is the primary: check if primary path is reported.
|
||
if _, found := reported[entry.Path]; !found {
|
||
delete(r.volumes, name)
|
||
delete(names, name)
|
||
// Also clean up replica entries from byServer.
|
||
for _, ri := range entry.Replicas {
|
||
r.removeFromServer(ri.Server, name)
|
||
}
|
||
}
|
||
} else {
|
||
// Server is a replica: check if replica path is reported.
|
||
ri := entry.ReplicaByServer(server)
|
||
if ri == nil {
|
||
// No replica record — stale byServer index, just clean up.
|
||
delete(names, name)
|
||
continue
|
||
}
|
||
if _, found := reported[ri.Path]; !found {
|
||
// Replica path not reported — remove this replica, NOT the whole volume.
|
||
r.removeReplicaLocked(entry, server, name)
|
||
delete(names, name)
|
||
glog.V(0).Infof("block registry: removed stale replica %s for %q (path %s not in heartbeat)",
|
||
server, name, ri.Path)
|
||
}
|
||
}
|
||
}
|
||
}
|
||
|
||
// Update or add entries for reported volumes.
|
||
for _, info := range infos {
|
||
// Find existing entry: search byServer index for matching path (primary or replica).
|
||
var existing *BlockVolumeEntry
|
||
var existingName string
|
||
if names, ok := r.byServer[server]; ok {
|
||
for vname := range names {
|
||
if e := r.volumes[vname]; e != nil {
|
||
if e.VolumeServer == server && e.Path == info.Path {
|
||
existing = e
|
||
existingName = vname
|
||
break
|
||
}
|
||
if ri := e.ReplicaByServer(server); ri != nil && ri.Path == info.Path {
|
||
existing = e
|
||
existingName = vname
|
||
break
|
||
}
|
||
}
|
||
}
|
||
}
|
||
// Also try lookup by name derived from path (handles post-restart).
|
||
if existing == nil {
|
||
name := nameFromPath(info.Path)
|
||
if name != "" {
|
||
if e, ok := r.volumes[name]; ok {
|
||
existing = e
|
||
existingName = name
|
||
}
|
||
}
|
||
}
|
||
|
||
if existing != nil {
|
||
isPrimary := existing.VolumeServer == server
|
||
isReplica := existing.ReplicaByServer(server) != nil
|
||
|
||
if isPrimary {
|
||
// Primary heartbeat: update primary fields.
|
||
existing.SizeBytes = info.VolumeSize
|
||
existing.Epoch = info.Epoch
|
||
existing.Role = info.Role
|
||
existing.Status = StatusActive
|
||
existing.LastLeaseGrant = time.Now()
|
||
existing.HealthScore = info.HealthScore
|
||
existing.ReplicaDegraded = info.ReplicaDegraded
|
||
existing.WALHeadLSN = info.WalHeadLsn
|
||
// F3: only update DurabilityMode when non-empty (prevents older VS from clearing strict mode).
|
||
if info.DurabilityMode != "" {
|
||
existing.DurabilityMode = info.DurabilityMode
|
||
}
|
||
// F5: update replica addresses from heartbeat info.
|
||
if info.ReplicaDataAddr != "" {
|
||
existing.ReplicaDataAddr = info.ReplicaDataAddr
|
||
}
|
||
if info.ReplicaCtrlAddr != "" {
|
||
existing.ReplicaCtrlAddr = info.ReplicaCtrlAddr
|
||
}
|
||
// NVMe publication: update NVMe fields from heartbeat.
|
||
// Required for master restart reconstruction and NVMe enable/disable.
|
||
existing.NvmeAddr = info.NvmeAddr
|
||
existing.NQN = info.Nqn
|
||
// Sync first replica's data addrs to Replicas[].
|
||
if info.ReplicaDataAddr != "" && len(existing.Replicas) > 0 {
|
||
existing.Replicas[0].DataAddr = info.ReplicaDataAddr
|
||
existing.Replicas[0].CtrlAddr = info.ReplicaCtrlAddr
|
||
}
|
||
} else if isReplica {
|
||
// Replica heartbeat: update ReplicaInfo fields.
|
||
for i := range existing.Replicas {
|
||
if existing.Replicas[i].Server == server {
|
||
existing.Replicas[i].Path = info.Path
|
||
existing.Replicas[i].WALHeadLSN = info.WalHeadLsn
|
||
existing.Replicas[i].HealthScore = info.HealthScore
|
||
existing.Replicas[i].LastHeartbeat = time.Now()
|
||
existing.Replicas[i].Role = info.Role
|
||
existing.Replicas[i].NvmeAddr = info.NvmeAddr
|
||
existing.Replicas[i].NQN = info.Nqn
|
||
if existing.WALHeadLSN > info.WalHeadLsn {
|
||
existing.Replicas[i].WALLag = existing.WALHeadLSN - info.WalHeadLsn
|
||
} else {
|
||
existing.Replicas[i].WALLag = 0
|
||
}
|
||
break
|
||
}
|
||
}
|
||
} else {
|
||
// Fix #3: Server reports a volume that exists but has no record of this server.
|
||
// This happens after master restart: primary heartbeat re-created the entry,
|
||
// but replica heartbeat arrives and isn't linked. Add as replica.
|
||
ri := ReplicaInfo{
|
||
Server: server,
|
||
Path: info.Path,
|
||
HealthScore: info.HealthScore,
|
||
WALHeadLSN: info.WalHeadLsn,
|
||
LastHeartbeat: time.Now(),
|
||
Role: info.Role,
|
||
NvmeAddr: info.NvmeAddr,
|
||
NQN: info.Nqn,
|
||
}
|
||
existing.Replicas = append(existing.Replicas, ri)
|
||
r.addToServer(server, existingName)
|
||
// Sync deprecated scalar fields.
|
||
if len(existing.Replicas) == 1 {
|
||
existing.ReplicaServer = ri.Server
|
||
existing.ReplicaPath = ri.Path
|
||
}
|
||
glog.V(0).Infof("block registry: attached replica %s for %q from heartbeat (path=%s)",
|
||
server, existingName, info.Path)
|
||
}
|
||
} else {
|
||
// Auto-register volumes reported by heartbeat but not in registry.
|
||
// This recovers state after master restart.
|
||
name := nameFromPath(info.Path)
|
||
if name == "" {
|
||
continue
|
||
}
|
||
if _, dup := r.volumes[name]; !dup {
|
||
entry := &BlockVolumeEntry{
|
||
Name: name,
|
||
VolumeServer: server,
|
||
Path: info.Path,
|
||
SizeBytes: info.VolumeSize,
|
||
Epoch: info.Epoch,
|
||
Role: info.Role,
|
||
Status: StatusActive,
|
||
LastLeaseGrant: time.Now(),
|
||
LeaseTTL: 30 * time.Second,
|
||
HealthScore: info.HealthScore,
|
||
WALHeadLSN: info.WalHeadLsn,
|
||
DurabilityMode: info.DurabilityMode,
|
||
}
|
||
if info.ReplicaDataAddr != "" {
|
||
entry.ReplicaDataAddr = info.ReplicaDataAddr
|
||
}
|
||
if info.ReplicaCtrlAddr != "" {
|
||
entry.ReplicaCtrlAddr = info.ReplicaCtrlAddr
|
||
}
|
||
// NVMe publication: propagate NVMe fields from heartbeat.
|
||
entry.NvmeAddr = info.NvmeAddr
|
||
entry.NQN = info.Nqn
|
||
r.volumes[name] = entry
|
||
r.addToServer(server, name)
|
||
glog.V(0).Infof("block registry: auto-registered %q from heartbeat (server=%s, path=%s, size=%d)",
|
||
name, server, info.Path, info.VolumeSize)
|
||
}
|
||
}
|
||
}
|
||
}
|
||
|
||
// UpdateDeltaHeartbeat processes incremental new/deleted block volumes.
|
||
// Called on subsequent heartbeats (not the first).
|
||
func (r *BlockVolumeRegistry) UpdateDeltaHeartbeat(server string, added []*master_pb.BlockVolumeShortInfoMessage, removed []*master_pb.BlockVolumeShortInfoMessage) {
|
||
r.mu.Lock()
|
||
defer r.mu.Unlock()
|
||
|
||
// Remove deleted volumes.
|
||
for _, rm := range removed {
|
||
if names, ok := r.byServer[server]; ok {
|
||
for name := range names {
|
||
if e := r.volumes[name]; e != nil && e.Path == rm.Path {
|
||
delete(r.volumes, name)
|
||
delete(names, name)
|
||
break
|
||
}
|
||
}
|
||
}
|
||
}
|
||
|
||
// Mark newly appeared volumes as active (if they exist in registry).
|
||
for _, add := range added {
|
||
if names, ok := r.byServer[server]; ok {
|
||
for name := range names {
|
||
if e := r.volumes[name]; e != nil && e.Path == add.Path {
|
||
e.Status = StatusActive
|
||
break
|
||
}
|
||
}
|
||
}
|
||
}
|
||
}
|
||
|
||
// PickServer returns the server address with the fewest block volumes.
|
||
// servers is the list of online volume server addresses.
|
||
// Returns error if no servers available.
|
||
func (r *BlockVolumeRegistry) PickServer(servers []string) (string, error) {
|
||
if len(servers) == 0 {
|
||
return "", fmt.Errorf("no block volume servers available")
|
||
}
|
||
r.mu.RLock()
|
||
defer r.mu.RUnlock()
|
||
|
||
best := servers[0]
|
||
bestCount := r.countForServer(best)
|
||
for _, s := range servers[1:] {
|
||
c := r.countForServer(s)
|
||
if c < bestCount {
|
||
best = s
|
||
bestCount = c
|
||
}
|
||
}
|
||
return best, nil
|
||
}
|
||
|
||
// AcquireInflight tries to acquire a per-name create lock.
|
||
// Returns true if acquired (caller must call ReleaseInflight when done).
|
||
// Returns false if another create is already in progress for this name.
|
||
func (r *BlockVolumeRegistry) AcquireInflight(name string) bool {
|
||
_, loaded := r.inflight.LoadOrStore(name, &inflightEntry{})
|
||
return !loaded // true = we stored it (acquired), false = already existed
|
||
}
|
||
|
||
// ReleaseInflight releases the per-name create lock.
|
||
func (r *BlockVolumeRegistry) ReleaseInflight(name string) {
|
||
r.inflight.Delete(name)
|
||
}
|
||
|
||
// countForServer returns the number of volumes on the given server.
|
||
// Caller must hold at least RLock.
|
||
func (r *BlockVolumeRegistry) countForServer(server string) int {
|
||
if names, ok := r.byServer[server]; ok {
|
||
return len(names)
|
||
}
|
||
return 0
|
||
}
|
||
|
||
func (r *BlockVolumeRegistry) addToServer(server, name string) {
|
||
if r.byServer[server] == nil {
|
||
r.byServer[server] = make(map[string]bool)
|
||
}
|
||
r.byServer[server][name] = true
|
||
}
|
||
|
||
func (r *BlockVolumeRegistry) removeFromServer(server, name string) {
|
||
if names, ok := r.byServer[server]; ok {
|
||
delete(names, name)
|
||
if len(names) == 0 {
|
||
delete(r.byServer, server)
|
||
}
|
||
}
|
||
}
|
||
|
||
// removeReplicaLocked removes a replica from an entry by server address.
|
||
// Caller must hold r.mu. Also syncs deprecated scalar fields.
|
||
func (r *BlockVolumeRegistry) removeReplicaLocked(entry *BlockVolumeEntry, server, name string) {
|
||
newReplicas := make([]ReplicaInfo, 0, len(entry.Replicas))
|
||
for _, ri := range entry.Replicas {
|
||
if ri.Server == server {
|
||
continue
|
||
}
|
||
newReplicas = append(newReplicas, ri)
|
||
}
|
||
entry.Replicas = newReplicas
|
||
// Sync deprecated scalar fields.
|
||
if len(entry.Replicas) > 0 {
|
||
r0 := &entry.Replicas[0]
|
||
entry.ReplicaServer = r0.Server
|
||
entry.ReplicaPath = r0.Path
|
||
entry.ReplicaISCSIAddr = r0.ISCSIAddr
|
||
entry.ReplicaIQN = r0.IQN
|
||
entry.ReplicaDataAddr = r0.DataAddr
|
||
entry.ReplicaCtrlAddr = r0.CtrlAddr
|
||
} else {
|
||
entry.ReplicaServer = ""
|
||
entry.ReplicaPath = ""
|
||
entry.ReplicaISCSIAddr = ""
|
||
entry.ReplicaIQN = ""
|
||
entry.ReplicaDataAddr = ""
|
||
entry.ReplicaCtrlAddr = ""
|
||
}
|
||
}
|
||
|
||
// SetReplica sets replica info for a registered volume.
|
||
// Deprecated: use AddReplica for new code. This method syncs both scalar and Replicas[].
|
||
func (r *BlockVolumeRegistry) SetReplica(name, server, path, iscsiAddr, iqn string) error {
|
||
r.mu.Lock()
|
||
defer r.mu.Unlock()
|
||
entry, ok := r.volumes[name]
|
||
if !ok {
|
||
return fmt.Errorf("block volume %q not found", name)
|
||
}
|
||
// Remove old replica from byServer index before replacing.
|
||
if entry.ReplicaServer != "" && entry.ReplicaServer != server {
|
||
r.removeFromServer(entry.ReplicaServer, name)
|
||
}
|
||
entry.ReplicaServer = server
|
||
entry.ReplicaPath = path
|
||
entry.ReplicaISCSIAddr = iscsiAddr
|
||
entry.ReplicaIQN = iqn
|
||
r.addToServer(server, name)
|
||
|
||
// CP8-2: also sync to Replicas[].
|
||
info := ReplicaInfo{Server: server, Path: path, ISCSIAddr: iscsiAddr, IQN: iqn}
|
||
replaced := false
|
||
for i := range entry.Replicas {
|
||
if entry.Replicas[i].Server == server {
|
||
// Preserve existing health/LSN data.
|
||
info.HealthScore = entry.Replicas[i].HealthScore
|
||
info.WALHeadLSN = entry.Replicas[i].WALHeadLSN
|
||
info.DataAddr = entry.Replicas[i].DataAddr
|
||
info.CtrlAddr = entry.Replicas[i].CtrlAddr
|
||
entry.Replicas[i] = info
|
||
replaced = true
|
||
break
|
||
}
|
||
}
|
||
if !replaced {
|
||
entry.Replicas = append(entry.Replicas, info)
|
||
}
|
||
return nil
|
||
}
|
||
|
||
// ClearReplica removes all replica info for a registered volume.
|
||
// Deprecated: use RemoveReplica for new code. This method clears both scalar and Replicas[].
|
||
func (r *BlockVolumeRegistry) ClearReplica(name string) error {
|
||
r.mu.Lock()
|
||
defer r.mu.Unlock()
|
||
entry, ok := r.volumes[name]
|
||
if !ok {
|
||
return fmt.Errorf("block volume %q not found", name)
|
||
}
|
||
if entry.ReplicaServer != "" {
|
||
r.removeFromServer(entry.ReplicaServer, name)
|
||
}
|
||
// Remove all replicas from byServer index.
|
||
for _, ri := range entry.Replicas {
|
||
if ri.Server != entry.ReplicaServer {
|
||
r.removeFromServer(ri.Server, name)
|
||
}
|
||
}
|
||
entry.ReplicaServer = ""
|
||
entry.ReplicaPath = ""
|
||
entry.ReplicaISCSIAddr = ""
|
||
entry.ReplicaIQN = ""
|
||
entry.ReplicaDataAddr = ""
|
||
entry.ReplicaCtrlAddr = ""
|
||
entry.Replicas = nil
|
||
return nil
|
||
}
|
||
|
||
// SwapPrimaryReplica promotes the replica to primary and clears the old replica.
|
||
// The old primary becomes the new replica (if it reconnects, rebuild will handle it).
|
||
// Epoch is atomically computed as entry.Epoch+1 inside the lock (R2-F5).
|
||
// Returns the new epoch for use in assignment messages.
|
||
func (r *BlockVolumeRegistry) SwapPrimaryReplica(name string) (uint64, error) {
|
||
r.mu.Lock()
|
||
defer r.mu.Unlock()
|
||
entry, ok := r.volumes[name]
|
||
if !ok {
|
||
return 0, fmt.Errorf("block volume %q not found", name)
|
||
}
|
||
if entry.ReplicaServer == "" {
|
||
return 0, fmt.Errorf("block volume %q has no replica", name)
|
||
}
|
||
|
||
// Remove old primary from byServer index.
|
||
r.removeFromServer(entry.VolumeServer, name)
|
||
|
||
oldPrimaryServer := entry.VolumeServer
|
||
oldPrimaryPath := entry.Path
|
||
oldPrimaryIQN := entry.IQN
|
||
oldPrimaryISCSI := entry.ISCSIAddr
|
||
|
||
// Atomically bump epoch inside lock (R2-F5: prevents race with heartbeat updates).
|
||
newEpoch := entry.Epoch + 1
|
||
|
||
// Promote replica to primary.
|
||
entry.VolumeServer = entry.ReplicaServer
|
||
entry.Path = entry.ReplicaPath
|
||
entry.IQN = entry.ReplicaIQN
|
||
entry.ISCSIAddr = entry.ReplicaISCSIAddr
|
||
entry.Epoch = newEpoch
|
||
entry.Role = blockvol.RoleToWire(blockvol.RolePrimary) // R2-F3
|
||
entry.LastLeaseGrant = time.Now()
|
||
|
||
// Old primary becomes stale replica (will be rebuilt when it reconnects).
|
||
entry.ReplicaServer = oldPrimaryServer
|
||
entry.ReplicaPath = oldPrimaryPath
|
||
entry.ReplicaIQN = oldPrimaryIQN
|
||
entry.ReplicaISCSIAddr = oldPrimaryISCSI
|
||
entry.ReplicaDataAddr = ""
|
||
entry.ReplicaCtrlAddr = ""
|
||
|
||
// Update byServer index: new primary server now hosts this volume.
|
||
r.addToServer(entry.VolumeServer, name)
|
||
return newEpoch, nil
|
||
}
|
||
|
||
// AddReplica adds or replaces a replica in the Replicas slice (by server).
|
||
// Also updates the byServer index and deprecated scalar fields for backward compat.
|
||
func (r *BlockVolumeRegistry) AddReplica(name string, info ReplicaInfo) error {
|
||
r.mu.Lock()
|
||
defer r.mu.Unlock()
|
||
entry, ok := r.volumes[name]
|
||
if !ok {
|
||
return fmt.Errorf("block volume %q not found", name)
|
||
}
|
||
// Replace if same server already exists.
|
||
replaced := false
|
||
for i := range entry.Replicas {
|
||
if entry.Replicas[i].Server == info.Server {
|
||
entry.Replicas[i] = info
|
||
replaced = true
|
||
break
|
||
}
|
||
}
|
||
if !replaced {
|
||
entry.Replicas = append(entry.Replicas, info)
|
||
}
|
||
r.addToServer(info.Server, name)
|
||
|
||
// Sync deprecated scalar fields (first replica → scalar).
|
||
if len(entry.Replicas) > 0 {
|
||
r0 := &entry.Replicas[0]
|
||
entry.ReplicaServer = r0.Server
|
||
entry.ReplicaPath = r0.Path
|
||
entry.ReplicaISCSIAddr = r0.ISCSIAddr
|
||
entry.ReplicaIQN = r0.IQN
|
||
entry.ReplicaDataAddr = r0.DataAddr
|
||
entry.ReplicaCtrlAddr = r0.CtrlAddr
|
||
}
|
||
return nil
|
||
}
|
||
|
||
// RemoveReplica removes a replica by server address.
|
||
func (r *BlockVolumeRegistry) RemoveReplica(name, server string) error {
|
||
r.mu.Lock()
|
||
defer r.mu.Unlock()
|
||
entry, ok := r.volumes[name]
|
||
if !ok {
|
||
return fmt.Errorf("block volume %q not found", name)
|
||
}
|
||
found := false
|
||
newReplicas := make([]ReplicaInfo, 0, len(entry.Replicas))
|
||
for _, ri := range entry.Replicas {
|
||
if ri.Server == server {
|
||
found = true
|
||
r.removeFromServer(server, name)
|
||
continue
|
||
}
|
||
newReplicas = append(newReplicas, ri)
|
||
}
|
||
if !found {
|
||
return fmt.Errorf("replica on %q not found for volume %q", server, name)
|
||
}
|
||
entry.Replicas = newReplicas
|
||
|
||
// Sync deprecated scalar fields.
|
||
if len(entry.Replicas) > 0 {
|
||
r0 := &entry.Replicas[0]
|
||
entry.ReplicaServer = r0.Server
|
||
entry.ReplicaPath = r0.Path
|
||
entry.ReplicaISCSIAddr = r0.ISCSIAddr
|
||
entry.ReplicaIQN = r0.IQN
|
||
entry.ReplicaDataAddr = r0.DataAddr
|
||
entry.ReplicaCtrlAddr = r0.CtrlAddr
|
||
} else {
|
||
entry.ReplicaServer = ""
|
||
entry.ReplicaPath = ""
|
||
entry.ReplicaISCSIAddr = ""
|
||
entry.ReplicaIQN = ""
|
||
entry.ReplicaDataAddr = ""
|
||
entry.ReplicaCtrlAddr = ""
|
||
}
|
||
return nil
|
||
}
|
||
|
||
// SetPromotionLSNTolerance configures the max WAL LSN lag for promotion eligibility.
|
||
func (r *BlockVolumeRegistry) SetPromotionLSNTolerance(tolerance uint64) {
|
||
r.mu.Lock()
|
||
defer r.mu.Unlock()
|
||
r.promotionLSNTolerance = tolerance
|
||
}
|
||
|
||
// PromotionLSNTolerance returns the current promotion LSN tolerance.
|
||
func (r *BlockVolumeRegistry) PromotionLSNTolerance() uint64 {
|
||
r.mu.RLock()
|
||
defer r.mu.RUnlock()
|
||
return r.promotionLSNTolerance
|
||
}
|
||
|
||
// PromoteBestReplica promotes the best eligible replica to primary.
|
||
// Eligibility: heartbeat fresh (within 2×LeaseTTL), WALHeadLSN within tolerance of primary,
|
||
// and role must be RoleReplica (not RoleRebuilding).
|
||
// The promoted replica is removed from Replicas[]. Other replicas stay.
|
||
// Old primary is NOT added to Replicas (needs rebuild).
|
||
// Returns the new epoch.
|
||
func (r *BlockVolumeRegistry) PromoteBestReplica(name string) (uint64, error) {
|
||
r.mu.Lock()
|
||
defer r.mu.Unlock()
|
||
entry, ok := r.volumes[name]
|
||
if !ok {
|
||
return 0, fmt.Errorf("block volume %q not found", name)
|
||
}
|
||
if len(entry.Replicas) == 0 {
|
||
return 0, fmt.Errorf("block volume %q has no replicas", name)
|
||
}
|
||
|
||
// Filter eligible replicas.
|
||
now := time.Now()
|
||
freshnessCutoff := 2 * entry.LeaseTTL
|
||
if freshnessCutoff == 0 {
|
||
freshnessCutoff = 60 * time.Second // default if LeaseTTL not set
|
||
}
|
||
primaryLSN := entry.WALHeadLSN
|
||
|
||
bestIdx := -1
|
||
for i := range entry.Replicas {
|
||
ri := &entry.Replicas[i]
|
||
// Gate 1: heartbeat freshness.
|
||
if !ri.LastHeartbeat.IsZero() && now.Sub(ri.LastHeartbeat) > freshnessCutoff {
|
||
continue
|
||
}
|
||
// Gate 2: WAL LSN recency (skip if primary LSN is 0 — no data yet, all eligible).
|
||
if primaryLSN > 0 && ri.WALHeadLSN+r.promotionLSNTolerance < primaryLSN {
|
||
continue
|
||
}
|
||
// Gate 3: role must be RoleReplica (not rebuilding/stale).
|
||
if ri.Role != 0 && blockvol.RoleFromWire(ri.Role) != blockvol.RoleReplica {
|
||
continue
|
||
}
|
||
// Eligible — pick best by health score, tie-break by WALHeadLSN.
|
||
if bestIdx == -1 {
|
||
bestIdx = i
|
||
} else if ri.HealthScore > entry.Replicas[bestIdx].HealthScore {
|
||
bestIdx = i
|
||
} else if ri.HealthScore == entry.Replicas[bestIdx].HealthScore &&
|
||
ri.WALHeadLSN > entry.Replicas[bestIdx].WALHeadLSN {
|
||
bestIdx = i
|
||
}
|
||
}
|
||
|
||
if bestIdx == -1 {
|
||
return 0, fmt.Errorf("block volume %q: no eligible replicas for promotion", name)
|
||
}
|
||
|
||
promoted := entry.Replicas[bestIdx]
|
||
|
||
// Remove old primary from byServer index.
|
||
r.removeFromServer(entry.VolumeServer, name)
|
||
|
||
// Bump epoch atomically.
|
||
newEpoch := entry.Epoch + 1
|
||
|
||
// Promote replica to primary.
|
||
entry.VolumeServer = promoted.Server
|
||
entry.Path = promoted.Path
|
||
entry.IQN = promoted.IQN
|
||
entry.ISCSIAddr = promoted.ISCSIAddr
|
||
entry.NvmeAddr = promoted.NvmeAddr
|
||
entry.NQN = promoted.NQN
|
||
entry.Epoch = newEpoch
|
||
entry.Role = blockvol.RoleToWire(blockvol.RolePrimary)
|
||
entry.LastLeaseGrant = time.Now()
|
||
|
||
// Remove promoted from Replicas. Others stay.
|
||
entry.Replicas = append(entry.Replicas[:bestIdx], entry.Replicas[bestIdx+1:]...)
|
||
|
||
// Sync deprecated scalar fields.
|
||
if len(entry.Replicas) > 0 {
|
||
r0 := &entry.Replicas[0]
|
||
entry.ReplicaServer = r0.Server
|
||
entry.ReplicaPath = r0.Path
|
||
entry.ReplicaISCSIAddr = r0.ISCSIAddr
|
||
entry.ReplicaIQN = r0.IQN
|
||
entry.ReplicaDataAddr = r0.DataAddr
|
||
entry.ReplicaCtrlAddr = r0.CtrlAddr
|
||
} else {
|
||
entry.ReplicaServer = ""
|
||
entry.ReplicaPath = ""
|
||
entry.ReplicaISCSIAddr = ""
|
||
entry.ReplicaIQN = ""
|
||
entry.ReplicaDataAddr = ""
|
||
entry.ReplicaCtrlAddr = ""
|
||
}
|
||
|
||
// Update byServer index: new primary server now hosts this volume.
|
||
r.addToServer(entry.VolumeServer, name)
|
||
|
||
return newEpoch, nil
|
||
}
|
||
|
||
// MarkBlockCapable records that the given server supports block volumes.
|
||
func (r *BlockVolumeRegistry) MarkBlockCapable(server string) {
|
||
r.mu.Lock()
|
||
r.blockServers[server] = true
|
||
r.mu.Unlock()
|
||
}
|
||
|
||
// UnmarkBlockCapable removes a server from the block-capable set.
|
||
func (r *BlockVolumeRegistry) UnmarkBlockCapable(server string) {
|
||
r.mu.Lock()
|
||
delete(r.blockServers, server)
|
||
r.mu.Unlock()
|
||
}
|
||
|
||
// LeaseGrant holds the minimal fields for a lease renewal.
|
||
type LeaseGrant struct {
|
||
Path string
|
||
Epoch uint64
|
||
Role uint32
|
||
LeaseTtlMs uint32
|
||
}
|
||
|
||
// LeaseGrants generates lightweight lease renewals for all active primary
|
||
// volumes on a server. Only primaries need lease renewal — replicas are passive
|
||
// WAL receivers without a write lease. Grants carry path + epoch + role + TTL
|
||
// and are processed by HandleAssignment's same-role refresh path, which
|
||
// validates the epoch and calls lease.Grant().
|
||
// Volumes with a pending assignment are excluded (the full assignment handles lease).
|
||
func (r *BlockVolumeRegistry) LeaseGrants(server string, pendingPaths map[string]bool) []LeaseGrant {
|
||
r.mu.RLock()
|
||
defer r.mu.RUnlock()
|
||
names, ok := r.byServer[server]
|
||
if !ok {
|
||
return nil
|
||
}
|
||
var grants []LeaseGrant
|
||
for name := range names {
|
||
e := r.volumes[name]
|
||
if e == nil || e.Status != StatusActive {
|
||
continue
|
||
}
|
||
// Only primaries need lease renewal. Replicas are passive WAL receivers
|
||
// and don't hold a write lease.
|
||
if blockvol.RoleFromWire(e.Role) != blockvol.RolePrimary {
|
||
continue
|
||
}
|
||
// Primary must be on this server.
|
||
if e.VolumeServer != server {
|
||
continue
|
||
}
|
||
if pendingPaths[e.Path] {
|
||
continue
|
||
}
|
||
grants = append(grants, LeaseGrant{
|
||
Path: e.Path,
|
||
Epoch: e.Epoch,
|
||
Role: e.Role,
|
||
LeaseTtlMs: blockvol.LeaseTTLToWire(e.LeaseTTL),
|
||
})
|
||
}
|
||
return grants
|
||
}
|
||
|
||
// ListAll returns all registered block volume entries, sorted by name.
|
||
func (r *BlockVolumeRegistry) ListAll() []*BlockVolumeEntry {
|
||
r.mu.RLock()
|
||
defer r.mu.RUnlock()
|
||
entries := make([]*BlockVolumeEntry, 0, len(r.volumes))
|
||
for _, e := range r.volumes {
|
||
entries = append(entries, e)
|
||
}
|
||
sort.Slice(entries, func(i, j int) bool { return entries[i].Name < entries[j].Name })
|
||
return entries
|
||
}
|
||
|
||
// BlockServerSummary summarizes a block-capable volume server.
|
||
type BlockServerSummary struct {
|
||
Address string
|
||
VolumeCount int
|
||
BlockCapable bool
|
||
}
|
||
|
||
// ServerSummaries returns a summary for each block-capable server.
|
||
func (r *BlockVolumeRegistry) ServerSummaries() []BlockServerSummary {
|
||
r.mu.RLock()
|
||
defer r.mu.RUnlock()
|
||
summaries := make([]BlockServerSummary, 0, len(r.blockServers))
|
||
for addr := range r.blockServers {
|
||
count := 0
|
||
if names, ok := r.byServer[addr]; ok {
|
||
count = len(names)
|
||
}
|
||
summaries = append(summaries, BlockServerSummary{
|
||
Address: addr,
|
||
VolumeCount: count,
|
||
BlockCapable: true,
|
||
})
|
||
}
|
||
sort.Slice(summaries, func(i, j int) bool { return summaries[i].Address < summaries[j].Address })
|
||
return summaries
|
||
}
|
||
|
||
// BlockCapableServers returns the list of servers known to support block volumes.
|
||
func (r *BlockVolumeRegistry) BlockCapableServers() []string {
|
||
r.mu.RLock()
|
||
defer r.mu.RUnlock()
|
||
servers := make([]string, 0, len(r.blockServers))
|
||
for s := range r.blockServers {
|
||
servers = append(servers, s)
|
||
}
|
||
return servers
|
||
}
|
||
|
||
// MaxBarrierLagLSN returns the maximum WAL lag across all volumes and replicas.
|
||
// This is the primary durability-risk metric: primary WALHeadLSN minus
|
||
// replica's WALHeadLSN. SLO threshold: < 100 under normal load.
|
||
func (r *BlockVolumeRegistry) MaxBarrierLagLSN() uint64 {
|
||
r.mu.RLock()
|
||
defer r.mu.RUnlock()
|
||
var maxLag uint64
|
||
for _, entry := range r.volumes {
|
||
for _, ri := range entry.Replicas {
|
||
if ri.WALLag > maxLag {
|
||
maxLag = ri.WALLag
|
||
}
|
||
}
|
||
}
|
||
return maxLag
|
||
}
|
||
|
||
// AssignmentQueueDepth returns the total number of pending assignments across all servers.
|
||
func (r *BlockVolumeRegistry) AssignmentQueueDepth() int {
|
||
// Delegated to the assignment queue, not the registry.
|
||
// Placeholder: the queue tracks its own depth.
|
||
return 0
|
||
}
|
||
|
||
// nameFromPath extracts the volume name from a .blk file path.
|
||
// e.g. "/opt/data/block/my-volume.blk" -> "my-volume"
|
||
func nameFromPath(path string) string {
|
||
base := filepath.Base(path)
|
||
if strings.HasSuffix(base, ".blk") {
|
||
return strings.TrimSuffix(base, ".blk")
|
||
}
|
||
return base
|
||
}
|
||
|