mirror of
https://github.com/seaweedfs/seaweedfs.git
synced 2026-11-19 02:15:02 +01:00
refactor: rebind bounded replica-ready heartbeat truth
Make the heartbeat/master boundary carry explicit replica readiness truth so the registry no longer depends only on replica transport-address presence as a readiness proxy. Keep backward compatibility by falling back to the old address heuristic when older heartbeats do not send the field. Made-with: Cursor
This commit is contained in:
1 parent
43dbebfa04
commit
69b41a7f16
12 files changed
+710
-346
No files matched your search
@@ -714,6 +714,9 @@ func replicaReadyObservedFromHeartbeat(info *master_pb.BlockVolumeInfoMessage) b
|
||||
if info == nil {
|
||||
return false
|
||||
}
|
||||
if info.ReplicaReady != nil {
|
||||
return info.GetReplicaReady()
|
||||
}
|
||||
return info.ReplicaDataAddr != "" && info.ReplicaCtrlAddr != ""
|
||||
}
|
||||
|
||||
@@ -1271,10 +1274,10 @@ type PromotionPreflightResult struct {
|
||||
// evaluatePromotionLocked evaluates promotion candidates for a volume.
|
||||
// Caller must hold r.mu (read or write). Returns a preflight result without
|
||||
// mutating the registry. The four gates:
|
||||
// 1. Heartbeat freshness (within 2×LeaseTTL)
|
||||
// 2. WAL LSN recency (within promotionLSNTolerance of primary)
|
||||
// 3. Role must be RoleReplica (not RoleRebuilding)
|
||||
// 4. Server must be in blockServers (alive) — fixes B-12
|
||||
// 1. Heartbeat freshness (within 2×LeaseTTL)
|
||||
// 2. WAL LSN recency (within promotionLSNTolerance of primary)
|
||||
// 3. Role must be RoleReplica (not RoleRebuilding)
|
||||
// 4. Server must be in blockServers (alive) — fixes B-12
|
||||
func (r *BlockVolumeRegistry) evaluatePromotionLocked(entry *BlockVolumeEntry) PromotionPreflightResult {
|
||||
result := PromotionPreflightResult{
|
||||
VolumeName: entry.Name,
|
||||
|
||||
@@ -2103,10 +2103,18 @@ func TestRegistry_UpdateFullHeartbeat_ConsumesCoreInfluencedReplicaReady(t *test
|
||||
if hb.ReplicaDataAddr == "" || hb.ReplicaCtrlAddr == "" {
|
||||
t.Fatalf("expected core-influenced replica addresses on heartbeat, hb=%+v", hb)
|
||||
}
|
||||
if !hb.ReplicaReady {
|
||||
t.Fatalf("expected explicit replica_ready truth on heartbeat, hb=%+v", hb)
|
||||
}
|
||||
if hb.ReplicaDegraded {
|
||||
t.Fatalf("did not expect degraded heartbeat on ready path, hb=%+v", hb)
|
||||
}
|
||||
|
||||
// Prove master-side readiness now follows the explicit heartbeat truth even
|
||||
// when transport addresses are absent on the consume path.
|
||||
hb.ReplicaDataAddr = ""
|
||||
hb.ReplicaCtrlAddr = ""
|
||||
|
||||
r := NewBlockVolumeRegistry()
|
||||
if err := r.Register(&BlockVolumeEntry{
|
||||
Name: "vol-master-ready",
|
||||
@@ -2139,3 +2147,35 @@ func TestRegistry_UpdateFullHeartbeat_ConsumesCoreInfluencedReplicaReady(t *test
|
||||
t.Fatalf("expected publish_healthy after ready consume, got %q", entry.VolumeMode)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRegistry_UpdateFullHeartbeat_ReplicaReadyFallsBackToAddressesWhenFieldAbsent(t *testing.T) {
|
||||
r := NewBlockVolumeRegistry()
|
||||
if err := r.Register(&BlockVolumeEntry{
|
||||
Name: "vol-master-ready-fallback",
|
||||
VolumeServer: "primary-server:8080",
|
||||
Path: "/blocks/vol-master-ready-fallback-primary.blk",
|
||||
Status: StatusActive,
|
||||
Role: blockvol.RoleToWire(blockvol.RolePrimary),
|
||||
ReplicaFactor: 2,
|
||||
Replicas: []ReplicaInfo{{
|
||||
Server: "replica-server:8080",
|
||||
Path: "/blocks/vol-master-ready-fallback.blk",
|
||||
}},
|
||||
}); err != nil {
|
||||
t.Fatalf("register: %v", err)
|
||||
}
|
||||
|
||||
r.UpdateFullHeartbeat("replica-server:8080", []*master_pb.BlockVolumeInfoMessage{{
|
||||
Path: "/blocks/vol-master-ready-fallback.blk",
|
||||
ReplicaDataAddr: "10.0.0.2:4260",
|
||||
ReplicaCtrlAddr: "10.0.0.2:4261",
|
||||
}}, "")
|
||||
|
||||
entry, _ := r.Lookup("vol-master-ready-fallback")
|
||||
if !entry.Replicas[0].Ready {
|
||||
t.Fatalf("expected backward-compatible ready fallback from addresses, entry=%+v", entry)
|
||||
}
|
||||
if !entry.ReplicaReady {
|
||||
t.Fatalf("expected aggregate replica ready from fallback consume, entry=%+v", entry)
|
||||
}
|
||||
}
|
||||
@@ -38,9 +38,9 @@ type volReplState struct {
|
||||
// not by blockvol's local storage mechanics.
|
||||
//
|
||||
// Important:
|
||||
// PublishHealthy here is still an adapter-local publication bit used by current
|
||||
// `weed/server` surfaces. It is NOT the semantic owner for Phase 14 core
|
||||
// publication health; that owner is `engine.PublicationView`.
|
||||
// PublishHealthy here is the server-boundary mirror of the core-owned
|
||||
// publication health when a core projection exists. Adapter-local fallback
|
||||
// remains only for paths where the core is absent.
|
||||
type BlockReadinessSnapshot struct {
|
||||
RoleApplied bool
|
||||
ReceiverReady bool
|
||||
@@ -140,8 +140,7 @@ func (bs *BlockService) ExecutedCoreCommands(path string) []string {
|
||||
}
|
||||
|
||||
// CoreProjectionMismatches reports fields that should already agree on the
|
||||
// narrow Phase 15A path but do not. It intentionally excludes adapter-local
|
||||
// `PublishHealthy`, which is not yet rebound to the core publication owner.
|
||||
// current bounded core-present path but do not.
|
||||
func (bs *BlockService) CoreProjectionMismatches(path string) []string {
|
||||
proj, ok := bs.CoreProjection(path)
|
||||
if !ok {
|
||||
@@ -161,6 +160,9 @@ func (bs *BlockService) CoreProjectionMismatches(path string) []string {
|
||||
if readiness.ShipperConnected != proj.Readiness.ShipperConnected {
|
||||
mismatches = append(mismatches, "shipper_connected")
|
||||
}
|
||||
if readiness.PublishHealthy != proj.Publication.Healthy {
|
||||
mismatches = append(mismatches, "publish_healthy")
|
||||
}
|
||||
return mismatches
|
||||
}
|
||||
|
||||
@@ -963,6 +965,7 @@ func (bs *BlockService) CollectBlockVolumeHeartbeat() []blockvol.BlockVolumeInfo
|
||||
for i := range msgs {
|
||||
if s, ok := bs.replStates[msgs[i].Path]; ok {
|
||||
msgs[i].ReplicaDataAddr, msgs[i].ReplicaCtrlAddr = bs.heartbeatReplicaAddrs(msgs[i].Path, s)
|
||||
msgs[i].ReplicaReady = bs.heartbeatReplicaReady(msgs[i].Path, s)
|
||||
}
|
||||
msgs[i].ReplicaDegraded = bs.heartbeatReplicaDegraded(msgs[i].Path, msgs[i].ReplicaDegraded)
|
||||
// NVMe publication: report nvme_addr and nqn if NVMe target is running.
|
||||
@@ -1004,6 +1007,20 @@ func (bs *BlockService) heartbeatReplicaAddrs(path string, state *volReplState)
|
||||
return "", ""
|
||||
}
|
||||
|
||||
// heartbeatReplicaReady returns the explicit replica-readiness truth that should
|
||||
// be exposed on the current heartbeat surface. On the core-present path it
|
||||
// prefers the core-owned readiness projection; older paths fall back to the
|
||||
// adapter-local publish gate for compatibility.
|
||||
func (bs *BlockService) heartbeatReplicaReady(path string, state *volReplState) bool {
|
||||
if state == nil {
|
||||
return false
|
||||
}
|
||||
if proj, ok := bs.CoreProjection(path); ok {
|
||||
return proj.Readiness.ReplicaReady
|
||||
}
|
||||
return state.publishHealthy
|
||||
}
|
||||
|
||||
// heartbeatReplicaDegraded returns the bounded degraded bit for the current
|
||||
// heartbeat surface. On the Phase 15 live path it prefers the core mode when
|
||||
// present, then falls back to the runtime-local status bit.
|
||||
@@ -1118,10 +1135,10 @@ func (bs *BlockService) markReceiverReady(path, dataAddr, ctrlAddr string) {
|
||||
state.allReplicas = nil
|
||||
}
|
||||
|
||||
// ReadinessSnapshot reports the service-owned assignment/readiness closure for
|
||||
// one volume. On the Phase 15 live path it prefers the explicit core projection
|
||||
// for the aligned readiness subset, while `PublishHealthy` remains adapter-local
|
||||
// until publication ownership is fully rebound.
|
||||
// ReadinessSnapshot reports the current server-boundary readiness/publication
|
||||
// closure for one volume. On the core-present path it prefers the explicit core
|
||||
// projection for aligned readiness plus publication health, while retaining
|
||||
// adapter-local fallback only when the core is absent.
|
||||
func (bs *BlockService) ReadinessSnapshot(path string) BlockReadinessSnapshot {
|
||||
snap := BlockReadinessSnapshot{}
|
||||
bs.replMu.RLock()
|
||||
@@ -1141,6 +1158,7 @@ func (bs *BlockService) ReadinessSnapshot(path string) BlockReadinessSnapshot {
|
||||
snap.ShipperConfigured = proj.Readiness.ShipperConfigured
|
||||
snap.ShipperConnected = proj.Readiness.ShipperConnected
|
||||
snap.ReplicaEligible = proj.Readiness.ReplicaReady
|
||||
snap.PublishHealthy = proj.Publication.Healthy
|
||||
}
|
||||
return snap
|
||||
}
|
||||
@@ -1154,6 +1172,7 @@ func (bs *BlockService) ReadinessSnapshot(path string) BlockReadinessSnapshot {
|
||||
snap.ShipperConfigured = proj.Readiness.ShipperConfigured
|
||||
snap.ShipperConnected = proj.Readiness.ShipperConnected
|
||||
snap.ReplicaEligible = proj.Readiness.ReplicaReady
|
||||
snap.PublishHealthy = proj.Publication.Healthy
|
||||
}
|
||||
return snap
|
||||
}
|
||||
|
||||
@@ -937,8 +937,8 @@ func TestBlockService_DebugInfoForVolume_UsesCoreProjectionPrimaryPath(t *testin
|
||||
}
|
||||
|
||||
readiness := bs.ReadinessSnapshot(path)
|
||||
if !readiness.PublishHealthy {
|
||||
t.Fatalf("expected adapter-local readiness to still report publish healthy, got %+v", readiness)
|
||||
if readiness.PublishHealthy {
|
||||
t.Fatalf("readiness snapshot must follow core publication truth on primary path without durable boundary, got %+v", readiness)
|
||||
}
|
||||
|
||||
vol, ok := bs.blockStore.GetBlockVolume(path)
|
||||
@@ -1064,6 +1064,9 @@ func TestBlockService_CollectBlockVolumeHeartbeat_PrimaryUsesCoreReadinessGate(t
|
||||
if msg.ReplicaCtrlAddr != "10.0.0.2:4261" {
|
||||
t.Fatalf("ReplicaCtrlAddr=%q", msg.ReplicaCtrlAddr)
|
||||
}
|
||||
if msg.ReplicaReady {
|
||||
t.Fatalf("primary heartbeat must not claim replica-ready truth, msg=%+v", msg)
|
||||
}
|
||||
}
|
||||
|
||||
func TestBlockService_CollectBlockVolumeHeartbeat_ReplicaUsesCoreReadinessGate(t *testing.T) {
|
||||
@@ -1117,6 +1120,9 @@ func TestBlockService_CollectBlockVolumeHeartbeat_ReplicaUsesCoreReadinessGate(t
|
||||
if msg.ReplicaCtrlAddr != expectedCtrl {
|
||||
t.Fatalf("ReplicaCtrlAddr=%q expected=%q", msg.ReplicaCtrlAddr, expectedCtrl)
|
||||
}
|
||||
if !msg.ReplicaReady {
|
||||
t.Fatalf("replica heartbeat should carry explicit replica-ready truth, msg=%+v", msg)
|
||||
}
|
||||
}
|
||||
|
||||
func TestBlockService_ReadinessSnapshot_PrefersCoreProjectionPrimaryFields(t *testing.T) {
|
||||
@@ -1157,7 +1163,47 @@ func TestBlockService_ReadinessSnapshot_PrefersCoreProjectionPrimaryFields(t *te
|
||||
t.Fatalf("expected snapshot shipper_configured from core projection, got %+v", snap)
|
||||
}
|
||||
if snap.PublishHealthy {
|
||||
t.Fatalf("publish_healthy should remain adapter-local on readiness snapshot, got %+v", snap)
|
||||
t.Fatalf("publish_healthy should follow core publication truth on primary path without durable boundary, got %+v", snap)
|
||||
}
|
||||
}
|
||||
|
||||
func TestBlockService_ReadinessSnapshot_PrefersCorePublicationHealth(t *testing.T) {
|
||||
bs := newTestBlockServiceDirect(t)
|
||||
path := createTestVolDirect(t, bs, "vol-readiness-publish-healthy")
|
||||
|
||||
errs := bs.ApplyAssignments([]blockvol.BlockVolumeAssignment{
|
||||
{
|
||||
Path: path,
|
||||
Epoch: 1,
|
||||
Role: blockvol.RoleToWire(blockvol.RolePrimary),
|
||||
LeaseTtlMs: 30000,
|
||||
ReplicaServerID: "vs-2",
|
||||
ReplicaDataAddr: "10.0.0.2:4260",
|
||||
ReplicaCtrlAddr: "10.0.0.2:4261",
|
||||
},
|
||||
})
|
||||
if len(errs) != 1 || errs[0] != nil {
|
||||
t.Fatalf("apply assignment errs=%v", errs)
|
||||
}
|
||||
|
||||
bs.applyCoreEvent(engine.ShipperConnectedObserved{ID: path})
|
||||
bs.applyCoreEvent(engine.BarrierAccepted{ID: path, FlushedLSN: 12})
|
||||
|
||||
bs.replMu.Lock()
|
||||
state := bs.replStates[path]
|
||||
if state == nil {
|
||||
bs.replMu.Unlock()
|
||||
t.Fatal("missing repl state")
|
||||
}
|
||||
state.publishHealthy = false
|
||||
bs.replMu.Unlock()
|
||||
|
||||
snap := bs.ReadinessSnapshot(path)
|
||||
if !snap.PublishHealthy {
|
||||
t.Fatalf("publish_healthy should follow core publication truth once healthy, got %+v", snap)
|
||||
}
|
||||
if mismatches := bs.CoreProjectionMismatches(path); len(mismatches) != 0 {
|
||||
t.Fatalf("readiness/core mismatches=%v", mismatches)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1198,7 +1244,7 @@ func TestBlockService_ReadinessSnapshot_PrefersCoreProjectionReplicaFields(t *te
|
||||
t.Fatalf("expected snapshot replica_eligible from core projection, got %+v", snap)
|
||||
}
|
||||
if snap.PublishHealthy {
|
||||
t.Fatalf("publish_healthy should remain adapter-local on readiness snapshot, got %+v", snap)
|
||||
t.Fatalf("publish_healthy should follow core publication truth on replica path, got %+v", snap)
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
Reference in new issue
Block a user