refactor: rebind bounded replica-ready heartbeat truth

Make the heartbeat/master boundary carry explicit replica readiness truth so the registry no longer depends only on replica transport-address presence as a readiness proxy. Keep backward compatibility by falling back to the old address heuristic when older heartbeats do not send the field.

Made-with: Cursor
This commit is contained in:
pingqiu committed 2026-04-04 12:06:53 -07:00
1 parent 43dbebfa04
commit 69b41a7f16
12 files changed
+710 -346

No files matched your search

+7 -4
View File
@@ -714,6 +714,9 @@ func replicaReadyObservedFromHeartbeat(info *master_pb.BlockVolumeInfoMessage) b
if info == nil {
return false
}
if info.ReplicaReady != nil {
return info.GetReplicaReady()
}
return info.ReplicaDataAddr != "" && info.ReplicaCtrlAddr != ""
}
@@ -1271,10 +1274,10 @@ type PromotionPreflightResult struct {
// evaluatePromotionLocked evaluates promotion candidates for a volume.
// Caller must hold r.mu (read or write). Returns a preflight result without
// mutating the registry. The four gates:
// 1. Heartbeat freshness (within 2×LeaseTTL)
// 2. WAL LSN recency (within promotionLSNTolerance of primary)
// 3. Role must be RoleReplica (not RoleRebuilding)
// 4. Server must be in blockServers (alive) — fixes B-12
// 1. Heartbeat freshness (within 2×LeaseTTL)
// 2. WAL LSN recency (within promotionLSNTolerance of primary)
// 3. Role must be RoleReplica (not RoleRebuilding)
// 4. Server must be in blockServers (alive) — fixes B-12
func (r *BlockVolumeRegistry) evaluatePromotionLocked(entry *BlockVolumeEntry) PromotionPreflightResult {
result := PromotionPreflightResult{
VolumeName: entry.Name,
+40
View File
@@ -2103,10 +2103,18 @@ func TestRegistry_UpdateFullHeartbeat_ConsumesCoreInfluencedReplicaReady(t *test
if hb.ReplicaDataAddr == "" || hb.ReplicaCtrlAddr == "" {
t.Fatalf("expected core-influenced replica addresses on heartbeat, hb=%+v", hb)
}
if !hb.ReplicaReady {
t.Fatalf("expected explicit replica_ready truth on heartbeat, hb=%+v", hb)
}
if hb.ReplicaDegraded {
t.Fatalf("did not expect degraded heartbeat on ready path, hb=%+v", hb)
}
// Prove master-side readiness now follows the explicit heartbeat truth even
// when transport addresses are absent on the consume path.
hb.ReplicaDataAddr = ""
hb.ReplicaCtrlAddr = ""
r := NewBlockVolumeRegistry()
if err := r.Register(&BlockVolumeEntry{
Name: "vol-master-ready",
@@ -2139,3 +2147,35 @@ func TestRegistry_UpdateFullHeartbeat_ConsumesCoreInfluencedReplicaReady(t *test
t.Fatalf("expected publish_healthy after ready consume, got %q", entry.VolumeMode)
}
}
func TestRegistry_UpdateFullHeartbeat_ReplicaReadyFallsBackToAddressesWhenFieldAbsent(t *testing.T) {
r := NewBlockVolumeRegistry()
if err := r.Register(&BlockVolumeEntry{
Name: "vol-master-ready-fallback",
VolumeServer: "primary-server:8080",
Path: "/blocks/vol-master-ready-fallback-primary.blk",
Status: StatusActive,
Role: blockvol.RoleToWire(blockvol.RolePrimary),
ReplicaFactor: 2,
Replicas: []ReplicaInfo{{
Server: "replica-server:8080",
Path: "/blocks/vol-master-ready-fallback.blk",
}},
}); err != nil {
t.Fatalf("register: %v", err)
}
r.UpdateFullHeartbeat("replica-server:8080", []*master_pb.BlockVolumeInfoMessage{{
Path: "/blocks/vol-master-ready-fallback.blk",
ReplicaDataAddr: "10.0.0.2:4260",
ReplicaCtrlAddr: "10.0.0.2:4261",
}}, "")
entry, _ := r.Lookup("vol-master-ready-fallback")
if !entry.Replicas[0].Ready {
t.Fatalf("expected backward-compatible ready fallback from addresses, entry=%+v", entry)
}
if !entry.ReplicaReady {
t.Fatalf("expected aggregate replica ready from fallback consume, entry=%+v", entry)
}
}
+28 -9
View File
@@ -38,9 +38,9 @@ type volReplState struct {
// not by blockvol's local storage mechanics.
//
// Important:
// PublishHealthy here is still an adapter-local publication bit used by current
// `weed/server` surfaces. It is NOT the semantic owner for Phase 14 core
// publication health; that owner is `engine.PublicationView`.
// PublishHealthy here is the server-boundary mirror of the core-owned
// publication health when a core projection exists. Adapter-local fallback
// remains only for paths where the core is absent.
type BlockReadinessSnapshot struct {
RoleApplied bool
ReceiverReady bool
@@ -140,8 +140,7 @@ func (bs *BlockService) ExecutedCoreCommands(path string) []string {
}
// CoreProjectionMismatches reports fields that should already agree on the
// narrow Phase 15A path but do not. It intentionally excludes adapter-local
// `PublishHealthy`, which is not yet rebound to the core publication owner.
// current bounded core-present path but do not.
func (bs *BlockService) CoreProjectionMismatches(path string) []string {
proj, ok := bs.CoreProjection(path)
if !ok {
@@ -161,6 +160,9 @@ func (bs *BlockService) CoreProjectionMismatches(path string) []string {
if readiness.ShipperConnected != proj.Readiness.ShipperConnected {
mismatches = append(mismatches, "shipper_connected")
}
if readiness.PublishHealthy != proj.Publication.Healthy {
mismatches = append(mismatches, "publish_healthy")
}
return mismatches
}
@@ -963,6 +965,7 @@ func (bs *BlockService) CollectBlockVolumeHeartbeat() []blockvol.BlockVolumeInfo
for i := range msgs {
if s, ok := bs.replStates[msgs[i].Path]; ok {
msgs[i].ReplicaDataAddr, msgs[i].ReplicaCtrlAddr = bs.heartbeatReplicaAddrs(msgs[i].Path, s)
msgs[i].ReplicaReady = bs.heartbeatReplicaReady(msgs[i].Path, s)
}
msgs[i].ReplicaDegraded = bs.heartbeatReplicaDegraded(msgs[i].Path, msgs[i].ReplicaDegraded)
// NVMe publication: report nvme_addr and nqn if NVMe target is running.
@@ -1004,6 +1007,20 @@ func (bs *BlockService) heartbeatReplicaAddrs(path string, state *volReplState)
return "", ""
}
// heartbeatReplicaReady returns the explicit replica-readiness truth that should
// be exposed on the current heartbeat surface. On the core-present path it
// prefers the core-owned readiness projection; older paths fall back to the
// adapter-local publish gate for compatibility.
func (bs *BlockService) heartbeatReplicaReady(path string, state *volReplState) bool {
if state == nil {
return false
}
if proj, ok := bs.CoreProjection(path); ok {
return proj.Readiness.ReplicaReady
}
return state.publishHealthy
}
// heartbeatReplicaDegraded returns the bounded degraded bit for the current
// heartbeat surface. On the Phase 15 live path it prefers the core mode when
// present, then falls back to the runtime-local status bit.
@@ -1118,10 +1135,10 @@ func (bs *BlockService) markReceiverReady(path, dataAddr, ctrlAddr string) {
state.allReplicas = nil
}
// ReadinessSnapshot reports the service-owned assignment/readiness closure for
// one volume. On the Phase 15 live path it prefers the explicit core projection
// for the aligned readiness subset, while `PublishHealthy` remains adapter-local
// until publication ownership is fully rebound.
// ReadinessSnapshot reports the current server-boundary readiness/publication
// closure for one volume. On the core-present path it prefers the explicit core
// projection for aligned readiness plus publication health, while retaining
// adapter-local fallback only when the core is absent.
func (bs *BlockService) ReadinessSnapshot(path string) BlockReadinessSnapshot {
snap := BlockReadinessSnapshot{}
bs.replMu.RLock()
@@ -1141,6 +1158,7 @@ func (bs *BlockService) ReadinessSnapshot(path string) BlockReadinessSnapshot {
snap.ShipperConfigured = proj.Readiness.ShipperConfigured
snap.ShipperConnected = proj.Readiness.ShipperConnected
snap.ReplicaEligible = proj.Readiness.ReplicaReady
snap.PublishHealthy = proj.Publication.Healthy
}
return snap
}
@@ -1154,6 +1172,7 @@ func (bs *BlockService) ReadinessSnapshot(path string) BlockReadinessSnapshot {
snap.ShipperConfigured = proj.Readiness.ShipperConfigured
snap.ShipperConnected = proj.Readiness.ShipperConnected
snap.ReplicaEligible = proj.Readiness.ReplicaReady
snap.PublishHealthy = proj.Publication.Healthy
}
return snap
}
+50 -4
View File
@@ -937,8 +937,8 @@ func TestBlockService_DebugInfoForVolume_UsesCoreProjectionPrimaryPath(t *testin
}
readiness := bs.ReadinessSnapshot(path)
if !readiness.PublishHealthy {
t.Fatalf("expected adapter-local readiness to still report publish healthy, got %+v", readiness)
if readiness.PublishHealthy {
t.Fatalf("readiness snapshot must follow core publication truth on primary path without durable boundary, got %+v", readiness)
}
vol, ok := bs.blockStore.GetBlockVolume(path)
@@ -1064,6 +1064,9 @@ func TestBlockService_CollectBlockVolumeHeartbeat_PrimaryUsesCoreReadinessGate(t
if msg.ReplicaCtrlAddr != "10.0.0.2:4261" {
t.Fatalf("ReplicaCtrlAddr=%q", msg.ReplicaCtrlAddr)
}
if msg.ReplicaReady {
t.Fatalf("primary heartbeat must not claim replica-ready truth, msg=%+v", msg)
}
}
func TestBlockService_CollectBlockVolumeHeartbeat_ReplicaUsesCoreReadinessGate(t *testing.T) {
@@ -1117,6 +1120,9 @@ func TestBlockService_CollectBlockVolumeHeartbeat_ReplicaUsesCoreReadinessGate(t
if msg.ReplicaCtrlAddr != expectedCtrl {
t.Fatalf("ReplicaCtrlAddr=%q expected=%q", msg.ReplicaCtrlAddr, expectedCtrl)
}
if !msg.ReplicaReady {
t.Fatalf("replica heartbeat should carry explicit replica-ready truth, msg=%+v", msg)
}
}
func TestBlockService_ReadinessSnapshot_PrefersCoreProjectionPrimaryFields(t *testing.T) {
@@ -1157,7 +1163,47 @@ func TestBlockService_ReadinessSnapshot_PrefersCoreProjectionPrimaryFields(t *te
t.Fatalf("expected snapshot shipper_configured from core projection, got %+v", snap)
}
if snap.PublishHealthy {
t.Fatalf("publish_healthy should remain adapter-local on readiness snapshot, got %+v", snap)
t.Fatalf("publish_healthy should follow core publication truth on primary path without durable boundary, got %+v", snap)
}
}
func TestBlockService_ReadinessSnapshot_PrefersCorePublicationHealth(t *testing.T) {
bs := newTestBlockServiceDirect(t)
path := createTestVolDirect(t, bs, "vol-readiness-publish-healthy")
errs := bs.ApplyAssignments([]blockvol.BlockVolumeAssignment{
{
Path: path,
Epoch: 1,
Role: blockvol.RoleToWire(blockvol.RolePrimary),
LeaseTtlMs: 30000,
ReplicaServerID: "vs-2",
ReplicaDataAddr: "10.0.0.2:4260",
ReplicaCtrlAddr: "10.0.0.2:4261",
},
})
if len(errs) != 1 || errs[0] != nil {
t.Fatalf("apply assignment errs=%v", errs)
}
bs.applyCoreEvent(engine.ShipperConnectedObserved{ID: path})
bs.applyCoreEvent(engine.BarrierAccepted{ID: path, FlushedLSN: 12})
bs.replMu.Lock()
state := bs.replStates[path]
if state == nil {
bs.replMu.Unlock()
t.Fatal("missing repl state")
}
state.publishHealthy = false
bs.replMu.Unlock()
snap := bs.ReadinessSnapshot(path)
if !snap.PublishHealthy {
t.Fatalf("publish_healthy should follow core publication truth once healthy, got %+v", snap)
}
if mismatches := bs.CoreProjectionMismatches(path); len(mismatches) != 0 {
t.Fatalf("readiness/core mismatches=%v", mismatches)
}
}
@@ -1198,7 +1244,7 @@ func TestBlockService_ReadinessSnapshot_PrefersCoreProjectionReplicaFields(t *te
t.Fatalf("expected snapshot replica_eligible from core projection, got %+v", snap)
}
if snap.PublishHealthy {
t.Fatalf("publish_healthy should remain adapter-local on readiness snapshot, got %+v", snap)
t.Fatalf("publish_healthy should follow core publication truth on replica path, got %+v", snap)
}
}