diag: add sender registry verification after installSession

The accepted ack from the replica is rejected with "sender not found"
even though installSession succeeds. Add diagnostic logging to verify
the sender exists in the orchestrator registry immediately after
installSession, and dump all registry IDs if not found.

This will reveal whether the sender is removed between installSession
and the ack arrival (by syncProtocolExecutionState, evaluateActivationGate,
or another ProcessAssignment that reconciles with a stale replica list).

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
pingqiuandClaude Opus 4.6 committed 2026-04-09 15:53:33 -07:00
1 parent 943000ae8e
commit 0faf93a152
1 file changed
+18
+18
View File
@@ -212,6 +212,20 @@ func (rm *RecoveryManager) StartRebuildFromProbe(replicaID string) {
glog.Warningf("recovery: install rebuild session for probe %s: %v", replicaID, err)
return
}
// Diagnostic: verify sender exists after installSession.
if s := rm.bs.v2Orchestrator.Registry.Sender(replicaID); s != nil {
snap := s.SessionSnapshot()
glog.V(0).Infof("recovery: post-install sender %s: state=%s session=%+v", replicaID, s.State(), snap)
} else {
glog.Warningf("recovery: post-install sender %s NOT FOUND in registry", replicaID)
// List all senders for diagnosis.
all := rm.bs.v2Orchestrator.Registry.All()
ids := make([]string, len(all))
for i, s := range all {
ids[i] = s.ReplicaID()
}
glog.Warningf("recovery: registry contains: %v", ids)
}
rm.cancelAndDrain(replicaID, false)
rm.startTask(replicaID, nil)
}
@@ -588,6 +602,10 @@ func (rm *RecoveryManager) runRebuild(ctx context.Context, replicaID string, ass
if rm.OnPendingExecution != nil {
rm.OnPendingExecution(rctx.volPath, pe)
}
// Diagnostic: log the engine's DesiredReplicas so we can verify replicaID match.
if proj, ok := bs.CoreProjection(rctx.volPath); ok {
glog.V(0).Infof("recovery: pre-RebuildStarted replicaID=%q engine_replicas=%v", replicaID, proj.ReplicaIDs)
}
bs.applyCoreEvent(engine.RebuildStarted{ID: rctx.volPath, ReplicaID: replicaID, TargetLSN: plan.RebuildTargetLSN})
if rm.coord.Has(replicaID) {
rm.coord.Cancel(replicaID, "start_rebuild_not_emitted")