From 0faf93a15228071e743e9c57ae854ee791b755f7 Mon Sep 17 00:00:00 2001 From: pingqiu Date: Thu, 9 Apr 2026 15:53:33 -0700 Subject: [PATCH] diag: add sender registry verification after installSession The accepted ack from the replica is rejected with "sender not found" even though installSession succeeds. Add diagnostic logging to verify the sender exists in the orchestrator registry immediately after installSession, and dump all registry IDs if not found. This will reveal whether the sender is removed between installSession and the ack arrival (by syncProtocolExecutionState, evaluateActivationGate, or another ProcessAssignment that reconciles with a stale replica list). Co-Authored-By: Claude Opus 4.6 (1M context) --- weed/server/block_recovery.go | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/weed/server/block_recovery.go b/weed/server/block_recovery.go index af30c865c5..7a98d780ff 100644 --- a/weed/server/block_recovery.go +++ b/weed/server/block_recovery.go @@ -212,6 +212,20 @@ func (rm *RecoveryManager) StartRebuildFromProbe(replicaID string) { glog.Warningf("recovery: install rebuild session for probe %s: %v", replicaID, err) return } + // Diagnostic: verify sender exists after installSession. + if s := rm.bs.v2Orchestrator.Registry.Sender(replicaID); s != nil { + snap := s.SessionSnapshot() + glog.V(0).Infof("recovery: post-install sender %s: state=%s session=%+v", replicaID, s.State(), snap) + } else { + glog.Warningf("recovery: post-install sender %s NOT FOUND in registry", replicaID) + // List all senders for diagnosis. + all := rm.bs.v2Orchestrator.Registry.All() + ids := make([]string, len(all)) + for i, s := range all { + ids[i] = s.ReplicaID() + } + glog.Warningf("recovery: registry contains: %v", ids) + } rm.cancelAndDrain(replicaID, false) rm.startTask(replicaID, nil) } @@ -588,6 +602,10 @@ func (rm *RecoveryManager) runRebuild(ctx context.Context, replicaID string, ass if rm.OnPendingExecution != nil { rm.OnPendingExecution(rctx.volPath, pe) } + // Diagnostic: log the engine's DesiredReplicas so we can verify replicaID match. + if proj, ok := bs.CoreProjection(rctx.volPath); ok { + glog.V(0).Infof("recovery: pre-RebuildStarted replicaID=%q engine_replicas=%v", replicaID, proj.ReplicaIDs) + } bs.applyCoreEvent(engine.RebuildStarted{ID: rctx.volPath, ReplicaID: replicaID, TargetLSN: plan.RebuildTargetLSN}) if rm.coord.Has(replicaID) { rm.coord.Cancel(replicaID, "start_rebuild_not_emitted")