From 6f036c7015c6cd5a6419c4ae4bd88c568875b941 Mon Sep 17 00:00:00 2001
From: Chris Lu <chrislusf@users.noreply.github.com>
Date: Wed, 8 Apr 2026 21:08:50 -0700
Subject: [PATCH] fix(master): skip redundant DoJoinCommand on resumeState to
 prevent deadlock (#8998)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

* fix(master): skip redundant DoJoinCommand on resumeState to prevent deadlock

When fastResume is active (single-master + resumeState + non-empty log),
the raft server becomes leader within ~1ms. DoJoinCommand then enters
the leaderLoop's processCommand path, which calls setCommitIndex to
commit all pending entries. The goraft setCommitIndex implementation
returns early when it encounters a JoinCommand entry (to recalculate
quorum), which can prevent the new entry's event channel from being
notified — leaving DoJoinCommand blocked forever.

Each restart appends a new raft:join entry to the log, while the conf
file's commitIndex (only persisted on AddPeer) lags behind. After 3-4
restarts the uncommitted range contains old JoinCommand entries that
trigger the early return before the new entry is reached.

Fix: skip DoJoinCommand when the raft log already has entries (the
server was already joined in a previous run). The fastResume mechanism
handles leader election independently.

* fix(master): handle Hashicorp Raft in HasExistingState

Add Hashicorp Raft support to HasExistingState by checking
AppliedIndex, consistent with how other RaftServer methods
handle both raft implementations.

* fix(master): use LastIndex() instead of AppliedIndex() for Hashicorp Raft

AppliedIndex() reflects in-memory FSM state which starts at 0 before
log replay completes. LastIndex() reads from persisted stable storage,
correctly mirroring the non-Hashicorp IsLogEmpty() check.
---
 weed/command/master.go     |  9 +++++++--
 weed/server/raft_server.go | 13 +++++++++++++
 2 files changed, 20 insertions(+), 2 deletions(-)

diff --git a/weed/command/master.go b/weed/command/master.go
index 3ea8c96fc..189a38a35 100644
--- a/weed/command/master.go
+++ b/weed/command/master.go
@@ -224,8 +224,13 @@ func startMaster(masterOption MasterOptions, masterWhiteList []string) {
 		if raftServer == nil {
 			glog.Fatalf("please verify %s is writable, see https://github.com/seaweedfs/seaweedfs/issues/717: %s", *masterOption.metaFolder, err)
 		}
-		// For single-master mode, initialize cluster immediately without waiting
-		if isSingleMaster {
+		// For single-master mode with a fresh log, initialize cluster immediately.
+		// When resuming with existing state, the server is already a member and
+		// will self-elect via fastResume — sending another JoinCommand would block
+		// because goraft's setCommitIndex returns early on JoinCommand entries,
+		// preventing the new entry's event from being notified when old uncommitted
+		// JoinCommands exist in the log.
+		if isSingleMaster && !raftServer.HasExistingState() {
 			glog.V(0).Infof("Single-master mode: initializing cluster immediately")
 			raftServer.DoJoinCommand()
 		}
diff --git a/weed/server/raft_server.go b/weed/server/raft_server.go
index b42ac0671..f3b84c690 100644
--- a/weed/server/raft_server.go
+++ b/weed/server/raft_server.go
@@ -290,6 +290,19 @@ func recoverTopologyIdFromSnapshot(dataDir string, topo *topology.Topology) {
 	recoverTopologyIdFromState(snap.State, topo)
 }
 
+// HasExistingState returns true when the raft log already contains entries,
+// indicating this server was previously joined and does not need a new
+// JoinCommand on startup.
+func (s *RaftServer) HasExistingState() bool {
+	if s.raftServer != nil {
+		return !s.raftServer.IsLogEmpty()
+	}
+	if s.RaftHashicorp != nil {
+		return s.RaftHashicorp.LastIndex() > 0
+	}
+	return false
+}
+
 func (s *RaftServer) DoJoinCommand() {
 
 	glog.V(0).Infoln("Initializing new cluster")