mirror of
https://github.com/seaweedfs/seaweedfs.git
synced 2026-10-26 15:00:48 +01:00
Chris LuandDevin
da1d7444b8
* master: release the post-mint allocation hold after a quiet pulse DeferAllocation armed a fixed warmup window whenever the leader minted a fresh topology id, so the first write on a brand-new cluster blocked for the whole window even though no prior state could exist. The window still has to cover a rebuilt master's veteran servers, which report on their first post-reconnect heartbeat. Track the last heartbeat that could carry prior cluster state — a server joining, held volumes or shards, a nonzero id bound, or a foreign topology id — and lift the hold one pulse after it goes quiet, capped by the original window. * master: make the post-mint allocation hold configurable master.volume_growth.allocation_deferral overrides the warmup window a leader holds volume id allocation for after minting a fresh topology id. Unset keeps the existing window; "0s" disables the hold for setups that know the cluster is fresh (test harnesses, single-node labs); any other duration replaces the cap while the quiet-pulse early release still applies. * master: re-check the allocation deadline while it is held NextVolumeId slept for the whole computed delay, so a heartbeat landing mid-sleep that pushed the quiet deadline out was ignored — the pending allocation could still run ahead of a straggler's report. Wait in short slices, re-evaluating the deadline and the growth freeze each round. * master: count only ids above the issued bound as prior-state evidence Heartbeat echoes of volumes and shards this master allocated during the hold refreshed the quiet window, so each growth allocation re-extended the deferral until the hard cap. Evidence now requires an id beyond the bound this leader has issued, a higher reported bound, or a foreign topology id; a joining server (dn == nil) still counts once. * master: reject an unparseable allocation_deferral instead of disabling the hold Generated with [Devin](https://devin.ai) Co-Authored-By: Devin <158243242+devin-ai-integration[bot]@users.noreply.github.com> * master: do not count a skipped heartbeat as a joining server Generated with [Devin](https://devin.ai) Co-Authored-By: Devin <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test: assert the allocation hold extends past the original quiet deadline Generated with [Devin](https://devin.ai) Co-Authored-By: Devin <158243242+devin-ai-integration[bot]@users.noreply.github.com> * master: release the post-mint hold after two quiet pulses One pulse proved short for a veteran that reconnects on a staggered restart: with only five seconds of quiet, allocation could resume and reuse ids before a slower server landed its first heartbeat. Two pulses still halves the fresh-cluster first-write stall versus the full warmup cap, and known-fresh setups keep the "0s" override. * master: pick once more when assign times out as growth concludes The pick loop's final attempt can predate the growth it triggered concluding: growth registers its volumes in the layout before clearing the pending flag, so a timeout landing on that boundary used to shed a hard 406 over volumes that had just arrived. Pick once more after the loop in both the http and grpc assign paths before deciding how to shed. --------- Co-authored-by: Devin <158243242+devin-ai-integration[bot]@users.noreply.github.com>
72 lines
2.2 KiB
Go
72 lines
2.2 KiB
Go
package weed_server
|
|
|
|
import (
|
|
"encoding/json"
|
|
"time"
|
|
|
|
"github.com/google/uuid"
|
|
"github.com/seaweedfs/seaweedfs/weed/glog"
|
|
"github.com/seaweedfs/seaweedfs/weed/topology"
|
|
)
|
|
|
|
// recoverTopologyIdFromState restores the TopologyId from serialized FSM
|
|
// state bytes (JSON-encoded MaxVolumeIdCommand). Both raft implementations
|
|
// call this after reading their snapshot in their own format.
|
|
func recoverTopologyIdFromState(fsmState []byte, topo *topology.Topology) {
|
|
if topo.GetTopologyId() != "" {
|
|
return
|
|
}
|
|
var cmd topology.MaxVolumeIdCommand
|
|
if err := json.Unmarshal(fsmState, &cmd); err != nil {
|
|
return
|
|
}
|
|
if cmd.TopologyId != "" {
|
|
topo.SetTopologyId(cmd.TopologyId)
|
|
glog.V(0).Infof("Recovered TopologyId from snapshot: %s", cmd.TopologyId)
|
|
}
|
|
}
|
|
|
|
// EnsureTopologyId ensures that a TopologyId is generated and persisted if it's currently missing.
|
|
// It uses the provided checkLeaderFn to verify leadership and persistFn to save the new ID.
|
|
func EnsureTopologyId(topo *topology.Topology, checkLeaderFn func() bool, persistFn func(string) error) {
|
|
if topo.GetTopologyId() != "" {
|
|
// The id arrived through raft replay or the mirror file; reports
|
|
// stashed while it was unknown can now be checked.
|
|
topo.SetTopologyInitPending(false)
|
|
return
|
|
}
|
|
|
|
topo.SetTopologyInitPending(true)
|
|
defer topo.SetTopologyInitPending(false)
|
|
|
|
topologyId := uuid.New().String()
|
|
for {
|
|
if !checkLeaderFn() {
|
|
glog.V(0).Infof("lost leadership while saving topologyId")
|
|
return
|
|
}
|
|
|
|
// Another concurrent operation may have set the ID between generation and now.
|
|
if latestId := topo.GetTopologyId(); latestId != "" {
|
|
glog.V(1).Infof("topologyId was set concurrently to %s, aborting generation", latestId)
|
|
return
|
|
}
|
|
|
|
if err := persistFn(topologyId); err != nil {
|
|
glog.Errorf("failed to save topologyId, will retry: %v", err)
|
|
time.Sleep(time.Second)
|
|
continue
|
|
}
|
|
|
|
// Verify that the topology ID was actually applied as expected.
|
|
appliedId := topo.GetTopologyId()
|
|
if appliedId != "" && appliedId != topologyId {
|
|
glog.V(0).Infof("TopologyId generation race: expected %s, but current TopologyId is %s", topologyId, appliedId)
|
|
} else {
|
|
glog.V(0).Infof("TopologyId generated: %s", topologyId)
|
|
}
|
|
topo.DeferAllocation(topo.AllocationDeferralDuration())
|
|
break
|
|
}
|
|
}
|