mirror of
https://github.com/seaweedfs/seaweedfs.git
synced 2026-10-31 17:30:42 +01:00
Chris LuandDevin
eec2452e30
* master: replicate the recovery-wait growth unfreeze through raft state_loss_recovery_wait lifted the volume-growth freeze only in memory, while the freeze itself was replicated. A restart replayed the frozen flag and growth stayed refused until the wait expired again, on every restart, unless an operator ran volume.growth.resume. The lift now applies GrowthFrozen=false like ResumeVolumeGrowth, from a goroutine since IsGrowthFrozen is a read path that cannot block on raft. The apply runs under reportedTopologyIdsLock so a freeze landing after the lift keeps its command last in the log; the stale unfreeze skips itself when the flag is frozen again. Generated with [Devin](https://devin.ai) Co-Authored-By: Devin <158243242+devin-ai-integration[bot]@users.noreply.github.com> * master: cover raft replication of the recovery-wait unfreeze Generated with [Devin](https://devin.ai) Co-Authored-By: Devin <158243242+devin-ai-integration[bot]@users.noreply.github.com> * master: retry and re-arm a growth unfreeze raft failed to record applyGrowthFreeze swallowed the raft error, so a rejected lift during a leader change left the flag clear while the log still held the freeze; nothing retried and a restart re-froze growth anyway. The apply now returns the error, replicateGrowthUnfreeze retries it, and on persistent failure re-arms the flag so the next IsGrowthFrozen attempts the lift again instead of diverging from the log. Generated with [Devin](https://devin.ai) Co-Authored-By: Devin <158243242+devin-ai-integration[bot]@users.noreply.github.com> * master: release the topology-id lock before the freeze raft write CheckReportedTopologyId and SetTopologyInitPending held reportedTopologyIdsLock across the blocking raft apply, so a slow apply stalled every later report — including the heartbeat that should trigger the next freeze. The lock now covers only the mismatch decision; the replicate step runs after it is released. * master: bind the recovery unfreeze to the freeze instant it lifts An unfreeze submitted while a newer freeze sat committed but not yet applied — e.g. after this node won an election — could land behind that freeze in the log and erase it. The command now carries the freeze instant cluster-wide (GrowthFrozenAt) and the lift is bound to the instant its submitter observed (GrowthUnfreezeFor): the state machine no-ops a lift while a strictly newer freeze exists, which also lets the unfreeze run without holding the topology-id lock at all. Concurrent freeze writes keep their own mutex so log order cannot regress the instant. * master: a superseded unfreeze retry leaves the resumed flag alone replicateGrowthUnfreeze broke out of its retry loop when a newer\nfreeze superseded the lift, but the break fell through to the\nre-arm: an operator resume in between was silently undone. Return\ninstead — the newer freeze generation owns the flag. * master: replayed freeze entries without an instant stay ageless Freeze commands written before the instant field existed carry zero.\nSubstituting the replay time made each restart stamp the freeze as\nnew, so a committed bound unfreeze always looked older and was\nrejected — the freeze came back on every restart. Keep the zero\ninstant: the recovery window reads as elapsed and bound lifts apply. * master: pack the growth freeze flag and generation into one atomic Separate flag and timestamp atomics left three holes: a resume's\nload-and-clear could lift a generation stamped in between, a bound\nlift compared stamps numerically although masters' clocks need not\nbe monotonic, and an ageless generation had no identity a bound lift\ncould match without colliding with the unconditional encoding.\n\nOne value now carries the state — 0 unfrozen, -1 an ageless freeze,\nelse the freeze instant — so every transition is a single atomic\nswap or compare-and-swap bound to an exact generation. * master: a repeat mismatch report keeps the armed freeze generation Re-stamping while already frozen never reaches the raft log, so the local generation diverges from the replicated one and every bound unfreeze becomes a no-op on replay — the freeze returns after restart or failover. Generated with [Devin](https://devin.ai) Co-Authored-By: Devin <158243242+devin-ai-integration[bot]@users.noreply.github.com> * master: snapshots read the freeze flag and generation in one load Save, Snapshot, and PersistMaxVolumeId read IsGrowthFrozen and the generation separately, so a resume landing between them persisted frozen=true with stamp 0 — restoring an ageless freeze a committed bound lift cannot clear. Generated with [Devin](https://devin.ai) Co-Authored-By: Devin <158243242+devin-ai-integration[bot]@users.noreply.github.com> --------- Co-authored-by: Devin <158243242+devin-ai-integration[bot]@users.noreply.github.com>
86 lines
2.9 KiB
Go
86 lines
2.9 KiB
Go
package topology
|
|
|
|
import (
|
|
"encoding/json"
|
|
"fmt"
|
|
|
|
hashicorpRaft "github.com/hashicorp/raft"
|
|
"github.com/seaweedfs/raft"
|
|
"github.com/seaweedfs/seaweedfs/weed/glog"
|
|
"github.com/seaweedfs/seaweedfs/weed/storage/needle"
|
|
)
|
|
|
|
type MaxVolumeIdCommand struct {
|
|
MaxVolumeId needle.VolumeId `json:"maxVolumeId"`
|
|
TopologyId string `json:"topologyId"`
|
|
// GrowthFrozen replicates the volume-growth freeze through raft so a
|
|
// leader change does not silently lift it. nil means no change.
|
|
GrowthFrozen *bool `json:"growthFrozen,omitempty"`
|
|
// GrowthFrozenAt pins the freeze instant so every master measures the
|
|
// same recovery window and can tell one freeze generation from the next.
|
|
// Zero on entries written before the field existed.
|
|
GrowthFrozenAt int64 `json:"growthFrozenAt,omitempty"`
|
|
// GrowthUnfreezeFor binds GrowthFrozen=false to the freeze generation
|
|
// the submitter observed: the lift applies only while that exact
|
|
// generation is armed, so a stale unfreeze cannot erase a freeze that
|
|
// reached the log first. Zero lifts unconditionally, for entries
|
|
// written before the binding existed; -1 binds the ageless generation
|
|
// left by freeze entries written before instants existed.
|
|
GrowthUnfreezeFor int64 `json:"growthUnfreezeFor,omitempty"`
|
|
}
|
|
|
|
func NewMaxVolumeIdCommand(value needle.VolumeId, topologyId string) *MaxVolumeIdCommand {
|
|
return &MaxVolumeIdCommand{
|
|
MaxVolumeId: value,
|
|
TopologyId: topologyId,
|
|
}
|
|
}
|
|
|
|
func (c *MaxVolumeIdCommand) CommandName() string {
|
|
return "MaxVolumeId"
|
|
}
|
|
|
|
// deprecatedCommandApply represents the old interface to apply a command to the server.
|
|
func (c *MaxVolumeIdCommand) Apply(server raft.Server) (interface{}, error) {
|
|
topo := server.Context().(*Topology)
|
|
before := topo.GetMaxVolumeId()
|
|
topo.UpAdjustMaxVolumeId(c.MaxVolumeId)
|
|
if c.TopologyId != "" {
|
|
prevTopologyId := topo.GetTopologyId()
|
|
topo.SetTopologyId(c.TopologyId)
|
|
// Log when TopologyId is set for the first time, with different messages for leader and follower.
|
|
if prevTopologyId == "" {
|
|
if server.State() == raft.Leader {
|
|
glog.V(0).Infof("TopologyId generated and applied on leader: %s", c.TopologyId)
|
|
} else {
|
|
glog.V(0).Infof("TopologyId applied on follower: %s", c.TopologyId)
|
|
}
|
|
}
|
|
}
|
|
if c.GrowthFrozen != nil {
|
|
topo.SetGrowthFrozen(*c.GrowthFrozen, c.GrowthFrozenAt, c.GrowthUnfreezeFor)
|
|
}
|
|
if err := topo.PersistMaxVolumeId(c.MaxVolumeId); err != nil {
|
|
glog.Warningf("persist max volume id %d: %v", c.MaxVolumeId, err)
|
|
}
|
|
glog.V(1).Infoln("max volume id", before, "==>", topo.GetMaxVolumeId())
|
|
|
|
return nil, nil
|
|
}
|
|
|
|
func (s *MaxVolumeIdCommand) Persist(sink hashicorpRaft.SnapshotSink) error {
|
|
b, err := json.Marshal(s)
|
|
if err != nil {
|
|
return fmt.Errorf("marshal: %w", err)
|
|
}
|
|
_, err = sink.Write(b)
|
|
if err != nil {
|
|
sink.Cancel()
|
|
return fmt.Errorf("sink.Write(): %w", err)
|
|
}
|
|
return sink.Close()
|
|
}
|
|
|
|
func (s *MaxVolumeIdCommand) Release() {
|
|
}
|