diff --git a/weed/server/raft_server.go b/weed/server/raft_server.go index f274a821d2..ae81d30ada 100644 --- a/weed/server/raft_server.go +++ b/weed/server/raft_server.go @@ -60,9 +60,10 @@ func (s StateMachine) Save() ([]byte, error) { MaxVolumeId: s.topo.GetMaxVolumeId(), TopologyId: s.topo.GetTopologyId(), } - if s.topo.IsGrowthFrozen() { + if frozen, frozenAt := s.topo.GrowthFreezeState(); frozen { state.GrowthFrozen = new(bool) *state.GrowthFrozen = true + state.GrowthFrozenAt = frozenAt } glog.V(1).Infof("Save raft state %+v", state) return json.Marshal(state) @@ -81,7 +82,7 @@ func (s StateMachine) Recovery(data []byte) error { glog.V(0).Infof("Recovered TopologyId: %s", state.TopologyId) } if state.GrowthFrozen != nil { - s.topo.SetGrowthFrozen(*state.GrowthFrozen) + s.topo.SetGrowthFrozen(*state.GrowthFrozen, state.GrowthFrozenAt, state.GrowthUnfreezeFor) } if err := s.topo.PersistMaxVolumeId(state.MaxVolumeId); err != nil { glog.Warningf("persist max volume id %d: %v", state.MaxVolumeId, err) @@ -106,7 +107,7 @@ func (s *StateMachine) Apply(l *hashicorpRaft.Log) interface{} { } } if state.GrowthFrozen != nil { - s.topo.SetGrowthFrozen(*state.GrowthFrozen) + s.topo.SetGrowthFrozen(*state.GrowthFrozen, state.GrowthFrozenAt, state.GrowthUnfreezeFor) } if err := s.topo.PersistMaxVolumeId(state.MaxVolumeId); err != nil { glog.Warningf("persist max volume id %d: %v", state.MaxVolumeId, err) @@ -117,11 +118,12 @@ func (s *StateMachine) Apply(l *hashicorpRaft.Log) interface{} { } func (s *StateMachine) Snapshot() (hashicorpRaft.FSMSnapshot, error) { - frozen := s.topo.IsGrowthFrozen() + frozen, frozenAt := s.topo.GrowthFreezeState() state := &topology.MaxVolumeIdCommand{ - MaxVolumeId: s.topo.GetMaxVolumeId(), - TopologyId: s.topo.GetTopologyId(), - GrowthFrozen: &frozen, + MaxVolumeId: s.topo.GetMaxVolumeId(), + TopologyId: s.topo.GetTopologyId(), + GrowthFrozen: &frozen, + GrowthFrozenAt: frozenAt, } return state, nil } diff --git a/weed/topology/cluster_commands.go b/weed/topology/cluster_commands.go index a4d7527863..c12528a2db 100644 --- a/weed/topology/cluster_commands.go +++ b/weed/topology/cluster_commands.go @@ -16,6 +16,17 @@ type MaxVolumeIdCommand struct { // GrowthFrozen replicates the volume-growth freeze through raft so a // leader change does not silently lift it. nil means no change. GrowthFrozen *bool `json:"growthFrozen,omitempty"` + // GrowthFrozenAt pins the freeze instant so every master measures the + // same recovery window and can tell one freeze generation from the next. + // Zero on entries written before the field existed. + GrowthFrozenAt int64 `json:"growthFrozenAt,omitempty"` + // GrowthUnfreezeFor binds GrowthFrozen=false to the freeze generation + // the submitter observed: the lift applies only while that exact + // generation is armed, so a stale unfreeze cannot erase a freeze that + // reached the log first. Zero lifts unconditionally, for entries + // written before the binding existed; -1 binds the ageless generation + // left by freeze entries written before instants existed. + GrowthUnfreezeFor int64 `json:"growthUnfreezeFor,omitempty"` } func NewMaxVolumeIdCommand(value needle.VolumeId, topologyId string) *MaxVolumeIdCommand { @@ -47,7 +58,7 @@ func (c *MaxVolumeIdCommand) Apply(server raft.Server) (interface{}, error) { } } if c.GrowthFrozen != nil { - topo.SetGrowthFrozen(*c.GrowthFrozen) + topo.SetGrowthFrozen(*c.GrowthFrozen, c.GrowthFrozenAt, c.GrowthUnfreezeFor) } if err := topo.PersistMaxVolumeId(c.MaxVolumeId); err != nil { glog.Warningf("persist max volume id %d: %v", c.MaxVolumeId, err) diff --git a/weed/topology/growth_freeze_test.go b/weed/topology/growth_freeze_test.go index 65e6e5812c..2404c809f2 100644 --- a/weed/topology/growth_freeze_test.go +++ b/weed/topology/growth_freeze_test.go @@ -1,10 +1,36 @@ package topology import ( + "errors" "testing" "time" + + "github.com/seaweedfs/raft" ) +// appliedRaftServer answers as leader and records each command Do applies. +type appliedRaftServer struct { + raft.Server + applied chan raft.Command +} + +func (appliedRaftServer) State() string { return raft.Leader } + +func (s appliedRaftServer) Do(c raft.Command) (interface{}, error) { + s.applied <- c + return nil, nil +} + +type failingRaftServer struct { + raft.Server +} + +func (failingRaftServer) State() string { return raft.Leader } + +func (failingRaftServer) Do(raft.Command) (interface{}, error) { + return nil, errors.New("not the leader") +} + func TestGrowthFreezeOnTopologyMismatch(t *testing.T) { topo := NewTopology("topo", nil, 128, 3, false) topo.SetGrowthFreezeConfig(true, 0) @@ -58,6 +84,120 @@ func TestGrowthFreezeAutoRecovery(t *testing.T) { } } +func TestGrowthFreezeAutoRecoveryReplicates(t *testing.T) { + applied := make(chan raft.Command, 2) + topo := NewTopology("topo", nil, 128, 3, false) + topo.RaftServer = appliedRaftServer{applied: applied} + topo.SetGrowthFreezeConfig(true, 50*time.Millisecond) + topo.SetTopologyId("cluster-a") + + topo.CheckReportedTopologyId("cluster-b") + if !topo.IsGrowthFrozen() { + t.Fatal("mismatching topology id should freeze growth") + } + cmd := (<-applied).(*MaxVolumeIdCommand) + if cmd.GrowthFrozen == nil || !*cmd.GrowthFrozen { + t.Fatal("the freeze should be replicated") + } + + time.Sleep(60 * time.Millisecond) + if topo.IsGrowthFrozen() { + t.Fatal("growth should auto-resume past the recovery wait") + } + select { + case c := <-applied: + lift := c.(*MaxVolumeIdCommand) + if lift.GrowthFrozen == nil || *lift.GrowthFrozen { + t.Fatal("the auto unfreeze should be replicated like volume.growth.resume") + } + if lift.GrowthUnfreezeFor != cmd.GrowthFrozenAt { + t.Fatal("the unfreeze should be bound to the freeze instant it lifts") + } + case <-time.After(2 * time.Second): + t.Fatal("auto unfreeze was not replicated; a restart would re-apply the freeze") + } +} + +// A repeat mismatch while already frozen must keep the generation the +// replicated freeze carries: re-stamping locally would bind every later +// lift to a generation followers never saw, and the freeze would return on +// replay. +func TestRepeatMismatchKeepsFreezeGeneration(t *testing.T) { + topo := NewTopology("topo", nil, 128, 3, false) + topo.SetGrowthFreezeConfig(true, 0) + topo.SetTopologyId("cluster-a") + + topo.CheckReportedTopologyId("cluster-b") + gen := topo.growthFrozenState.Load() + if gen == 0 { + t.Fatal("mismatching topology id should freeze growth") + } + topo.CheckReportedTopologyId("cluster-c") + if topo.growthFrozenState.Load() != gen { + t.Fatal("a repeat mismatch must not re-stamp the armed freeze") + } +} + +// Snapshots persist the flag and the generation together, so the pair must +// come from one read: a lifted freeze reports (false, 0) and an armed one +// reports its generation. +func TestGrowthFreezeStateConsistentPair(t *testing.T) { + topo := NewTopology("topo", nil, 128, 3, false) + topo.SetGrowthFreezeConfig(true, 0) + + topo.SetGrowthFrozen(true, time.Now().UnixNano(), 0) + armed := topo.growthFrozenState.Load() + if frozen, at := topo.GrowthFreezeState(); !frozen || at != armed { + t.Fatalf("armed freeze should report (true, %d), got (%v, %d)", armed, frozen, at) + } + topo.ResumeVolumeGrowth() + if frozen, at := topo.GrowthFreezeState(); frozen || at != 0 { + t.Fatalf("lifted freeze should report (false, 0), got (%v, %d)", frozen, at) + } +} + +func TestReplicateGrowthUnfreezeSkipsRefreeze(t *testing.T) { + applied := make(chan raft.Command, 1) + topo := NewTopology("topo", nil, 128, 3, false) + topo.RaftServer = appliedRaftServer{applied: applied} + + topo.SetGrowthFrozen(true, 1000, 0) + topo.SetGrowthFrozen(true, 2000, 0) + topo.replicateGrowthUnfreeze(1000) + select { + case <-applied: + t.Fatal("a stale unfreeze must not lift a freeze that landed after it") + default: + } + if topo.growthFrozenState.Load() != 2000 { + t.Fatal("the newer freeze must stay armed") + } +} + +func TestReplicateGrowthUnfreezeRearmsOnFailure(t *testing.T) { + topo := NewTopology("topo", nil, 128, 3, false) + topo.RaftServer = failingRaftServer{} + + topo.SetGrowthFrozen(true, 1000, 0) + topo.growthFrozenState.CompareAndSwap(1000, 0) // the local lift + topo.replicateGrowthUnfreeze(1000) + if topo.growthFrozenState.Load() != 1000 { + t.Fatal("a failed replication must re-arm the freeze so a later read retries the lift") + } +} + +func TestReplicateGrowthUnfreezeDoesNotRearmAfterResume(t *testing.T) { + topo := NewTopology("topo", nil, 128, 3, false) + + topo.SetGrowthFrozen(true, 1000, 0) + topo.SetGrowthFrozen(true, 2000, 0) + topo.ResumeVolumeGrowth() + topo.replicateGrowthUnfreeze(1000) + if topo.growthFrozenState.Load() != 0 { + t.Fatal("a superseded lift must not re-arm the freeze an operator resumed") + } +} + func TestCheckReportedTopologyIdBeforeTopologyIdKnown(t *testing.T) { topo := NewTopology("topo", nil, 128, 3, false) topo.SetGrowthFreezeConfig(true, 0) @@ -118,17 +258,57 @@ func TestGrowthFreezeCommandApply(t *testing.T) { topo.SetGrowthFreezeConfig(true, 0) frozen := true - topo.SetGrowthFrozen(frozen) + topo.SetGrowthFrozen(frozen, 0, 0) if !topo.IsGrowthFrozen() { t.Fatal("replicated freeze should set the flag") } - unfrozen := false - topo.SetGrowthFrozen(unfrozen) + topo.SetGrowthFrozen(false, 0, 0) if topo.IsGrowthFrozen() { t.Fatal("replicated unfreeze should clear the flag") } } +// Freeze entries written before the instant field existed replay as their +// own stable generation: a lift bound to it still applies, a lift bound to a +// different generation cannot clear it, and the recovery window reads as +// already elapsed — a replay-time stamp would shadow every committed +// unfreeze and re-apply the freeze on each restart. +func TestLegacyFreezeReplayStaysAgeless(t *testing.T) { + topo := NewTopology("topo", nil, 128, 3, false) + topo.SetGrowthFreezeConfig(true, time.Minute) + + topo.SetGrowthFrozen(true, 0, 0) + topo.SetGrowthFrozen(false, 0, 1234) + if topo.growthFrozenState.Load() != -1 { + t.Fatal("a lift bound to a different generation must not clear the ageless freeze") + } + topo.SetGrowthFrozen(false, 0, -1) + if topo.growthFrozenState.Load() != 0 { + t.Fatal("a lift bound to the ageless generation should clear it") + } + + topo.SetGrowthFrozen(true, 0, 0) + if topo.IsGrowthFrozen() { + t.Fatal("an ageless freeze should read its recovery window as elapsed") + } +} + +// A lift bound to a freeze instant must not erase a newer freeze, so a stale +// unfreeze cannot overwrite a freeze that committed first but applied later. +func TestGrowthUnfreezeBoundToFreezeInstant(t *testing.T) { + topo := NewTopology("topo", nil, 128, 3, false) + + topo.SetGrowthFrozen(true, 1000, 0) + topo.SetGrowthFrozen(false, 0, 999) + if !topo.IsGrowthFrozen() { + t.Fatal("a lift for an older freeze instant must not clear a newer freeze") + } + topo.SetGrowthFrozen(false, 0, 1000) + if topo.IsGrowthFrozen() { + t.Fatal("a lift bound to the current freeze instant should clear it") + } +} + func TestAllocationDelay(t *testing.T) { topo := NewTopology("topo", nil, 128, 3, false) diff --git a/weed/topology/topology.go b/weed/topology/topology.go index 177c66a63d..5559240385 100644 --- a/weed/topology/topology.go +++ b/weed/topology/topology.go @@ -92,8 +92,14 @@ type Topology struct { lastPersistedTopologyId atomic.Value lastPersistedGrowthFrozen atomic.Bool - growthFrozen atomic.Bool - growthFrozenAt atomic.Int64 + // growthFrozenState packs the freeze flag and its generation into one + // value: 0 = unfrozen, -1 = frozen without a recorded instant (entries + // written before the field existed), >0 = frozen at that instant. Every + // transition is a single atomic swap or compare-and-swap, so a lift can + // only clear the exact generation it was bound to — a stale unfreeze + // cannot erase a newer freeze, a resume cannot lift a generation it did + // not observe, and stamps need not be monotonic across masters. + growthFrozenState atomic.Int64 metaReadOnly atomic.Bool freezeOnMismatch atomic.Bool stateLossRecoveryWait atomic.Int64 @@ -122,6 +128,11 @@ type Topology struct { reportedTopologyIdsLock sync.Mutex pendingReportedTopologyIds map[string]struct{} + // growthFreezeApplyLock serializes replicated freeze writes so the log + // order of concurrent mismatch reports cannot land an older freeze + // instant after a newer one. Unfreezes skip it: the state machine + // applies a lift only against the freeze generation it targeted. + growthFreezeApplyLock sync.Mutex // quarantinedVolumeIds are (node, vid) pairs force-read-only because the // vid collided across layouts. Kept separate from the reported flag so @@ -1591,15 +1602,22 @@ func (t *Topology) CheckReportedTopologyId(reported string) { return } t.reportedTopologyIdsLock.Lock() - defer t.reportedTopologyIdsLock.Unlock() - t.checkReportedTopologyIdLocked(reported) + freeze := t.checkReportedTopologyIdLocked(reported) + t.reportedTopologyIdsLock.Unlock() + if freeze { + // The decision landed under the lock; the raft wait must not, or a + // slow apply would stall every later report — including one that + // needs to trigger the next freeze. + t.applyGrowthFreeze() + } } // checkReportedTopologyIdLocked runs the mismatch check inside // reportedTopologyIdsLock, the same critical section the init gate drains // under: a report received before the gate opens is either stashed or frozen -// before the gate can release. -func (t *Topology) checkReportedTopologyIdLocked(reported string) { +// before the gate can release. It returns whether the freeze must be +// replicated, which the caller does after releasing the lock. +func (t *Topology) checkReportedTopologyIdLocked(reported string) (freeze bool) { own := t.GetTopologyId() if own == "" { // The local topology id is still being minted. Keep the report so a @@ -1609,82 +1627,175 @@ func (t *Topology) checkReportedTopologyIdLocked(reported string) { t.pendingReportedTopologyIds = make(map[string]struct{}) } t.pendingReportedTopologyIds[reported] = struct{}{} - return + return false } if reported == own { - return + return false } - t.growthFrozenAt.Store(time.Now().UnixNano()) - if t.growthFrozen.CompareAndSwap(false, true) { + // Only the first mismatch stamps a generation: a re-stamp never reaches + // the log, so bound lifts would target a generation followers never saw. + if t.growthFrozenState.CompareAndSwap(0, time.Now().UnixNano()) { stats.MasterVolumeGrowthFrozen.Set(1) glog.Errorf("volume growth frozen: a volume server reports topology id %s but this cluster is %s; the master state was rebuilt. Verify the cluster inventory, then run: volume.growth.resume", reported, own) - t.applyGrowthFreeze(true) + return true } + return false } func (t *Topology) IsGrowthFrozen() bool { - if !t.growthFrozen.Load() { - return false + frozen, _ := t.GrowthFreezeState() + return frozen +} + +// GrowthFreezeState returns the freeze flag and its generation from one +// load, so snapshots persisting both cannot split the pair across a lift. +func (t *Topology) GrowthFreezeState() (frozen bool, frozenAt int64) { + state := t.growthFrozenState.Load() + if state == 0 { + return false, 0 } if wait := t.stateLossRecoveryWait.Load(); wait > 0 { - if time.Duration(time.Now().UnixNano()-t.growthFrozenAt.Load()) > time.Duration(wait) { - if t.growthFrozen.CompareAndSwap(true, false) { + // A legacy generation carries no instant; its window counts as + // already elapsed. + if state < 0 || time.Duration(time.Now().UnixNano()-state) > time.Duration(wait) { + if t.growthFrozenState.CompareAndSwap(state, 0) { stats.MasterVolumeGrowthFrozen.Set(0) glog.V(0).Infof("volume growth unfrozen after %v", time.Duration(wait)) + go t.replicateGrowthUnfreeze(state) + return false, 0 } - return false + state = t.growthFrozenState.Load() } } - return true + return state != 0, state +} + +// replicateGrowthUnfreeze pushes the recovery-wait lift through raft like +// ResumeVolumeGrowth, so a restart does not replay the freeze. The command +// carries the generation it lifts: a newer freeze — even one committed but +// not yet applied when this node won an election — makes the apply a no-op +// instead of being erased. If the write never lands, the generation is +// re-armed so a later IsGrowthFrozen retries the lift. +func (t *Topology) replicateGrowthUnfreeze(frozenAt int64) { + for attempt := 0; attempt < 3; attempt++ { + if attempt > 0 { + time.Sleep(time.Duration(attempt) * time.Second) + } + // The caller's lift already cleared the flag, so unfrozen is the + // state to expect; anything else armed is a newer generation this + // lift must not touch — and falling through to the re-arm would + // undo a resume that followed it. + if s := t.growthFrozenState.Load(); s != 0 && s != frozenAt { + return + } + if err := t.applyGrowthUnfreeze(frozenAt); err == nil { + return + } + } + if frozenAt != 0 && t.growthFrozenState.CompareAndSwap(0, frozenAt) { + stats.MasterVolumeGrowthFrozen.Set(1) + glog.Warningf("volume growth unfreeze not replicated; freeze re-armed for retry") + } } func (t *Topology) ResumeVolumeGrowth() bool { - if !t.growthFrozen.CompareAndSwap(true, false) { - return false + for { + // The compare binds the resume to the generation it observed: a + // freeze landing between the load and the swap just moves to the + // next iteration, which lifts the generation actually armed. + frozenAt := t.growthFrozenState.Load() + if frozenAt == 0 { + return false + } + if t.growthFrozenState.CompareAndSwap(frozenAt, 0) { + stats.MasterVolumeGrowthFrozen.Set(0) + glog.V(0).Infof("volume growth resumed by operator") + t.applyGrowthUnfreeze(frozenAt) + return true + } } - stats.MasterVolumeGrowthFrozen.Set(0) - glog.V(0).Infof("volume growth resumed by operator") - t.applyGrowthFreeze(false) - return true } -// SetGrowthFrozen applies a replicated freeze state on every master. The -// timestamp lands before the flag so a reader can never see frozen with a -// zero time and misjudge the auto-resume window. -func (t *Topology) SetGrowthFrozen(frozen bool) { +// SetGrowthFrozen applies a replicated freeze transition on every master. A +// positive frozenAt pins the instant so all nodes share one recovery +// window; a non-positive one — entries written before the field existed — +// becomes the ageless generation, a stable identity a bound lift can still +// match and the recovery check reads as already elapsed. An unfreeze bound +// to unfreezeFor applies only while that exact generation is armed, so a +// stale lift cannot erase a newer freeze; zero lifts unconditionally, for +// entries written before the binding existed. +func (t *Topology) SetGrowthFrozen(frozen bool, frozenAt int64, unfreezeFor int64) { if frozen { - t.growthFrozenAt.Store(time.Now().UnixNano()) - if t.growthFrozen.CompareAndSwap(false, true) { + gen := frozenAt + if gen == 0 { + gen = -1 + } + if t.growthFrozenState.Swap(gen) == 0 { stats.MasterVolumeGrowthFrozen.Set(1) glog.V(0).Infof("volume growth freeze applied from raft") } return } - if t.growthFrozen.CompareAndSwap(true, false) { + if unfreezeFor == 0 { + if t.growthFrozenState.Swap(0) != 0 { + stats.MasterVolumeGrowthFrozen.Set(0) + } + return + } + if t.growthFrozenState.CompareAndSwap(unfreezeFor, 0) { stats.MasterVolumeGrowthFrozen.Set(0) } } -// applyGrowthFreeze replicates the freeze flag so followers keep it across a -// leader change. No raft server (single-master test topologies) just sets it. -func (t *Topology) applyGrowthFreeze(frozen bool) { +// applyGrowthFreeze replicates the current freeze flag and instant so +// followers keep both across a leader change. +func (t *Topology) applyGrowthFreeze() error { + t.growthFreezeApplyLock.Lock() + defer t.growthFreezeApplyLock.Unlock() cmd := NewMaxVolumeIdCommand(t.GetMaxVolumeId(), t.GetTopologyId()) + frozen := true cmd.GrowthFrozen = &frozen + cmd.GrowthFrozenAt = t.growthFrozenState.Load() + return t.applyGrowthFreezeCommand(cmd) +} + +// applyGrowthUnfreeze replicates a lift bound to the freeze instant the +// caller observed. +func (t *Topology) applyGrowthUnfreeze(frozenAt int64) error { + cmd := NewMaxVolumeIdCommand(t.GetMaxVolumeId(), t.GetTopologyId()) + frozen := false + cmd.GrowthFrozen = &frozen + cmd.GrowthUnfreezeFor = frozenAt + return t.applyGrowthFreezeCommand(cmd) +} + +// applyGrowthFreezeCommand sends the transition through raft. With no raft +// server (single-master test topologies) it applies locally. +func (t *Topology) applyGrowthFreezeCommand(cmd *MaxVolumeIdCommand) error { + frozen := cmd.GrowthFrozen != nil && *cmd.GrowthFrozen t.RaftServerAccessLock.RLock() defer t.RaftServerAccessLock.RUnlock() if t.RaftServer != nil { if _, err := t.RaftServer.Do(cmd); err != nil { glog.Warningf("apply growth freeze %v: %v", frozen, err) + return err } - } else if t.HashicorpRaft != nil { - if b, err := json.Marshal(cmd); err != nil { - glog.Warningf("marshal growth freeze %v: %v", frozen, err) - } else if future := t.HashicorpRaft.Apply(b, time.Second); future.Error() != nil { - glog.Warningf("apply growth freeze %v: %v", frozen, future.Error()) - } - } else { - t.SetGrowthFrozen(frozen) + return nil } + if t.HashicorpRaft != nil { + b, err := json.Marshal(cmd) + if err != nil { + glog.Warningf("marshal growth freeze %v: %v", frozen, err) + return err + } + if future := t.HashicorpRaft.Apply(b, time.Second); future.Error() != nil { + glog.Warningf("apply growth freeze %v: %v", frozen, future.Error()) + return future.Error() + } + return nil + } + t.SetGrowthFrozen(*cmd.GrowthFrozen, cmd.GrowthFrozenAt, cmd.GrowthUnfreezeFor) + return nil } // MarkFreshMint records that this boot minted the TopologyId rather than @@ -1713,13 +1824,17 @@ func (t *Topology) SetTopologyInitPending(pending bool) { // a mismatching report checked here, or still in flight under the same // lock, lands its freeze before any allocation can observe the open gate. t.reportedTopologyIdsLock.Lock() - defer t.reportedTopologyIdsLock.Unlock() reported := t.pendingReportedTopologyIds t.pendingReportedTopologyIds = nil + freeze := false for id := range reported { - t.checkReportedTopologyIdLocked(id) + freeze = t.checkReportedTopologyIdLocked(id) || freeze } t.topologyInitPending.Store(false) + t.reportedTopologyIdsLock.Unlock() + if freeze { + t.applyGrowthFreeze() + } } // ReleaseAllocationOnFirstReport opts the topology into lifting the post-mint @@ -1879,7 +1994,7 @@ func (t *Topology) SetMaxVolumeIdFile(file string) { } if cmd.GrowthFrozen != nil { t.lastPersistedGrowthFrozen.Store(*cmd.GrowthFrozen) - t.SetGrowthFrozen(*cmd.GrowthFrozen) + t.SetGrowthFrozen(*cmd.GrowthFrozen, cmd.GrowthFrozenAt, 0) } } @@ -1888,7 +2003,7 @@ func (t *Topology) SetMaxVolumeIdFile(file string) { // recovery, so follower mirrors stay current too. func (t *Topology) PersistMaxVolumeId(vid needle.VolumeId) error { topoId := t.GetTopologyId() - frozen := t.IsGrowthFrozen() + frozen, frozenAt := t.GrowthFreezeState() mirrored := func() bool { return t.lastPersistedMaxVolumeId.Load() >= uint32(vid) && t.lastPersistedTopologyId.Load() == topoId && t.lastPersistedGrowthFrozen.Load() == frozen } @@ -1908,6 +2023,9 @@ func (t *Topology) PersistMaxVolumeId(vid needle.VolumeId) error { } cmd := NewMaxVolumeIdCommand(vid, topoId) cmd.GrowthFrozen = &frozen + if frozen { + cmd.GrowthFrozenAt = frozenAt + } b, err := json.Marshal(cmd) if err != nil { return err