Freeze volume growth on topology id mismatch (#11763) (#11820)

* pb: ResumeVolumeGrowth rpc for operator-driven growth resume (#11763)

* master: freeze volume growth on topology id mismatch (#11763)

* shell: volume.growth.resume command (#11763)

* master: replicate the growth freeze and gate it on veteran heartbeats (#11763)

* master: block allocation while the topology id is being minted

DeferAllocation only armed after the topology id persisted, so growth
requests arriving during init ran ahead of the warmup window. Gate
NextVolumeId on an init-pending flag spanning the whole mint.

* master: stash reported topology ids until the local id is minted (#11763)

* drain stashed topology id reports when the id arrives outside the mint path

* persist the growth-freeze flag in the mirror file and raft snapshots

* log hashicorp raft marshal and apply errors in applyGrowthFreeze

weed/server/master_grpc_server.go

@@ -184,6 +184,9 @@ func (ms *MasterServer) SendHeartbeat(stream master_pb.Seaweed_SendHeartbeatServ
 
 		ms.Topo.Sequence.SetMax(heartbeat.MaxFileKey)
 		ms.Topo.UpAdjustMaxVolumeId(needle.VolumeId(heartbeat.MaxVolumeIdSeen))
+		if len(heartbeat.Volumes) > 0 {
+			ms.Topo.CheckReportedTopologyId(heartbeat.LastSeenTopologyId)
+		}
 		if dn == nil {
 			// Skip delta heartbeat for volume server versions  better than 3.28 https://github.com/seaweedfs/seaweedfs/pull/3630
 			if heartbeat.Ip == "" {

weed/server/master_grpc_server_volume.go

@@ -471,6 +471,13 @@ func (ms *MasterServer) VolumeMarkReadonly(ctx context.Context, req *master_pb.V
 	return resp, nil
 }
 
+func (ms *MasterServer) ResumeVolumeGrowth(ctx context.Context, req *master_pb.ResumeVolumeGrowthRequest) (*master_pb.ResumeVolumeGrowthResponse, error) {
+	if !ms.Topo.IsLeader() {
+		return nil, raft.NotLeaderError
+	}
+	return &master_pb.ResumeVolumeGrowthResponse{WasFrozen: ms.Topo.ResumeVolumeGrowth()}, nil
+}
+
 func (ms *MasterServer) VolumeGrow(ctx context.Context, req *master_pb.VolumeGrowRequest) (*master_pb.VolumeGrowResponse, error) {
 	if !ms.Topo.IsLeader() {
 		return nil, raft.NotLeaderError

weed/server/master_server.go

@@ -122,6 +122,8 @@ func NewMasterServer(r *mux.Router, option *MasterOption, peers map[string]pb.Se
 	v.SetDefault("master.volume_growth.reservation_timeout", "5m")
 	v.SetDefault("master.volume_growth.disable", false)
 	option.VolumeGrowthDisabled = v.GetBool("master.volume_growth.disable")
+	v.SetDefault("master.volume_growth.freeze_on_topology_mismatch", true)
+	v.SetDefault("master.volume_growth.state_loss_recovery_wait", "0s")
 
 	topology.VolumeGrowStrategy.Copy1Count = v.GetUint32("master.volume_growth.copy_1")
 	topology.VolumeGrowStrategy.Copy2Count = v.GetUint32("master.volume_growth.copy_2")
@@ -157,6 +159,9 @@ func NewMasterServer(r *mux.Router, option *MasterOption, peers map[string]pb.Se
 		glog.Fatalf("create sequencer failed.")
 	}
 	ms.Topo = topology.NewTopology("topo", seq, uint64(ms.option.VolumeSizeLimitMB)*1024*1024, 5, replicationAsMin)
+	ms.Topo.SetGrowthFreezeConfig(
+		v.GetBool("master.volume_growth.freeze_on_topology_mismatch"),
+		v.GetDuration("master.volume_growth.state_loss_recovery_wait"))
 	ms.vg = topology.NewDefaultVolumeGrowth()
 	glog.V(0).Infoln("Volume Size Limit is", ms.option.VolumeSizeLimitMB, "MB")
 

weed/server/raft_common.go

@@ -30,9 +30,15 @@ func recoverTopologyIdFromState(fsmState []byte, topo *topology.Topology) {
 // It uses the provided checkLeaderFn to verify leadership and persistFn to save the new ID.
 func EnsureTopologyId(topo *topology.Topology, checkLeaderFn func() bool, persistFn func(string) error) {
 	if topo.GetTopologyId() != "" {
+		// The id arrived through raft replay or the mirror file; reports
+		// stashed while it was unknown can now be checked.
+		topo.SetTopologyInitPending(false)
 		return
 	}
 
+	topo.SetTopologyInitPending(true)
+	defer topo.SetTopologyInitPending(false)
+
 	topologyId := uuid.New().String()
 	for {
 		if !checkLeaderFn() {
@@ -59,6 +65,7 @@ func EnsureTopologyId(topo *topology.Topology, checkLeaderFn func() bool, persis
 		} else {
 			glog.V(0).Infof("TopologyId generated: %s", topologyId)
 		}
+		topo.DeferAllocation(topo.WarmupDuration())
 		break
 	}
 }

weed/server/raft_server.go

@@ -60,6 +60,10 @@ func (s StateMachine) Save() ([]byte, error) {
 		MaxVolumeId: s.topo.GetMaxVolumeId(),
 		TopologyId:  s.topo.GetTopologyId(),
 	}
+	if s.topo.IsGrowthFrozen() {
+		state.GrowthFrozen = new(bool)
+		*state.GrowthFrozen = true
+	}
 	glog.V(1).Infof("Save raft state %+v", state)
 	return json.Marshal(state)
 }
@@ -76,6 +80,9 @@ func (s StateMachine) Recovery(data []byte) error {
 		s.topo.SetTopologyId(state.TopologyId)
 		glog.V(0).Infof("Recovered TopologyId: %s", state.TopologyId)
 	}
+	if state.GrowthFrozen != nil {
+		s.topo.SetGrowthFrozen(*state.GrowthFrozen)
+	}
 	if err := s.topo.PersistMaxVolumeId(state.MaxVolumeId); err != nil {
 		glog.Warningf("persist max volume id %d: %v", state.MaxVolumeId, err)
 	}
@@ -98,6 +105,9 @@ func (s *StateMachine) Apply(l *hashicorpRaft.Log) interface{} {
 			glog.V(0).Infof("Set TopologyId from raft log: %s", state.TopologyId)
 		}
 	}
+	if state.GrowthFrozen != nil {
+		s.topo.SetGrowthFrozen(*state.GrowthFrozen)
+	}
 	if err := s.topo.PersistMaxVolumeId(state.MaxVolumeId); err != nil {
 		glog.Warningf("persist max volume id %d: %v", state.MaxVolumeId, err)
 	}
@@ -107,10 +117,13 @@ func (s *StateMachine) Apply(l *hashicorpRaft.Log) interface{} {
 }
 
 func (s *StateMachine) Snapshot() (hashicorpRaft.FSMSnapshot, error) {
-	return &topology.MaxVolumeIdCommand{
-		MaxVolumeId: s.topo.GetMaxVolumeId(),
-		TopologyId:  s.topo.GetTopologyId(),
-	}, nil
+	frozen := s.topo.IsGrowthFrozen()
+	state := &topology.MaxVolumeIdCommand{
+		MaxVolumeId:  s.topo.GetMaxVolumeId(),
+		TopologyId:   s.topo.GetTopologyId(),
+		GrowthFrozen: &frozen,
+	}
+	return state, nil
 }
 
 func (s *StateMachine) Restore(r io.ReadCloser) error {