Federated volume-id bound via heartbeat stamps (#11763) (#11819)

* pb: heartbeat fields for the federated volume-id bound (#11763)

Heartbeat carries max_volume_id_seen and last_seen_topology_id up, and
HeartbeatResponse echoes max_volume_id and topology_id down, so the
allocation bound is replicated to every volume server.

* master: echo the volume-id bound and fold reported bounds in (#11763)

Every heartbeat response carries the leader current max volume id and
topology id; every heartbeat reported max_volume_id_seen is folded into
the topology bound via UpAdjustMaxVolumeId.

* volume: persist a per-dir vid stamp of the highest id ever seen (#11763)

The store keeps max_volume_id_seen + last_seen_topology_id in a
.vid_stamp file per volume dir, fed by heartbeat echoes, AllocateVolume,
and local volume/EC creation, seeded at boot from stamps and local
holdings, and reported in every heartbeat. A rebuilt master recovers
the bound from the first surviving veteran report.

* volume-rust: mirror the vid stamp and heartbeat echo handling (#11763)

Same mechanism as the Go volume server: .vid_stamp per dir, bound folded
in from add_location scans and allocate/mount calls, echoed fields
applied on heartbeat responses, reported on every heartbeat.

* volume: serialize vid stamp writes and fsync the directory (#11763)

* volume-rust: serialize vid stamp writes and fsync the directory (#11763)

* master: echo the allocation bound to connected servers on change

The bound was only sent at connect time or on digest mismatch, so a
healthy stream never learned later increases. If the high-id server then
went offline and the master lost state, survivors restored a stale bound
and the id could be reissued. Resend the heartbeat response whenever the
bound or topology id moved.

* volume: retry failed vid stamp saves on later updates

A transient write failure left the in-memory bound raised but the stamp
stale, and repeat updates at the same bound never retried. Track a dirty
flag so any later update re-attempts the save.

* volume: cover stamp bootstrap for pre-stamp dirs in test

* volume-rust: retry failed vid stamp saves, mirroring Go

* volume-rust: stamp every dir with the combined bound on add_location

A dir whose own stamp sat below the combined bound kept its stale value
after fold; losing the higher-stamped disk then restarted the server
below ids it had already learned. Rewrite all stamps after each location
folds in.

* volume: stamp mounted volume and ec shard ids (#11763)

* volume-rust: stamp mounted volume ids, mirroring Go (#11763)

* volume: retry dirty vid stamp save on unchanged topology id (#11763)

* volume-rust: retry dirty vid stamp save on unchanged topology id (#11763)

* master: record echoed bound from the sent heartbeat response (#11763)

weed/server/master_grpc_server.go

@@ -46,6 +46,8 @@ func (ms *MasterServer) heartbeatResponse() *master_pb.HeartbeatResponse {
 	return &master_pb.HeartbeatResponse{
 		VolumeSizeLimit: uint64(ms.option.VolumeSizeLimitMB) * 1024 * 1024,
 		Preallocate:     ms.preallocateSize > 0,
+		MaxVolumeId:     uint32(ms.Topo.GetMaxVolumeId()),
+		TopologyId:      ms.Topo.GetTopologyId(),
 	}
 }
 
@@ -137,6 +139,8 @@ func (ms *MasterServer) SendHeartbeat(stream master_pb.Seaweed_SendHeartbeatServ
 		}
 	}()
 
+	var echoedVid needle.VolumeId
+	var echoedTopoId string
 	for {
 		heartbeat, err := stream.Recv()
 		if err != nil {
@@ -179,6 +183,7 @@ func (ms *MasterServer) SendHeartbeat(stream master_pb.Seaweed_SendHeartbeatServ
 		}
 
 		ms.Topo.Sequence.SetMax(heartbeat.MaxFileKey)
+		ms.Topo.UpAdjustMaxVolumeId(needle.VolumeId(heartbeat.MaxVolumeIdSeen))
 		if dn == nil {
 			// Skip delta heartbeat for volume server versions  better than 3.28 https://github.com/seaweedfs/seaweedfs/pull/3630
 			if heartbeat.Ip == "" {
@@ -206,6 +211,7 @@ func (ms *MasterServer) SendHeartbeat(stream master_pb.Seaweed_SendHeartbeatServ
 				glog.Warningf("SendHeartbeat.Send volume size to %s:%d %v", dn.Ip, dn.Port, err)
 				return err
 			}
+			echoedVid, echoedTopoId = needle.VolumeId(response.MaxVolumeId), response.TopologyId
 			stats.MasterReceivedHeartbeatCounter.WithLabelValues("dataNode").Inc()
 			dn.Counter++
 		}
@@ -342,6 +348,19 @@ func (ms *MasterServer) SendHeartbeat(stream master_pb.Seaweed_SendHeartbeatServ
 				glog.Warningf("SendHeartbeat.Send resend request to %s:%d %v", dn.Ip, dn.Port, err)
 				return err
 			}
+			echoedVid, echoedTopoId = needle.VolumeId(response.MaxVolumeId), response.TopologyId
+		}
+
+		// Every connected server keeps a durable copy of the bound. Echo it
+		// whenever it moved so one veteran can restore it after state loss,
+		// even while the offline server that held the high ids never rejoins.
+		if vid, topoId := ms.Topo.GetMaxVolumeId(), ms.Topo.GetTopologyId(); vid != echoedVid || topoId != echoedTopoId {
+			response := ms.heartbeatResponse()
+			if err := stream.Send(response); err != nil {
+				glog.Warningf("SendHeartbeat.Send bound echo %v", err)
+				return err
+			}
+			echoedVid, echoedTopoId = needle.VolumeId(response.MaxVolumeId), response.TopologyId
 		}
 	}
 }

weed/server/volume_grpc_client_to_master.go

@@ -19,6 +19,7 @@ import (
 	"github.com/seaweedfs/seaweedfs/weed/security"
 	"github.com/seaweedfs/seaweedfs/weed/storage/backend"
 	"github.com/seaweedfs/seaweedfs/weed/storage/erasure_coding"
+	"github.com/seaweedfs/seaweedfs/weed/storage/needle"
 
 	"github.com/seaweedfs/seaweedfs/weed/glog"
 	"github.com/seaweedfs/seaweedfs/weed/pb/master_pb"
@@ -217,6 +218,8 @@ func (vs *VolumeServer) doHeartbeatWithRetry(masterAddress pb.ServerAddress, grp
 					os.Exit(1)
 				}
 			}
+			vs.store.UpdateMaxVolumeIdSeen(needle.VolumeId(in.MaxVolumeId))
+			vs.store.UpdateLastSeenTopologyId(in.TopologyId)
 			volumeOptsChanged := false
 			if vs.store.GetPreallocate() != in.GetPreallocate() {
 				vs.store.SetPreallocate(in.GetPreallocate())