Federated volume-id bound via heartbeat stamps (#11763) (#11819)

* pb: heartbeat fields for the federated volume-id bound (#11763)

Heartbeat carries max_volume_id_seen and last_seen_topology_id up, and
HeartbeatResponse echoes max_volume_id and topology_id down, so the
allocation bound is replicated to every volume server.

* master: echo the volume-id bound and fold reported bounds in (#11763)

Every heartbeat response carries the leader current max volume id and
topology id; every heartbeat reported max_volume_id_seen is folded into
the topology bound via UpAdjustMaxVolumeId.

* volume: persist a per-dir vid stamp of the highest id ever seen (#11763)

The store keeps max_volume_id_seen + last_seen_topology_id in a
.vid_stamp file per volume dir, fed by heartbeat echoes, AllocateVolume,
and local volume/EC creation, seeded at boot from stamps and local
holdings, and reported in every heartbeat. A rebuilt master recovers
the bound from the first surviving veteran report.

* volume-rust: mirror the vid stamp and heartbeat echo handling (#11763)

Same mechanism as the Go volume server: .vid_stamp per dir, bound folded
in from add_location scans and allocate/mount calls, echoed fields
applied on heartbeat responses, reported on every heartbeat.

* volume: serialize vid stamp writes and fsync the directory (#11763)

* volume-rust: serialize vid stamp writes and fsync the directory (#11763)

* master: echo the allocation bound to connected servers on change

The bound was only sent at connect time or on digest mismatch, so a
healthy stream never learned later increases. If the high-id server then
went offline and the master lost state, survivors restored a stale bound
and the id could be reissued. Resend the heartbeat response whenever the
bound or topology id moved.

* volume: retry failed vid stamp saves on later updates

A transient write failure left the in-memory bound raised but the stamp
stale, and repeat updates at the same bound never retried. Track a dirty
flag so any later update re-attempts the save.

* volume: cover stamp bootstrap for pre-stamp dirs in test

* volume-rust: retry failed vid stamp saves, mirroring Go

* volume-rust: stamp every dir with the combined bound on add_location

A dir whose own stamp sat below the combined bound kept its stale value
after fold; losing the higher-stamped disk then restarted the server
below ids it had already learned. Rewrite all stamps after each location
folds in.

* volume: stamp mounted volume and ec shard ids (#11763)

* volume-rust: stamp mounted volume ids, mirroring Go (#11763)

* volume: retry dirty vid stamp save on unchanged topology id (#11763)

* volume-rust: retry dirty vid stamp save on unchanged topology id (#11763)

* master: record echoed bound from the sent heartbeat response (#11763)

weed/pb/master.proto

@@ -119,6 +119,12 @@ message Heartbeat {
   // place of `volumes`. A master that does not understand this never sets
   // volume_digest_supported, so it keeps being sent the whole list.
   repeated VolumeInformationMessage changed_volumes = 28;
+
+  // Highest volume id this server has ever observed, persisted locally. Fed
+  // by HeartbeatResponse.max_volume_id echoes and by AllocateVolume requests.
+  uint32 max_volume_id_seen = 29;
+  // Topology id of the master cluster this server last heartbeated to.
+  string last_seen_topology_id = 30;
 }
 
 message HeartbeatResponse {
@@ -135,6 +141,12 @@ message HeartbeatResponse {
   // The master compares volume digests, so a server that reports one may send
   // changed_volumes in place of its whole list.
   bool volume_digest_supported = 9;
+  // The leader's current allocation bound; servers persist it and report it
+  // back as max_volume_id_seen so the bound survives master state loss.
+  uint32 max_volume_id = 10;
+  // The leader's cluster identity; a different value reported back later
+  // means the master's state was rebuilt.
+  string topology_id = 11;
 }
 
 message VolumeInformationMessage {

weed/pb/master_pb/master.pb.go

@@ -130,8 +130,13 @@ type Heartbeat struct {
 	// place of `volumes`. A master that does not understand this never sets
 	// volume_digest_supported, so it keeps being sent the whole list.
 	ChangedVolumes []*VolumeInformationMessage `protobuf:"bytes,28,rep,name=changed_volumes,json=changedVolumes,proto3" json:"changed_volumes,omitempty"`
-	unknownFields  protoimpl.UnknownFields
-	sizeCache      protoimpl.SizeCache
+	// Highest volume id this server has ever observed, persisted locally. Fed
+	// by HeartbeatResponse.max_volume_id echoes and by AllocateVolume requests.
+	MaxVolumeIdSeen uint32 `protobuf:"varint,29,opt,name=max_volume_id_seen,json=maxVolumeIdSeen,proto3" json:"max_volume_id_seen,omitempty"`
+	// Topology id of the master cluster this server last heartbeated to.
+	LastSeenTopologyId string `protobuf:"bytes,30,opt,name=last_seen_topology_id,json=lastSeenTopologyId,proto3" json:"last_seen_topology_id,omitempty"`
+	unknownFields      protoimpl.UnknownFields
+	sizeCache          protoimpl.SizeCache
 }
 
 func (x *Heartbeat) Reset() {
@@ -339,6 +344,20 @@ func (x *Heartbeat) GetChangedVolumes() []*VolumeInformationMessage {
 	return nil
 }
 
+func (x *Heartbeat) GetMaxVolumeIdSeen() uint32 {
+	if x != nil {
+		return x.MaxVolumeIdSeen
+	}
+	return 0
+}
+
+func (x *Heartbeat) GetLastSeenTopologyId() string {
+	if x != nil {
+		return x.LastSeenTopologyId
+	}
+	return ""
+}
+
 type HeartbeatResponse struct {
 	state                  protoimpl.MessageState `protogen:"open.v1"`
 	VolumeSizeLimit        uint64                 `protobuf:"varint,1,opt,name=volume_size_limit,json=volumeSizeLimit,proto3" json:"volume_size_limit,omitempty"`
@@ -354,8 +373,14 @@ type HeartbeatResponse struct {
 	// The master compares volume digests, so a server that reports one may send
 	// changed_volumes in place of its whole list.
 	VolumeDigestSupported bool `protobuf:"varint,9,opt,name=volume_digest_supported,json=volumeDigestSupported,proto3" json:"volume_digest_supported,omitempty"`
-	unknownFields         protoimpl.UnknownFields
-	sizeCache             protoimpl.SizeCache
+	// The leader's current allocation bound; servers persist it and report it
+	// back as max_volume_id_seen so the bound survives master state loss.
+	MaxVolumeId uint32 `protobuf:"varint,10,opt,name=max_volume_id,json=maxVolumeId,proto3" json:"max_volume_id,omitempty"`
+	// The leader's cluster identity; a different value reported back later
+	// means the master's state was rebuilt.
+	TopologyId    string `protobuf:"bytes,11,opt,name=topology_id,json=topologyId,proto3" json:"topology_id,omitempty"`
+	unknownFields protoimpl.UnknownFields
+	sizeCache     protoimpl.SizeCache
 }
 
 func (x *HeartbeatResponse) Reset() {
@@ -451,6 +476,20 @@ func (x *HeartbeatResponse) GetVolumeDigestSupported() bool {
 	return false
 }
 
+func (x *HeartbeatResponse) GetMaxVolumeId() uint32 {
+	if x != nil {
+		return x.MaxVolumeId
+	}
+	return 0
+}
+
+func (x *HeartbeatResponse) GetTopologyId() string {
+	if x != nil {
+		return x.TopologyId
+	}
+	return ""
+}
+
 type VolumeInformationMessage struct {
 	state             protoimpl.MessageState `protogen:"open.v1"`
 	Id                uint32                 `protobuf:"varint,1,opt,name=id,proto3" json:"id,omitempty"`
@@ -5058,7 +5097,7 @@ const file_master_proto_rawDesc = "" +
 	"\adisk_id\x18\x01 \x01(\rR\x06diskId\x12\x12\n" +
 	"\x04tags\x18\x02 \x03(\tR\x04tags\x12\x12\n" +
 	"\x04type\x18\x03 \x01(\tR\x04type\x12(\n" +
-	"\x10max_volume_count\x18\x04 \x01(\x03R\x0emaxVolumeCount\"\xf0\v\n" +
+	"\x10max_volume_count\x18\x04 \x01(\x03R\x0emaxVolumeCount\"\xd0\f\n" +
 	"\tHeartbeat\x12\x0e\n" +
 	"\x02ip\x18\x01 \x01(\tR\x02ip\x12\x12\n" +
 	"\x04port\x18\x02 \x01(\rR\x04port\x12\x1d\n" +
@@ -5090,7 +5129,9 @@ const file_master_proto_rawDesc = "" +
 	"\x10disk_total_bytes\x18\x19 \x03(\v2(.master_pb.Heartbeat.DiskTotalBytesEntryR\x0ediskTotalBytes\x12O\n" +
 	"\x0fdisk_free_bytes\x18\x1a \x03(\v2'.master_pb.Heartbeat.DiskFreeBytesEntryR\rdiskFreeBytes\x12(\n" +
 	"\rvolume_digest\x18\x1b \x01(\x04H\x00R\fvolumeDigest\x88\x01\x01\x12L\n" +
-	"\x0fchanged_volumes\x18\x1c \x03(\v2#.master_pb.VolumeInformationMessageR\x0echangedVolumes\x1aB\n" +
+	"\x0fchanged_volumes\x18\x1c \x03(\v2#.master_pb.VolumeInformationMessageR\x0echangedVolumes\x12+\n" +
+	"\x12max_volume_id_seen\x18\x1d \x01(\rR\x0fmaxVolumeIdSeen\x121\n" +
+	"\x15last_seen_topology_id\x18\x1e \x01(\tR\x12lastSeenTopologyId\x1aB\n" +
 	"\x14MaxVolumeCountsEntry\x12\x10\n" +
 	"\x03key\x18\x01 \x01(\tR\x03key\x12\x14\n" +
 	"\x05value\x18\x02 \x01(\rR\x05value:\x028\x01\x1aA\n" +
@@ -5100,7 +5141,7 @@ const file_master_proto_rawDesc = "" +
 	"\x12DiskFreeBytesEntry\x12\x10\n" +
 	"\x03key\x18\x01 \x01(\tR\x03key\x12\x14\n" +
 	"\x05value\x18\x02 \x01(\x04R\x05value:\x028\x01B\x10\n" +
-	"\x0e_volume_digest\"\xbc\x03\n" +
+	"\x0e_volume_digest\"\x81\x04\n" +
 	"\x11HeartbeatResponse\x12*\n" +
 	"\x11volume_size_limit\x18\x01 \x01(\x04R\x0fvolumeSizeLimit\x12\x16\n" +
 	"\x06leader\x18\x02 \x01(\tR\x06leader\x12'\n" +
@@ -5110,7 +5151,11 @@ const file_master_proto_rawDesc = "" +
 	"\x10duplicated_uuids\x18\x06 \x03(\tR\x0fduplicatedUuids\x12 \n" +
 	"\vpreallocate\x18\a \x01(\bR\vpreallocate\x125\n" +
 	"\x17resend_full_volume_list\x18\b \x01(\bR\x14resendFullVolumeList\x126\n" +
-	"\x17volume_digest_supported\x18\t \x01(\bR\x15volumeDigestSupported\"\xe2\x04\n" +
+	"\x17volume_digest_supported\x18\t \x01(\bR\x15volumeDigestSupported\x12\"\n" +
+	"\rmax_volume_id\x18\n" +
+	" \x01(\rR\vmaxVolumeId\x12\x1f\n" +
+	"\vtopology_id\x18\v \x01(\tR\n" +
+	"topologyId\"\xe2\x04\n" +
 	"\x18VolumeInformationMessage\x12\x0e\n" +
 	"\x02id\x18\x01 \x01(\rR\x02id\x12\x12\n" +
 	"\x04size\x18\x02 \x01(\x04R\x04size\x12\x1e\n" +

weed/server/master_grpc_server.go

@@ -46,6 +46,8 @@ func (ms *MasterServer) heartbeatResponse() *master_pb.HeartbeatResponse {
 	return &master_pb.HeartbeatResponse{
 		VolumeSizeLimit: uint64(ms.option.VolumeSizeLimitMB) * 1024 * 1024,
 		Preallocate:     ms.preallocateSize > 0,
+		MaxVolumeId:     uint32(ms.Topo.GetMaxVolumeId()),
+		TopologyId:      ms.Topo.GetTopologyId(),
 	}
 }
 
@@ -137,6 +139,8 @@ func (ms *MasterServer) SendHeartbeat(stream master_pb.Seaweed_SendHeartbeatServ
 		}
 	}()
 
+	var echoedVid needle.VolumeId
+	var echoedTopoId string
 	for {
 		heartbeat, err := stream.Recv()
 		if err != nil {
@@ -179,6 +183,7 @@ func (ms *MasterServer) SendHeartbeat(stream master_pb.Seaweed_SendHeartbeatServ
 		}
 
 		ms.Topo.Sequence.SetMax(heartbeat.MaxFileKey)
+		ms.Topo.UpAdjustMaxVolumeId(needle.VolumeId(heartbeat.MaxVolumeIdSeen))
 		if dn == nil {
 			// Skip delta heartbeat for volume server versions  better than 3.28 https://github.com/seaweedfs/seaweedfs/pull/3630
 			if heartbeat.Ip == "" {
@@ -206,6 +211,7 @@ func (ms *MasterServer) SendHeartbeat(stream master_pb.Seaweed_SendHeartbeatServ
 				glog.Warningf("SendHeartbeat.Send volume size to %s:%d %v", dn.Ip, dn.Port, err)
 				return err
 			}
+			echoedVid, echoedTopoId = needle.VolumeId(response.MaxVolumeId), response.TopologyId
 			stats.MasterReceivedHeartbeatCounter.WithLabelValues("dataNode").Inc()
 			dn.Counter++
 		}
@@ -342,6 +348,19 @@ func (ms *MasterServer) SendHeartbeat(stream master_pb.Seaweed_SendHeartbeatServ
 				glog.Warningf("SendHeartbeat.Send resend request to %s:%d %v", dn.Ip, dn.Port, err)
 				return err
 			}
+			echoedVid, echoedTopoId = needle.VolumeId(response.MaxVolumeId), response.TopologyId
+		}
+
+		// Every connected server keeps a durable copy of the bound. Echo it
+		// whenever it moved so one veteran can restore it after state loss,
+		// even while the offline server that held the high ids never rejoins.
+		if vid, topoId := ms.Topo.GetMaxVolumeId(), ms.Topo.GetTopologyId(); vid != echoedVid || topoId != echoedTopoId {
+			response := ms.heartbeatResponse()
+			if err := stream.Send(response); err != nil {
+				glog.Warningf("SendHeartbeat.Send bound echo %v", err)
+				return err
+			}
+			echoedVid, echoedTopoId = needle.VolumeId(response.MaxVolumeId), response.TopologyId
 		}
 	}
 }

weed/server/volume_grpc_client_to_master.go

@@ -19,6 +19,7 @@ import (
 	"github.com/seaweedfs/seaweedfs/weed/security"
 	"github.com/seaweedfs/seaweedfs/weed/storage/backend"
 	"github.com/seaweedfs/seaweedfs/weed/storage/erasure_coding"
+	"github.com/seaweedfs/seaweedfs/weed/storage/needle"
 
 	"github.com/seaweedfs/seaweedfs/weed/glog"
 	"github.com/seaweedfs/seaweedfs/weed/pb/master_pb"
@@ -217,6 +218,8 @@ func (vs *VolumeServer) doHeartbeatWithRetry(masterAddress pb.ServerAddress, grp
 					os.Exit(1)
 				}
 			}
+			vs.store.UpdateMaxVolumeIdSeen(needle.VolumeId(in.MaxVolumeId))
+			vs.store.UpdateLastSeenTopologyId(in.TopologyId)
 			volumeOptsChanged := false
 			if vs.store.GetPreallocate() != in.GetPreallocate() {
 				vs.store.SetPreallocate(in.GetPreallocate())

weed/storage/store.go

@@ -97,6 +97,11 @@ type Store struct {
 	// the heartbeat that saw it. Written only from the heartbeat goroutine.
 	reportedCollections   map[string]struct{}
 	reportedEcCollections map[string]struct{}
+	maxVolumeIdSeen       atomic.Uint32
+	lastSeenTopologyId    atomic.Value // string
+	vidStampReady         atomic.Bool
+	vidStampDirty         atomic.Bool
+	vidStampLock          sync.Mutex
 }
 
 func (s *Store) String() (str string) {
@@ -145,6 +150,7 @@ func NewStore(
 		diskId := uint32(i) // Track disk ID
 
 		location.ecShardNotifyHandler = func(collection string, vid needle.VolumeId, shardId erasure_coding.ShardId, ecVolume *erasure_coding.EcVolume) {
+			s.UpdateMaxVolumeIdSeen(vid)
 			si := erasure_coding.NewShardsInfo()
 			si.Set(erasure_coding.NewShardInfo(shardId, erasure_coding.ShardSize(ecVolume.ShardSize())))
 
@@ -211,6 +217,8 @@ func NewStore(
 		glog.Fatalf("failed to resolve state for volume %s: %v", id, err)
 	}
 
+	s.bootstrapMaxVolumeIdSeen()
+
 	return
 }
 
@@ -320,6 +328,7 @@ func (s *Store) FindFreeLocation(filterFn func(location *DiskLocation) bool, rep
 	return ret
 }
 func (s *Store) addVolume(vid needle.VolumeId, collection string, needleMapKind NeedleMapKind, replicaPlacement *super_block.ReplicaPlacement, ttl *needle.TTL, preallocate int64, ver needle.Version, memoryMapMaxSizeMb uint32, diskType DiskType, ldbTimeout int64) error {
+	s.UpdateMaxVolumeIdSeen(vid)
 	if s.findVolume(vid) != nil {
 		return fmt.Errorf("Volume Id %d already exists!", vid)
 	}
@@ -686,27 +695,29 @@ func (s *Store) CollectHeartbeat() *master_pb.Heartbeat {
 	stats.VolumeServerIoQuarantineGauge.WithLabelValues("ec_shard").Set(float64(quarantinedEcShards))
 
 	return &master_pb.Heartbeat{
-		Ip:              s.Ip,
-		Port:            uint32(s.Port),
-		GrpcPort:        uint32(s.GrpcPort),
-		PublicUrl:       s.PublicUrl,
-		Id:              s.Id,
-		MaxVolumeCounts: maxVolumeCounts,
-		DiskTotalBytes:  diskTotalBytes,
-		DiskFreeBytes:   diskFreeBytes,
-		MaxFileKey:      NeedleIdToUint64(maxFileKey),
-		DataCenter:      s.dataCenter,
-		Rack:            s.rack,
-		Volumes:         heartbeatVolumes,
-		ChangedVolumes:  changedVolumes,
-		DeletedVolumes:  departedVolumes,
-		VolumeDigest:    &volumeDigest,
-		DeletedEcShards: deletedEcVolumes,
-		HasNoVolumes:    hasNoVolumes,
-		HasNoEcShards:   len(ecVolumeMessages) == 0,
-		LocationUuids:   uuidList,
-		DiskTags:        diskTags,
-		State:           state,
+		Ip:                 s.Ip,
+		Port:               uint32(s.Port),
+		GrpcPort:           uint32(s.GrpcPort),
+		PublicUrl:          s.PublicUrl,
+		Id:                 s.Id,
+		MaxVolumeCounts:    maxVolumeCounts,
+		DiskTotalBytes:     diskTotalBytes,
+		DiskFreeBytes:      diskFreeBytes,
+		MaxFileKey:         NeedleIdToUint64(maxFileKey),
+		DataCenter:         s.dataCenter,
+		Rack:               s.rack,
+		Volumes:            heartbeatVolumes,
+		ChangedVolumes:     changedVolumes,
+		DeletedVolumes:     departedVolumes,
+		VolumeDigest:       &volumeDigest,
+		DeletedEcShards:    deletedEcVolumes,
+		HasNoVolumes:       hasNoVolumes,
+		HasNoEcShards:      len(ecVolumeMessages) == 0,
+		LocationUuids:      uuidList,
+		DiskTags:           diskTags,
+		State:              state,
+		MaxVolumeIdSeen:    uint32(s.MaxVolumeIdSeen()),
+		LastSeenTopologyId: s.LastSeenTopologyId(),
 	}
 
 }
@@ -1027,6 +1038,7 @@ func (s *Store) mountVolume(i needle.VolumeId, collection *string, validators ..
 				ReadOnly:          readOnly,
 				ReadOnlyCanDelete: readOnlyCanDelete,
 			}
+			s.UpdateMaxVolumeIdSeen(i)
 			return nil
 		}
 	}

weed/storage/store_ec.go

@@ -304,6 +304,7 @@ func (s *Store) MountEcShards(collection string, vid needle.VolumeId, shardId er
 				DiskId:      uint32(diskId),
 				EncodeTsNs:  ecVolume.EncodeTsNs,
 			}
+			s.UpdateMaxVolumeIdSeen(vid)
 			return nil
 		}
 		if errors.Is(err, os.ErrNotExist) {

weed/storage/vid_stamp.go

@@ -0,0 +1,148 @@
+package storage
+
+import (
+	"encoding/json"
+	"os"
+	"path/filepath"
+
+	"github.com/seaweedfs/seaweedfs/weed/glog"
+	"github.com/seaweedfs/seaweedfs/weed/storage/needle"
+	"github.com/seaweedfs/seaweedfs/weed/util"
+)
+
+const vidStampFileName = ".vid_stamp"
+
+type vidStamp struct {
+	MaxVolumeId uint32 `json:"maxVolumeId"`
+	TopologyId  string `json:"topologyId,omitempty"`
+}
+
+func (s *Store) MaxVolumeIdSeen() needle.VolumeId {
+	return needle.VolumeId(s.maxVolumeIdSeen.Load())
+}
+
+func (s *Store) LastSeenTopologyId() string {
+	if v := s.lastSeenTopologyId.Load(); v != nil {
+		return v.(string)
+	}
+	return ""
+}
+
+// UpdateMaxVolumeIdSeen raises the server's persisted bound and stamps every
+// volume dir when it grows. Feeds: heartbeat echoes, AllocateVolume, local
+// volume and EC shard creation.
+func (s *Store) UpdateMaxVolumeIdSeen(vid needle.VolumeId) {
+	grew := false
+	for {
+		current := s.maxVolumeIdSeen.Load()
+		if current >= uint32(vid) {
+			break
+		}
+		if s.maxVolumeIdSeen.CompareAndSwap(current, uint32(vid)) {
+			grew = true
+			break
+		}
+	}
+	if s.vidStampReady.Load() && (grew || s.vidStampDirty.Load()) {
+		s.saveVidStamps()
+	}
+}
+
+func (s *Store) UpdateLastSeenTopologyId(id string) {
+	if id == "" || (s.LastSeenTopologyId() == id && !s.vidStampDirty.Load()) {
+		return
+	}
+	s.lastSeenTopologyId.Store(id)
+	if s.vidStampReady.Load() {
+		s.saveVidStamps()
+	}
+}
+
+// bootstrapMaxVolumeIdSeen seeds the bound from persisted stamps plus the max
+// of every volume and EC shard id on disk, so a server upgraded mid-incident
+// still reports a bound covering its own holdings.
+func (s *Store) bootstrapMaxVolumeIdSeen() {
+	var maxVid needle.VolumeId
+	for _, location := range s.Locations {
+		if stamp, err := readVidStamp(location.Directory); err == nil {
+			if needle.VolumeId(stamp.MaxVolumeId) > maxVid {
+				maxVid = needle.VolumeId(stamp.MaxVolumeId)
+			}
+			if stamp.TopologyId != "" && s.LastSeenTopologyId() == "" {
+				s.lastSeenTopologyId.Store(stamp.TopologyId)
+			}
+		}
+		location.volumesLock.RLock()
+		for vid := range location.volumes {
+			if vid > maxVid {
+				maxVid = vid
+			}
+		}
+		location.volumesLock.RUnlock()
+		location.ecVolumesLock.RLock()
+		for vid := range location.ecVolumes {
+			if vid > maxVid {
+				maxVid = vid
+			}
+		}
+		location.ecVolumesLock.RUnlock()
+	}
+	s.maxVolumeIdSeen.Store(uint32(maxVid))
+	s.vidStampReady.Store(true)
+	s.saveVidStamps()
+}
+
+func (s *Store) saveVidStamps() {
+	s.vidStampLock.Lock()
+	defer s.vidStampLock.Unlock()
+	stamp, err := json.Marshal(vidStamp{
+		MaxVolumeId: s.maxVolumeIdSeen.Load(),
+		TopologyId:  s.LastSeenTopologyId(),
+	})
+	if err != nil {
+		return
+	}
+	failed := false
+	for _, location := range s.Locations {
+		if err := writeVidStamp(location.Directory, stamp); err != nil {
+			glog.Warningf("write vid stamp in %s: %v", location.Directory, err)
+			failed = true
+		}
+	}
+	s.vidStampDirty.Store(failed)
+}
+
+func readVidStamp(dir string) (*vidStamp, error) {
+	data, err := os.ReadFile(filepath.Join(dir, vidStampFileName))
+	if err != nil {
+		return nil, err
+	}
+	stamp := &vidStamp{}
+	if err := json.Unmarshal(data, stamp); err != nil {
+		return nil, err
+	}
+	return stamp, nil
+}
+
+func writeVidStamp(dir string, stamp []byte) error {
+	tmp := filepath.Join(dir, vidStampFileName+".tmp")
+	f, err := os.OpenFile(tmp, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0644)
+	if err != nil {
+		return err
+	}
+	if _, err := f.Write(stamp); err != nil {
+		f.Close()
+		return err
+	}
+	if err := f.Sync(); err != nil {
+		f.Close()
+		return err
+	}
+	if err := f.Close(); err != nil {
+		return err
+	}
+	if err := os.Rename(tmp, filepath.Join(dir, vidStampFileName)); err != nil {
+		return err
+	}
+	return util.FsyncDir(dir)
+}

weed/storage/vid_stamp_test.go

@@ -0,0 +1,94 @@
+package storage
+
+import (
+	"os"
+	"path/filepath"
+	"testing"
+
+	"github.com/seaweedfs/seaweedfs/weed/stats"
+	"github.com/seaweedfs/seaweedfs/weed/storage/needle"
+	"github.com/seaweedfs/seaweedfs/weed/storage/types"
+	"github.com/seaweedfs/seaweedfs/weed/util"
+)
+
+func newVidStampStore(t *testing.T, dir string) *Store {
+	t.Helper()
+	return NewStore(nil, "localhost", 8080, 18080, "http://localhost:8080", "store-id",
+		[]string{dir}, []int32{100}, []util.MinFreeSpace{{}}, "",
+		NeedleMapInMemory, []types.DiskType{types.HardDriveType}, nil, 3,
+		stats.DefaultDiskIOProbeConfig())
+}
+
+func TestVidStampWrittenOnBoundIncrease(t *testing.T) {
+	dir := t.TempDir()
+	store := newVidStampStore(t, dir)
+	defer store.Close()
+
+	store.UpdateMaxVolumeIdSeen(needle.VolumeId(42))
+
+	data, err := os.ReadFile(filepath.Join(dir, vidStampFileName))
+	if err != nil {
+		t.Fatalf("read stamp: %v", err)
+	}
+	stamp, err := readVidStamp(dir)
+	if err != nil {
+		t.Fatalf("parse stamp: %v", err)
+	}
+	if stamp.MaxVolumeId != 42 {
+		t.Fatalf("stamp max = %d, want 42", stamp.MaxVolumeId)
+	}
+	if len(data) == 0 {
+		t.Fatal("empty stamp")
+	}
+}
+
+func TestVidStampSurvivesRestart(t *testing.T) {
+	dir := t.TempDir()
+
+	store := newVidStampStore(t, dir)
+	store.UpdateMaxVolumeIdSeen(needle.VolumeId(42))
+	store.UpdateLastSeenTopologyId("cluster-a")
+	store.Close()
+
+	restarted := newVidStampStore(t, dir)
+	defer restarted.Close()
+	if got := restarted.MaxVolumeIdSeen(); got != needle.VolumeId(42) {
+		t.Fatalf("restored bound = %d, want 42", got)
+	}
+	if got := restarted.LastSeenTopologyId(); got != "cluster-a" {
+		t.Fatalf("restored topology id = %q, want cluster-a", got)
+	}
+}
+
+func TestVidStampMonotonic(t *testing.T) {
+	dir := t.TempDir()
+	store := newVidStampStore(t, dir)
+	defer store.Close()
+
+	store.UpdateMaxVolumeIdSeen(needle.VolumeId(10))
+	store.UpdateMaxVolumeIdSeen(needle.VolumeId(5))
+	if got := store.MaxVolumeIdSeen(); got != needle.VolumeId(10) {
+		t.Fatalf("bound regressed to %d, want 10", got)
+	}
+}
+
+func TestVidStampBootstrapFromVolumes(t *testing.T) {
+	dir := t.TempDir()
+
+	store := newVidStampStore(t, dir)
+	if err := store.AddVolume(17, "", NeedleMapInMemory, "000", "", 0, needle.GetCurrentVersion(), 0, types.HardDriveType, 0); err != nil {
+		t.Fatalf("add volume: %v", err)
+	}
+	store.Close()
+
+	// upgrading a pre-stamp dir: volumes exist but no stamp yet
+	if err := os.Remove(filepath.Join(dir, vidStampFileName)); err != nil {
+		t.Fatalf("remove stamp: %v", err)
+	}
+
+	restarted := newVidStampStore(t, dir)
+	defer restarted.Close()
+	if got := restarted.MaxVolumeIdSeen(); got != needle.VolumeId(17) {
+		t.Fatalf("bootstrapped bound = %d, want 17", got)
+	}
+}