Federated volume-id bound via heartbeat stamps (#11763) (#11819)

* pb: heartbeat fields for the federated volume-id bound (#11763)

Heartbeat carries max_volume_id_seen and last_seen_topology_id up, and
HeartbeatResponse echoes max_volume_id and topology_id down, so the
allocation bound is replicated to every volume server.

* master: echo the volume-id bound and fold reported bounds in (#11763)

Every heartbeat response carries the leader current max volume id and
topology id; every heartbeat reported max_volume_id_seen is folded into
the topology bound via UpAdjustMaxVolumeId.

* volume: persist a per-dir vid stamp of the highest id ever seen (#11763)

The store keeps max_volume_id_seen + last_seen_topology_id in a
.vid_stamp file per volume dir, fed by heartbeat echoes, AllocateVolume,
and local volume/EC creation, seeded at boot from stamps and local
holdings, and reported in every heartbeat. A rebuilt master recovers
the bound from the first surviving veteran report.

* volume-rust: mirror the vid stamp and heartbeat echo handling (#11763)

Same mechanism as the Go volume server: .vid_stamp per dir, bound folded
in from add_location scans and allocate/mount calls, echoed fields
applied on heartbeat responses, reported on every heartbeat.

* volume: serialize vid stamp writes and fsync the directory (#11763)

* volume-rust: serialize vid stamp writes and fsync the directory (#11763)

* master: echo the allocation bound to connected servers on change

The bound was only sent at connect time or on digest mismatch, so a
healthy stream never learned later increases. If the high-id server then
went offline and the master lost state, survivors restored a stale bound
and the id could be reissued. Resend the heartbeat response whenever the
bound or topology id moved.

* volume: retry failed vid stamp saves on later updates

A transient write failure left the in-memory bound raised but the stamp
stale, and repeat updates at the same bound never retried. Track a dirty
flag so any later update re-attempts the save.

* volume: cover stamp bootstrap for pre-stamp dirs in test

* volume-rust: retry failed vid stamp saves, mirroring Go

* volume-rust: stamp every dir with the combined bound on add_location

A dir whose own stamp sat below the combined bound kept its stale value
after fold; losing the higher-stamped disk then restarted the server
below ids it had already learned. Rewrite all stamps after each location
folds in.

* volume: stamp mounted volume and ec shard ids (#11763)

* volume-rust: stamp mounted volume ids, mirroring Go (#11763)

* volume: retry dirty vid stamp save on unchanged topology id (#11763)

* volume-rust: retry dirty vid stamp save on unchanged topology id (#11763)

* master: record echoed bound from the sent heartbeat response (#11763)

weed/storage/store.go

@@ -97,6 +97,11 @@ type Store struct {
 	// the heartbeat that saw it. Written only from the heartbeat goroutine.
 	reportedCollections   map[string]struct{}
 	reportedEcCollections map[string]struct{}
+	maxVolumeIdSeen       atomic.Uint32
+	lastSeenTopologyId    atomic.Value // string
+	vidStampReady         atomic.Bool
+	vidStampDirty         atomic.Bool
+	vidStampLock          sync.Mutex
 }
 
 func (s *Store) String() (str string) {
@@ -145,6 +150,7 @@ func NewStore(
 		diskId := uint32(i) // Track disk ID
 
 		location.ecShardNotifyHandler = func(collection string, vid needle.VolumeId, shardId erasure_coding.ShardId, ecVolume *erasure_coding.EcVolume) {
+			s.UpdateMaxVolumeIdSeen(vid)
 			si := erasure_coding.NewShardsInfo()
 			si.Set(erasure_coding.NewShardInfo(shardId, erasure_coding.ShardSize(ecVolume.ShardSize())))
 
@@ -211,6 +217,8 @@ func NewStore(
 		glog.Fatalf("failed to resolve state for volume %s: %v", id, err)
 	}
 
+	s.bootstrapMaxVolumeIdSeen()
+
 	return
 }
 
@@ -320,6 +328,7 @@ func (s *Store) FindFreeLocation(filterFn func(location *DiskLocation) bool, rep
 	return ret
 }
 func (s *Store) addVolume(vid needle.VolumeId, collection string, needleMapKind NeedleMapKind, replicaPlacement *super_block.ReplicaPlacement, ttl *needle.TTL, preallocate int64, ver needle.Version, memoryMapMaxSizeMb uint32, diskType DiskType, ldbTimeout int64) error {
+	s.UpdateMaxVolumeIdSeen(vid)
 	if s.findVolume(vid) != nil {
 		return fmt.Errorf("Volume Id %d already exists!", vid)
 	}
@@ -686,27 +695,29 @@ func (s *Store) CollectHeartbeat() *master_pb.Heartbeat {
 	stats.VolumeServerIoQuarantineGauge.WithLabelValues("ec_shard").Set(float64(quarantinedEcShards))
 
 	return &master_pb.Heartbeat{
-		Ip:              s.Ip,
-		Port:            uint32(s.Port),
-		GrpcPort:        uint32(s.GrpcPort),
-		PublicUrl:       s.PublicUrl,
-		Id:              s.Id,
-		MaxVolumeCounts: maxVolumeCounts,
-		DiskTotalBytes:  diskTotalBytes,
-		DiskFreeBytes:   diskFreeBytes,
-		MaxFileKey:      NeedleIdToUint64(maxFileKey),
-		DataCenter:      s.dataCenter,
-		Rack:            s.rack,
-		Volumes:         heartbeatVolumes,
-		ChangedVolumes:  changedVolumes,
-		DeletedVolumes:  departedVolumes,
-		VolumeDigest:    &volumeDigest,
-		DeletedEcShards: deletedEcVolumes,
-		HasNoVolumes:    hasNoVolumes,
-		HasNoEcShards:   len(ecVolumeMessages) == 0,
-		LocationUuids:   uuidList,
-		DiskTags:        diskTags,
-		State:           state,
+		Ip:                 s.Ip,
+		Port:               uint32(s.Port),
+		GrpcPort:           uint32(s.GrpcPort),
+		PublicUrl:          s.PublicUrl,
+		Id:                 s.Id,
+		MaxVolumeCounts:    maxVolumeCounts,
+		DiskTotalBytes:     diskTotalBytes,
+		DiskFreeBytes:      diskFreeBytes,
+		MaxFileKey:         NeedleIdToUint64(maxFileKey),
+		DataCenter:         s.dataCenter,
+		Rack:               s.rack,
+		Volumes:            heartbeatVolumes,
+		ChangedVolumes:     changedVolumes,
+		DeletedVolumes:     departedVolumes,
+		VolumeDigest:       &volumeDigest,
+		DeletedEcShards:    deletedEcVolumes,
+		HasNoVolumes:       hasNoVolumes,
+		HasNoEcShards:      len(ecVolumeMessages) == 0,
+		LocationUuids:      uuidList,
+		DiskTags:           diskTags,
+		State:              state,
+		MaxVolumeIdSeen:    uint32(s.MaxVolumeIdSeen()),
+		LastSeenTopologyId: s.LastSeenTopologyId(),
 	}
 
 }
@@ -1027,6 +1038,7 @@ func (s *Store) mountVolume(i needle.VolumeId, collection *string, validators ..
 				ReadOnly:          readOnly,
 				ReadOnlyCanDelete: readOnlyCanDelete,
 			}
+			s.UpdateMaxVolumeIdSeen(i)
 			return nil
 		}
 	}

weed/storage/store_ec.go

@@ -304,6 +304,7 @@ func (s *Store) MountEcShards(collection string, vid needle.VolumeId, shardId er
 				DiskId:      uint32(diskId),
 				EncodeTsNs:  ecVolume.EncodeTsNs,
 			}
+			s.UpdateMaxVolumeIdSeen(vid)
 			return nil
 		}
 		if errors.Is(err, os.ErrNotExist) {

weed/storage/vid_stamp.go

@@ -0,0 +1,148 @@
+package storage
+
+import (
+	"encoding/json"
+	"os"
+	"path/filepath"
+
+	"github.com/seaweedfs/seaweedfs/weed/glog"
+	"github.com/seaweedfs/seaweedfs/weed/storage/needle"
+	"github.com/seaweedfs/seaweedfs/weed/util"
+)
+
+const vidStampFileName = ".vid_stamp"
+
+type vidStamp struct {
+	MaxVolumeId uint32 `json:"maxVolumeId"`
+	TopologyId  string `json:"topologyId,omitempty"`
+}
+
+func (s *Store) MaxVolumeIdSeen() needle.VolumeId {
+	return needle.VolumeId(s.maxVolumeIdSeen.Load())
+}
+
+func (s *Store) LastSeenTopologyId() string {
+	if v := s.lastSeenTopologyId.Load(); v != nil {
+		return v.(string)
+	}
+	return ""
+}
+
+// UpdateMaxVolumeIdSeen raises the server's persisted bound and stamps every
+// volume dir when it grows. Feeds: heartbeat echoes, AllocateVolume, local
+// volume and EC shard creation.
+func (s *Store) UpdateMaxVolumeIdSeen(vid needle.VolumeId) {
+	grew := false
+	for {
+		current := s.maxVolumeIdSeen.Load()
+		if current >= uint32(vid) {
+			break
+		}
+		if s.maxVolumeIdSeen.CompareAndSwap(current, uint32(vid)) {
+			grew = true
+			break
+		}
+	}
+	if s.vidStampReady.Load() && (grew || s.vidStampDirty.Load()) {
+		s.saveVidStamps()
+	}
+}
+
+func (s *Store) UpdateLastSeenTopologyId(id string) {
+	if id == "" || (s.LastSeenTopologyId() == id && !s.vidStampDirty.Load()) {
+		return
+	}
+	s.lastSeenTopologyId.Store(id)
+	if s.vidStampReady.Load() {
+		s.saveVidStamps()
+	}
+}
+
+// bootstrapMaxVolumeIdSeen seeds the bound from persisted stamps plus the max
+// of every volume and EC shard id on disk, so a server upgraded mid-incident
+// still reports a bound covering its own holdings.
+func (s *Store) bootstrapMaxVolumeIdSeen() {
+	var maxVid needle.VolumeId
+	for _, location := range s.Locations {
+		if stamp, err := readVidStamp(location.Directory); err == nil {
+			if needle.VolumeId(stamp.MaxVolumeId) > maxVid {
+				maxVid = needle.VolumeId(stamp.MaxVolumeId)
+			}
+			if stamp.TopologyId != "" && s.LastSeenTopologyId() == "" {
+				s.lastSeenTopologyId.Store(stamp.TopologyId)
+			}
+		}
+		location.volumesLock.RLock()
+		for vid := range location.volumes {
+			if vid > maxVid {
+				maxVid = vid
+			}
+		}
+		location.volumesLock.RUnlock()
+		location.ecVolumesLock.RLock()
+		for vid := range location.ecVolumes {
+			if vid > maxVid {
+				maxVid = vid
+			}
+		}
+		location.ecVolumesLock.RUnlock()
+	}
+	s.maxVolumeIdSeen.Store(uint32(maxVid))
+	s.vidStampReady.Store(true)
+	s.saveVidStamps()
+}
+
+func (s *Store) saveVidStamps() {
+	s.vidStampLock.Lock()
+	defer s.vidStampLock.Unlock()
+	stamp, err := json.Marshal(vidStamp{
+		MaxVolumeId: s.maxVolumeIdSeen.Load(),
+		TopologyId:  s.LastSeenTopologyId(),
+	})
+	if err != nil {
+		return
+	}
+	failed := false
+	for _, location := range s.Locations {
+		if err := writeVidStamp(location.Directory, stamp); err != nil {
+			glog.Warningf("write vid stamp in %s: %v", location.Directory, err)
+			failed = true
+		}
+	}
+	s.vidStampDirty.Store(failed)
+}
+
+func readVidStamp(dir string) (*vidStamp, error) {
+	data, err := os.ReadFile(filepath.Join(dir, vidStampFileName))
+	if err != nil {
+		return nil, err
+	}
+	stamp := &vidStamp{}
+	if err := json.Unmarshal(data, stamp); err != nil {
+		return nil, err
+	}
+	return stamp, nil
+}
+
+func writeVidStamp(dir string, stamp []byte) error {
+	tmp := filepath.Join(dir, vidStampFileName+".tmp")
+	f, err := os.OpenFile(tmp, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0644)
+	if err != nil {
+		return err
+	}
+	if _, err := f.Write(stamp); err != nil {
+		f.Close()
+		return err
+	}
+	if err := f.Sync(); err != nil {
+		f.Close()
+		return err
+	}
+	if err := f.Close(); err != nil {
+		return err
+	}
+	if err := os.Rename(tmp, filepath.Join(dir, vidStampFileName)); err != nil {
+		return err
+	}
+	return util.FsyncDir(dir)
+}

weed/storage/vid_stamp_test.go

@@ -0,0 +1,94 @@
+package storage
+
+import (
+	"os"
+	"path/filepath"
+	"testing"
+
+	"github.com/seaweedfs/seaweedfs/weed/stats"
+	"github.com/seaweedfs/seaweedfs/weed/storage/needle"
+	"github.com/seaweedfs/seaweedfs/weed/storage/types"
+	"github.com/seaweedfs/seaweedfs/weed/util"
+)
+
+func newVidStampStore(t *testing.T, dir string) *Store {
+	t.Helper()
+	return NewStore(nil, "localhost", 8080, 18080, "http://localhost:8080", "store-id",
+		[]string{dir}, []int32{100}, []util.MinFreeSpace{{}}, "",
+		NeedleMapInMemory, []types.DiskType{types.HardDriveType}, nil, 3,
+		stats.DefaultDiskIOProbeConfig())
+}
+
+func TestVidStampWrittenOnBoundIncrease(t *testing.T) {
+	dir := t.TempDir()
+	store := newVidStampStore(t, dir)
+	defer store.Close()
+
+	store.UpdateMaxVolumeIdSeen(needle.VolumeId(42))
+
+	data, err := os.ReadFile(filepath.Join(dir, vidStampFileName))
+	if err != nil {
+		t.Fatalf("read stamp: %v", err)
+	}
+	stamp, err := readVidStamp(dir)
+	if err != nil {
+		t.Fatalf("parse stamp: %v", err)
+	}
+	if stamp.MaxVolumeId != 42 {
+		t.Fatalf("stamp max = %d, want 42", stamp.MaxVolumeId)
+	}
+	if len(data) == 0 {
+		t.Fatal("empty stamp")
+	}
+}
+
+func TestVidStampSurvivesRestart(t *testing.T) {
+	dir := t.TempDir()
+
+	store := newVidStampStore(t, dir)
+	store.UpdateMaxVolumeIdSeen(needle.VolumeId(42))
+	store.UpdateLastSeenTopologyId("cluster-a")
+	store.Close()
+
+	restarted := newVidStampStore(t, dir)
+	defer restarted.Close()
+	if got := restarted.MaxVolumeIdSeen(); got != needle.VolumeId(42) {
+		t.Fatalf("restored bound = %d, want 42", got)
+	}
+	if got := restarted.LastSeenTopologyId(); got != "cluster-a" {
+		t.Fatalf("restored topology id = %q, want cluster-a", got)
+	}
+}
+
+func TestVidStampMonotonic(t *testing.T) {
+	dir := t.TempDir()
+	store := newVidStampStore(t, dir)
+	defer store.Close()
+
+	store.UpdateMaxVolumeIdSeen(needle.VolumeId(10))
+	store.UpdateMaxVolumeIdSeen(needle.VolumeId(5))
+	if got := store.MaxVolumeIdSeen(); got != needle.VolumeId(10) {
+		t.Fatalf("bound regressed to %d, want 10", got)
+	}
+}
+
+func TestVidStampBootstrapFromVolumes(t *testing.T) {
+	dir := t.TempDir()
+
+	store := newVidStampStore(t, dir)
+	if err := store.AddVolume(17, "", NeedleMapInMemory, "000", "", 0, needle.GetCurrentVersion(), 0, types.HardDriveType, 0); err != nil {
+		t.Fatalf("add volume: %v", err)
+	}
+	store.Close()
+
+	// upgrading a pre-stamp dir: volumes exist but no stamp yet
+	if err := os.Remove(filepath.Join(dir, vidStampFileName)); err != nil {
+		t.Fatalf("remove stamp: %v", err)
+	}
+
+	restarted := newVidStampStore(t, dir)
+	defer restarted.Close()
+	if got := restarted.MaxVolumeIdSeen(); got != needle.VolumeId(17) {
+		t.Fatalf("bootstrapped bound = %d, want 17", got)
+	}
+}