seaweedfs/weed/storage/store_volume_report.go

package storage

import (
	"sync"
	"unique"

	"github.com/seaweedfs/seaweedfs/weed/pb/master_pb"
)

// volumeReportKey identifies one reported copy. Keyed by disk as well as id
// because a volume id can be mounted on two disks, and reporting one of them
// would leave the other's changes untold.
type volumeReportKey struct {
	diskId   uint32
	volumeId uint32
}

// reportedIdentity is what naming a departure needs beyond the volume and disk
// ids the key already holds. Volumes share very few distinct values here — one
// per collection, placement, version, ttl and disk type in use — so entries
// hold a handle to a shared copy, which keeps them small enough that a server
// with millions of volumes is not paying for identity per volume.
//
// The handle is what keeps the shared copy alive, and dropping the last one
// clears the entry. Handing out the value and letting the handle go, as
// internVolumeString warns against, would have the next volume make a second
// copy.
type reportedIdentity struct {
	collection        string
	diskType          string
	replicaPlacement  uint32
	version           uint32
	ttl               uint32
	readOnly          bool
	readOnlyCanDelete bool
}

// reportedVolume is what the master was told about one volume copy: the hash
// that detects change, the heartbeat pass that last found the copy held, and
// the identity that names the volume if it departs. The departure message
// itself is built on the way out, since almost no volume ever leaves.
type reportedVolume struct {
	hash     uint64
	pass     uint64
	identity unique.Handle[reportedIdentity]
}

// volumeReportState remembers what the master was last told about each volume,
// so a heartbeat can carry only what moved since.
//
// It is per-connection: a server that reconnects, or reaches a different
// master, knows nothing about what that master holds and starts again from the
// full list. The zero value has told no master anything, so it sends the whole
// list until one accepts changes.
type volumeReportState struct {
	mu sync.Mutex
	// deltasAccepted is set once the master says it compares digests. Until
	// then the whole list goes every time, which is what an older master needs.
	deltasAccepted bool
	fullListNeeded bool
	// fullListGeneration counts requests for the whole list, so one arriving
	// while a heartbeat is being built is not marked satisfied by it.
	fullListGeneration uint64
	// pass numbers heartbeats, so one can mark the copies it finds held without
	// building a second map of them.
	pass         uint64
	lastReported map[volumeReportKey]reportedVolume
}

// reset drops everything known about the master's view.
func (s *volumeReportState) reset() {
	s.mu.Lock()
	defer s.mu.Unlock()
	s.deltasAccepted = false
	s.fullListNeeded = true
	s.fullListGeneration++
	s.lastReported = nil
}

func (s *volumeReportState) acceptDeltas() {
	s.mu.Lock()
	defer s.mu.Unlock()
	s.deltasAccepted = true
}

func (s *volumeReportState) requestFullList() {
	s.mu.Lock()
	defer s.mu.Unlock()
	s.fullListNeeded = true
	s.fullListGeneration++
}

// begin opens a heartbeat: whether it must carry the whole list, the request it
// answers, and the pass number that marks the copies it finds still held.
func (s *volumeReportState) begin() (full bool, generation uint64, pass uint64) {
	s.mu.Lock()
	defer s.mu.Unlock()
	s.pass++
	return s.fullListNeeded || !s.deltasAccepted, s.fullListGeneration, s.pass
}

// record marks one volume copy as held by the heartbeat being built, and reports
// whether the master needs telling about it. It updates the entry already held
// rather than build a second map beside it, so a server whose volumes are quiet
// allocates nothing per volume per heartbeat.
func (s *volumeReportState) record(m *master_pb.VolumeInformationMessage, hash uint64, pass uint64) bool {
	key := volumeReportKey{diskId: m.DiskId, volumeId: m.Id}
	s.mu.Lock()
	defer s.mu.Unlock()
	if previous, known := s.lastReported[key]; known {
		changed := previous.hash != hash
		if changed {
			previous.identity = identityOf(m)
		}
		previous.hash, previous.pass = hash, pass
		s.lastReported[key] = previous
		return changed
	}
	if s.lastReported == nil {
		s.lastReported = make(map[volumeReportKey]reportedVolume)
	}
	s.lastReported[key] = reportedVolume{hash: hash, pass: pass, identity: identityOf(m)}
	return true
}

func identityOf(m *master_pb.VolumeInformationMessage) unique.Handle[reportedIdentity] {
	return unique.Make(reportedIdentity{
		collection:        m.Collection,
		diskType:          m.DiskType,
		replicaPlacement:  m.ReplicaPlacement,
		version:           m.Version,
		ttl:               m.Ttl,
		readOnly:          m.ReadOnly,
		readOnlyCanDelete: m.ReadOnlyCanDelete,
	})
}

func (held reportedVolume) toShortInformation(key volumeReportKey) *master_pb.VolumeShortInformationMessage {
	identity := held.identity.Value()
	return &master_pb.VolumeShortInformationMessage{
		Id:                key.volumeId,
		Collection:        identity.collection,
		ReplicaPlacement:  identity.replicaPlacement,
		Version:           identity.version,
		Ttl:               identity.ttl,
		DiskType:          identity.diskType,
		DiskId:            key.diskId,
		ReadOnly:          identity.readOnly,
		ReadOnlyCanDelete: identity.readOnlyCanDelete,
	}
}

// commit closes the heartbeat. Copies this pass did not find are forgotten, so
// one that comes back is reported again, and those whose volume left the server
// altogether are returned. A delta heartbeat says nothing through silence, so
// they must be named or the master keeps counting them until a digest mismatch
// buys it a full list — long enough for a busy cluster to run its free-slot
// accounting dry. A volume that moved disks is still held, so it is not a
// departure; a full list is already the whole truth, so it names none.
func (s *volumeReportState) commit(pass uint64, generation uint64, full bool) []*master_pb.VolumeShortInformationMessage {
	s.mu.Lock()
	defer s.mu.Unlock()
	// A request that arrived while this heartbeat was being built asked about a
	// later state than it carries, so it stands.
	if s.fullListGeneration == generation {
		s.fullListNeeded = false
	}
	var goneKeys []volumeReportKey
	for key, prior := range s.lastReported {
		if prior.pass != pass {
			goneKeys = append(goneKeys, key)
		}
	}
	if len(goneKeys) == 0 {
		return nil
	}
	goneIds := make(map[uint32]bool, len(goneKeys))
	for _, key := range goneKeys {
		goneIds[key.volumeId] = true
	}
	for key, prior := range s.lastReported {
		if prior.pass == pass {
			delete(goneIds, key.volumeId)
		}
	}
	var gone []*master_pb.VolumeShortInformationMessage
	for _, key := range goneKeys {
		if !full && goneIds[key.volumeId] {
			gone = append(gone, s.lastReported[key].toShortInformation(key))
		}
		delete(s.lastReported, key)
	}
	return gone
}