Detect and quarantine duplicate volume ids (#11763) (#11828)

* master: detect and quarantine duplicate volume ids across layouts (#11763)

* shell: volume.check.duplicate_ids audit command (#11763)

* master: persist duplicate-vid quarantine across heartbeats (#11763)

* master: quarantine all copies of a duplicated volume id

Near-simultaneous registrations let each side quarantine only the other,
so arrival order decided which copy stayed writable. A confirmed
cross-layout collision means either copy could be the impostor, so flag
every colliding copy read-only regardless of registration order.

* topology: return a copy from VolumeLayout.Lookup

Callers iterating the returned slice raced with registrations and
removals replacing the location list in place.

* master: release duplicate-vid quarantine when the conflict resolves

A cross-node volume.move briefly shows the same id on two nodes under
different disk layouts; permanent quarantine would pin a valid target
read-only. Keep the first-reported state per quarantined copy and lift
the marks once the conflicting copy unregisters, so only a persistent
collision stays read-only.

* shell: include ec shards in volume.check.duplicate_ids

Ec-only volumes share the global id space. Compare shard collections
against regular ones while allowing shards and regular copies of the
same encoded volume.

* master: keep quarantine release current and cover all removal paths

Track the latest server-reported read-only flag while quarantined so a
release restores the server s true state, and run the release check on
collection deletion and data-node removal too, not only explicit volume
unregistration.

* shell: do not flag ec-only volumes as duplicates

An ec-only volume has no regular copy, so its shard collections had
nothing to compare against and always read as foreign.

* master: quarantine every conflicting layout at the layout level (#11763)

- collect all conflicting layouts, not just the first, so a vid spanning
  three layouts does not leave one writable
- mark quarantine on the location list only: folding it into the stored
  report made the volume digest permanently disagree with the server and
  forced a full volume list every heartbeat
- isAllWritable now honors the location read-only mark so a quarantined
  copy stays out of writables through resyncs and re-registration
- re-check the conflict before keeping marks, and do not clear a node's
  quarantine while it still holds the vid in another layout
- log the full layout keys on both sides of the conflict

* master: push duplicate-vid quarantine to the volume server (#11763)

A quarantined copy stayed writable on the volume server itself for a
client holding an existing file id, since the master-only mark never
reached the server's local write path. Quarantine now also calls
VolumeMarkReadonly (can_delete so expiry still drains), is re-pushed if
the server comes back reporting it writable, and VolumeMarkWritable
runs when the conflict resolves and the server had reported writable.

* master: retry unenforced quarantine marks and restore server writability on release (#11763)

Dial the advertised grpc port, return mark errors, and keep a
quarantine entry until the writable push succeeds so a released
copy does not stay read-only on the server.

* master: treat rp-only layout differences as an in-flight replication change (#11763)

volume.configure.replication rolls through nodes one at a time, so a
volume legitimately reports the old and new replica placement while it
settles. Layouts differing only in rp within the same collection are a
supported transition, not a reused id.

* shell: group ec shards by collection in the duplicate id audit (#11763)

EC shards of one volume legitimately span disk types when
DiskTypePrefer spills, so grouping by disk type reports healthy
volumes as duplicates.

weed/shell/command_volume_check_duplicate_ids.go

@@ -0,0 +1,140 @@
+package shell
+
+import (
+	"fmt"
+	"io"
+	"slices"
+
+	"github.com/seaweedfs/seaweedfs/weed/storage/needle"
+	"github.com/seaweedfs/seaweedfs/weed/storage/super_block"
+)
+
+func init() {
+	Commands = append(Commands, &commandVolumeCheckDuplicateIds{})
+}
+
+type commandVolumeCheckDuplicateIds struct {
+}
+
+func (c *commandVolumeCheckDuplicateIds) Name() string {
+	return "volume.check.duplicate_ids"
+}
+
+func (c *commandVolumeCheckDuplicateIds) Help() string {
+	return `find volume ids registered under more than one layout
+
+	volume.check.duplicate_ids
+
+	Volume ids are global. The same id under two collections, replica
+	placements, TTLs, or disk types means the allocator reused an id, for
+	example after master state loss. See volume.growth.resume.
+
+`
+}
+
+func (c *commandVolumeCheckDuplicateIds) HasTag(CommandTag) bool {
+	return false
+}
+
+func (c *commandVolumeCheckDuplicateIds) Do(args []string, commandEnv *CommandEnv, writer io.Writer) (err error) {
+	topoInfo, _, err := collectTopologyInfo(commandEnv, 0)
+	if err != nil {
+		return err
+	}
+
+	// vid -> layout key -> locations; ec shards tracked per collection since
+	// shards plus regular copies of the same encoded volume are legitimate
+	seen := make(map[needle.VolumeId]map[string][]string)
+	ecSeen := make(map[needle.VolumeId]map[string][]string)
+	collections := make(map[needle.VolumeId]map[string]bool)
+	ecCollections := make(map[needle.VolumeId]map[string]bool)
+	add := func(m map[needle.VolumeId]map[string][]string, id uint32, key, location string) {
+		layouts := m[needle.VolumeId(id)]
+		if layouts == nil {
+			layouts = make(map[string][]string)
+			m[needle.VolumeId(id)] = layouts
+		}
+		layouts[key] = append(layouts[key], location)
+	}
+	addCollection := func(m map[needle.VolumeId]map[string]bool, id uint32, collection string) {
+		if m[needle.VolumeId(id)] == nil {
+			m[needle.VolumeId(id)] = make(map[string]bool)
+		}
+		m[needle.VolumeId(id)][collection] = true
+	}
+	for _, dc := range topoInfo.DataCenterInfos {
+		for _, r := range dc.RackInfos {
+			for _, dn := range r.DataNodeInfos {
+				for diskKey, di := range dn.DiskInfos {
+					for _, vi := range di.VolumeInfos {
+						rp, _ := super_block.NewReplicaPlacementFromByte(byte(vi.ReplicaPlacement))
+						add(seen, vi.Id, fmt.Sprintf("collection=%q replication=%s ttl=%s disk=%s",
+							vi.Collection, rp.String(), needle.LoadTTLFromUint32(vi.Ttl), vi.DiskType),
+							fmt.Sprintf("%s %s", dn.Id, diskKey))
+						addCollection(collections, vi.Id, vi.Collection)
+					}
+					for _, si := range di.EcShardInfos {
+						add(ecSeen, si.Id, fmt.Sprintf("collection=%q ec_shards", si.Collection),
+							fmt.Sprintf("%s %s disk=%s", dn.Id, diskKey, si.DiskType))
+						addCollection(ecCollections, si.Id, si.Collection)
+					}
+				}
+			}
+		}
+	}
+
+	vids := make(map[needle.VolumeId]struct{}, len(seen)+len(ecSeen))
+	for vid := range seen {
+		vids[vid] = struct{}{}
+	}
+	for vid := range ecSeen {
+		vids[vid] = struct{}{}
+	}
+
+	duplicates := 0
+	for _, vid := range sortedKeys(vids) {
+		layouts := seen[vid]
+		shardLayouts := ecSeen[vid]
+		ecForeign := false
+		if len(collections[vid]) > 0 {
+			for collection := range ecCollections[vid] {
+				if !collections[vid][collection] {
+					ecForeign = true
+				}
+			}
+		}
+		if len(layouts) < 2 && len(shardLayouts) < 2 && !ecForeign {
+			continue
+		}
+		duplicates++
+		fmt.Fprintf(writer, "volume %d is registered under %d layouts:\n", vid, len(layouts)+len(shardLayouts))
+		for _, key := range sortedKeysString(layouts) {
+			fmt.Fprintf(writer, "  %-60s %v\n", key, layouts[key])
+		}
+		for _, key := range sortedKeysString(shardLayouts) {
+			fmt.Fprintf(writer, "  %-60s %v\n", key, shardLayouts[key])
+		}
+	}
+	if duplicates == 0 {
+		fmt.Fprintf(writer, "no duplicate volume ids found\n")
+	}
+	return nil
+}
+
+func sortedKeys[V any](m map[needle.VolumeId]V) []needle.VolumeId {
+	keys := make([]needle.VolumeId, 0, len(m))
+	for k := range m {
+		keys = append(keys, k)
+	}
+	slices.Sort(keys)
+	return keys
+}
+
+func sortedKeysString[V any](m map[string]V) []string {
+	keys := make([]string, 0, len(m))
+	for k := range m {
+		keys = append(keys, k)
+	}
+	slices.Sort(keys)
+	return keys
+}