master: keep periodic volume growth to the data centers a layout lives in (#11060)

* master: keep the periodic growth scan to data centers hosting the layout The rack-aware scan planned growth for every data center in the topology, so a collection pinned to one DC (fs.configure -dataCenter) sprouted volumes in all the others within one scan cycle. Plan only for data centers already hosting the layout's volumes; an empty DC gets its volumes from the DC-constrained assign that first asks for them. The lastGrowCount divisor likewise counts only the racks the scan can plan for. Claude-Session: https://claude.ai/code/session_01J22TVTyoCMzdHyJirsLMG5 * master: pin periodic must-grow growth to a single-DC layout's data center The must-grow and crowded paths of the periodic loop grow with no DataCenter, so even with the scan fixed a pinned collection's volumes could still land in any DC once lastGrowCount demands more writables. Stamp the grow request with the layout's data center when its volumes all live in one; layouts spanning DCs keep unconstrained growth. Claude-Session: https://claude.ai/code/session_01J22TVTyoCMzdHyJirsLMG5 * master: never pin growth of a cross-DC-replicated layout A layout whose replication spans data centers cannot legitimately live in one DC; observing a single hosting DC there means the other DCs are down. Do not encode that outage as a placement constraint. Claude-Session: https://claude.ai/code/session_01J22TVTyoCMzdHyJirsLMG5 * master: bound the hosting-DC walk by the answer it needs listVolumeDataCenters walked every location of the layout under accessLock — ~190ms for a million volumes, twice per layout per cycle, stalling assigns behind the read lock. Stop once enough distinct DCs answer the caller's question: two for the single-DC check, the topology's DC count for the scan. A spanning million-volume layout now finishes in microseconds; only a layout truly confined to fewer DCs still pays a full walk, the same cost class as the under-replication count this loop already takes each cycle. Claude-Session: https://claude.ai/code/session_01J22TVTyoCMzdHyJirsLMG5
2026-09-08 15:41:15 +02:00 · 2026-09-01 00:40:50 -07:00
parent 8873f9775c
commit 9d5525e747
2 changed files with 165 additions and 6 deletions
@@ -758,6 +758,28 @@ func (vl *VolumeLayout) ShouldGrowVolumesByDcAndRack(writables *[]needle.VolumeI
 	return true
 }

+// listVolumeDataCenters returns the data centers hosting at least one volume
+// of this layout, stopping once limit distinct DCs are seen (0 = no limit).
+// The walk holds accessLock and a layout can span millions of volumes, so
+// callers pass the smallest limit that answers their question and only a
+// layout truly confined to fewer DCs pays for a full walk.
+func (vl *VolumeLayout) listVolumeDataCenters(limit int) map[NodeId]struct{} {
+	vl.accessLock.RLock()
+	defer vl.accessLock.RUnlock()
+	dataCenters := make(map[NodeId]struct{})
+	for _, location := range vl.vid2location {
+		for _, dn := range location.list {
+			if dcId := dn.GetDataCenterId(); dcId != "" {
+				dataCenters[NodeId(dcId)] = struct{}{}
+				if limit > 0 && len(dataCenters) >= limit {
+					return dataCenters
+				}
+			}
+		}
+	}
+	return dataCenters
+}
+
 // RackGrowPlan is one volume grow action produced by the periodic rack-aware
 // growth scan. An empty Rack means the grow is DC-wide.
 type RackGrowPlan struct {
@@ -766,8 +788,8 @@ type RackGrowPlan struct {
 	WritableVolumeCount uint32
 }

-// PlanRackAwareGrowth returns the grow actions needed so every location that
-// can serve writes keeps a non-crowded writable volume. stepCount is the
+// PlanRackAwareGrowth returns the grow actions needed so every data center
+// this layout lives in keeps a non-crowded writable volume. stepCount is the
 // default per-event increment.
 //
 // For rack-spanning replication (DiffRackCount > 0) a single logical volume
@@ -782,21 +804,33 @@ func (vl *VolumeLayout) PlanRackAwareGrowth(dcs map[NodeId][]NodeId, lastGrowCou
 		stepCount = c
 	}
 	growOncePerDc := vl.rp.DiffRackCount > 0
+	// Only maintain data centers already hosting this layout's volumes: a
+	// collection pinned to one DC (fs.configure -dataCenter) must not sprout
+	// volumes in every other DC, and a DC that does need this layout gets its
+	// volumes from the DC-constrained assign that first asks for them.
+	hostingDCs := vl.listVolumeDataCenters(len(dcs))
 	// Spread lastGrowCount evenly across all grow targets. Summing every rack
 	// up front keeps the divisor global, so DCs with different rack counts do
 	// not each over-grow from a per-DC divisor.
-	var rackPairs uint32
-	for _, racks := range dcs {
+	var rackPairs, dcCount uint32
+	for dcId, racks := range dcs {
+		if _, hosting := hostingDCs[dcId]; !hosting {
+			continue
+		}
+		dcCount++
 		rackPairs += uint32(len(racks))
 	}
 	for dcId, racks := range dcs {
+		if _, hosting := hostingDCs[dcId]; !hosting {
+			continue
+		}
 		if growOncePerDc {
 			if !vl.ShouldGrowVolumesByDcAndRack(&writables, dcId, "") {
 				continue
 			}
 			count := stepCount
 			if lastGrowCount > 0 {
-				count = ceilDiv(lastGrowCount, uint32(len(dcs)))
+				count = ceilDiv(lastGrowCount, dcCount)
 			}
 			plans = append(plans, RackGrowPlan{DataCenter: string(dcId), WritableVolumeCount: count})
 			continue
@@ -1046,12 +1080,26 @@ func (vl *VolumeLayout) ToInfo() (info VolumeLayoutInfo) {
 }

 func (vlc *VolumeLayoutCollection) ToVolumeGrowRequest() *master_pb.VolumeGrowRequest {
-	return &master_pb.VolumeGrowRequest{
+	vgr := &master_pb.VolumeGrowRequest{
 		Collection:  vlc.Collection,
 		Replication: vlc.VolumeLayout.rp.String(),
 		Ttl:         vlc.VolumeLayout.ttl.String(),
 		DiskType:    vlc.VolumeLayout.diskType.String(),
 	}
+	// A layout living in exactly one data center is either pinned there or on
+	// a single-DC cluster; either way unconstrained growth has no business
+	// placing its volumes elsewhere. Cross-DC replication can never
+	// legitimately live in one DC — there a single hosting DC means an
+	// outage, not a pin.
+	if vlc.VolumeLayout.rp.DiffDataCenterCount > 0 {
+		return vgr
+	}
+	if dcs := vlc.VolumeLayout.listVolumeDataCenters(2); len(dcs) == 1 {
+		for dc := range dcs {
+			vgr.DataCenter = string(dc)
+		}
+	}
+	return vgr
 }

 func (vl *VolumeLayout) Stats() *VolumeLayoutStats {
@@ -1,6 +1,7 @@
 package topology

 import (
+	"fmt"
 	"testing"

 	"github.com/seaweedfs/seaweedfs/weed/storage/needle"
@@ -240,6 +241,116 @@ func TestPlanRackAwareGrowth_EvenDistributionAcrossUnevenDCs(t *testing.T) {
 	}
 }

+// A collection pinned to one data center (fs.configure -dataCenter) lives only
+// there, yet the scan used to plan growth for every data center in the
+// topology, spreading the collection's volumes across all of them. Data
+// centers hosting none of the layout's volumes are not the scan's to fill.
+func TestPlanRackAwareGrowth_SkipsDataCentersWithoutLayoutVolumes(t *testing.T) {
+	layout := `
+{
+  "dc1":{ "rack1":{ "node-a":{ "ip":"10.0.0.1", "volumes":[ {"id":1, "size":%d, "replication":"000", "collection":"pinned"} ], "limit":30 } } },
+  "dc2":{ "rack2":{ "node-b":{ "ip":"10.0.0.2", "volumes":[], "limit":30 } } },
+  "dc3":{ "rack3":{ "node-c":{ "ip":"10.0.0.3", "volumes":[], "limit":30 } } },
+  "dc4":{ "rack4":{ "node-d":{ "ip":"10.0.0.4", "volumes":[], "limit":30 } } }
+}
+`
+	rp, _ := super_block.NewReplicaPlacementFromString("000")
+
+	topo := setupWithLimit(t, fmt.Sprintf(layout, 1000), 30000)
+	vl := topo.GetVolumeLayout("pinned", rp, needle.EMPTY_TTL, types.HardDriveType)
+	if plans := vl.PlanRackAwareGrowth(topo.ListDCAndRacks(), 0, 2); len(plans) != 0 {
+		t.Fatalf("healthy dc1 volume: expected no growth, got %+v", plans)
+	}
+
+	topo = setupWithLimit(t, fmt.Sprintf(layout, 28650), 30000)
+	vl = topo.GetVolumeLayout("pinned", rp, needle.EMPTY_TTL, types.HardDriveType)
+	plans := vl.PlanRackAwareGrowth(topo.ListDCAndRacks(), 0, 2)
+	if len(plans) != 1 {
+		t.Fatalf("crowded dc1 volume: expected 1 grow, got %d: %+v", len(plans), plans)
+	}
+	if plans[0].DataCenter != "dc1" || plans[0].Rack != "rack1" {
+		t.Errorf("expected grow pinned to dc1/rack1, got %s/%s", plans[0].DataCenter, plans[0].Rack)
+	}
+}
+
+// The lastGrowCount divisor counts only the racks the scan can actually plan
+// for; racks of skipped data centers would dilute every grow.
+func TestPlanRackAwareGrowth_DivisorCountsOnlyHostingRacks(t *testing.T) {
+	layout := `
+{
+  "dc1":{
+    "rack1":{ "node-a":{ "ip":"10.0.0.1", "volumes":[ {"id":1, "size":28650, "replication":"000", "collection":"pinned"} ], "limit":30 } },
+    "rack2":{ "node-b":{ "ip":"10.0.0.2", "volumes":[ {"id":2, "size":28650, "replication":"000", "collection":"pinned"} ], "limit":30 } }
+  },
+  "dc2":{
+    "rack3":{ "node-c":{ "ip":"10.0.0.3", "volumes":[], "limit":30 } },
+    "rack4":{ "node-d":{ "ip":"10.0.0.4", "volumes":[], "limit":30 } }
+  }
+}
+`
+	topo := setupWithLimit(t, layout, 30000)
+	rp, _ := super_block.NewReplicaPlacementFromString("000")
+	vl := topo.GetVolumeLayout("pinned", rp, needle.EMPTY_TTL, types.HardDriveType)
+
+	plans := vl.PlanRackAwareGrowth(topo.ListDCAndRacks(), 4, 2)
+	if len(plans) != 2 {
+		t.Fatalf("expected a grow per crowded dc1 rack, got %d: %+v", len(plans), plans)
+	}
+	for _, p := range plans {
+		if p.DataCenter != "dc1" {
+			t.Errorf("expected grows pinned to dc1, got %s/%s", p.DataCenter, p.Rack)
+		}
+		if p.WritableVolumeCount != 2 { // ceilDiv(4, 2 hosting racks), not ceilDiv(4, 4)
+			t.Errorf("expected per-rack count 2, got %d", p.WritableVolumeCount)
+		}
+	}
+}
+
+// The periodic must-grow and crowded growth paths build their request from
+// ToVolumeGrowRequest. For a layout living in exactly one data center the
+// request carries that DC, so those paths do not scatter a pinned collection
+// either; a layout spanning DCs keeps unconstrained growth.
+func TestToVolumeGrowRequest_SingleDataCenterLayout(t *testing.T) {
+	layout := `
+{
+  "dc1":{ "rack1":{ "node-a":{ "ip":"10.0.0.1", "volumes":[ {"id":1, "size":1000, "replication":"000", "collection":"pinned"} ], "limit":30 } } },
+  "dc2":{ "rack2":{ "node-b":{ "ip":"10.0.0.2", "volumes":[ {"id":2, "size":1000, "replication":"000", "collection":"spread"} ], "limit":30 } } },
+  "dc3":{ "rack3":{ "node-c":{ "ip":"10.0.0.3", "volumes":[ {"id":3, "size":1000, "replication":"000", "collection":"spread"} ], "limit":30 } } }
+}
+`
+	topo := setupWithLimit(t, layout, 30000)
+	rp, _ := super_block.NewReplicaPlacementFromString("000")
+
+	pinned := &VolumeLayoutCollection{"pinned", topo.GetVolumeLayout("pinned", rp, needle.EMPTY_TTL, types.HardDriveType)}
+	if vgr := pinned.ToVolumeGrowRequest(); vgr.DataCenter != "dc1" {
+		t.Errorf("expected growth pinned to dc1, got %q", vgr.DataCenter)
+	}
+
+	spread := &VolumeLayoutCollection{"spread", topo.GetVolumeLayout("spread", rp, needle.EMPTY_TTL, types.HardDriveType)}
+	if vgr := spread.ToVolumeGrowRequest(); vgr.DataCenter != "" {
+		t.Errorf("expected unconstrained growth for a multi-DC layout, got %q", vgr.DataCenter)
+	}
+}
+
+// Cross-DC replication can never legitimately live in one data center: a
+// single hosting DC there means the other DCs are down, and pinning growth to
+// the survivor would encode the outage as a placement constraint.
+func TestToVolumeGrowRequest_CrossDCReplicationNeverPinned(t *testing.T) {
+	layout := `
+{
+  "dc1":{ "rack1":{ "node-a":{ "ip":"10.0.0.1", "volumes":[ {"id":1, "size":1000, "replication":"100", "collection":"c"} ], "limit":30 } } },
+  "dc2":{ "rack2":{ "node-b":{ "ip":"10.0.0.2", "volumes":[], "limit":30 } } }
+}
+`
+	topo := setupWithLimit(t, layout, 30000)
+	rp, _ := super_block.NewReplicaPlacementFromString("100")
+
+	vlc := &VolumeLayoutCollection{"c", topo.GetVolumeLayout("c", rp, needle.EMPTY_TTL, types.HardDriveType)}
+	if vgr := vlc.ToVolumeGrowRequest(); vgr.DataCenter != "" {
+		t.Errorf("expected unconstrained growth for cross-DC replication, got %q", vgr.DataCenter)
+	}
+}
+
 // Volumes packed to capacity (e.g. by fs.mergeVolumes) go crowded and then
 // unwritable, but stay in the crowded map. ShouldGrowVolumes must count only
 // writable crowded volumes, or those leftovers keep writable <= crowded true