diff --git a/weed/topology/volume_layout.go b/weed/topology/volume_layout.go index 93d226a7b..979e9623a 100644 --- a/weed/topology/volume_layout.go +++ b/weed/topology/volume_layout.go @@ -758,6 +758,28 @@ func (vl *VolumeLayout) ShouldGrowVolumesByDcAndRack(writables *[]needle.VolumeI return true } +// listVolumeDataCenters returns the data centers hosting at least one volume +// of this layout, stopping once limit distinct DCs are seen (0 = no limit). +// The walk holds accessLock and a layout can span millions of volumes, so +// callers pass the smallest limit that answers their question and only a +// layout truly confined to fewer DCs pays for a full walk. +func (vl *VolumeLayout) listVolumeDataCenters(limit int) map[NodeId]struct{} { + vl.accessLock.RLock() + defer vl.accessLock.RUnlock() + dataCenters := make(map[NodeId]struct{}) + for _, location := range vl.vid2location { + for _, dn := range location.list { + if dcId := dn.GetDataCenterId(); dcId != "" { + dataCenters[NodeId(dcId)] = struct{}{} + if limit > 0 && len(dataCenters) >= limit { + return dataCenters + } + } + } + } + return dataCenters +} + // RackGrowPlan is one volume grow action produced by the periodic rack-aware // growth scan. An empty Rack means the grow is DC-wide. type RackGrowPlan struct { @@ -766,8 +788,8 @@ type RackGrowPlan struct { WritableVolumeCount uint32 } -// PlanRackAwareGrowth returns the grow actions needed so every location that -// can serve writes keeps a non-crowded writable volume. stepCount is the +// PlanRackAwareGrowth returns the grow actions needed so every data center +// this layout lives in keeps a non-crowded writable volume. stepCount is the // default per-event increment. // // For rack-spanning replication (DiffRackCount > 0) a single logical volume @@ -782,21 +804,33 @@ func (vl *VolumeLayout) PlanRackAwareGrowth(dcs map[NodeId][]NodeId, lastGrowCou stepCount = c } growOncePerDc := vl.rp.DiffRackCount > 0 + // Only maintain data centers already hosting this layout's volumes: a + // collection pinned to one DC (fs.configure -dataCenter) must not sprout + // volumes in every other DC, and a DC that does need this layout gets its + // volumes from the DC-constrained assign that first asks for them. + hostingDCs := vl.listVolumeDataCenters(len(dcs)) // Spread lastGrowCount evenly across all grow targets. Summing every rack // up front keeps the divisor global, so DCs with different rack counts do // not each over-grow from a per-DC divisor. - var rackPairs uint32 - for _, racks := range dcs { + var rackPairs, dcCount uint32 + for dcId, racks := range dcs { + if _, hosting := hostingDCs[dcId]; !hosting { + continue + } + dcCount++ rackPairs += uint32(len(racks)) } for dcId, racks := range dcs { + if _, hosting := hostingDCs[dcId]; !hosting { + continue + } if growOncePerDc { if !vl.ShouldGrowVolumesByDcAndRack(&writables, dcId, "") { continue } count := stepCount if lastGrowCount > 0 { - count = ceilDiv(lastGrowCount, uint32(len(dcs))) + count = ceilDiv(lastGrowCount, dcCount) } plans = append(plans, RackGrowPlan{DataCenter: string(dcId), WritableVolumeCount: count}) continue @@ -1046,12 +1080,26 @@ func (vl *VolumeLayout) ToInfo() (info VolumeLayoutInfo) { } func (vlc *VolumeLayoutCollection) ToVolumeGrowRequest() *master_pb.VolumeGrowRequest { - return &master_pb.VolumeGrowRequest{ + vgr := &master_pb.VolumeGrowRequest{ Collection: vlc.Collection, Replication: vlc.VolumeLayout.rp.String(), Ttl: vlc.VolumeLayout.ttl.String(), DiskType: vlc.VolumeLayout.diskType.String(), } + // A layout living in exactly one data center is either pinned there or on + // a single-DC cluster; either way unconstrained growth has no business + // placing its volumes elsewhere. Cross-DC replication can never + // legitimately live in one DC — there a single hosting DC means an + // outage, not a pin. + if vlc.VolumeLayout.rp.DiffDataCenterCount > 0 { + return vgr + } + if dcs := vlc.VolumeLayout.listVolumeDataCenters(2); len(dcs) == 1 { + for dc := range dcs { + vgr.DataCenter = string(dc) + } + } + return vgr } func (vl *VolumeLayout) Stats() *VolumeLayoutStats { diff --git a/weed/topology/volume_layout_grow_test.go b/weed/topology/volume_layout_grow_test.go index 83fb2d6f2..c27af5a31 100644 --- a/weed/topology/volume_layout_grow_test.go +++ b/weed/topology/volume_layout_grow_test.go @@ -1,6 +1,7 @@ package topology import ( + "fmt" "testing" "github.com/seaweedfs/seaweedfs/weed/storage/needle" @@ -240,6 +241,116 @@ func TestPlanRackAwareGrowth_EvenDistributionAcrossUnevenDCs(t *testing.T) { } } +// A collection pinned to one data center (fs.configure -dataCenter) lives only +// there, yet the scan used to plan growth for every data center in the +// topology, spreading the collection's volumes across all of them. Data +// centers hosting none of the layout's volumes are not the scan's to fill. +func TestPlanRackAwareGrowth_SkipsDataCentersWithoutLayoutVolumes(t *testing.T) { + layout := ` +{ + "dc1":{ "rack1":{ "node-a":{ "ip":"10.0.0.1", "volumes":[ {"id":1, "size":%d, "replication":"000", "collection":"pinned"} ], "limit":30 } } }, + "dc2":{ "rack2":{ "node-b":{ "ip":"10.0.0.2", "volumes":[], "limit":30 } } }, + "dc3":{ "rack3":{ "node-c":{ "ip":"10.0.0.3", "volumes":[], "limit":30 } } }, + "dc4":{ "rack4":{ "node-d":{ "ip":"10.0.0.4", "volumes":[], "limit":30 } } } +} +` + rp, _ := super_block.NewReplicaPlacementFromString("000") + + topo := setupWithLimit(t, fmt.Sprintf(layout, 1000), 30000) + vl := topo.GetVolumeLayout("pinned", rp, needle.EMPTY_TTL, types.HardDriveType) + if plans := vl.PlanRackAwareGrowth(topo.ListDCAndRacks(), 0, 2); len(plans) != 0 { + t.Fatalf("healthy dc1 volume: expected no growth, got %+v", plans) + } + + topo = setupWithLimit(t, fmt.Sprintf(layout, 28650), 30000) + vl = topo.GetVolumeLayout("pinned", rp, needle.EMPTY_TTL, types.HardDriveType) + plans := vl.PlanRackAwareGrowth(topo.ListDCAndRacks(), 0, 2) + if len(plans) != 1 { + t.Fatalf("crowded dc1 volume: expected 1 grow, got %d: %+v", len(plans), plans) + } + if plans[0].DataCenter != "dc1" || plans[0].Rack != "rack1" { + t.Errorf("expected grow pinned to dc1/rack1, got %s/%s", plans[0].DataCenter, plans[0].Rack) + } +} + +// The lastGrowCount divisor counts only the racks the scan can actually plan +// for; racks of skipped data centers would dilute every grow. +func TestPlanRackAwareGrowth_DivisorCountsOnlyHostingRacks(t *testing.T) { + layout := ` +{ + "dc1":{ + "rack1":{ "node-a":{ "ip":"10.0.0.1", "volumes":[ {"id":1, "size":28650, "replication":"000", "collection":"pinned"} ], "limit":30 } }, + "rack2":{ "node-b":{ "ip":"10.0.0.2", "volumes":[ {"id":2, "size":28650, "replication":"000", "collection":"pinned"} ], "limit":30 } } + }, + "dc2":{ + "rack3":{ "node-c":{ "ip":"10.0.0.3", "volumes":[], "limit":30 } }, + "rack4":{ "node-d":{ "ip":"10.0.0.4", "volumes":[], "limit":30 } } + } +} +` + topo := setupWithLimit(t, layout, 30000) + rp, _ := super_block.NewReplicaPlacementFromString("000") + vl := topo.GetVolumeLayout("pinned", rp, needle.EMPTY_TTL, types.HardDriveType) + + plans := vl.PlanRackAwareGrowth(topo.ListDCAndRacks(), 4, 2) + if len(plans) != 2 { + t.Fatalf("expected a grow per crowded dc1 rack, got %d: %+v", len(plans), plans) + } + for _, p := range plans { + if p.DataCenter != "dc1" { + t.Errorf("expected grows pinned to dc1, got %s/%s", p.DataCenter, p.Rack) + } + if p.WritableVolumeCount != 2 { // ceilDiv(4, 2 hosting racks), not ceilDiv(4, 4) + t.Errorf("expected per-rack count 2, got %d", p.WritableVolumeCount) + } + } +} + +// The periodic must-grow and crowded growth paths build their request from +// ToVolumeGrowRequest. For a layout living in exactly one data center the +// request carries that DC, so those paths do not scatter a pinned collection +// either; a layout spanning DCs keeps unconstrained growth. +func TestToVolumeGrowRequest_SingleDataCenterLayout(t *testing.T) { + layout := ` +{ + "dc1":{ "rack1":{ "node-a":{ "ip":"10.0.0.1", "volumes":[ {"id":1, "size":1000, "replication":"000", "collection":"pinned"} ], "limit":30 } } }, + "dc2":{ "rack2":{ "node-b":{ "ip":"10.0.0.2", "volumes":[ {"id":2, "size":1000, "replication":"000", "collection":"spread"} ], "limit":30 } } }, + "dc3":{ "rack3":{ "node-c":{ "ip":"10.0.0.3", "volumes":[ {"id":3, "size":1000, "replication":"000", "collection":"spread"} ], "limit":30 } } } +} +` + topo := setupWithLimit(t, layout, 30000) + rp, _ := super_block.NewReplicaPlacementFromString("000") + + pinned := &VolumeLayoutCollection{"pinned", topo.GetVolumeLayout("pinned", rp, needle.EMPTY_TTL, types.HardDriveType)} + if vgr := pinned.ToVolumeGrowRequest(); vgr.DataCenter != "dc1" { + t.Errorf("expected growth pinned to dc1, got %q", vgr.DataCenter) + } + + spread := &VolumeLayoutCollection{"spread", topo.GetVolumeLayout("spread", rp, needle.EMPTY_TTL, types.HardDriveType)} + if vgr := spread.ToVolumeGrowRequest(); vgr.DataCenter != "" { + t.Errorf("expected unconstrained growth for a multi-DC layout, got %q", vgr.DataCenter) + } +} + +// Cross-DC replication can never legitimately live in one data center: a +// single hosting DC there means the other DCs are down, and pinning growth to +// the survivor would encode the outage as a placement constraint. +func TestToVolumeGrowRequest_CrossDCReplicationNeverPinned(t *testing.T) { + layout := ` +{ + "dc1":{ "rack1":{ "node-a":{ "ip":"10.0.0.1", "volumes":[ {"id":1, "size":1000, "replication":"100", "collection":"c"} ], "limit":30 } } }, + "dc2":{ "rack2":{ "node-b":{ "ip":"10.0.0.2", "volumes":[], "limit":30 } } } +} +` + topo := setupWithLimit(t, layout, 30000) + rp, _ := super_block.NewReplicaPlacementFromString("100") + + vlc := &VolumeLayoutCollection{"c", topo.GetVolumeLayout("c", rp, needle.EMPTY_TTL, types.HardDriveType)} + if vgr := vlc.ToVolumeGrowRequest(); vgr.DataCenter != "" { + t.Errorf("expected unconstrained growth for cross-DC replication, got %q", vgr.DataCenter) + } +} + // Volumes packed to capacity (e.g. by fs.mergeVolumes) go crowded and then // unwritable, but stay in the crowded map. ShouldGrowVolumes must count only // writable crowded volumes, or those leftovers keep writable <= crowded true