From 646549eee1a08e5352b3983404c3972b2f2cdca3 Mon Sep 17 00:00:00 2001 From: Konstantin Lebedev <9497591+kmlebedev@users.noreply.github.com> Date: Thu, 19 Nov 2026 04:51:13 +0500 Subject: [PATCH] volume growth: spread writable volumes of a layout over servers and disks (#12749) * volume growth: spread writable volumes of a layout over servers and disks Volume growth picked every server, and the volume server picked every disk, independently: the server weighted by free slots, the disk by the lowest total volume count. A grow request therefore stacks several writable volumes of one collection on some disks while most disks get none (a Poisson spread: with 160 replicas on 479 disks about 20 disks get two or three), and their writes share those disks. Data centers and racks are still picked weighted by free slots. Inside a rack the server is now one of those holding the fewest writable volumes of the layout per disk, weighted by free slots among them; this covers the main server, same-rack replicas and replicas walked into other racks and data centers. On the volume server the disk is the one with the fewest writable volumes of the collection, then the lowest volume count as before. A grow request registers each volume before placing the next, so a request smaller than the number of disks lands on distinct disks. Fixes #12746 * volume growth: scan the layout load once per grow request Placing each volume rescanned all writable volumes of the layout under its read lock. RecordAssign takes that lock exclusively on every assign, so with thousands of writables (0.8 ms for 10000) every volume of a grow request stalled assigns once more. GrowByCountAndType now scans once and findAndGrow adds every volume it creates to the counts. Per volume, placement costs about 130-160 us on 480 servers whatever the number of writables, against 120 us before the spread change; the single scan is 0.17 ms for 2000 writables and 0.8 ms for 10000. * volume growth: address review of the spread placement - Count only disks that report volume slots in a server's load. An unavailable disk, or one low on space with no volumes, is still listed in DiskTags with no slots; counting it understated the load of a server with failed disks and piled volumes onto its remaining ones. - On the volume server, judge a volume full by its .dat size, which the master compares with the limit, not by ContentSize. - A server that refuses an allocation ranks last for the rest of the grow request, and the request retries the volume up to 3 times. The least loaded server is picked first deterministically, so without this one server that cannot fit a preallocated volume stopped the layout from growing. - Scan the layout load after warmup, so a grow request issued right after a leader change sees the writable volumes the returning servers report. * volume growth: keep the request-local placement load ahead of the request fields The JSON request fields stay together at the end of VolumeGrowOption, and the comment now says the load lives for the whole grow request. Co-Authored-By: Claude Opus 5.5 * volume(rs): place a new volume on the disk with the fewest writable volumes of its collection Store::add_volume picked the disk with the most free slots, so the Rust volume server ignored the collection spread rule the Go server now applies. find_free_location ranks disks by the writable volumes of the collection first (not read-only, .dat below the volume size limit), then by free slots. Co-Authored-By: Claude Opus 5.5 * volume growth: leave a refusing server out of every placement tier A server that refused an allocation only ranked last among the servers of its rack, so the weighted data center and rack picks could still choose its rack on every retry while other racks had room. It now offers no free slots for the rest of the grow request, which takes its slots out of the rack and data center weights too. If only refusing servers are left, they are tried again. Co-Authored-By: Claude Opus 5.5 * volume growth: require the full same-rack count only in the main rack The data center check asked every rack for SameRackCount+1 free servers, but only the main rack holds the same-rack copies; each other rack takes one. With a refused server left out of its rack, the check rejected a feasible placement, the retry put the refused server back, and it was picked again. Co-Authored-By: Claude Opus 5.5 --------- Co-authored-by: Konstantin Lebedev Co-authored-by: Chris Lu Co-authored-by: Claude Opus 5.5 --- seaweed-volume/src/storage/disk_location.rs | 18 ++ seaweed-volume/src/storage/store.rs | 108 +++++++++- weed/storage/disk_location.go | 23 +++ weed/storage/store.go | 11 +- weed/storage/store_collection_spread_test.go | 89 ++++++++ weed/topology/node.go | 12 ++ weed/topology/volume_growth.go | 85 +++++++- weed/topology/volume_growth_spread.go | 124 +++++++++++ .../volume_growth_spread_bench_test.go | 83 ++++++++ weed/topology/volume_growth_spread_test.go | 192 ++++++++++++++++++ 10 files changed, 725 insertions(+), 20 deletions(-) create mode 100644 weed/storage/store_collection_spread_test.go create mode 100644 weed/topology/volume_growth_spread.go create mode 100644 weed/topology/volume_growth_spread_bench_test.go create mode 100644 weed/topology/volume_growth_spread_test.go diff --git a/seaweed-volume/src/storage/disk_location.rs b/seaweed-volume/src/storage/disk_location.rs index ad8578b7bf..fb668a39fc 100644 --- a/seaweed-volume/src/storage/disk_location.rs +++ b/seaweed-volume/src/storage/disk_location.rs @@ -829,6 +829,24 @@ impl DiskLocation { self.volumes.len() } + /// Number of local volumes of `collection` that still take writes: not + /// read-only and, when `volume_size_limit` is known (non zero), with a .dat + /// below it. The .dat size is what the master compares with the limit to + /// stop writes. Mirrors Go's WritableCollectionVolumesLen. + pub fn writable_collection_volumes_len( + &self, + collection: &str, + volume_size_limit: u64, + ) -> usize { + self.volumes + .values() + .filter(|v| v.collection == collection && !v.has_remote_file() && !v.is_read_only()) + .filter(|v| { + volume_size_limit == 0 || v.dat_file_size().unwrap_or(0) < volume_size_limit + }) + .count() + } + /// Get all volume IDs, sorted. pub fn volume_ids(&self) -> Vec { let mut ids: Vec = self.volumes.keys().copied().collect(); diff --git a/seaweed-volume/src/storage/store.rs b/seaweed-volume/src/storage/store.rs index 4f9d32c793..9976900a4a 100644 --- a/seaweed-volume/src/storage/store.rs +++ b/seaweed-volume/src/storage/store.rs @@ -292,12 +292,20 @@ impl Store { // ---- Volume lifecycle ---- - /// Find the location with fewest volumes (load-balance) of the given disk type. - /// Matches Go's FindFreeLocation: accounts for EC shards when computing free slots. - fn find_free_location(&self, disk_type: &DiskType, preallocate: u64) -> Option { + /// Find the location with the fewest writable volumes of `collection`, so + /// they do not share one disk's write bandwidth, then the most free slots + /// (load-balance) of the given disk type. Matches Go's FindFreeLocation: + /// accounts for EC shards when computing free slots. + fn find_free_location( + &self, + disk_type: &DiskType, + collection: &str, + preallocate: u64, + ) -> Option { use crate::storage::erasure_coding::ec_shard::DATA_SHARDS_COUNT; - let mut best: Option<(usize, i64)> = None; // (index, effective_free) + let volume_size_limit = self.volume_size_limit.load(Ordering::Relaxed); + let mut best: Option<(usize, usize, i64)> = None; // (index, writable, effective_free) for (i, loc) in self.locations.iter().enumerate() { if &loc.disk_type != disk_type { continue; @@ -327,11 +335,14 @@ impl Store { ) { continue; } - if best.is_none() || effective_free > best.unwrap().1 { - best = Some((i, effective_free)); + let writable = loc.writable_collection_volumes_len(collection, volume_size_limit); + if best.is_none_or(|(_, min_writable, max_free)| { + writable < min_writable || (writable == min_writable && effective_free > max_free) + }) { + best = Some((i, writable, effective_free)); } } - best.map(|(i, _)| i) + best.map(|(i, _, _)| i) } /// Find a free location matching a predicate. @@ -601,7 +612,7 @@ impl Store { return Err(VolumeError::AlreadyExists); } let loc_idx = self - .find_free_location(&disk_type, spec.preallocate) + .find_free_location(&disk_type, spec.collection, spec.preallocate) .ok_or_else(|| { VolumeError::Io(io::Error::other(format!( "no free location for disk type {:?}", @@ -3039,6 +3050,87 @@ mod tests { assert_eq!(store.locations[1].volumes_len(), 1); } + fn add_spread_test_volume(store: &mut Store, vid: u32, collection: &str) { + let spec = VolumeSpec { + collection, + ..VolumeSpec::default() + }; + store + .add_volume(VolumeId(vid), DiskType::HardDrive, &spec) + .unwrap(); + } + + fn location_of(store: &Store, vid: u32) -> Option { + store + .locations + .iter() + .position(|loc| loc.find_volume(VolumeId(vid)).is_some()) + } + + // Mirrors Go's TestAddVolumeSpreadsCollectionOverDisks. + #[test] + fn test_add_volume_spreads_collection_over_disks() { + let tmp1 = TempDir::new().unwrap(); + let tmp2 = TempDir::new().unwrap(); + let mut store = + make_test_store(&[tmp1.path().to_str().unwrap(), tmp2.path().to_str().unwrap()]); + + add_spread_test_volume(&mut store, 1, "other"); + add_spread_test_volume(&mut store, 2, "other"); + add_spread_test_volume(&mut store, 3, "spread"); + let first = location_of(&store, 3); + // Leave the disk with the "spread" volume holding fewer volumes overall. + let other_on_first = if first == Some(1) { 2 } else { 1 }; + store + .delete_volume(VolumeId(other_on_first), false, false, false) + .unwrap(); + + add_spread_test_volume(&mut store, 4, "spread"); + assert_ne!(location_of(&store, 4), first); + } + + // Mirrors Go's TestAddVolumeIgnoresFullVolumesOfCollection. + #[test] + fn test_add_volume_ignores_full_volumes_of_collection() { + let tmp1 = TempDir::new().unwrap(); + let tmp2 = TempDir::new().unwrap(); + let mut store = + make_test_store(&[tmp1.path().to_str().unwrap(), tmp2.path().to_str().unwrap()]); + // Every .dat holds at least its superblock, so all volumes are full. + store.volume_size_limit.store(1, Ordering::Relaxed); + + add_spread_test_volume(&mut store, 1, "spread"); + let full_disk = location_of(&store, 1); + add_spread_test_volume(&mut store, 2, "other"); + add_spread_test_volume(&mut store, 3, "spread"); + assert_eq!(location_of(&store, 3), full_disk); + } + + // Mirrors Go's TestAddVolumeIgnoresReadOnlyVolumesOfCollection. + #[test] + fn test_add_volume_ignores_read_only_volumes_of_collection() { + let tmp1 = TempDir::new().unwrap(); + let tmp2 = TempDir::new().unwrap(); + let mut store = + make_test_store(&[tmp1.path().to_str().unwrap(), tmp2.path().to_str().unwrap()]); + + add_spread_test_volume(&mut store, 1, "spread"); + let writable_disk = location_of(&store, 1); + add_spread_test_volume(&mut store, 2, "other"); + add_spread_test_volume(&mut store, 3, "spread"); + assert_ne!(location_of(&store, 3), writable_disk); + store + .find_volume(VolumeId(3)) + .unwrap() + .1 + .set_read_only() + .unwrap(); + + // The other disk holds more volumes, but its "spread" volume is read-only. + add_spread_test_volume(&mut store, 4, "spread"); + assert_ne!(location_of(&store, 4), writable_disk); + } + #[test] fn test_store_delete_collection() { let tmp = TempDir::new().unwrap(); diff --git a/weed/storage/disk_location.go b/weed/storage/disk_location.go index c80bb8ea2d..a1887980dd 100644 --- a/weed/storage/disk_location.go +++ b/weed/storage/disk_location.go @@ -708,6 +708,29 @@ func (l *DiskLocation) LocalVolumesLen() int { return count } +// WritableCollectionVolumesLen counts the local volumes of collection that +// still take writes: not read-only and, when volumeSizeLimit is known (non +// zero), with a .dat below it. The .dat size is what the master compares with +// the limit to stop writes. +func (l *DiskLocation) WritableCollectionVolumesLen(collection string, volumeSizeLimit uint64) int { + l.volumesLock.RLock() + defer l.volumesLock.RUnlock() + + count := 0 + for _, v := range l.volumes { + if v.Collection != collection || v.HasRemoteFile() || v.IsReadOnly() { + continue + } + if volumeSizeLimit > 0 { + if datSize, _, _ := v.FileStat(); datSize >= volumeSizeLimit { + continue + } + } + count++ + } + return count +} + func (l *DiskLocation) SetStopping() { l.volumesLock.Lock() for _, v := range l.volumes { diff --git a/weed/storage/store.go b/weed/storage/store.go index 864ea925c2..6c4a259da2 100644 --- a/weed/storage/store.go +++ b/weed/storage/store.go @@ -420,16 +420,21 @@ func (s *Store) addVolume(vid needle.VolumeId, collection string, needleMapKind return fmt.Errorf("Volume Id %d already exists!", vid) } - // Find location with lowest local volume count (load balancing) + // Find the location with the fewest writable volumes of this collection, so + // they do not share one disk's write bandwidth, then the lowest local + // volume count (load balancing) var location *DiskLocation var diskId uint32 - var minVolCount int + var minWritable, minVolCount int + volumeSizeLimit := s.GetVolumeSizeLimit() for i, loc := range s.Locations { if loc.DiskType == diskType && s.hasFreeDiskLocation(loc) && hasRoomForNewVolume(loc, preallocate) { + writable := loc.WritableCollectionVolumesLen(collection, volumeSizeLimit) volCount := loc.LocalVolumesLen() - if location == nil || volCount < minVolCount { + if location == nil || writable < minWritable || (writable == minWritable && volCount < minVolCount) { location = loc diskId = uint32(i) + minWritable = writable minVolCount = volCount } } diff --git a/weed/storage/store_collection_spread_test.go b/weed/storage/store_collection_spread_test.go new file mode 100644 index 0000000000..b998bdc1f1 --- /dev/null +++ b/weed/storage/store_collection_spread_test.go @@ -0,0 +1,89 @@ +package storage + +import ( + "testing" + + "github.com/seaweedfs/seaweedfs/weed/storage/needle" + "github.com/seaweedfs/seaweedfs/weed/storage/types" +) + +func addSpreadTestVolume(t *testing.T, store *Store, vid needle.VolumeId, collection string) { + t.Helper() + if err := store.AddVolume(vid, collection, NeedleMapInMemory, "000", "", + 0, needle.GetCurrentVersion(), 0, types.HardDriveType, 3); err != nil { + t.Fatalf("add volume %d: %v", vid, err) + } +} + +func locationOf(store *Store, vid needle.VolumeId) int { + for i, location := range store.Locations { + if _, found := location.FindVolume(vid); found { + return i + } + } + return -1 +} + +// A new volume goes to the disk with the fewest writable volumes of its own +// collection, even when that disk holds more volumes overall: two writable +// volumes of one collection on a disk split its write bandwidth. +func TestAddVolumeSpreadsCollectionOverDisks(t *testing.T) { + store := newTestStore(t, 2) + + addSpreadTestVolume(t, store, 1, "other") // disk 0 + addSpreadTestVolume(t, store, 2, "other") // disk 1 + addSpreadTestVolume(t, store, 3, "spread") + first := locationOf(store, 3) + // Leave the disk with the "spread" volume holding fewer volumes overall. + otherOnFirst := needle.VolumeId(1) + if first == 1 { + otherOnFirst = 2 + } + if err := store.DeleteVolume(otherOnFirst, false, false, false); err != nil { + t.Fatalf("delete volume %d: %v", otherOnFirst, err) + } + + addSpreadTestVolume(t, store, 4, "spread") + if got := locationOf(store, 4); got == first { + t.Fatalf("volume 4 landed on disk %d next to volume 3 of the same collection; the other disk has none", got) + } +} + +// A volume whose .dat reached the size limit takes no more writes even though +// it is not read-only, so it does not count against its disk. +func TestAddVolumeIgnoresFullVolumesOfCollection(t *testing.T) { + store := newTestStore(t, 2) + // Every .dat holds at least its superblock, so all volumes are full. + store.SetVolumeSizeLimit(1) + + addSpreadTestVolume(t, store, 1, "spread") + fullDisk := locationOf(store, 1) + addSpreadTestVolume(t, store, 2, "other") // the other disk + // Both disks hold one volume and no writable "spread" volume. + addSpreadTestVolume(t, store, 3, "spread") + if got := locationOf(store, 3); got != fullDisk { + t.Fatalf("volume 3 went to disk %d; the full volume 1 on disk %d should not count", got, fullDisk) + } +} + +// Read-only volumes no longer take writes, so they do not count against a disk. +func TestAddVolumeIgnoresReadOnlyVolumesOfCollection(t *testing.T) { + store := newTestStore(t, 2) + + addSpreadTestVolume(t, store, 1, "spread") + writableDisk := locationOf(store, 1) + addSpreadTestVolume(t, store, 2, "other") // the other disk + addSpreadTestVolume(t, store, 3, "spread") + if got := locationOf(store, 3); got == writableDisk { + t.Fatalf("volume 3 landed next to volume 1 of the same collection") + } + if err := store.MarkVolumeReadonly(3, false, false); err != nil { + t.Fatalf("mark volume 3 read-only: %v", err) + } + + // The other disk holds more volumes, but its "spread" volume is read-only. + addSpreadTestVolume(t, store, 4, "spread") + if got := locationOf(store, 4); got == writableDisk { + t.Fatalf("volume 4 landed next to writable volume 1; the read-only volume 3 should not count") + } +} diff --git a/weed/topology/node.go b/weed/topology/node.go index 4e2132c6a1..000a8d4e4f 100644 --- a/weed/topology/node.go +++ b/weed/topology/node.go @@ -4,6 +4,7 @@ import ( "errors" "fmt" "math/rand/v2" + "sort" "strings" "sync" "sync/atomic" @@ -244,6 +245,11 @@ func (n *NodeImpl) PickNodesByWeight(numberOfNodes int, option *VolumeGrowOption lastWeights += weights } } + // Servers holding fewer writable volumes of the layout come first; the + // weighted order above only breaks ties among them. + sort.SliceStable(sortedCandidates, func(i, j int) bool { + return option.lessLoaded(sortedCandidates[i], sortedCandidates[j]) + }) restNodes = make([]Node, 0, numberOfNodes-1) ret := false @@ -334,6 +340,9 @@ func (n *NodeImpl) AvailableSpaceFor(option *VolumeGrowOption) int64 { } return free } + if option.hasRefused(n) { + return 0 + } t := n.diskUsages.getDisk(option.DiskType) if t == nil { return 0 @@ -447,6 +456,9 @@ func (n *NodeImpl) reserveOneVolumeInternal(r int64, option *VolumeGrowOption, u } return node.AvailableSpaceFor(option) } + if option.layoutLoad != nil && n.IsRack() { + return n.reserveLeastLoaded(r, option, freeSpaceOf) + } // The caller drew r from this node's free slots, which can exceed what the // children offer once their reservations or a newer heartbeat are counted. // Fold r into the space that is actually on offer so it cannot walk off the diff --git a/weed/topology/volume_growth.go b/weed/topology/volume_growth.go index e50b6a726d..ff1c284529 100644 --- a/weed/topology/volume_growth.go +++ b/weed/topology/volume_growth.go @@ -2,6 +2,7 @@ package topology import ( "encoding/json" + "errors" "fmt" "math/rand/v2" "reflect" @@ -69,6 +70,13 @@ var ( ) type VolumeGrowOption struct { + // layoutLoad is set only during one grow request: the writable + // volumes of the layout being grown on each data node. See lessLoaded. + layoutLoad map[*DataNode]int + // refused holds the data nodes that refused an allocation earlier in the + // same grow request; they offer no free slots for the rest of it. + refused map[*DataNode]bool + Collection string `json:"collection,omitempty"` ReplicaPlacement *super_block.ReplicaPlacement `json:"replication,omitempty"` Ttl *needle.TTL `json:"ttl,omitempty"` @@ -145,19 +153,51 @@ func (vg *VolumeGrowth) GrowByCountAndType(grpcDialOption grpc.DialOption, targe vg.accessLock.Lock() defer vg.accessLock.Unlock() + // findAndGrow scans the layout for its load once per request, after + // warmup, and adds each volume it creates, so the scan does not hold the + // layout's lock, which assigns need, once per volume. The copy keeps that + // load out of the caller's option. + placed := *option + placed.layoutLoad = nil + placed.refused = nil + option = &placed + retries := 0 for i := uint32(0); i < targetCount; i++ { - if res, e := vg.findAndGrow(grpcDialOption, topo, option); e == nil { + res, e := vg.findAndGrow(grpcDialOption, topo, option) + if e == nil { result = append(result, res...) stats.MasterVolumeCreationCounter.WithLabelValues("success").Inc() - } else { - glog.V(0).Infof("create %d volume, created %d: %v", targetCount, len(result), e) - stats.MasterVolumeCreationCounter.WithLabelValues("failure").Inc() - return result, e + continue } + stats.MasterVolumeCreationCounter.WithLabelValues("failure").Inc() + // A server that refused the allocation is left out of the next + // attempt, so it places the volume elsewhere if it can. + var refused *allocateError + if errors.As(e, &refused) && retries < maxAllocateRetries { + retries++ + i-- + continue + } + glog.V(0).Infof("create %d volume, created %d: %v", targetCount, len(result), e) + return result, e } return } +// maxAllocateRetries bounds how many refused allocations one grow request +// retries on other servers. +const maxAllocateRetries = 3 + +// allocateError is a volume allocation that one server refused. +type allocateError struct { + server *DataNode + err error +} + +func (e *allocateError) Error() string { return e.err.Error() } + +func (e *allocateError) Unwrap() error { return e.err } + func (vg *VolumeGrowth) findAndGrow(grpcDialOption grpc.DialOption, topo *Topology, option *VolumeGrowOption) (result []*master_pb.VolumeLocation, err error) { // Wait for warmup to complete before searching for slots so the // topology has all volume servers registered. @@ -165,8 +205,17 @@ func (vg *VolumeGrowth) findAndGrow(grpcDialOption grpc.DialOption, topo *Topolo glog.V(0).Infof("wait for volume servers to join back") time.Sleep(topo.WarmupDuration() / WarmupPulseMultiplier / 2) } + if option.layoutLoad == nil { + option.layoutLoad = option.withLayoutLoad(topo).layoutLoad + } servers, reservation, e := vg.findEmptySlotsForOneVolume(topo, option, true) // use reservations + if e != nil && len(option.refused) > 0 { + // Only servers that refused are left, so try them again. + retry := *option + retry.refused = nil + servers, reservation, e = vg.findEmptySlotsForOneVolume(topo, &retry, true) + } if e != nil { return nil, e } @@ -182,6 +231,9 @@ func (vg *VolumeGrowth) findAndGrow(grpcDialOption grpc.DialOption, topo *Topolo } if err = vg.grow(grpcDialOption, topo, vid, option, reservation, servers...); err == nil { for _, server := range servers { + if option.layoutLoad != nil { + option.layoutLoad[server]++ + } result = append(result, &master_pb.VolumeLocation{ Url: server.Url(), PublicUrl: server.PublicUrl, @@ -190,6 +242,11 @@ func (vg *VolumeGrowth) findAndGrow(grpcDialOption grpc.DialOption, topo *Topolo NewVids: []uint32{uint32(vid)}, }) } + } else if refused := (*allocateError)(nil); errors.As(err, &refused) { + if option.refused == nil { + option.refused = make(map[*DataNode]bool) + } + option.refused[refused.server] = true } return } @@ -201,6 +258,10 @@ func (vg *VolumeGrowth) findAndGrow(grpcDialOption grpc.DialOption, topo *Topolo // 2. find rest data nodes // If useReservations is true, reserves capacity on each server and returns reservation info func (vg *VolumeGrowth) findEmptySlotsForOneVolume(topo *Topology, option *VolumeGrowOption, useReservations bool) (servers []*DataNode, reservation *VolumeGrowReservation, err error) { + if option.layoutLoad == nil { + option = option.withLayoutLoad(topo) + } + //find main datacenter and other data centers rp := option.ReplicaPlacement @@ -251,7 +312,7 @@ func (vg *VolumeGrowth) findEmptySlotsForOneVolume(topo *Topology, option *Volum if availableSpaceFunc(node, option) < int64(rp.DiffRackCount+rp.SameRackCount+1) { return fmt.Errorf("Free:%d < Expected:%d", availableSpaceFunc(node, option), rp.DiffRackCount+rp.SameRackCount+1) } - possibleRacksCount := 0 + possibleRacksCount, possibleMainRacksCount := 0, 0 for _, rack := range node.Children() { possibleDataNodesCount := 0 for _, n := range rack.Children() { @@ -259,12 +320,18 @@ func (vg *VolumeGrowth) findEmptySlotsForOneVolume(topo *Topology, option *Volum possibleDataNodesCount++ } } - if possibleDataNodesCount >= rp.SameRackCount+1 { + if possibleDataNodesCount >= 1 { possibleRacksCount++ } + if possibleDataNodesCount >= rp.SameRackCount+1 { + possibleMainRacksCount++ + } + } + if possibleMainRacksCount == 0 { + return fmt.Errorf("No rack has %d free data nodes.", rp.SameRackCount+1) } if possibleRacksCount < rp.DiffRackCount+1 { - return fmt.Errorf("Only has %d racks with more than %d free data nodes, not enough for %d.", possibleRacksCount, rp.SameRackCount+1, rp.DiffRackCount+1) + return fmt.Errorf("Only has %d racks with a free data node, not enough for %d.", possibleRacksCount, rp.DiffRackCount+1) } return nil }) @@ -395,7 +462,7 @@ func (vg *VolumeGrowth) grow(grpcDialOption grpc.DialOption, topo *Topology, vid glog.V(0).Infof("Created Volume %d on %s", vid, server.NodeImpl.String()) } else { glog.Warningf("Failed to assign volume %d on %s: %v", vid, server.NodeImpl.String(), err) - growErr = fmt.Errorf("failed to assign volume %d on %s: %v", vid, server.NodeImpl.String(), err) + growErr = &allocateError{server: server, err: fmt.Errorf("failed to assign volume %d on %s: %v", vid, server.NodeImpl.String(), err)} break } } diff --git a/weed/topology/volume_growth_spread.go b/weed/topology/volume_growth_spread.go new file mode 100644 index 0000000000..0dfd763701 --- /dev/null +++ b/weed/topology/volume_growth_spread.go @@ -0,0 +1,124 @@ +package topology + +import ( + "errors" + + "github.com/seaweedfs/seaweedfs/weed/storage/types" +) + +// Volume growth picks data centers and racks weighted by free slots, so an +// emptier location still takes more of the new volumes. The server inside a +// rack is picked differently: among the servers with a free slot, one of +// those holding the fewest writable volumes of the layout per disk, weighted +// by free slots among them. Picking every server independently by free slots +// stacks several writable volumes of one collection on some disks while most +// get none, and their writes pile up on those disks. + +// withLayoutLoad returns a copy of option carrying the writable volumes of +// its layout on each data node. The copy keeps the load out of the option the +// caller holds, which grow requests compare with reflect.DeepEqual. +func (o *VolumeGrowOption) withLayoutLoad(topo *Topology) *VolumeGrowOption { + placed := *o + placed.layoutLoad = make(map[*DataNode]int) + if o.ReplicaPlacement == nil { + return &placed + } + if vl := topo.FindVolumeLayout(o.Collection, o.ReplicaPlacement, o.Ttl, o.DiskType); vl != nil { + placed.layoutLoad = vl.writableCountByNode() + } + return &placed +} + +// writableCountByNode counts, for each data node, the writable volumes of this +// layout it holds a replica of. +func (vl *VolumeLayout) writableCountByNode() map[*DataNode]int { + vl.accessLock.RLock() + defer vl.accessLock.RUnlock() + counts := make(map[*DataNode]int) + for _, vid := range vl.writables { + if locations, ok := vl.vid2location[vid]; ok { + for _, dn := range locations.list { + counts[dn]++ + } + } + } + return counts +} + +// diskCount is the number of disks of diskType on the node that can take new +// volumes, from the heartbeat DiskTags. An unavailable disk, or one low on +// space with no volumes, reports no volume slots and is left out. A node that +// reported none counts as one disk. +func (dn *DataNode) diskCount(diskType types.DiskType) int { + dn.RLock() + defer dn.RUnlock() + count := 0 + for _, meta := range dn.diskMetas { + if meta.diskType == diskType && meta.maxVolumeCount > 0 { + count++ + } + } + return max(count, 1) +} + +// hasRefused reports whether data node n refused an allocation earlier in +// this grow request. +func (o *VolumeGrowOption) hasRefused(n *NodeImpl) bool { + if len(o.refused) == 0 { + return false + } + dn, ok := n.value.(*DataNode) + return ok && o.refused[dn] +} + +// lessLoaded reports whether a holds fewer writable volumes of the layout per +// disk than b. Only data nodes carry a load, so for racks and data centers it +// is false and a stable sort keeps their weighted order. +func (o *VolumeGrowOption) lessLoaded(a, b Node) bool { + if o.layoutLoad == nil { + return false + } + da, okA := a.(*DataNode) + db, okB := b.(*DataNode) + if !okA || !okB { + return false + } + return o.layoutLoad[da]*db.diskCount(o.DiskType) < o.layoutLoad[db]*da.diskCount(o.DiskType) +} + +// reserveLeastLoaded picks one of this rack's data nodes for a replica: among +// those with a free slot, the least loaded ones, weighted by free slots with r +// the way reserveOneVolumeInternal walks the tree. The caller holds n's lock. +func (n *NodeImpl) reserveLeastLoaded(r int64, option *VolumeGrowOption, freeSpaceOf func(Node) int64) (*DataNode, error) { + var least []*DataNode + var weight int64 + for _, node := range n.children { + dn, ok := node.(*DataNode) + if !ok || dn.IsTerminating { + continue + } + free := freeSpaceOf(dn) + if free <= 0 { + continue + } + switch { + case len(least) == 0 || option.lessLoaded(dn, least[0]): + least, weight = []*DataNode{dn}, free + case !option.lessLoaded(least[0], dn): + least = append(least, dn) + weight += free + } + } + if len(least) == 0 { + return nil, errors.New("No free volume slot found!") + } + r %= weight + for _, dn := range least { + free := freeSpaceOf(dn) + if r < free { + return dn, nil + } + r -= free + } + return least[len(least)-1], nil +} diff --git a/weed/topology/volume_growth_spread_bench_test.go b/weed/topology/volume_growth_spread_bench_test.go new file mode 100644 index 0000000000..1a4e8476d7 --- /dev/null +++ b/weed/topology/volume_growth_spread_bench_test.go @@ -0,0 +1,83 @@ +package topology + +import ( + "fmt" + "testing" + + "github.com/seaweedfs/seaweedfs/weed/storage" + "github.com/seaweedfs/seaweedfs/weed/storage/needle" + "github.com/seaweedfs/seaweedfs/weed/storage/super_block" + "github.com/seaweedfs/seaweedfs/weed/storage/types" +) + +// benchSpreadTopology builds 4 data centers of 120 single-disk servers and a +// layout with `writables` writable rp=100 volumes spread over them. +func benchSpreadTopology(b *testing.B, writables int) (*Topology, *VolumeGrowOption) { + b.Helper() + topo, racks := newSpreadTopology(4, 120, 1000) + rp, _ := super_block.NewReplicaPlacementFromString("100") + option := &VolumeGrowOption{Collection: "bench", ReplicaPlacement: rp, Ttl: needle.EMPTY_TTL, DiskType: types.HardDriveType} + var nodes [][]*DataNode + for _, rack := range racks { + var dcNodes []*DataNode + for _, c := range rack.Children() { + dcNodes = append(dcNodes, c.(*DataNode)) + } + nodes = append(nodes, dcNodes) + } + for i := 0; i < writables; i++ { + vi := storage.VolumeInfo{Id: needle.VolumeId(i + 1), Collection: "bench", ReplicaPlacement: rp, Ttl: needle.EMPTY_TTL, Version: needle.GetCurrentVersion()} + for c := 0; c < 2; c++ { + dn := nodes[(i+c)%4][i%120] + dn.AddOrUpdateVolume(vi) + topo.RegisterVolumeLayout(vi, dn) + } + } + return topo, option +} + +func BenchmarkFindEmptySlotsForOneVolume(b *testing.B) { + for _, writables := range []int{0, 100, 2000, 10000} { + b.Run(fmt.Sprintf("writables=%d", writables), func(b *testing.B) { + topo, option := benchSpreadTopology(b, writables) + vg := NewDefaultVolumeGrowth() + b.ResetTimer() + for i := 0; i < b.N; i++ { + if _, _, err := vg.findEmptySlotsForOneVolume(topo, option, false); err != nil { + b.Fatal(err) + } + } + }) + } +} + +// BenchmarkFindEmptySlotsWithRequestLoad is the per-volume cost inside a grow +// request, which scans the layout once and reuses the load. +func BenchmarkFindEmptySlotsWithRequestLoad(b *testing.B) { + for _, writables := range []int{0, 2000, 10000} { + b.Run(fmt.Sprintf("writables=%d", writables), func(b *testing.B) { + topo, option := benchSpreadTopology(b, writables) + option = option.withLayoutLoad(topo) + vg := NewDefaultVolumeGrowth() + b.ResetTimer() + for i := 0; i < b.N; i++ { + if _, _, err := vg.findEmptySlotsForOneVolume(topo, option, false); err != nil { + b.Fatal(err) + } + } + }) + } +} + +func BenchmarkWritableCountByNode(b *testing.B) { + for _, writables := range []int{100, 2000, 10000} { + b.Run(fmt.Sprintf("writables=%d", writables), func(b *testing.B) { + topo, option := benchSpreadTopology(b, writables) + vl := topo.FindVolumeLayout(option.Collection, option.ReplicaPlacement, option.Ttl, option.DiskType) + b.ResetTimer() + for i := 0; i < b.N; i++ { + vl.writableCountByNode() + } + }) + } +} diff --git a/weed/topology/volume_growth_spread_test.go b/weed/topology/volume_growth_spread_test.go new file mode 100644 index 0000000000..5d37b2bfd1 --- /dev/null +++ b/weed/topology/volume_growth_spread_test.go @@ -0,0 +1,192 @@ +package topology + +import ( + "fmt" + "testing" + + "github.com/seaweedfs/seaweedfs/weed/pb/master_pb" + "github.com/seaweedfs/seaweedfs/weed/storage" + "github.com/seaweedfs/seaweedfs/weed/storage/needle" + "github.com/seaweedfs/seaweedfs/weed/storage/super_block" + "github.com/seaweedfs/seaweedfs/weed/storage/types" +) + +// growSpreadVolumes places count volumes one by one the way a grow request +// does: each one is registered in its layout before the next is placed. +// It returns how many replicas of the layout each data node received. +func growSpreadVolumes(t *testing.T, topo *Topology, option *VolumeGrowOption, count int) map[string]int { + t.Helper() + vg := NewDefaultVolumeGrowth() + perNode := make(map[string]int) + for i := 0; i < count; i++ { + servers, _, err := vg.findEmptySlotsForOneVolume(topo, option, false) + if err != nil { + t.Fatalf("volume %d: %v", i, err) + } + vi := storage.VolumeInfo{ + Id: needle.VolumeId(i + 1), + Collection: option.Collection, + ReplicaPlacement: option.ReplicaPlacement, + Ttl: option.Ttl, + Version: needle.GetCurrentVersion(), + } + for _, dn := range servers { + dn.AddOrUpdateVolume(vi) + topo.RegisterVolumeLayout(vi, dn) + perNode[string(dn.Id())]++ + } + } + return perNode +} + +// newSpreadTopology builds dcs data centers with one rack each, holding +// nodesPerDc single-disk servers, and returns the racks. +func newSpreadTopology(dcs, nodesPerDc int, maxVol int64) (*Topology, []*Rack) { + topo, _ := newMaintenanceTestTopology() + topo.UnlinkChildNode("dc1") + var racks []*Rack + for d := 0; d < dcs; d++ { + dc := NewDataCenter(fmt.Sprintf("dc%d", d)) + topo.LinkChildNode(dc) + rack := NewRack("rack1") + dc.LinkChildNode(rack) + racks = append(racks, rack) + for n := 0; n < nodesPerDc; n++ { + addMaintenanceTestNode(rack, fmt.Sprintf("dc%d-server%d", d, n), maxVol) + } + } + return topo, racks +} + +// A grow request smaller than the number of servers must not put two writable +// volumes of the layout on one server. Free slots alone would pick servers +// independently at random and collide. +func TestVolumeGrowthSpreadsWritableVolumesOverServers(t *testing.T) { + rp, _ := super_block.NewReplicaPlacementFromString("000") + option := &VolumeGrowOption{Collection: "spread", ReplicaPlacement: rp, Ttl: needle.EMPTY_TTL, DiskType: types.HardDriveType} + for trial := 0; trial < 20; trial++ { + topo, _ := newSpreadTopology(1, 16, 100) + perNode := growSpreadVolumes(t, topo, option, 16) + for node, n := range perNode { + if n != 1 { + t.Fatalf("trial %d: %s holds %d writable volumes of the layout, want 1: %v", trial, node, n, perNode) + } + } + } +} + +// Replicas placed in another data center go through the same rule. +func TestVolumeGrowthSpreadsReplicasInOtherDataCenters(t *testing.T) { + rp, _ := super_block.NewReplicaPlacementFromString("100") + option := &VolumeGrowOption{Collection: "spread", ReplicaPlacement: rp, Ttl: needle.EMPTY_TTL, DiskType: types.HardDriveType} + for trial := 0; trial < 20; trial++ { + topo, _ := newSpreadTopology(2, 8, 100) + perNode := growSpreadVolumes(t, topo, option, 8) + if len(perNode) != 16 { + t.Fatalf("trial %d: replicas on %d servers, want all 16: %v", trial, len(perNode), perNode) + } + } +} + +// A server with several disks takes a share of the layout's writable volumes +// in proportion to its disks: load is counted per disk, not per server. +func TestVolumeGrowthCountsLoadPerDisk(t *testing.T) { + rp, _ := super_block.NewReplicaPlacementFromString("000") + option := &VolumeGrowOption{Collection: "spread", ReplicaPlacement: rp, Ttl: needle.EMPTY_TTL, DiskType: types.HardDriveType} + for trial := 0; trial < 20; trial++ { + topo, racks := newSpreadTopology(1, 3, 100) + multi := addMaintenanceTestNode(racks[0], "multi", 300) + multi.UpdateDiskTags([]*master_pb.DiskTag{{DiskId: 0, MaxVolumeCount: 100}, {DiskId: 1, MaxVolumeCount: 100}, {DiskId: 2, MaxVolumeCount: 100}}) + perNode := growSpreadVolumes(t, topo, option, 6) + if perNode["multi"] != 3 { + t.Fatalf("trial %d: the 3-disk server got %d volumes of 6, want 3: %v", trial, perNode["multi"], perNode) + } + } +} + +// Disks that report no volume slots (unavailable, or low on space) do not +// count: a server with one usable disk of four takes one disk's share. +func TestVolumeGrowthSkipsDisksWithoutSlots(t *testing.T) { + rp, _ := super_block.NewReplicaPlacementFromString("000") + option := &VolumeGrowOption{Collection: "spread", ReplicaPlacement: rp, Ttl: needle.EMPTY_TTL, DiskType: types.HardDriveType} + for trial := 0; trial < 20; trial++ { + topo, racks := newSpreadTopology(1, 3, 100) + damaged := addMaintenanceTestNode(racks[0], "damaged", 100) + damaged.UpdateDiskTags([]*master_pb.DiskTag{{DiskId: 0, MaxVolumeCount: 100}, {DiskId: 1}, {DiskId: 2}, {DiskId: 3}}) + perNode := growSpreadVolumes(t, topo, option, 8) + if perNode["damaged"] != 2 { + t.Fatalf("trial %d: the server with one usable disk got %d volumes of 8, want 2: %v", trial, perNode["damaged"], perNode) + } + } +} + +// A server that refused an allocation in this grow request offers no slots +// for the rest of it, so the retry places the volume elsewhere. +func TestVolumeGrowthSkipsRefusingServer(t *testing.T) { + rp, _ := super_block.NewReplicaPlacementFromString("000") + option := &VolumeGrowOption{Collection: "spread", ReplicaPlacement: rp, Ttl: needle.EMPTY_TTL, DiskType: types.HardDriveType} + topo, racks := newSpreadTopology(1, 4, 100) + refusing := racks[0].Children()[0].(*DataNode) + placed := option.withLayoutLoad(topo) + placed.refused = map[*DataNode]bool{refusing: true} + vg := NewDefaultVolumeGrowth() + for i := 0; i < 50; i++ { + servers, _, err := vg.findEmptySlotsForOneVolume(topo, placed, false) + if err != nil { + t.Fatal(err) + } + if servers[0] == refusing { + t.Fatalf("attempt %d picked %s, which refused an allocation", i, refusing.Id()) + } + } +} + +// The refusing server's free slots also leave the rack and data center +// weights, so a much emptier rack holding only that server is not picked. +func TestVolumeGrowthSkipsRefusingServerInOtherRack(t *testing.T) { + rp, _ := super_block.NewReplicaPlacementFromString("000") + option := &VolumeGrowOption{Collection: "spread", ReplicaPlacement: rp, Ttl: needle.EMPTY_TTL, DiskType: types.HardDriveType} + topo, racks := newSpreadTopology(1, 0, 0) + refusing := addMaintenanceTestNode(racks[0], "refusing", 100) + otherRack := NewRack("rack2") + racks[0].Parent().LinkChildNode(otherRack) + healthy := addMaintenanceTestNode(otherRack, "healthy", 1) + placed := option.withLayoutLoad(topo) + placed.refused = map[*DataNode]bool{refusing: true} + vg := NewDefaultVolumeGrowth() + for i := 0; i < 50; i++ { + servers, _, err := vg.findEmptySlotsForOneVolume(topo, placed, false) + if err != nil { + t.Fatal(err) + } + if servers[0] != healthy { + t.Fatalf("attempt %d picked %s, want %s", i, servers[0].Id(), healthy.Id()) + } + } +} + +func TestVolumeGrowthSkipsRefusingServerWhenItsRackKeepsOneServer(t *testing.T) { + rp, _ := super_block.NewReplicaPlacementFromString("011") + option := &VolumeGrowOption{Collection: "spread", ReplicaPlacement: rp, Ttl: needle.EMPTY_TTL, DiskType: types.HardDriveType} + topo, racks := newSpreadTopology(1, 0, 0) + refusing := addMaintenanceTestNode(racks[0], "refusing", 100) + addMaintenanceTestNode(racks[0], "healthy1", 100) + otherRack := NewRack("rack2") + racks[0].Parent().LinkChildNode(otherRack) + addMaintenanceTestNode(otherRack, "healthy2", 100) + addMaintenanceTestNode(otherRack, "healthy3", 100) + placed := option.withLayoutLoad(topo) + placed.refused = map[*DataNode]bool{refusing: true} + vg := NewDefaultVolumeGrowth() + for i := 0; i < 50; i++ { + servers, _, err := vg.findEmptySlotsForOneVolume(topo, placed, false) + if err != nil { + t.Fatalf("attempt %d: %v", i, err) + } + for _, s := range servers { + if s == refusing { + t.Fatalf("attempt %d picked the refusing server", i) + } + } + } +}