diff --git a/seaweed-volume/src/storage/disk_location.rs b/seaweed-volume/src/storage/disk_location.rs index ad8578b7bf..fb668a39fc 100644 --- a/seaweed-volume/src/storage/disk_location.rs +++ b/seaweed-volume/src/storage/disk_location.rs @@ -829,6 +829,24 @@ impl DiskLocation { self.volumes.len() } + /// Number of local volumes of `collection` that still take writes: not + /// read-only and, when `volume_size_limit` is known (non zero), with a .dat + /// below it. The .dat size is what the master compares with the limit to + /// stop writes. Mirrors Go's WritableCollectionVolumesLen. + pub fn writable_collection_volumes_len( + &self, + collection: &str, + volume_size_limit: u64, + ) -> usize { + self.volumes + .values() + .filter(|v| v.collection == collection && !v.has_remote_file() && !v.is_read_only()) + .filter(|v| { + volume_size_limit == 0 || v.dat_file_size().unwrap_or(0) < volume_size_limit + }) + .count() + } + /// Get all volume IDs, sorted. pub fn volume_ids(&self) -> Vec { let mut ids: Vec = self.volumes.keys().copied().collect(); diff --git a/seaweed-volume/src/storage/store.rs b/seaweed-volume/src/storage/store.rs index 4f9d32c793..9976900a4a 100644 --- a/seaweed-volume/src/storage/store.rs +++ b/seaweed-volume/src/storage/store.rs @@ -292,12 +292,20 @@ impl Store { // ---- Volume lifecycle ---- - /// Find the location with fewest volumes (load-balance) of the given disk type. - /// Matches Go's FindFreeLocation: accounts for EC shards when computing free slots. - fn find_free_location(&self, disk_type: &DiskType, preallocate: u64) -> Option { + /// Find the location with the fewest writable volumes of `collection`, so + /// they do not share one disk's write bandwidth, then the most free slots + /// (load-balance) of the given disk type. Matches Go's FindFreeLocation: + /// accounts for EC shards when computing free slots. + fn find_free_location( + &self, + disk_type: &DiskType, + collection: &str, + preallocate: u64, + ) -> Option { use crate::storage::erasure_coding::ec_shard::DATA_SHARDS_COUNT; - let mut best: Option<(usize, i64)> = None; // (index, effective_free) + let volume_size_limit = self.volume_size_limit.load(Ordering::Relaxed); + let mut best: Option<(usize, usize, i64)> = None; // (index, writable, effective_free) for (i, loc) in self.locations.iter().enumerate() { if &loc.disk_type != disk_type { continue; @@ -327,11 +335,14 @@ impl Store { ) { continue; } - if best.is_none() || effective_free > best.unwrap().1 { - best = Some((i, effective_free)); + let writable = loc.writable_collection_volumes_len(collection, volume_size_limit); + if best.is_none_or(|(_, min_writable, max_free)| { + writable < min_writable || (writable == min_writable && effective_free > max_free) + }) { + best = Some((i, writable, effective_free)); } } - best.map(|(i, _)| i) + best.map(|(i, _, _)| i) } /// Find a free location matching a predicate. @@ -601,7 +612,7 @@ impl Store { return Err(VolumeError::AlreadyExists); } let loc_idx = self - .find_free_location(&disk_type, spec.preallocate) + .find_free_location(&disk_type, spec.collection, spec.preallocate) .ok_or_else(|| { VolumeError::Io(io::Error::other(format!( "no free location for disk type {:?}", @@ -3039,6 +3050,87 @@ mod tests { assert_eq!(store.locations[1].volumes_len(), 1); } + fn add_spread_test_volume(store: &mut Store, vid: u32, collection: &str) { + let spec = VolumeSpec { + collection, + ..VolumeSpec::default() + }; + store + .add_volume(VolumeId(vid), DiskType::HardDrive, &spec) + .unwrap(); + } + + fn location_of(store: &Store, vid: u32) -> Option { + store + .locations + .iter() + .position(|loc| loc.find_volume(VolumeId(vid)).is_some()) + } + + // Mirrors Go's TestAddVolumeSpreadsCollectionOverDisks. + #[test] + fn test_add_volume_spreads_collection_over_disks() { + let tmp1 = TempDir::new().unwrap(); + let tmp2 = TempDir::new().unwrap(); + let mut store = + make_test_store(&[tmp1.path().to_str().unwrap(), tmp2.path().to_str().unwrap()]); + + add_spread_test_volume(&mut store, 1, "other"); + add_spread_test_volume(&mut store, 2, "other"); + add_spread_test_volume(&mut store, 3, "spread"); + let first = location_of(&store, 3); + // Leave the disk with the "spread" volume holding fewer volumes overall. + let other_on_first = if first == Some(1) { 2 } else { 1 }; + store + .delete_volume(VolumeId(other_on_first), false, false, false) + .unwrap(); + + add_spread_test_volume(&mut store, 4, "spread"); + assert_ne!(location_of(&store, 4), first); + } + + // Mirrors Go's TestAddVolumeIgnoresFullVolumesOfCollection. + #[test] + fn test_add_volume_ignores_full_volumes_of_collection() { + let tmp1 = TempDir::new().unwrap(); + let tmp2 = TempDir::new().unwrap(); + let mut store = + make_test_store(&[tmp1.path().to_str().unwrap(), tmp2.path().to_str().unwrap()]); + // Every .dat holds at least its superblock, so all volumes are full. + store.volume_size_limit.store(1, Ordering::Relaxed); + + add_spread_test_volume(&mut store, 1, "spread"); + let full_disk = location_of(&store, 1); + add_spread_test_volume(&mut store, 2, "other"); + add_spread_test_volume(&mut store, 3, "spread"); + assert_eq!(location_of(&store, 3), full_disk); + } + + // Mirrors Go's TestAddVolumeIgnoresReadOnlyVolumesOfCollection. + #[test] + fn test_add_volume_ignores_read_only_volumes_of_collection() { + let tmp1 = TempDir::new().unwrap(); + let tmp2 = TempDir::new().unwrap(); + let mut store = + make_test_store(&[tmp1.path().to_str().unwrap(), tmp2.path().to_str().unwrap()]); + + add_spread_test_volume(&mut store, 1, "spread"); + let writable_disk = location_of(&store, 1); + add_spread_test_volume(&mut store, 2, "other"); + add_spread_test_volume(&mut store, 3, "spread"); + assert_ne!(location_of(&store, 3), writable_disk); + store + .find_volume(VolumeId(3)) + .unwrap() + .1 + .set_read_only() + .unwrap(); + + // The other disk holds more volumes, but its "spread" volume is read-only. + add_spread_test_volume(&mut store, 4, "spread"); + assert_ne!(location_of(&store, 4), writable_disk); + } + #[test] fn test_store_delete_collection() { let tmp = TempDir::new().unwrap(); diff --git a/weed/storage/disk_location.go b/weed/storage/disk_location.go index c80bb8ea2d..a1887980dd 100644 --- a/weed/storage/disk_location.go +++ b/weed/storage/disk_location.go @@ -708,6 +708,29 @@ func (l *DiskLocation) LocalVolumesLen() int { return count } +// WritableCollectionVolumesLen counts the local volumes of collection that +// still take writes: not read-only and, when volumeSizeLimit is known (non +// zero), with a .dat below it. The .dat size is what the master compares with +// the limit to stop writes. +func (l *DiskLocation) WritableCollectionVolumesLen(collection string, volumeSizeLimit uint64) int { + l.volumesLock.RLock() + defer l.volumesLock.RUnlock() + + count := 0 + for _, v := range l.volumes { + if v.Collection != collection || v.HasRemoteFile() || v.IsReadOnly() { + continue + } + if volumeSizeLimit > 0 { + if datSize, _, _ := v.FileStat(); datSize >= volumeSizeLimit { + continue + } + } + count++ + } + return count +} + func (l *DiskLocation) SetStopping() { l.volumesLock.Lock() for _, v := range l.volumes { diff --git a/weed/storage/store.go b/weed/storage/store.go index 864ea925c2..6c4a259da2 100644 --- a/weed/storage/store.go +++ b/weed/storage/store.go @@ -420,16 +420,21 @@ func (s *Store) addVolume(vid needle.VolumeId, collection string, needleMapKind return fmt.Errorf("Volume Id %d already exists!", vid) } - // Find location with lowest local volume count (load balancing) + // Find the location with the fewest writable volumes of this collection, so + // they do not share one disk's write bandwidth, then the lowest local + // volume count (load balancing) var location *DiskLocation var diskId uint32 - var minVolCount int + var minWritable, minVolCount int + volumeSizeLimit := s.GetVolumeSizeLimit() for i, loc := range s.Locations { if loc.DiskType == diskType && s.hasFreeDiskLocation(loc) && hasRoomForNewVolume(loc, preallocate) { + writable := loc.WritableCollectionVolumesLen(collection, volumeSizeLimit) volCount := loc.LocalVolumesLen() - if location == nil || volCount < minVolCount { + if location == nil || writable < minWritable || (writable == minWritable && volCount < minVolCount) { location = loc diskId = uint32(i) + minWritable = writable minVolCount = volCount } } diff --git a/weed/storage/store_collection_spread_test.go b/weed/storage/store_collection_spread_test.go new file mode 100644 index 0000000000..b998bdc1f1 --- /dev/null +++ b/weed/storage/store_collection_spread_test.go @@ -0,0 +1,89 @@ +package storage + +import ( + "testing" + + "github.com/seaweedfs/seaweedfs/weed/storage/needle" + "github.com/seaweedfs/seaweedfs/weed/storage/types" +) + +func addSpreadTestVolume(t *testing.T, store *Store, vid needle.VolumeId, collection string) { + t.Helper() + if err := store.AddVolume(vid, collection, NeedleMapInMemory, "000", "", + 0, needle.GetCurrentVersion(), 0, types.HardDriveType, 3); err != nil { + t.Fatalf("add volume %d: %v", vid, err) + } +} + +func locationOf(store *Store, vid needle.VolumeId) int { + for i, location := range store.Locations { + if _, found := location.FindVolume(vid); found { + return i + } + } + return -1 +} + +// A new volume goes to the disk with the fewest writable volumes of its own +// collection, even when that disk holds more volumes overall: two writable +// volumes of one collection on a disk split its write bandwidth. +func TestAddVolumeSpreadsCollectionOverDisks(t *testing.T) { + store := newTestStore(t, 2) + + addSpreadTestVolume(t, store, 1, "other") // disk 0 + addSpreadTestVolume(t, store, 2, "other") // disk 1 + addSpreadTestVolume(t, store, 3, "spread") + first := locationOf(store, 3) + // Leave the disk with the "spread" volume holding fewer volumes overall. + otherOnFirst := needle.VolumeId(1) + if first == 1 { + otherOnFirst = 2 + } + if err := store.DeleteVolume(otherOnFirst, false, false, false); err != nil { + t.Fatalf("delete volume %d: %v", otherOnFirst, err) + } + + addSpreadTestVolume(t, store, 4, "spread") + if got := locationOf(store, 4); got == first { + t.Fatalf("volume 4 landed on disk %d next to volume 3 of the same collection; the other disk has none", got) + } +} + +// A volume whose .dat reached the size limit takes no more writes even though +// it is not read-only, so it does not count against its disk. +func TestAddVolumeIgnoresFullVolumesOfCollection(t *testing.T) { + store := newTestStore(t, 2) + // Every .dat holds at least its superblock, so all volumes are full. + store.SetVolumeSizeLimit(1) + + addSpreadTestVolume(t, store, 1, "spread") + fullDisk := locationOf(store, 1) + addSpreadTestVolume(t, store, 2, "other") // the other disk + // Both disks hold one volume and no writable "spread" volume. + addSpreadTestVolume(t, store, 3, "spread") + if got := locationOf(store, 3); got != fullDisk { + t.Fatalf("volume 3 went to disk %d; the full volume 1 on disk %d should not count", got, fullDisk) + } +} + +// Read-only volumes no longer take writes, so they do not count against a disk. +func TestAddVolumeIgnoresReadOnlyVolumesOfCollection(t *testing.T) { + store := newTestStore(t, 2) + + addSpreadTestVolume(t, store, 1, "spread") + writableDisk := locationOf(store, 1) + addSpreadTestVolume(t, store, 2, "other") // the other disk + addSpreadTestVolume(t, store, 3, "spread") + if got := locationOf(store, 3); got == writableDisk { + t.Fatalf("volume 3 landed next to volume 1 of the same collection") + } + if err := store.MarkVolumeReadonly(3, false, false); err != nil { + t.Fatalf("mark volume 3 read-only: %v", err) + } + + // The other disk holds more volumes, but its "spread" volume is read-only. + addSpreadTestVolume(t, store, 4, "spread") + if got := locationOf(store, 4); got == writableDisk { + t.Fatalf("volume 4 landed next to writable volume 1; the read-only volume 3 should not count") + } +} diff --git a/weed/topology/node.go b/weed/topology/node.go index 4e2132c6a1..000a8d4e4f 100644 --- a/weed/topology/node.go +++ b/weed/topology/node.go @@ -4,6 +4,7 @@ import ( "errors" "fmt" "math/rand/v2" + "sort" "strings" "sync" "sync/atomic" @@ -244,6 +245,11 @@ func (n *NodeImpl) PickNodesByWeight(numberOfNodes int, option *VolumeGrowOption lastWeights += weights } } + // Servers holding fewer writable volumes of the layout come first; the + // weighted order above only breaks ties among them. + sort.SliceStable(sortedCandidates, func(i, j int) bool { + return option.lessLoaded(sortedCandidates[i], sortedCandidates[j]) + }) restNodes = make([]Node, 0, numberOfNodes-1) ret := false @@ -334,6 +340,9 @@ func (n *NodeImpl) AvailableSpaceFor(option *VolumeGrowOption) int64 { } return free } + if option.hasRefused(n) { + return 0 + } t := n.diskUsages.getDisk(option.DiskType) if t == nil { return 0 @@ -447,6 +456,9 @@ func (n *NodeImpl) reserveOneVolumeInternal(r int64, option *VolumeGrowOption, u } return node.AvailableSpaceFor(option) } + if option.layoutLoad != nil && n.IsRack() { + return n.reserveLeastLoaded(r, option, freeSpaceOf) + } // The caller drew r from this node's free slots, which can exceed what the // children offer once their reservations or a newer heartbeat are counted. // Fold r into the space that is actually on offer so it cannot walk off the diff --git a/weed/topology/volume_growth.go b/weed/topology/volume_growth.go index e50b6a726d..ff1c284529 100644 --- a/weed/topology/volume_growth.go +++ b/weed/topology/volume_growth.go @@ -2,6 +2,7 @@ package topology import ( "encoding/json" + "errors" "fmt" "math/rand/v2" "reflect" @@ -69,6 +70,13 @@ var ( ) type VolumeGrowOption struct { + // layoutLoad is set only during one grow request: the writable + // volumes of the layout being grown on each data node. See lessLoaded. + layoutLoad map[*DataNode]int + // refused holds the data nodes that refused an allocation earlier in the + // same grow request; they offer no free slots for the rest of it. + refused map[*DataNode]bool + Collection string `json:"collection,omitempty"` ReplicaPlacement *super_block.ReplicaPlacement `json:"replication,omitempty"` Ttl *needle.TTL `json:"ttl,omitempty"` @@ -145,19 +153,51 @@ func (vg *VolumeGrowth) GrowByCountAndType(grpcDialOption grpc.DialOption, targe vg.accessLock.Lock() defer vg.accessLock.Unlock() + // findAndGrow scans the layout for its load once per request, after + // warmup, and adds each volume it creates, so the scan does not hold the + // layout's lock, which assigns need, once per volume. The copy keeps that + // load out of the caller's option. + placed := *option + placed.layoutLoad = nil + placed.refused = nil + option = &placed + retries := 0 for i := uint32(0); i < targetCount; i++ { - if res, e := vg.findAndGrow(grpcDialOption, topo, option); e == nil { + res, e := vg.findAndGrow(grpcDialOption, topo, option) + if e == nil { result = append(result, res...) stats.MasterVolumeCreationCounter.WithLabelValues("success").Inc() - } else { - glog.V(0).Infof("create %d volume, created %d: %v", targetCount, len(result), e) - stats.MasterVolumeCreationCounter.WithLabelValues("failure").Inc() - return result, e + continue } + stats.MasterVolumeCreationCounter.WithLabelValues("failure").Inc() + // A server that refused the allocation is left out of the next + // attempt, so it places the volume elsewhere if it can. + var refused *allocateError + if errors.As(e, &refused) && retries < maxAllocateRetries { + retries++ + i-- + continue + } + glog.V(0).Infof("create %d volume, created %d: %v", targetCount, len(result), e) + return result, e } return } +// maxAllocateRetries bounds how many refused allocations one grow request +// retries on other servers. +const maxAllocateRetries = 3 + +// allocateError is a volume allocation that one server refused. +type allocateError struct { + server *DataNode + err error +} + +func (e *allocateError) Error() string { return e.err.Error() } + +func (e *allocateError) Unwrap() error { return e.err } + func (vg *VolumeGrowth) findAndGrow(grpcDialOption grpc.DialOption, topo *Topology, option *VolumeGrowOption) (result []*master_pb.VolumeLocation, err error) { // Wait for warmup to complete before searching for slots so the // topology has all volume servers registered. @@ -165,8 +205,17 @@ func (vg *VolumeGrowth) findAndGrow(grpcDialOption grpc.DialOption, topo *Topolo glog.V(0).Infof("wait for volume servers to join back") time.Sleep(topo.WarmupDuration() / WarmupPulseMultiplier / 2) } + if option.layoutLoad == nil { + option.layoutLoad = option.withLayoutLoad(topo).layoutLoad + } servers, reservation, e := vg.findEmptySlotsForOneVolume(topo, option, true) // use reservations + if e != nil && len(option.refused) > 0 { + // Only servers that refused are left, so try them again. + retry := *option + retry.refused = nil + servers, reservation, e = vg.findEmptySlotsForOneVolume(topo, &retry, true) + } if e != nil { return nil, e } @@ -182,6 +231,9 @@ func (vg *VolumeGrowth) findAndGrow(grpcDialOption grpc.DialOption, topo *Topolo } if err = vg.grow(grpcDialOption, topo, vid, option, reservation, servers...); err == nil { for _, server := range servers { + if option.layoutLoad != nil { + option.layoutLoad[server]++ + } result = append(result, &master_pb.VolumeLocation{ Url: server.Url(), PublicUrl: server.PublicUrl, @@ -190,6 +242,11 @@ func (vg *VolumeGrowth) findAndGrow(grpcDialOption grpc.DialOption, topo *Topolo NewVids: []uint32{uint32(vid)}, }) } + } else if refused := (*allocateError)(nil); errors.As(err, &refused) { + if option.refused == nil { + option.refused = make(map[*DataNode]bool) + } + option.refused[refused.server] = true } return } @@ -201,6 +258,10 @@ func (vg *VolumeGrowth) findAndGrow(grpcDialOption grpc.DialOption, topo *Topolo // 2. find rest data nodes // If useReservations is true, reserves capacity on each server and returns reservation info func (vg *VolumeGrowth) findEmptySlotsForOneVolume(topo *Topology, option *VolumeGrowOption, useReservations bool) (servers []*DataNode, reservation *VolumeGrowReservation, err error) { + if option.layoutLoad == nil { + option = option.withLayoutLoad(topo) + } + //find main datacenter and other data centers rp := option.ReplicaPlacement @@ -251,7 +312,7 @@ func (vg *VolumeGrowth) findEmptySlotsForOneVolume(topo *Topology, option *Volum if availableSpaceFunc(node, option) < int64(rp.DiffRackCount+rp.SameRackCount+1) { return fmt.Errorf("Free:%d < Expected:%d", availableSpaceFunc(node, option), rp.DiffRackCount+rp.SameRackCount+1) } - possibleRacksCount := 0 + possibleRacksCount, possibleMainRacksCount := 0, 0 for _, rack := range node.Children() { possibleDataNodesCount := 0 for _, n := range rack.Children() { @@ -259,12 +320,18 @@ func (vg *VolumeGrowth) findEmptySlotsForOneVolume(topo *Topology, option *Volum possibleDataNodesCount++ } } - if possibleDataNodesCount >= rp.SameRackCount+1 { + if possibleDataNodesCount >= 1 { possibleRacksCount++ } + if possibleDataNodesCount >= rp.SameRackCount+1 { + possibleMainRacksCount++ + } + } + if possibleMainRacksCount == 0 { + return fmt.Errorf("No rack has %d free data nodes.", rp.SameRackCount+1) } if possibleRacksCount < rp.DiffRackCount+1 { - return fmt.Errorf("Only has %d racks with more than %d free data nodes, not enough for %d.", possibleRacksCount, rp.SameRackCount+1, rp.DiffRackCount+1) + return fmt.Errorf("Only has %d racks with a free data node, not enough for %d.", possibleRacksCount, rp.DiffRackCount+1) } return nil }) @@ -395,7 +462,7 @@ func (vg *VolumeGrowth) grow(grpcDialOption grpc.DialOption, topo *Topology, vid glog.V(0).Infof("Created Volume %d on %s", vid, server.NodeImpl.String()) } else { glog.Warningf("Failed to assign volume %d on %s: %v", vid, server.NodeImpl.String(), err) - growErr = fmt.Errorf("failed to assign volume %d on %s: %v", vid, server.NodeImpl.String(), err) + growErr = &allocateError{server: server, err: fmt.Errorf("failed to assign volume %d on %s: %v", vid, server.NodeImpl.String(), err)} break } } diff --git a/weed/topology/volume_growth_spread.go b/weed/topology/volume_growth_spread.go new file mode 100644 index 0000000000..0dfd763701 --- /dev/null +++ b/weed/topology/volume_growth_spread.go @@ -0,0 +1,124 @@ +package topology + +import ( + "errors" + + "github.com/seaweedfs/seaweedfs/weed/storage/types" +) + +// Volume growth picks data centers and racks weighted by free slots, so an +// emptier location still takes more of the new volumes. The server inside a +// rack is picked differently: among the servers with a free slot, one of +// those holding the fewest writable volumes of the layout per disk, weighted +// by free slots among them. Picking every server independently by free slots +// stacks several writable volumes of one collection on some disks while most +// get none, and their writes pile up on those disks. + +// withLayoutLoad returns a copy of option carrying the writable volumes of +// its layout on each data node. The copy keeps the load out of the option the +// caller holds, which grow requests compare with reflect.DeepEqual. +func (o *VolumeGrowOption) withLayoutLoad(topo *Topology) *VolumeGrowOption { + placed := *o + placed.layoutLoad = make(map[*DataNode]int) + if o.ReplicaPlacement == nil { + return &placed + } + if vl := topo.FindVolumeLayout(o.Collection, o.ReplicaPlacement, o.Ttl, o.DiskType); vl != nil { + placed.layoutLoad = vl.writableCountByNode() + } + return &placed +} + +// writableCountByNode counts, for each data node, the writable volumes of this +// layout it holds a replica of. +func (vl *VolumeLayout) writableCountByNode() map[*DataNode]int { + vl.accessLock.RLock() + defer vl.accessLock.RUnlock() + counts := make(map[*DataNode]int) + for _, vid := range vl.writables { + if locations, ok := vl.vid2location[vid]; ok { + for _, dn := range locations.list { + counts[dn]++ + } + } + } + return counts +} + +// diskCount is the number of disks of diskType on the node that can take new +// volumes, from the heartbeat DiskTags. An unavailable disk, or one low on +// space with no volumes, reports no volume slots and is left out. A node that +// reported none counts as one disk. +func (dn *DataNode) diskCount(diskType types.DiskType) int { + dn.RLock() + defer dn.RUnlock() + count := 0 + for _, meta := range dn.diskMetas { + if meta.diskType == diskType && meta.maxVolumeCount > 0 { + count++ + } + } + return max(count, 1) +} + +// hasRefused reports whether data node n refused an allocation earlier in +// this grow request. +func (o *VolumeGrowOption) hasRefused(n *NodeImpl) bool { + if len(o.refused) == 0 { + return false + } + dn, ok := n.value.(*DataNode) + return ok && o.refused[dn] +} + +// lessLoaded reports whether a holds fewer writable volumes of the layout per +// disk than b. Only data nodes carry a load, so for racks and data centers it +// is false and a stable sort keeps their weighted order. +func (o *VolumeGrowOption) lessLoaded(a, b Node) bool { + if o.layoutLoad == nil { + return false + } + da, okA := a.(*DataNode) + db, okB := b.(*DataNode) + if !okA || !okB { + return false + } + return o.layoutLoad[da]*db.diskCount(o.DiskType) < o.layoutLoad[db]*da.diskCount(o.DiskType) +} + +// reserveLeastLoaded picks one of this rack's data nodes for a replica: among +// those with a free slot, the least loaded ones, weighted by free slots with r +// the way reserveOneVolumeInternal walks the tree. The caller holds n's lock. +func (n *NodeImpl) reserveLeastLoaded(r int64, option *VolumeGrowOption, freeSpaceOf func(Node) int64) (*DataNode, error) { + var least []*DataNode + var weight int64 + for _, node := range n.children { + dn, ok := node.(*DataNode) + if !ok || dn.IsTerminating { + continue + } + free := freeSpaceOf(dn) + if free <= 0 { + continue + } + switch { + case len(least) == 0 || option.lessLoaded(dn, least[0]): + least, weight = []*DataNode{dn}, free + case !option.lessLoaded(least[0], dn): + least = append(least, dn) + weight += free + } + } + if len(least) == 0 { + return nil, errors.New("No free volume slot found!") + } + r %= weight + for _, dn := range least { + free := freeSpaceOf(dn) + if r < free { + return dn, nil + } + r -= free + } + return least[len(least)-1], nil +} diff --git a/weed/topology/volume_growth_spread_bench_test.go b/weed/topology/volume_growth_spread_bench_test.go new file mode 100644 index 0000000000..1a4e8476d7 --- /dev/null +++ b/weed/topology/volume_growth_spread_bench_test.go @@ -0,0 +1,83 @@ +package topology + +import ( + "fmt" + "testing" + + "github.com/seaweedfs/seaweedfs/weed/storage" + "github.com/seaweedfs/seaweedfs/weed/storage/needle" + "github.com/seaweedfs/seaweedfs/weed/storage/super_block" + "github.com/seaweedfs/seaweedfs/weed/storage/types" +) + +// benchSpreadTopology builds 4 data centers of 120 single-disk servers and a +// layout with `writables` writable rp=100 volumes spread over them. +func benchSpreadTopology(b *testing.B, writables int) (*Topology, *VolumeGrowOption) { + b.Helper() + topo, racks := newSpreadTopology(4, 120, 1000) + rp, _ := super_block.NewReplicaPlacementFromString("100") + option := &VolumeGrowOption{Collection: "bench", ReplicaPlacement: rp, Ttl: needle.EMPTY_TTL, DiskType: types.HardDriveType} + var nodes [][]*DataNode + for _, rack := range racks { + var dcNodes []*DataNode + for _, c := range rack.Children() { + dcNodes = append(dcNodes, c.(*DataNode)) + } + nodes = append(nodes, dcNodes) + } + for i := 0; i < writables; i++ { + vi := storage.VolumeInfo{Id: needle.VolumeId(i + 1), Collection: "bench", ReplicaPlacement: rp, Ttl: needle.EMPTY_TTL, Version: needle.GetCurrentVersion()} + for c := 0; c < 2; c++ { + dn := nodes[(i+c)%4][i%120] + dn.AddOrUpdateVolume(vi) + topo.RegisterVolumeLayout(vi, dn) + } + } + return topo, option +} + +func BenchmarkFindEmptySlotsForOneVolume(b *testing.B) { + for _, writables := range []int{0, 100, 2000, 10000} { + b.Run(fmt.Sprintf("writables=%d", writables), func(b *testing.B) { + topo, option := benchSpreadTopology(b, writables) + vg := NewDefaultVolumeGrowth() + b.ResetTimer() + for i := 0; i < b.N; i++ { + if _, _, err := vg.findEmptySlotsForOneVolume(topo, option, false); err != nil { + b.Fatal(err) + } + } + }) + } +} + +// BenchmarkFindEmptySlotsWithRequestLoad is the per-volume cost inside a grow +// request, which scans the layout once and reuses the load. +func BenchmarkFindEmptySlotsWithRequestLoad(b *testing.B) { + for _, writables := range []int{0, 2000, 10000} { + b.Run(fmt.Sprintf("writables=%d", writables), func(b *testing.B) { + topo, option := benchSpreadTopology(b, writables) + option = option.withLayoutLoad(topo) + vg := NewDefaultVolumeGrowth() + b.ResetTimer() + for i := 0; i < b.N; i++ { + if _, _, err := vg.findEmptySlotsForOneVolume(topo, option, false); err != nil { + b.Fatal(err) + } + } + }) + } +} + +func BenchmarkWritableCountByNode(b *testing.B) { + for _, writables := range []int{100, 2000, 10000} { + b.Run(fmt.Sprintf("writables=%d", writables), func(b *testing.B) { + topo, option := benchSpreadTopology(b, writables) + vl := topo.FindVolumeLayout(option.Collection, option.ReplicaPlacement, option.Ttl, option.DiskType) + b.ResetTimer() + for i := 0; i < b.N; i++ { + vl.writableCountByNode() + } + }) + } +} diff --git a/weed/topology/volume_growth_spread_test.go b/weed/topology/volume_growth_spread_test.go new file mode 100644 index 0000000000..5d37b2bfd1 --- /dev/null +++ b/weed/topology/volume_growth_spread_test.go @@ -0,0 +1,192 @@ +package topology + +import ( + "fmt" + "testing" + + "github.com/seaweedfs/seaweedfs/weed/pb/master_pb" + "github.com/seaweedfs/seaweedfs/weed/storage" + "github.com/seaweedfs/seaweedfs/weed/storage/needle" + "github.com/seaweedfs/seaweedfs/weed/storage/super_block" + "github.com/seaweedfs/seaweedfs/weed/storage/types" +) + +// growSpreadVolumes places count volumes one by one the way a grow request +// does: each one is registered in its layout before the next is placed. +// It returns how many replicas of the layout each data node received. +func growSpreadVolumes(t *testing.T, topo *Topology, option *VolumeGrowOption, count int) map[string]int { + t.Helper() + vg := NewDefaultVolumeGrowth() + perNode := make(map[string]int) + for i := 0; i < count; i++ { + servers, _, err := vg.findEmptySlotsForOneVolume(topo, option, false) + if err != nil { + t.Fatalf("volume %d: %v", i, err) + } + vi := storage.VolumeInfo{ + Id: needle.VolumeId(i + 1), + Collection: option.Collection, + ReplicaPlacement: option.ReplicaPlacement, + Ttl: option.Ttl, + Version: needle.GetCurrentVersion(), + } + for _, dn := range servers { + dn.AddOrUpdateVolume(vi) + topo.RegisterVolumeLayout(vi, dn) + perNode[string(dn.Id())]++ + } + } + return perNode +} + +// newSpreadTopology builds dcs data centers with one rack each, holding +// nodesPerDc single-disk servers, and returns the racks. +func newSpreadTopology(dcs, nodesPerDc int, maxVol int64) (*Topology, []*Rack) { + topo, _ := newMaintenanceTestTopology() + topo.UnlinkChildNode("dc1") + var racks []*Rack + for d := 0; d < dcs; d++ { + dc := NewDataCenter(fmt.Sprintf("dc%d", d)) + topo.LinkChildNode(dc) + rack := NewRack("rack1") + dc.LinkChildNode(rack) + racks = append(racks, rack) + for n := 0; n < nodesPerDc; n++ { + addMaintenanceTestNode(rack, fmt.Sprintf("dc%d-server%d", d, n), maxVol) + } + } + return topo, racks +} + +// A grow request smaller than the number of servers must not put two writable +// volumes of the layout on one server. Free slots alone would pick servers +// independently at random and collide. +func TestVolumeGrowthSpreadsWritableVolumesOverServers(t *testing.T) { + rp, _ := super_block.NewReplicaPlacementFromString("000") + option := &VolumeGrowOption{Collection: "spread", ReplicaPlacement: rp, Ttl: needle.EMPTY_TTL, DiskType: types.HardDriveType} + for trial := 0; trial < 20; trial++ { + topo, _ := newSpreadTopology(1, 16, 100) + perNode := growSpreadVolumes(t, topo, option, 16) + for node, n := range perNode { + if n != 1 { + t.Fatalf("trial %d: %s holds %d writable volumes of the layout, want 1: %v", trial, node, n, perNode) + } + } + } +} + +// Replicas placed in another data center go through the same rule. +func TestVolumeGrowthSpreadsReplicasInOtherDataCenters(t *testing.T) { + rp, _ := super_block.NewReplicaPlacementFromString("100") + option := &VolumeGrowOption{Collection: "spread", ReplicaPlacement: rp, Ttl: needle.EMPTY_TTL, DiskType: types.HardDriveType} + for trial := 0; trial < 20; trial++ { + topo, _ := newSpreadTopology(2, 8, 100) + perNode := growSpreadVolumes(t, topo, option, 8) + if len(perNode) != 16 { + t.Fatalf("trial %d: replicas on %d servers, want all 16: %v", trial, len(perNode), perNode) + } + } +} + +// A server with several disks takes a share of the layout's writable volumes +// in proportion to its disks: load is counted per disk, not per server. +func TestVolumeGrowthCountsLoadPerDisk(t *testing.T) { + rp, _ := super_block.NewReplicaPlacementFromString("000") + option := &VolumeGrowOption{Collection: "spread", ReplicaPlacement: rp, Ttl: needle.EMPTY_TTL, DiskType: types.HardDriveType} + for trial := 0; trial < 20; trial++ { + topo, racks := newSpreadTopology(1, 3, 100) + multi := addMaintenanceTestNode(racks[0], "multi", 300) + multi.UpdateDiskTags([]*master_pb.DiskTag{{DiskId: 0, MaxVolumeCount: 100}, {DiskId: 1, MaxVolumeCount: 100}, {DiskId: 2, MaxVolumeCount: 100}}) + perNode := growSpreadVolumes(t, topo, option, 6) + if perNode["multi"] != 3 { + t.Fatalf("trial %d: the 3-disk server got %d volumes of 6, want 3: %v", trial, perNode["multi"], perNode) + } + } +} + +// Disks that report no volume slots (unavailable, or low on space) do not +// count: a server with one usable disk of four takes one disk's share. +func TestVolumeGrowthSkipsDisksWithoutSlots(t *testing.T) { + rp, _ := super_block.NewReplicaPlacementFromString("000") + option := &VolumeGrowOption{Collection: "spread", ReplicaPlacement: rp, Ttl: needle.EMPTY_TTL, DiskType: types.HardDriveType} + for trial := 0; trial < 20; trial++ { + topo, racks := newSpreadTopology(1, 3, 100) + damaged := addMaintenanceTestNode(racks[0], "damaged", 100) + damaged.UpdateDiskTags([]*master_pb.DiskTag{{DiskId: 0, MaxVolumeCount: 100}, {DiskId: 1}, {DiskId: 2}, {DiskId: 3}}) + perNode := growSpreadVolumes(t, topo, option, 8) + if perNode["damaged"] != 2 { + t.Fatalf("trial %d: the server with one usable disk got %d volumes of 8, want 2: %v", trial, perNode["damaged"], perNode) + } + } +} + +// A server that refused an allocation in this grow request offers no slots +// for the rest of it, so the retry places the volume elsewhere. +func TestVolumeGrowthSkipsRefusingServer(t *testing.T) { + rp, _ := super_block.NewReplicaPlacementFromString("000") + option := &VolumeGrowOption{Collection: "spread", ReplicaPlacement: rp, Ttl: needle.EMPTY_TTL, DiskType: types.HardDriveType} + topo, racks := newSpreadTopology(1, 4, 100) + refusing := racks[0].Children()[0].(*DataNode) + placed := option.withLayoutLoad(topo) + placed.refused = map[*DataNode]bool{refusing: true} + vg := NewDefaultVolumeGrowth() + for i := 0; i < 50; i++ { + servers, _, err := vg.findEmptySlotsForOneVolume(topo, placed, false) + if err != nil { + t.Fatal(err) + } + if servers[0] == refusing { + t.Fatalf("attempt %d picked %s, which refused an allocation", i, refusing.Id()) + } + } +} + +// The refusing server's free slots also leave the rack and data center +// weights, so a much emptier rack holding only that server is not picked. +func TestVolumeGrowthSkipsRefusingServerInOtherRack(t *testing.T) { + rp, _ := super_block.NewReplicaPlacementFromString("000") + option := &VolumeGrowOption{Collection: "spread", ReplicaPlacement: rp, Ttl: needle.EMPTY_TTL, DiskType: types.HardDriveType} + topo, racks := newSpreadTopology(1, 0, 0) + refusing := addMaintenanceTestNode(racks[0], "refusing", 100) + otherRack := NewRack("rack2") + racks[0].Parent().LinkChildNode(otherRack) + healthy := addMaintenanceTestNode(otherRack, "healthy", 1) + placed := option.withLayoutLoad(topo) + placed.refused = map[*DataNode]bool{refusing: true} + vg := NewDefaultVolumeGrowth() + for i := 0; i < 50; i++ { + servers, _, err := vg.findEmptySlotsForOneVolume(topo, placed, false) + if err != nil { + t.Fatal(err) + } + if servers[0] != healthy { + t.Fatalf("attempt %d picked %s, want %s", i, servers[0].Id(), healthy.Id()) + } + } +} + +func TestVolumeGrowthSkipsRefusingServerWhenItsRackKeepsOneServer(t *testing.T) { + rp, _ := super_block.NewReplicaPlacementFromString("011") + option := &VolumeGrowOption{Collection: "spread", ReplicaPlacement: rp, Ttl: needle.EMPTY_TTL, DiskType: types.HardDriveType} + topo, racks := newSpreadTopology(1, 0, 0) + refusing := addMaintenanceTestNode(racks[0], "refusing", 100) + addMaintenanceTestNode(racks[0], "healthy1", 100) + otherRack := NewRack("rack2") + racks[0].Parent().LinkChildNode(otherRack) + addMaintenanceTestNode(otherRack, "healthy2", 100) + addMaintenanceTestNode(otherRack, "healthy3", 100) + placed := option.withLayoutLoad(topo) + placed.refused = map[*DataNode]bool{refusing: true} + vg := NewDefaultVolumeGrowth() + for i := 0; i < 50; i++ { + servers, _, err := vg.findEmptySlotsForOneVolume(topo, placed, false) + if err != nil { + t.Fatalf("attempt %d: %v", i, err) + } + for _, s := range servers { + if s == refusing { + t.Fatalf("attempt %d picked the refusing server", i) + } + } + } +}