mirror of
https://github.com/seaweedfs/seaweedfs.git
synced 2026-11-19 02:15:02 +01:00
* volume growth: spread writable volumes of a layout over servers and disks Volume growth picked every server, and the volume server picked every disk, independently: the server weighted by free slots, the disk by the lowest total volume count. A grow request therefore stacks several writable volumes of one collection on some disks while most disks get none (a Poisson spread: with 160 replicas on 479 disks about 20 disks get two or three), and their writes share those disks. Data centers and racks are still picked weighted by free slots. Inside a rack the server is now one of those holding the fewest writable volumes of the layout per disk, weighted by free slots among them; this covers the main server, same-rack replicas and replicas walked into other racks and data centers. On the volume server the disk is the one with the fewest writable volumes of the collection, then the lowest volume count as before. A grow request registers each volume before placing the next, so a request smaller than the number of disks lands on distinct disks. Fixes #12746 * volume growth: scan the layout load once per grow request Placing each volume rescanned all writable volumes of the layout under its read lock. RecordAssign takes that lock exclusively on every assign, so with thousands of writables (0.8 ms for 10000) every volume of a grow request stalled assigns once more. GrowByCountAndType now scans once and findAndGrow adds every volume it creates to the counts. Per volume, placement costs about 130-160 us on 480 servers whatever the number of writables, against 120 us before the spread change; the single scan is 0.17 ms for 2000 writables and 0.8 ms for 10000. * volume growth: address review of the spread placement - Count only disks that report volume slots in a server's load. An unavailable disk, or one low on space with no volumes, is still listed in DiskTags with no slots; counting it understated the load of a server with failed disks and piled volumes onto its remaining ones. - On the volume server, judge a volume full by its .dat size, which the master compares with the limit, not by ContentSize. - A server that refuses an allocation ranks last for the rest of the grow request, and the request retries the volume up to 3 times. The least loaded server is picked first deterministically, so without this one server that cannot fit a preallocated volume stopped the layout from growing. - Scan the layout load after warmup, so a grow request issued right after a leader change sees the writable volumes the returning servers report. * volume growth: keep the request-local placement load ahead of the request fields The JSON request fields stay together at the end of VolumeGrowOption, and the comment now says the load lives for the whole grow request. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * volume(rs): place a new volume on the disk with the fewest writable volumes of its collection Store::add_volume picked the disk with the most free slots, so the Rust volume server ignored the collection spread rule the Go server now applies. find_free_location ranks disks by the writable volumes of the collection first (not read-only, .dat below the volume size limit), then by free slots. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * volume growth: leave a refusing server out of every placement tier A server that refused an allocation only ranked last among the servers of its rack, so the weighted data center and rack picks could still choose its rack on every retry while other racks had room. It now offers no free slots for the rest of the grow request, which takes its slots out of the rack and data center weights too. If only refusing servers are left, they are tried again. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * volume growth: require the full same-rack count only in the main rack The data center check asked every rack for SameRackCount+1 free servers, but only the main rack holds the same-rack copies; each other rack takes one. With a refused server left out of its rack, the check rejected a feasible placement, the retry put the refused server back, and it was picked again. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> --------- Co-authored-by: Konstantin Lebedev <whitefox@mayflower.work> Co-authored-by: Chris Lu <chrislusf@users.noreply.github.com> Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
640 lines
20 KiB
Go
640 lines
20 KiB
Go
package topology
|
|
|
|
import (
|
|
"errors"
|
|
"fmt"
|
|
"math/rand/v2"
|
|
"sort"
|
|
"strings"
|
|
"sync"
|
|
"sync/atomic"
|
|
"time"
|
|
|
|
"github.com/seaweedfs/seaweedfs/weed/glog"
|
|
"github.com/seaweedfs/seaweedfs/weed/stats"
|
|
"github.com/seaweedfs/seaweedfs/weed/storage/erasure_coding"
|
|
"github.com/seaweedfs/seaweedfs/weed/storage/needle"
|
|
"github.com/seaweedfs/seaweedfs/weed/storage/types"
|
|
)
|
|
|
|
type NodeId string
|
|
|
|
// CapacityReservation represents a temporary reservation of capacity
|
|
type CapacityReservation struct {
|
|
reservationId string
|
|
diskType types.DiskType
|
|
count int64
|
|
createdAt time.Time
|
|
}
|
|
|
|
// CapacityReservations manages capacity reservations for a node
|
|
type CapacityReservations struct {
|
|
sync.RWMutex
|
|
reservations map[string]*CapacityReservation
|
|
reservedCounts map[types.DiskType]int64
|
|
}
|
|
|
|
func newCapacityReservations() *CapacityReservations {
|
|
return &CapacityReservations{
|
|
reservations: make(map[string]*CapacityReservation),
|
|
reservedCounts: make(map[types.DiskType]int64),
|
|
}
|
|
}
|
|
|
|
func (cr *CapacityReservations) removeReservation(reservationId string) bool {
|
|
cr.Lock()
|
|
defer cr.Unlock()
|
|
|
|
if reservation, exists := cr.reservations[reservationId]; exists {
|
|
delete(cr.reservations, reservationId)
|
|
cr.decrementCount(reservation.diskType, reservation.count)
|
|
return true
|
|
}
|
|
return false
|
|
}
|
|
|
|
func (cr *CapacityReservations) getReservedCount(diskType types.DiskType) int64 {
|
|
cr.RLock()
|
|
defer cr.RUnlock()
|
|
|
|
return cr.reservedCounts[diskType]
|
|
}
|
|
|
|
// decrementCount is a helper to decrement reserved count and clean up zero entries
|
|
func (cr *CapacityReservations) decrementCount(diskType types.DiskType, count int64) {
|
|
cr.reservedCounts[diskType] -= count
|
|
// Clean up zero counts to prevent map growth
|
|
if cr.reservedCounts[diskType] <= 0 {
|
|
delete(cr.reservedCounts, diskType)
|
|
}
|
|
}
|
|
|
|
// doAddReservation is a helper to add a reservation, assuming the lock is already held
|
|
func (cr *CapacityReservations) doAddReservation(diskType types.DiskType, count int64) string {
|
|
now := time.Now()
|
|
reservationId := fmt.Sprintf("%s-%d-%d-%d", diskType, count, now.UnixNano(), rand.Int64())
|
|
cr.reservations[reservationId] = &CapacityReservation{
|
|
reservationId: reservationId,
|
|
diskType: diskType,
|
|
count: count,
|
|
createdAt: now,
|
|
}
|
|
cr.reservedCounts[diskType] += count
|
|
return reservationId
|
|
}
|
|
|
|
// tryReserveAtomic atomically checks available space and reserves if possible
|
|
func (cr *CapacityReservations) tryReserveAtomic(diskType types.DiskType, count int64, availableSpaceFunc func() int64) (reservationId string, success bool) {
|
|
cr.Lock()
|
|
defer cr.Unlock()
|
|
|
|
// Check available space under lock
|
|
currentReserved := cr.reservedCounts[diskType]
|
|
availableSpace := availableSpaceFunc() - currentReserved
|
|
|
|
if availableSpace >= count {
|
|
// Create and add reservation atomically
|
|
return cr.doAddReservation(diskType, count), true
|
|
}
|
|
|
|
return "", false
|
|
}
|
|
|
|
func (cr *CapacityReservations) cleanExpiredReservations(expirationDuration time.Duration) {
|
|
cr.Lock()
|
|
defer cr.Unlock()
|
|
|
|
now := time.Now()
|
|
for id, reservation := range cr.reservations {
|
|
if now.Sub(reservation.createdAt) > expirationDuration {
|
|
delete(cr.reservations, id)
|
|
cr.decrementCount(reservation.diskType, reservation.count)
|
|
glog.V(1).Infof("Cleaned up expired capacity reservation: %s", id)
|
|
}
|
|
}
|
|
}
|
|
|
|
type Node interface {
|
|
Id() NodeId
|
|
String() string
|
|
AvailableSpaceFor(option *VolumeGrowOption) int64
|
|
ReserveOneVolume(r int64, option *VolumeGrowOption) (*DataNode, error)
|
|
ReserveOneVolumeForReservation(r int64, option *VolumeGrowOption) (*DataNode, error)
|
|
UpAdjustDiskUsageDelta(diskType types.DiskType, diskUsage *DiskUsageCounts)
|
|
UpAdjustMaxVolumeId(vid needle.VolumeId)
|
|
GetDiskUsages() *DiskUsages
|
|
|
|
// Capacity reservation methods for avoiding race conditions
|
|
TryReserveCapacity(diskType types.DiskType, count int64) (reservationId string, success bool)
|
|
ReleaseReservedCapacity(reservationId string)
|
|
AvailableSpaceForReservation(option *VolumeGrowOption) int64
|
|
|
|
GetMaxVolumeId() needle.VolumeId
|
|
SetParent(Node)
|
|
LinkChildNode(node Node)
|
|
UnlinkChildNode(nodeId NodeId)
|
|
CollectDeadNodeAndFullVolumes(freshThreshHold int64, volumeSizeLimit uint64, growThreshold float64)
|
|
|
|
IsDataNode() bool
|
|
IsRack() bool
|
|
IsDataCenter() bool
|
|
IsLocked() bool
|
|
Children() []Node
|
|
Parent() Node
|
|
|
|
GetValue() interface{} //get reference to the topology,dc,rack,datanode
|
|
}
|
|
|
|
type NodeImpl struct {
|
|
diskUsages *DiskUsages
|
|
id NodeId
|
|
// parent is an atomic pointer because unlinking runs under the rack's
|
|
// lock while readers (vacuum sweeps, usage propagation, String) reach
|
|
// it with no shared lock.
|
|
parent atomic.Pointer[Node]
|
|
sync.RWMutex // lock children
|
|
children map[NodeId]Node
|
|
// maxVolumeId uses atomic ops so UpAdjustMaxVolumeId (called from the
|
|
// volume server heartbeat path) and GetMaxVolumeId (called from the
|
|
// master's assign / warmup checks) can run concurrently without a
|
|
// data race on NodeImpl.
|
|
maxVolumeId atomic.Uint32
|
|
|
|
//for rack, data center, topology
|
|
nodeType string
|
|
value interface{}
|
|
|
|
// capacity reservations to prevent race conditions during volume creation
|
|
capacityReservations *CapacityReservations
|
|
}
|
|
|
|
func (n *NodeImpl) GetDiskUsages() *DiskUsages {
|
|
return n.diskUsages
|
|
}
|
|
|
|
// nodeHost returns the host a node runs on, or "" for non-data-node tiers (data
|
|
// centers, racks).
|
|
func nodeHost(node Node) string {
|
|
if dn, ok := node.(*DataNode); ok {
|
|
return dn.Ip
|
|
}
|
|
return ""
|
|
}
|
|
|
|
// preferDistinctHosts reorders candidates so the first node of each not-yet-used
|
|
// host comes first (preserving weighted order), then the same-host leftovers, so a
|
|
// prefix covers the most distinct machines. usedHost seeds the set. No-op when
|
|
// hosts are empty (non-data-node tiers).
|
|
func preferDistinctHosts(usedHost string, candidates []Node) []Node {
|
|
used := map[string]bool{}
|
|
if usedHost != "" {
|
|
used[usedHost] = true
|
|
}
|
|
distinct := make([]Node, 0, len(candidates))
|
|
dup := make([]Node, 0, len(candidates))
|
|
for _, node := range candidates {
|
|
h := nodeHost(node)
|
|
if h != "" && !used[h] {
|
|
used[h] = true
|
|
distinct = append(distinct, node)
|
|
} else {
|
|
dup = append(dup, node)
|
|
}
|
|
}
|
|
return append(distinct, dup...)
|
|
}
|
|
|
|
// the first node must satisfy filterFirstNodeFn(), the rest nodes must have one free slot
|
|
func (n *NodeImpl) PickNodesByWeight(numberOfNodes int, option *VolumeGrowOption, filterFirstNodeFn func(dn Node) error) (firstNode Node, restNodes []Node, err error) {
|
|
var totalWeights int64
|
|
var errs []string
|
|
n.RLock()
|
|
candidates := make([]Node, 0, len(n.children))
|
|
candidatesWeights := make([]int64, 0, len(n.children))
|
|
//pick nodes which has enough free volumes as candidates, and use free volumes number as node weight.
|
|
for _, node := range n.children {
|
|
if node.AvailableSpaceFor(option) <= 0 {
|
|
continue
|
|
}
|
|
totalWeights += node.AvailableSpaceFor(option)
|
|
candidates = append(candidates, node)
|
|
candidatesWeights = append(candidatesWeights, node.AvailableSpaceFor(option))
|
|
}
|
|
n.RUnlock()
|
|
if len(candidates) < numberOfNodes {
|
|
glog.V(0).Infoln(n.Id(), "failed to pick", numberOfNodes, "from ", len(candidates), "node candidates")
|
|
return nil, nil, errors.New("Not enough data nodes found!")
|
|
}
|
|
|
|
//pick nodes randomly by weights, the node picked earlier has higher final weights
|
|
sortedCandidates := make([]Node, 0, len(candidates))
|
|
for i := 0; i < len(candidates); i++ {
|
|
// Break if no more weights available to prevent panic in rand.Int64N
|
|
if totalWeights <= 0 {
|
|
break
|
|
}
|
|
weightsInterval := rand.Int64N(totalWeights)
|
|
lastWeights := int64(0)
|
|
for k, weights := range candidatesWeights {
|
|
if (weightsInterval >= lastWeights) && (weightsInterval < lastWeights+weights) {
|
|
sortedCandidates = append(sortedCandidates, candidates[k])
|
|
candidatesWeights[k] = 0
|
|
totalWeights -= weights
|
|
break
|
|
}
|
|
lastWeights += weights
|
|
}
|
|
}
|
|
// Servers holding fewer writable volumes of the layout come first; the
|
|
// weighted order above only breaks ties among them.
|
|
sort.SliceStable(sortedCandidates, func(i, j int) bool {
|
|
return option.lessLoaded(sortedCandidates[i], sortedCandidates[j])
|
|
})
|
|
|
|
restNodes = make([]Node, 0, numberOfNodes-1)
|
|
ret := false
|
|
n.RLock()
|
|
for k, node := range sortedCandidates {
|
|
if err := filterFirstNodeFn(node); err == nil {
|
|
firstNode = node
|
|
// Fill the rest preferring not-yet-used hosts, so replicas spread across
|
|
// machines; falls back to same-host when too few. No-op for dc/rack tiers
|
|
// (empty host), which keep the weighted order.
|
|
pool := make([]Node, 0, len(sortedCandidates)-1)
|
|
pool = append(pool, sortedCandidates[:k]...)
|
|
pool = append(pool, sortedCandidates[k+1:]...)
|
|
pool = preferDistinctHosts(nodeHost(firstNode), pool)
|
|
if len(pool) > numberOfNodes-1 {
|
|
pool = pool[:numberOfNodes-1]
|
|
}
|
|
restNodes = pool
|
|
ret = true
|
|
break
|
|
} else {
|
|
errs = append(errs, string(node.Id())+":"+err.Error())
|
|
}
|
|
}
|
|
n.RUnlock()
|
|
if !ret {
|
|
return nil, nil, errors.New("No matching data node found! \n" + strings.Join(errs, "\n"))
|
|
}
|
|
return
|
|
}
|
|
|
|
func (n *NodeImpl) IsDataNode() bool {
|
|
return n.nodeType == "DataNode"
|
|
}
|
|
|
|
func (n *NodeImpl) IsRack() bool {
|
|
return n.nodeType == "Rack"
|
|
}
|
|
|
|
func (n *NodeImpl) IsDataCenter() bool {
|
|
return n.nodeType == "DataCenter"
|
|
}
|
|
|
|
func (n *NodeImpl) IsLocked() (isTryLock bool) {
|
|
if isTryLock = n.TryRLock(); isTryLock {
|
|
n.RUnlock()
|
|
}
|
|
return !isTryLock
|
|
}
|
|
|
|
func (n *NodeImpl) String() string {
|
|
if parent := n.Parent(); parent != nil {
|
|
return parent.String() + ":" + string(n.id)
|
|
}
|
|
return string(n.id)
|
|
}
|
|
|
|
func (n *NodeImpl) Id() NodeId {
|
|
return n.id
|
|
}
|
|
|
|
func (n *NodeImpl) getOrCreateDisk(diskType types.DiskType) *DiskUsageCounts {
|
|
return n.diskUsages.getOrCreateDisk(diskType)
|
|
}
|
|
|
|
// inMaintenanceMode is true for a data node whose volume server is in
|
|
// maintenance mode. Racks and data centers never are.
|
|
func (n *NodeImpl) inMaintenanceMode() bool {
|
|
dn, ok := n.value.(*DataNode)
|
|
return ok && dn.InMaintenanceMode()
|
|
}
|
|
|
|
// AvailableSpaceFor is the free volume slots on this node for the option's disk
|
|
// type. A data node in maintenance mode reports none: it is being drained, so
|
|
// it is neither a volume-growth candidate nor reservable. A data node with more
|
|
// volumes than slots also reports none, and racks, data centers and the
|
|
// topology sum their children, so it cannot hide the free slots of others.
|
|
func (n *NodeImpl) AvailableSpaceFor(option *VolumeGrowOption) int64 {
|
|
if n.inMaintenanceMode() {
|
|
return 0
|
|
}
|
|
if !n.IsDataNode() {
|
|
n.RLock()
|
|
defer n.RUnlock()
|
|
var free int64
|
|
for _, child := range n.children {
|
|
free += child.AvailableSpaceFor(option)
|
|
}
|
|
return free
|
|
}
|
|
if option.hasRefused(n) {
|
|
return 0
|
|
}
|
|
t := n.diskUsages.getDisk(option.DiskType)
|
|
if t == nil {
|
|
return 0
|
|
}
|
|
freeVolumeSlotCount := atomic.LoadInt64(&t.maxVolumeCount) + atomic.LoadInt64(&t.remoteVolumeCount) - atomic.LoadInt64(&t.volumeCount)
|
|
freeVolumeSlotCount -= erasure_coding.VolumeSlots(atomic.LoadInt64(&t.ecShardCount))
|
|
return max(0, freeVolumeSlotCount)
|
|
}
|
|
|
|
// CapacityFor is the total registered volume slots for the option's disk type;
|
|
// zero means no volume server has reported capacity for it yet.
|
|
func (n *NodeImpl) CapacityFor(option *VolumeGrowOption) int64 {
|
|
t := n.diskUsages.getDisk(option.DiskType)
|
|
if t == nil {
|
|
return 0
|
|
}
|
|
return atomic.LoadInt64(&t.maxVolumeCount) + atomic.LoadInt64(&t.remoteVolumeCount)
|
|
}
|
|
|
|
// CapacityForAnyDisk is the total registered volume slots across every disk
|
|
// type. CapacityFor answers zero both while a cluster is still starting and
|
|
// when it never serves the option's medium; this tells the two apart.
|
|
func (n *NodeImpl) CapacityForAnyDisk() (total int64) {
|
|
n.diskUsages.RLock()
|
|
defer n.diskUsages.RUnlock()
|
|
for _, t := range n.diskUsages.usages {
|
|
total += atomic.LoadInt64(&t.maxVolumeCount) + atomic.LoadInt64(&t.remoteVolumeCount)
|
|
}
|
|
return
|
|
}
|
|
|
|
// AvailableSpaceForReservation returns available space considering existing reservations
|
|
func (n *NodeImpl) AvailableSpaceForReservation(option *VolumeGrowOption) int64 {
|
|
// Expire here as well: a node whose reservations fill it is filtered out
|
|
// before TryReserveCapacity could clean them, stranding the capacity.
|
|
n.capacityReservations.cleanExpiredReservations(VolumeGrowStrategy.GetReservationTimeout())
|
|
baseAvailable := n.AvailableSpaceFor(option)
|
|
reservedCount := n.capacityReservations.getReservedCount(option.DiskType)
|
|
return baseAvailable - reservedCount
|
|
}
|
|
|
|
// TryReserveCapacity attempts to atomically reserve capacity for volume creation using the configured timeout
|
|
func (n *NodeImpl) TryReserveCapacity(diskType types.DiskType, count int64) (reservationId string, success bool) {
|
|
// Clean up any expired reservations first
|
|
n.capacityReservations.cleanExpiredReservations(VolumeGrowStrategy.GetReservationTimeout())
|
|
|
|
// Atomically check and reserve space
|
|
option := &VolumeGrowOption{DiskType: diskType}
|
|
reservationId, success = n.capacityReservations.tryReserveAtomic(diskType, count, func() int64 {
|
|
return n.AvailableSpaceFor(option)
|
|
})
|
|
|
|
if success {
|
|
glog.V(1).Infof("Reserved %d capacity for diskType %s on node %s: %s", count, diskType, n.Id(), reservationId)
|
|
}
|
|
|
|
return reservationId, success
|
|
}
|
|
|
|
// ReleaseReservedCapacity releases a previously reserved capacity
|
|
func (n *NodeImpl) ReleaseReservedCapacity(reservationId string) {
|
|
if n.capacityReservations.removeReservation(reservationId) {
|
|
glog.V(1).Infof("Released capacity reservation on node %s: %s", n.Id(), reservationId)
|
|
} else {
|
|
glog.V(1).Infof("Attempted to release non-existent reservation on node %s: %s", n.Id(), reservationId)
|
|
}
|
|
}
|
|
func (n *NodeImpl) SetParent(node Node) {
|
|
if node == nil {
|
|
n.parent.Store(nil)
|
|
return
|
|
}
|
|
n.parent.Store(&node)
|
|
}
|
|
|
|
func (n *NodeImpl) Children() (ret []Node) {
|
|
n.RLock()
|
|
defer n.RUnlock()
|
|
for _, c := range n.children {
|
|
ret = append(ret, c)
|
|
}
|
|
return ret
|
|
}
|
|
|
|
func (n *NodeImpl) Parent() Node {
|
|
if parent := n.parent.Load(); parent != nil {
|
|
return *parent
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func (n *NodeImpl) GetValue() interface{} {
|
|
return n.value
|
|
}
|
|
|
|
func (n *NodeImpl) ReserveOneVolume(r int64, option *VolumeGrowOption) (assignedNode *DataNode, err error) {
|
|
return n.reserveOneVolumeInternal(r, option, false)
|
|
}
|
|
|
|
// ReserveOneVolumeForReservation selects a node using reservation-aware capacity checks
|
|
func (n *NodeImpl) ReserveOneVolumeForReservation(r int64, option *VolumeGrowOption) (assignedNode *DataNode, err error) {
|
|
return n.reserveOneVolumeInternal(r, option, true)
|
|
}
|
|
|
|
func (n *NodeImpl) reserveOneVolumeInternal(r int64, option *VolumeGrowOption, useReservations bool) (assignedNode *DataNode, err error) {
|
|
n.RLock()
|
|
defer n.RUnlock()
|
|
freeSpaceOf := func(node Node) int64 {
|
|
if useReservations {
|
|
return node.AvailableSpaceForReservation(option)
|
|
}
|
|
return node.AvailableSpaceFor(option)
|
|
}
|
|
if option.layoutLoad != nil && n.IsRack() {
|
|
return n.reserveLeastLoaded(r, option, freeSpaceOf)
|
|
}
|
|
// The caller drew r from this node's free slots, which can exceed what the
|
|
// children offer once their reservations or a newer heartbeat are counted.
|
|
// Fold r into the space that is actually on offer so it cannot walk off the
|
|
// end and fail with slots still free.
|
|
var eligible int64
|
|
for _, node := range n.children {
|
|
if freeSpace := freeSpaceOf(node); freeSpace > 0 {
|
|
eligible += freeSpace
|
|
}
|
|
}
|
|
if eligible <= 0 {
|
|
return nil, errors.New("No free volume slot found!")
|
|
}
|
|
r %= eligible
|
|
for _, node := range n.children {
|
|
freeSpace := freeSpaceOf(node)
|
|
// fmt.Println("r =", r, ", node =", node, ", freeSpace =", freeSpace)
|
|
if freeSpace <= 0 {
|
|
continue
|
|
}
|
|
if r >= freeSpace {
|
|
r -= freeSpace
|
|
} else {
|
|
var hasSpace bool
|
|
if useReservations {
|
|
hasSpace = node.IsDataNode() && node.AvailableSpaceForReservation(option) > 0
|
|
} else {
|
|
hasSpace = node.IsDataNode() && node.AvailableSpaceFor(option) > 0
|
|
}
|
|
if hasSpace {
|
|
// fmt.Println("vid =", vid, " assigned to node =", node, ", freeSpace =", node.FreeSpace())
|
|
dn := node.(*DataNode)
|
|
if dn.IsTerminating {
|
|
continue
|
|
}
|
|
return dn, nil
|
|
}
|
|
if useReservations {
|
|
assignedNode, err = node.ReserveOneVolumeForReservation(r, option)
|
|
} else {
|
|
assignedNode, err = node.ReserveOneVolume(r, option)
|
|
}
|
|
if err == nil {
|
|
return
|
|
}
|
|
}
|
|
}
|
|
return nil, errors.New("No free volume slot found!")
|
|
}
|
|
|
|
func (n *NodeImpl) UpAdjustDiskUsageDelta(diskType types.DiskType, diskUsage *DiskUsageCounts) { //can be negative
|
|
existingDisk := n.getOrCreateDisk(diskType)
|
|
existingDisk.addDiskUsageCounts(diskUsage)
|
|
if parent := n.Parent(); parent != nil {
|
|
parent.UpAdjustDiskUsageDelta(diskType, diskUsage)
|
|
}
|
|
}
|
|
func (n *NodeImpl) UpAdjustMaxVolumeId(vid needle.VolumeId) {
|
|
target := uint32(vid)
|
|
for {
|
|
current := n.maxVolumeId.Load()
|
|
if current >= target {
|
|
return
|
|
}
|
|
if n.maxVolumeId.CompareAndSwap(current, target) {
|
|
break
|
|
}
|
|
}
|
|
if parent := n.Parent(); parent != nil {
|
|
parent.UpAdjustMaxVolumeId(vid)
|
|
}
|
|
}
|
|
func (n *NodeImpl) GetMaxVolumeId() needle.VolumeId {
|
|
return needle.VolumeId(n.maxVolumeId.Load())
|
|
}
|
|
|
|
func (n *NodeImpl) LinkChildNode(node Node) {
|
|
n.Lock()
|
|
defer n.Unlock()
|
|
n.doLinkChildNode(node)
|
|
}
|
|
|
|
func (n *NodeImpl) doLinkChildNode(node Node) {
|
|
if n.children[node.Id()] == nil {
|
|
n.children[node.Id()] = node
|
|
for dt, du := range node.GetDiskUsages().usages {
|
|
n.UpAdjustDiskUsageDelta(dt, du)
|
|
}
|
|
n.UpAdjustMaxVolumeId(node.GetMaxVolumeId())
|
|
node.SetParent(n)
|
|
// Maintain the topology's address index so Ping admission and other
|
|
// callers can resolve a data node from its address in O(1).
|
|
if dn, ok := node.GetValue().(*DataNode); ok {
|
|
if topo := n.GetTopology(); topo != nil {
|
|
topo.registerDataNodeAddress(dn)
|
|
}
|
|
}
|
|
glog.V(0).Infoln(n, "adds child", node.Id())
|
|
}
|
|
}
|
|
|
|
func (n *NodeImpl) UnlinkChildNode(nodeId NodeId) {
|
|
n.Lock()
|
|
defer n.Unlock()
|
|
node := n.children[nodeId]
|
|
if node != nil {
|
|
// Drop the topology address index before clearing the parent pointer
|
|
// so GetTopology() can still walk up to the root.
|
|
if dn, ok := node.GetValue().(*DataNode); ok {
|
|
if topo := n.GetTopology(); topo != nil {
|
|
topo.unregisterDataNodeAddress(dn.ServerAddress(), dn)
|
|
}
|
|
}
|
|
node.SetParent(nil)
|
|
delete(n.children, node.Id())
|
|
for dt, du := range node.GetDiskUsages().negative().usages {
|
|
n.UpAdjustDiskUsageDelta(dt, du)
|
|
}
|
|
glog.V(0).Infoln(n, "removes", node.Id())
|
|
}
|
|
}
|
|
|
|
func (n *NodeImpl) CollectDeadNodeAndFullVolumes(freshThreshHoldUnixTime int64, volumeSizeLimit uint64, growThreshold float64) {
|
|
if n.IsRack() {
|
|
for _, c := range n.Children() {
|
|
dn := c.(*DataNode) //can not cast n to DataNode
|
|
for _, v := range dn.GetVolumes() {
|
|
topo := n.GetTopology()
|
|
diskType := types.ToDiskType(v.DiskType)
|
|
vl := topo.GetVolumeLayout(v.Collection, v.ReplicaPlacement, v.Ttl, diskType)
|
|
|
|
if v.Size >= volumeSizeLimit {
|
|
vl.accessLock.RLock()
|
|
vacuumTime, ok := vl.vacuumedVolumes[v.Id]
|
|
vl.accessLock.RUnlock()
|
|
|
|
// If a volume has been vacuumed in the past 20 seconds, we do not check whether it has reached full capacity.
|
|
// After 20s(grpc timeout), theoretically all the heartbeats of the volume server have reached the master,
|
|
// the volume size should be correct, not the size before the vacuum.
|
|
if !ok || time.Now().Add(-20*time.Second).After(vacuumTime) {
|
|
//fmt.Println("volume",v.Id,"size",v.Size,">",volumeSizeLimit)
|
|
topo.chanFullVolumes <- v
|
|
}
|
|
} else if !v.ReadOnly && float64(v.Size) > float64(volumeSizeLimit)*growThreshold {
|
|
// Crowding asks for more room to write into, which a
|
|
// read-only volume can never provide. Growth already
|
|
// discounts them by intersecting with the writable list, so
|
|
// marking one only costs the entry.
|
|
topo.chanCrowdedVolumes <- v
|
|
}
|
|
copyCount := v.ReplicaPlacement.GetCopyCount()
|
|
if copyCount > 1 {
|
|
if copyCount > len(topo.Lookup(v.Collection, v.Id)) {
|
|
stats.MasterReplicaPlacementMismatch.WithLabelValues(v.Collection, v.Id.String()).Set(1)
|
|
} else {
|
|
stats.MasterReplicaPlacementMismatch.WithLabelValues(v.Collection, v.Id.String()).Set(0)
|
|
}
|
|
}
|
|
}
|
|
}
|
|
} else {
|
|
for _, c := range n.Children() {
|
|
c.CollectDeadNodeAndFullVolumes(freshThreshHoldUnixTime, volumeSizeLimit, growThreshold)
|
|
}
|
|
}
|
|
}
|
|
|
|
func (n *NodeImpl) GetTopology() *Topology {
|
|
var p Node = n
|
|
for p.Parent() != nil {
|
|
p = p.Parent()
|
|
}
|
|
// A detached subtree (no Topology root in scope) must not panic; the
|
|
// callers above check the returned value for nil and skip the
|
|
// address-index maintenance in that case.
|
|
topo, _ := p.GetValue().(*Topology)
|
|
return topo
|
|
}
|