mirror of
https://github.com/seaweedfs/seaweedfs.git
synced 2026-09-08 15:41:15 +02:00
Chris Lu
b77431c142
* master: estimate a hintless assign's size from the volume's average file size An assign that carries no dataSize hint charged a flat 1MB per file id against the volume's effective size. A small-file workload overpays by orders of magnitude: bulk-writing 4KB files marks volumes holding a few hundred MB of real data as crowded and then full, so the master grows unnecessary volumes and, once every volume is spuriously full, fails all assigns. Estimate from the volume's own average file size instead, and keep the 1MB fallback only for volumes with no history. * master: decay pending assign sizes for volumes gone quiet The decay that corrects pending assign estimates runs only when a heartbeat reports the volume, and a heartbeat only reports a volume whose content changed. A volume held out of the writable list takes no writes, so once inflated estimates mark every volume full, nothing is ever reported again, nothing decays, and the cluster refuses all writes until a restart. Run the decay from the master's periodic loop for volumes no heartbeat has reported within two pulses, feeding the last reported size back through the same path an unchanged heartbeat would take. * master: trim the comments on the assign size estimate * master: keep the periodic decay out of the replica-dedup window UpdateVolumeSize ignores a report arriving within two seconds of the last one, so replicas of the same volume do not each halve the pending estimate. The periodic decay went through the same path and stamped that window, so a real heartbeat landing right behind it was dropped along with its reported size and compact revision. Only a volume whose content changed is reported at all, so nothing would send that size again and the master kept a stale one. Let the dedup window belong to volume server reports alone. * master: let the decay read the size record under the lock it mutates The periodic decay picked its volumes under a read lock and replayed them under a write one, carrying the size it had read across the gap. A heartbeat landing in between was rolled back: the replay wrote the older size and compact revision over the fresh ones, and a compaction report lost that way is never resent, since only a volume whose content changed is reported. The decay has no size of its own to contribute, so it now reads the record under the same lock it mutates. * master: let a heartbeat that beat the decay stand for the cycle The decay chooses its volumes under a read lock and applies them under a write one. A heartbeat landing in that gap already did the halving the cycle owed, so applying the decay on top of it halved twice and forgot pending bytes the volume has not written yet - the double-halving the replica-dedup window exists to prevent. Both callers now give way to a report already handled for this cycle; only a real report still advances lastUpdateTime, so a quiet volume keeps decaying every pulse. * master: keep genuinely full volumes out of the decay pass A volume the disk really did fill keeps its fullSince set for good, so it was selected every pulse for a decay that cannot help it: UpdateVolumeSize refuses to recover a volume whose reported size is at the limit, and replaying a size that cannot move leaves the record as it found it. Full and quiet is the ordinary resting state of a cluster, so this was most of the pass, taking the layout write lock away from the heartbeats to do nothing. On a million tracked volumes with a hundredth of them phantom-full it costs ten thousand write locks a pulse instead of a million. * master: put the stale-replay test back on the path it guards Giving the decay the dedup window left this test short-circuiting there, so it no longer reached the locked read it was written for and passed with that read removed. Age the record past the window, which is the only case where reading it under the lock is what saves the report.
117 lines
3.7 KiB
Go
117 lines
3.7 KiB
Go
package topology
|
|
|
|
import (
|
|
"math/rand/v2"
|
|
"time"
|
|
|
|
"github.com/seaweedfs/seaweedfs/weed/stats"
|
|
"github.com/seaweedfs/seaweedfs/weed/storage/erasure_coding"
|
|
"github.com/seaweedfs/seaweedfs/weed/storage/types"
|
|
"google.golang.org/grpc"
|
|
|
|
"github.com/seaweedfs/seaweedfs/weed/glog"
|
|
"github.com/seaweedfs/seaweedfs/weed/storage"
|
|
)
|
|
|
|
func (t *Topology) StartRefreshWritableVolumes(grpcDialOption grpc.DialOption, garbageThreshold float64, concurrentVacuumLimitPerVolumeServer int, growThreshold float64, preallocate int64) {
|
|
go func() {
|
|
for {
|
|
if t.IsLeader() {
|
|
freshThreshHold := time.Now().Unix() - 3*t.pulse //3 times of sleep interval
|
|
t.CollectDeadNodeAndFullVolumes(freshThreshHold, t.volumeSizeLimit, growThreshold)
|
|
t.DecayQuietVolumeSizes()
|
|
}
|
|
time.Sleep(time.Duration(float32(t.pulse*1e3)*(1+rand.Float32())) * time.Millisecond)
|
|
}
|
|
}()
|
|
go func(garbageThreshold float64) {
|
|
for {
|
|
if t.IsLeader() {
|
|
// Safety net: if vacuum was disabled by the plugin monitor but the
|
|
// admin server is no longer connected, automatically re-enable.
|
|
// This handles the case where the admin server crashes without
|
|
// cleanup. Does NOT override an operator's intentional disable.
|
|
if t.IsVacuumDisabledByPlugin() && t.adminServerConnectedFunc != nil && !t.adminServerConnectedFunc() {
|
|
glog.V(0).Infof("Admin server disconnected while vacuum was disabled by plugin, clearing plugin disable")
|
|
t.EnableVacuumByPlugin()
|
|
}
|
|
if !t.IsVacuumDisabled() {
|
|
t.Vacuum(grpcDialOption, garbageThreshold, concurrentVacuumLimitPerVolumeServer, 0, "", preallocate, true)
|
|
}
|
|
} else {
|
|
stats.MasterReplicaPlacementMismatch.Reset()
|
|
}
|
|
time.Sleep(14*time.Minute + time.Duration(120*rand.Float32())*time.Second)
|
|
}
|
|
}(garbageThreshold)
|
|
go func() {
|
|
for {
|
|
select {
|
|
case fv := <-t.chanFullVolumes:
|
|
t.SetVolumeCapacityFull(fv)
|
|
case cv := <-t.chanCrowdedVolumes:
|
|
t.SetVolumeCrowded(cv)
|
|
}
|
|
}
|
|
}()
|
|
}
|
|
func (t *Topology) SetVolumeCapacityFull(volumeInfo storage.VolumeInfo) bool {
|
|
diskType := types.ToDiskType(volumeInfo.DiskType)
|
|
vl := t.GetVolumeLayout(volumeInfo.Collection, volumeInfo.ReplicaPlacement, volumeInfo.Ttl, diskType)
|
|
if !vl.SetVolumeCapacityFull(volumeInfo.Id) {
|
|
return false
|
|
}
|
|
|
|
vl.accessLock.RLock()
|
|
defer vl.accessLock.RUnlock()
|
|
|
|
vidLocations, found := vl.vid2location[volumeInfo.Id]
|
|
if !found {
|
|
return false
|
|
}
|
|
|
|
for _, dn := range vidLocations.list {
|
|
if !volumeInfo.ReadOnly {
|
|
|
|
disk := dn.getOrCreateDisk(volumeInfo.DiskType)
|
|
disk.UpAdjustDiskUsageDelta(types.ToDiskType(volumeInfo.DiskType), &DiskUsageCounts{
|
|
activeVolumeCount: -1,
|
|
})
|
|
|
|
}
|
|
}
|
|
return true
|
|
}
|
|
|
|
func (t *Topology) SetVolumeCrowded(volumeInfo storage.VolumeInfo) {
|
|
diskType := types.ToDiskType(volumeInfo.DiskType)
|
|
vl := t.GetVolumeLayout(volumeInfo.Collection, volumeInfo.ReplicaPlacement, volumeInfo.Ttl, diskType)
|
|
vl.SetVolumeCrowded(volumeInfo.Id)
|
|
}
|
|
|
|
func (t *Topology) UnRegisterDataNode(dn *DataNode) {
|
|
dn.IsTerminating = true
|
|
for _, v := range dn.GetVolumes() {
|
|
glog.V(0).Infoln("Removing Volume", v.Id, "from the dead volume server", dn.Id())
|
|
diskType := types.ToDiskType(v.DiskType)
|
|
vl := t.GetVolumeLayout(v.Collection, v.ReplicaPlacement, v.Ttl, diskType)
|
|
vl.SetVolumeUnavailable(dn, v.Id)
|
|
}
|
|
|
|
// unregister ec shards when volume server disconnected
|
|
for _, s := range dn.GetEcShards() {
|
|
t.UnRegisterEcShards(s, dn)
|
|
}
|
|
|
|
negativeUsages := dn.GetDiskUsages().negative()
|
|
for dt, du := range negativeUsages.usages {
|
|
dn.UpAdjustDiskUsageDelta(dt, du)
|
|
}
|
|
dn.DeltaUpdateVolumes([]storage.VolumeInfo{}, dn.GetVolumes())
|
|
dn.DeltaUpdateEcShards([]*erasure_coding.EcVolumeInfo{}, dn.GetEcShards())
|
|
t.unregisterDataNodeAddress(dn.ServerAddress(), dn)
|
|
if dn.Parent() != nil {
|
|
dn.Parent().UnlinkChildNode(dn.Id())
|
|
}
|
|
}
|