mirror of
https://github.com/seaweedfs/seaweedfs.git
synced 2026-11-19 02:15:02 +01:00
* topology: keep the oversized verdict after the location list is gone The per-replica oversized marks live in the volume's location list, which is deleted when its last location leaves and recreated flagless on the next registration. Between that recreation and the next heartbeat that re-measures the volume, ensureCorrectWritables saw no mark and re-admitted an oversized volume to the writable list; two other paths (quarantine release, mark-writable) never consulted the marks at all. VolumeLayout now remembers the oversized verdict per vid, written only by heartbeat sizes and forgotten only when a report measures the volume under the limit. The writable admission gate itself refuses a marked volume, and the vacuum commit path drops a verdict its live size check has just disproved. Generated with [Devin](https://devin.ai) Co-Authored-By: Devin <158243242+devin-ai-integration[bot]@users.noreply.github.com> * topology: mark oversized at registration when the report carries a size RegisterVolumeLayout ran EnsureCorrectWritables while the recreated location list still held no per-replica marks. When the registering report knows the volume's size, refresh the oversized state before the writable check instead of waiting for the next heartbeat pass. Generated with [Devin](https://devin.ai) Co-Authored-By: Devin <158243242+devin-ai-integration[bot]@users.noreply.github.com> * topology: test that oversized volumes stay out of writables across re-registration Generated with [Devin](https://devin.ai) Co-Authored-By: Devin <158243242+devin-ai-integration[bot]@users.noreply.github.com> * topology: remember the oversized verdict per replica A shared per-volume verdict was cleared by any single replica's under-limit report, so a still-oversized replica whose location entry had died could re-register from a size-less report and put the volume back on the writable list. Keep one verdict per reporting replica: an under-limit report clears only its own, and a replica that never returns cannot pin its volume out of writables forever because the verdict ages out. The vacuum commit now feeds the node's fresh answer through the same per-replica bookkeeping instead of erasing the whole verdict. Generated with [Devin](https://devin.ai) Co-Authored-By: Devin <158243242+devin-ai-integration[bot]@users.noreply.github.com> * topology: count a volume at the size limit as full on vacuum commit The layout's oversized test is >= the limit, but the vacuum commit and status checks used >, so a replica sitting exactly at the limit passed as not-full and the commit path could re-admit it. Generated with [Devin](https://devin.ai) Co-Authored-By: Devin <158243242+devin-ai-integration[bot]@users.noreply.github.com> * topology: cover per-replica oversized verdicts in tests One replica shrinking must not clear a still-oversized replica's remembered verdict; cover that plus a size-less re-registration before the second replica is re-measured. The vacuum-commit case now shrinks the stored record first, since the commit re-derives the mark from the node's own answer. Generated with [Devin](https://devin.ai) Co-Authored-By: Devin <158243242+devin-ai-integration[bot]@users.noreply.github.com> * topology: judge the vacuum commit by the size it just measured SetVolumeAvailable rebuilt the oversized verdict from the cached VolumeInfo, which still carries the pre-compaction size until the next heartbeat. A commit that shrank a volume under the limit re-marked the replica oversized and never restored writes. The commit path already asks every replica for its fresh size; pass it through per replica and record it on the mark, the verdict, and the size tracking seed. * topology: hold an expired oversized verdict while its replica is back The TTL forgives a replica that never returns, but a remount announces the volume without a size and registration skips the oversized check. Deleting the verdict on expiry then re-admitted a still-oversized volume until the next size-bearing heartbeat. Keep counting an expired verdict while its replica sits in the live location list, and keep sweeping verdicts whose replicas stayed gone. --------- Co-authored-by: Devin <158243242+devin-ai-integration[bot]@users.noreply.github.com> Co-authored-by: chrislusf <chrislusf@gmail.com>
333 lines
10 KiB
Go
333 lines
10 KiB
Go
package topology
|
|
|
|
import (
|
|
"testing"
|
|
"time"
|
|
|
|
"github.com/seaweedfs/seaweedfs/weed/storage/types"
|
|
)
|
|
|
|
// A volume removed by the heartbeat capacity path (not RecordAssign) must
|
|
// still recover once it shrinks — only works if SetVolumeCapacityFull stamps
|
|
// fullSince.
|
|
func TestSetVolumeCapacityFullStampsFullSinceAndRecovers(t *testing.T) {
|
|
layout := `
|
|
{
|
|
"dc1":{
|
|
"rack1":{
|
|
"server1":{
|
|
"volumes":[
|
|
{"id":1, "size":4000, "replication":"000"}
|
|
],
|
|
"limit":10
|
|
}
|
|
}
|
|
}
|
|
}
|
|
`
|
|
topo, vl := setupPickTest(t, layout, 10000)
|
|
|
|
initialActive := topo.diskUsages.usages[types.HardDriveType].activeVolumeCount
|
|
initialWritables, _ := vl.GetWritableVolumeCount()
|
|
if initialWritables != 1 {
|
|
t.Fatalf("expected 1 writable volume initially, got %d", initialWritables)
|
|
}
|
|
|
|
if !vl.SetVolumeCapacityFull(1) {
|
|
t.Fatalf("SetVolumeCapacityFull should report the volume was writable")
|
|
}
|
|
vl.AdjustActiveVolumeCountForFull(1)
|
|
if w, _ := vl.GetWritableVolumeCount(); w != 0 {
|
|
t.Fatalf("expected 0 writable after capacity full, got %d", w)
|
|
}
|
|
if vl.sizeTracking[1].fullSince.IsZero() {
|
|
t.Fatalf("expected SetVolumeCapacityFull to stamp fullSince")
|
|
}
|
|
|
|
// No recovery before capacityRecoveryDelay.
|
|
advanceSizeTrackingClock(vl, 1, 3*time.Second)
|
|
if vl.UpdateVolumeSize(1, 4000, 0, true) {
|
|
t.Fatalf("recovery should not fire before capacityRecoveryDelay")
|
|
}
|
|
|
|
// After the delay, a smaller size restores it.
|
|
advanceSizeTrackingClock(vl, 1, capacityRecoveryDelay)
|
|
if !vl.UpdateVolumeSize(1, 4000, 0, true) {
|
|
t.Fatalf("expected volume to recover to writable after shrinking")
|
|
}
|
|
vl.AdjustActiveVolumeCountAfterRecovery(1)
|
|
|
|
if w, _ := vl.GetWritableVolumeCount(); w != initialWritables {
|
|
t.Fatalf("expected %d writable after recovery, got %d", initialWritables, w)
|
|
}
|
|
if got := topo.diskUsages.usages[types.HardDriveType].activeVolumeCount; got != initialActive {
|
|
t.Fatalf("expected activeVolumeCount restored to %d, got %d", initialActive, got)
|
|
}
|
|
if !vl.sizeTracking[1].fullSince.IsZero() {
|
|
t.Fatalf("expected fullSince cleared after recovery")
|
|
}
|
|
}
|
|
|
|
func TestSetVolumeAvailableRestoresActiveCountForCapacityFullVolume(t *testing.T) {
|
|
layout := `
|
|
{
|
|
"dc1":{
|
|
"rack1":{
|
|
"server1":{
|
|
"volumes":[
|
|
{"id":1, "size":4000, "replication":"000"}
|
|
],
|
|
"limit":10
|
|
}
|
|
}
|
|
}
|
|
}
|
|
`
|
|
topo, vl := setupPickTest(t, layout, 10000)
|
|
|
|
initialActive := topo.diskUsages.usages[types.HardDriveType].activeVolumeCount
|
|
if !vl.SetVolumeCapacityFull(1) {
|
|
t.Fatalf("SetVolumeCapacityFull should report the volume was writable")
|
|
}
|
|
vl.AdjustActiveVolumeCountForFull(1)
|
|
if got := topo.diskUsages.usages[types.HardDriveType].activeVolumeCount; got != initialActive-1 {
|
|
t.Fatalf("expected activeVolumeCount decremented to %d, got %d", initialActive-1, got)
|
|
}
|
|
|
|
vl.accessLock.RLock()
|
|
dn := vl.vid2location[1].list[0]
|
|
vl.accessLock.RUnlock()
|
|
|
|
if !vl.SetVolumeAvailable(dn, 1, 4000, false, false, false) {
|
|
t.Fatalf("SetVolumeAvailable should report the volume became writable")
|
|
}
|
|
if got := topo.diskUsages.usages[types.HardDriveType].activeVolumeCount; got != initialActive {
|
|
t.Fatalf("expected SetVolumeAvailable to restore activeVolumeCount to %d, got %d", initialActive, got)
|
|
}
|
|
if !vl.sizeTracking[1].fullSince.IsZero() {
|
|
t.Fatalf("expected SetVolumeAvailable to clear fullSince after restoring activeVolumeCount")
|
|
}
|
|
|
|
advanceSizeTrackingClock(vl, 1, capacityRecoveryDelay+time.Second)
|
|
if vl.UpdateVolumeSize(1, 4000, 0, true) {
|
|
t.Fatalf("heartbeat recovery should not fire after SetVolumeAvailable already restored the volume")
|
|
}
|
|
if got := topo.diskUsages.usages[types.HardDriveType].activeVolumeCount; got != initialActive {
|
|
t.Fatalf("expected activeVolumeCount to remain %d, got %d", initialActive, got)
|
|
}
|
|
}
|
|
|
|
// A volume marked full by pending estimates takes no writes, so no heartbeat
|
|
// reports it again and only the periodic decay can bring it back.
|
|
func TestDecayQuietVolumeSizesRecoversPhantomFullVolume(t *testing.T) {
|
|
layout := `
|
|
{
|
|
"dc1":{
|
|
"rack1":{
|
|
"server1":{
|
|
"volumes":[
|
|
{"id":1, "size":4000, "replication":"000"}
|
|
],
|
|
"limit":10
|
|
}
|
|
}
|
|
}
|
|
}
|
|
`
|
|
topo, vl := setupPickTest(t, layout, 10000)
|
|
VolumeGrowStrategy.Threshold = 0.9
|
|
|
|
initialActive := topo.diskUsages.usages[types.HardDriveType].activeVolumeCount
|
|
if !vl.RecordAssign(1, 20000) {
|
|
t.Fatalf("expected RecordAssign to remove the volume from writable")
|
|
}
|
|
vl.AdjustActiveVolumeCountForFull(1)
|
|
if w, _ := vl.GetWritableVolumeCount(); w != 0 {
|
|
t.Fatalf("expected 0 writable after the pending estimate filled the volume, got %d", w)
|
|
}
|
|
|
|
recovered := false
|
|
for i := 0; i < 10 && !recovered; i++ {
|
|
advanceSizeTrackingClock(vl, 1, capacityRecoveryDelay+time.Second)
|
|
topo.DecayQuietVolumeSizes()
|
|
w, _ := vl.GetWritableVolumeCount()
|
|
recovered = w == 1
|
|
}
|
|
if !recovered {
|
|
t.Fatalf("the periodic decay never returned the volume to the writable list")
|
|
}
|
|
if got := topo.diskUsages.usages[types.HardDriveType].activeVolumeCount; got != initialActive {
|
|
t.Fatalf("expected activeVolumeCount restored to %d, got %d", initialActive, got)
|
|
}
|
|
if !vl.sizeTracking[1].fullSince.IsZero() {
|
|
t.Fatalf("expected fullSince cleared after recovery")
|
|
}
|
|
}
|
|
|
|
// The decay must not consume the replica-dedup window: a real report landing
|
|
// right behind it carries a size no volume server will send again, since only
|
|
// a volume whose content changed is reported at all.
|
|
func TestDecayQuietVolumeSizesKeepsTheNextHeartbeat(t *testing.T) {
|
|
layout := `
|
|
{
|
|
"dc1":{
|
|
"rack1":{
|
|
"server1":{
|
|
"volumes":[
|
|
{"id":1, "size":4000, "replication":"000"}
|
|
],
|
|
"limit":10
|
|
}
|
|
}
|
|
}
|
|
}
|
|
`
|
|
topo, vl := setupPickTest(t, layout, 10000)
|
|
|
|
vl.RecordAssign(1, 2000)
|
|
advanceSizeTrackingClock(vl, 1, 30*time.Second)
|
|
topo.DecayQuietVolumeSizes()
|
|
|
|
vl.UpdateVolumeSize(1, 6000, 7, true)
|
|
|
|
vl.accessLock.RLock()
|
|
defer vl.accessLock.RUnlock()
|
|
if got := vl.sizeTracking[1].reportedSize; got != 6000 {
|
|
t.Errorf("the heartbeat after a decay left reportedSize at %d, want the reported 6000", got)
|
|
}
|
|
if got := vl.sizeTracking[1].compactRevision; got != 7 {
|
|
t.Errorf("the heartbeat after a decay left compactRevision at %d, want the reported 7", got)
|
|
}
|
|
}
|
|
|
|
// The decay picks its volumes under a read lock and replays them under a write
|
|
// one. A compaction heartbeat landing in that gap must survive, so the replay
|
|
// takes the record as it stands rather than the size the pass set out with —
|
|
// the arguments below stand in for that older snapshot.
|
|
func TestDecayQuietVolumeSizesDoesNotRollBackAHeartbeat(t *testing.T) {
|
|
layout := `
|
|
{
|
|
"dc1":{
|
|
"rack1":{
|
|
"server1":{
|
|
"volumes":[
|
|
{"id":1, "size":9000, "replication":"000"}
|
|
],
|
|
"limit":10
|
|
}
|
|
}
|
|
}
|
|
}
|
|
`
|
|
_, vl := setupPickTest(t, layout, 10000)
|
|
|
|
vl.RecordAssign(1, 500)
|
|
// The heartbeat that wins the race: a compaction shrank the volume.
|
|
vl.UpdateVolumeSize(1, 2000, 3, true)
|
|
// Far enough behind that the dedup window no longer covers for the replay,
|
|
// which is the only case where reading the record under the lock is what
|
|
// saves the report.
|
|
advanceSizeTrackingClock(vl, 1, 3*time.Second)
|
|
|
|
vl.UpdateVolumeSize(1, 9000, 0, false)
|
|
|
|
vl.accessLock.RLock()
|
|
defer vl.accessLock.RUnlock()
|
|
if got := vl.sizeTracking[1].reportedSize; got != 2000 {
|
|
t.Errorf("the decay rolled reportedSize back to %d, want the compacted 2000", got)
|
|
}
|
|
if got := vl.sizeTracking[1].compactRevision; got != 3 {
|
|
t.Errorf("the decay rolled compactRevision back to %d, want the reported 3", got)
|
|
}
|
|
if got := vl.sizeTracking[1].effectiveSize; got > 2000 {
|
|
t.Errorf("effectiveSize %d outgrew the compacted size the heartbeat reported", got)
|
|
}
|
|
}
|
|
|
|
// The decay stands in for a report that never came, so a heartbeat arriving
|
|
// after the pass chose the volume takes its place rather than adding to it:
|
|
// halving twice in one cycle forgets pending bytes the volume has yet to write.
|
|
func TestDecayQuietVolumeSizesYieldsToAHeartbeatItRaced(t *testing.T) {
|
|
layout := `
|
|
{
|
|
"dc1":{
|
|
"rack1":{
|
|
"server1":{
|
|
"volumes":[
|
|
{"id":1, "size":4000, "replication":"000"}
|
|
],
|
|
"limit":10
|
|
}
|
|
}
|
|
}
|
|
}
|
|
`
|
|
_, vl := setupPickTest(t, layout, 10000)
|
|
|
|
vl.RecordAssign(1, 4000)
|
|
advanceSizeTrackingClock(vl, 1, 30*time.Second)
|
|
|
|
// The heartbeat wins the race and does the one decay this cycle owes.
|
|
vl.UpdateVolumeSize(1, 4000, 0, true)
|
|
vl.accessLock.RLock()
|
|
afterHeartbeat := vl.sizeTracking[1].effectiveSize
|
|
vl.accessLock.RUnlock()
|
|
if afterHeartbeat != 6000 {
|
|
t.Fatalf("the heartbeat left effectiveSize at %d, want 6000", afterHeartbeat)
|
|
}
|
|
|
|
vl.UpdateVolumeSize(1, 0, 0, false)
|
|
|
|
vl.accessLock.RLock()
|
|
defer vl.accessLock.RUnlock()
|
|
if got := vl.sizeTracking[1].effectiveSize; got != afterHeartbeat {
|
|
t.Errorf("the decay halved again to %d, want the heartbeat's %d left alone", got, afterHeartbeat)
|
|
}
|
|
}
|
|
|
|
// A volume the disk really did fill can never recover through the decay, so it
|
|
// must leave the candidate set instead of costing a write lock every pulse for
|
|
// the rest of its life.
|
|
func TestDecayQuietVolumeSizesSkipsAGenuinelyFullVolume(t *testing.T) {
|
|
layout := `
|
|
{
|
|
"dc1":{
|
|
"rack1":{
|
|
"server1":{
|
|
"volumes":[
|
|
{"id":1, "size":10000, "replication":"000"},
|
|
{"id":2, "size":4000, "replication":"000"}
|
|
],
|
|
"limit":10
|
|
}
|
|
}
|
|
}
|
|
}
|
|
`
|
|
_, vl := setupPickTest(t, layout, 10000)
|
|
|
|
// Volume 1 is full on disk; volume 2 only looks full to the estimate.
|
|
vl.SetVolumeCapacityFull(1)
|
|
vl.RecordAssign(2, 6000)
|
|
advanceSizeTrackingClock(vl, 1, 30*time.Second)
|
|
advanceSizeTrackingClock(vl, 2, 30*time.Second)
|
|
|
|
candidates := vl.quietDecayCandidates(10 * time.Second)
|
|
for _, vid := range candidates {
|
|
if vid == 1 {
|
|
t.Errorf("the decay keeps taking the write lock for volume 1, which the disk really did fill")
|
|
}
|
|
}
|
|
if len(candidates) != 1 || candidates[0] != 2 {
|
|
t.Errorf("candidates %v, want only the phantom-full volume 2", candidates)
|
|
}
|
|
|
|
vl.DecayQuietVolumeSizes(10 * time.Second)
|
|
|
|
vl.accessLock.RLock()
|
|
defer vl.accessLock.RUnlock()
|
|
if got := vl.sizeTracking[2].effectiveSize; got != 7000 {
|
|
t.Errorf("the phantom-full volume was left at %d, want it decayed to 7000", got)
|
|
}
|
|
}
|