seaweedfs/weed/topology/volume_layout_capacity_recovery_test.go

package topology

import (
	"testing"
	"time"

	"github.com/seaweedfs/seaweedfs/weed/storage/types"
)

// A volume removed by the heartbeat capacity path (not RecordAssign) must
// still recover once it shrinks — only works if SetVolumeCapacityFull stamps
// fullSince.
func TestSetVolumeCapacityFullStampsFullSinceAndRecovers(t *testing.T) {
	layout := `
{
  "dc1":{
    "rack1":{
      "server1":{
        "volumes":[
          {"id":1, "size":4000, "replication":"000"}
        ],
        "limit":10
      }
    }
  }
}
`
	topo, vl := setupPickTest(t, layout, 10000)

	initialActive := topo.diskUsages.usages[types.HardDriveType].activeVolumeCount
	initialWritables, _ := vl.GetWritableVolumeCount()
	if initialWritables != 1 {
		t.Fatalf("expected 1 writable volume initially, got %d", initialWritables)
	}

	if !vl.SetVolumeCapacityFull(1) {
		t.Fatalf("SetVolumeCapacityFull should report the volume was writable")
	}
	vl.AdjustActiveVolumeCountForFull(1)
	if w, _ := vl.GetWritableVolumeCount(); w != 0 {
		t.Fatalf("expected 0 writable after capacity full, got %d", w)
	}
	if vl.sizeTracking[1].fullSince.IsZero() {
		t.Fatalf("expected SetVolumeCapacityFull to stamp fullSince")
	}

	// No recovery before capacityRecoveryDelay.
	advanceSizeTrackingClock(vl, 1, 3*time.Second)
	if vl.UpdateVolumeSize(1, 4000, 0, true) {
		t.Fatalf("recovery should not fire before capacityRecoveryDelay")
	}

	// After the delay, a smaller size restores it.
	advanceSizeTrackingClock(vl, 1, capacityRecoveryDelay)
	if !vl.UpdateVolumeSize(1, 4000, 0, true) {
		t.Fatalf("expected volume to recover to writable after shrinking")
	}
	vl.AdjustActiveVolumeCountAfterRecovery(1)

	if w, _ := vl.GetWritableVolumeCount(); w != initialWritables {
		t.Fatalf("expected %d writable after recovery, got %d", initialWritables, w)
	}
	if got := topo.diskUsages.usages[types.HardDriveType].activeVolumeCount; got != initialActive {
		t.Fatalf("expected activeVolumeCount restored to %d, got %d", initialActive, got)
	}
	if !vl.sizeTracking[1].fullSince.IsZero() {
		t.Fatalf("expected fullSince cleared after recovery")
	}
}

func TestSetVolumeAvailableRestoresActiveCountForCapacityFullVolume(t *testing.T) {
	layout := `
{
  "dc1":{
    "rack1":{
      "server1":{
        "volumes":[
          {"id":1, "size":4000, "replication":"000"}
        ],
        "limit":10
      }
    }
  }
}
`
	topo, vl := setupPickTest(t, layout, 10000)

	initialActive := topo.diskUsages.usages[types.HardDriveType].activeVolumeCount
	if !vl.SetVolumeCapacityFull(1) {
		t.Fatalf("SetVolumeCapacityFull should report the volume was writable")
	}
	vl.AdjustActiveVolumeCountForFull(1)
	if got := topo.diskUsages.usages[types.HardDriveType].activeVolumeCount; got != initialActive-1 {
		t.Fatalf("expected activeVolumeCount decremented to %d, got %d", initialActive-1, got)
	}

	vl.accessLock.RLock()
	dn := vl.vid2location[1].list[0]
	vl.accessLock.RUnlock()

	if !vl.SetVolumeAvailable(dn, 1, false, false) {
		t.Fatalf("SetVolumeAvailable should report the volume became writable")
	}
	if got := topo.diskUsages.usages[types.HardDriveType].activeVolumeCount; got != initialActive {
		t.Fatalf("expected SetVolumeAvailable to restore activeVolumeCount to %d, got %d", initialActive, got)
	}
	if !vl.sizeTracking[1].fullSince.IsZero() {
		t.Fatalf("expected SetVolumeAvailable to clear fullSince after restoring activeVolumeCount")
	}

	advanceSizeTrackingClock(vl, 1, capacityRecoveryDelay+time.Second)
	if vl.UpdateVolumeSize(1, 4000, 0, true) {
		t.Fatalf("heartbeat recovery should not fire after SetVolumeAvailable already restored the volume")
	}
	if got := topo.diskUsages.usages[types.HardDriveType].activeVolumeCount; got != initialActive {
		t.Fatalf("expected activeVolumeCount to remain %d, got %d", initialActive, got)
	}
}

// A volume marked full by pending estimates takes no writes, so no heartbeat
// reports it again and only the periodic decay can bring it back.
func TestDecayQuietVolumeSizesRecoversPhantomFullVolume(t *testing.T) {
	layout := `
{
  "dc1":{
    "rack1":{
      "server1":{
        "volumes":[
          {"id":1, "size":4000, "replication":"000"}
        ],
        "limit":10
      }
    }
  }
}
`
	topo, vl := setupPickTest(t, layout, 10000)
	VolumeGrowStrategy.Threshold = 0.9

	initialActive := topo.diskUsages.usages[types.HardDriveType].activeVolumeCount
	if !vl.RecordAssign(1, 20000) {
		t.Fatalf("expected RecordAssign to remove the volume from writable")
	}
	vl.AdjustActiveVolumeCountForFull(1)
	if w, _ := vl.GetWritableVolumeCount(); w != 0 {
		t.Fatalf("expected 0 writable after the pending estimate filled the volume, got %d", w)
	}

	recovered := false
	for i := 0; i < 10 && !recovered; i++ {
		advanceSizeTrackingClock(vl, 1, capacityRecoveryDelay+time.Second)
		topo.DecayQuietVolumeSizes()
		w, _ := vl.GetWritableVolumeCount()
		recovered = w == 1
	}
	if !recovered {
		t.Fatalf("the periodic decay never returned the volume to the writable list")
	}
	if got := topo.diskUsages.usages[types.HardDriveType].activeVolumeCount; got != initialActive {
		t.Fatalf("expected activeVolumeCount restored to %d, got %d", initialActive, got)
	}
	if !vl.sizeTracking[1].fullSince.IsZero() {
		t.Fatalf("expected fullSince cleared after recovery")
	}
}

// The decay must not consume the replica-dedup window: a real report landing
// right behind it carries a size no volume server will send again, since only
// a volume whose content changed is reported at all.
func TestDecayQuietVolumeSizesKeepsTheNextHeartbeat(t *testing.T) {
	layout := `
{
  "dc1":{
    "rack1":{
      "server1":{
        "volumes":[
          {"id":1, "size":4000, "replication":"000"}
        ],
        "limit":10
      }
    }
  }
}
`
	topo, vl := setupPickTest(t, layout, 10000)

	vl.RecordAssign(1, 2000)
	advanceSizeTrackingClock(vl, 1, 30*time.Second)
	topo.DecayQuietVolumeSizes()

	vl.UpdateVolumeSize(1, 6000, 7, true)

	vl.accessLock.RLock()
	defer vl.accessLock.RUnlock()
	if got := vl.sizeTracking[1].reportedSize; got != 6000 {
		t.Errorf("the heartbeat after a decay left reportedSize at %d, want the reported 6000", got)
	}
	if got := vl.sizeTracking[1].compactRevision; got != 7 {
		t.Errorf("the heartbeat after a decay left compactRevision at %d, want the reported 7", got)
	}
}

// The decay picks its volumes under a read lock and replays them under a write
// one. A compaction heartbeat landing in that gap must survive, so the replay
// takes the record as it stands rather than the size the pass set out with —
// the arguments below stand in for that older snapshot.
func TestDecayQuietVolumeSizesDoesNotRollBackAHeartbeat(t *testing.T) {
	layout := `
{
  "dc1":{
    "rack1":{
      "server1":{
        "volumes":[
          {"id":1, "size":9000, "replication":"000"}
        ],
        "limit":10
      }
    }
  }
}
`
	_, vl := setupPickTest(t, layout, 10000)

	vl.RecordAssign(1, 500)
	// The heartbeat that wins the race: a compaction shrank the volume.
	vl.UpdateVolumeSize(1, 2000, 3, true)
	// Far enough behind that the dedup window no longer covers for the replay,
	// which is the only case where reading the record under the lock is what
	// saves the report.
	advanceSizeTrackingClock(vl, 1, 3*time.Second)

	vl.UpdateVolumeSize(1, 9000, 0, false)

	vl.accessLock.RLock()
	defer vl.accessLock.RUnlock()
	if got := vl.sizeTracking[1].reportedSize; got != 2000 {
		t.Errorf("the decay rolled reportedSize back to %d, want the compacted 2000", got)
	}
	if got := vl.sizeTracking[1].compactRevision; got != 3 {
		t.Errorf("the decay rolled compactRevision back to %d, want the reported 3", got)
	}
	if got := vl.sizeTracking[1].effectiveSize; got > 2000 {
		t.Errorf("effectiveSize %d outgrew the compacted size the heartbeat reported", got)
	}
}

// The decay stands in for a report that never came, so a heartbeat arriving
// after the pass chose the volume takes its place rather than adding to it:
// halving twice in one cycle forgets pending bytes the volume has yet to write.
func TestDecayQuietVolumeSizesYieldsToAHeartbeatItRaced(t *testing.T) {
	layout := `
{
  "dc1":{
    "rack1":{
      "server1":{
        "volumes":[
          {"id":1, "size":4000, "replication":"000"}
        ],
        "limit":10
      }
    }
  }
}
`
	_, vl := setupPickTest(t, layout, 10000)

	vl.RecordAssign(1, 4000)
	advanceSizeTrackingClock(vl, 1, 30*time.Second)

	// The heartbeat wins the race and does the one decay this cycle owes.
	vl.UpdateVolumeSize(1, 4000, 0, true)
	vl.accessLock.RLock()
	afterHeartbeat := vl.sizeTracking[1].effectiveSize
	vl.accessLock.RUnlock()
	if afterHeartbeat != 6000 {
		t.Fatalf("the heartbeat left effectiveSize at %d, want 6000", afterHeartbeat)
	}

	vl.UpdateVolumeSize(1, 0, 0, false)

	vl.accessLock.RLock()
	defer vl.accessLock.RUnlock()
	if got := vl.sizeTracking[1].effectiveSize; got != afterHeartbeat {
		t.Errorf("the decay halved again to %d, want the heartbeat's %d left alone", got, afterHeartbeat)
	}
}

// A volume the disk really did fill can never recover through the decay, so it
// must leave the candidate set instead of costing a write lock every pulse for
// the rest of its life.
func TestDecayQuietVolumeSizesSkipsAGenuinelyFullVolume(t *testing.T) {
	layout := `
{
  "dc1":{
    "rack1":{
      "server1":{
        "volumes":[
          {"id":1, "size":10000, "replication":"000"},
          {"id":2, "size":4000, "replication":"000"}
        ],
        "limit":10
      }
    }
  }
}
`
	_, vl := setupPickTest(t, layout, 10000)

	// Volume 1 is full on disk; volume 2 only looks full to the estimate.
	vl.SetVolumeCapacityFull(1)
	vl.RecordAssign(2, 6000)
	advanceSizeTrackingClock(vl, 1, 30*time.Second)
	advanceSizeTrackingClock(vl, 2, 30*time.Second)

	candidates := vl.quietDecayCandidates(10 * time.Second)
	for _, vid := range candidates {
		if vid == 1 {
			t.Errorf("the decay keeps taking the write lock for volume 1, which the disk really did fill")
		}
	}
	if len(candidates) != 1 || candidates[0] != 2 {
		t.Errorf("candidates %v, want only the phantom-full volume 2", candidates)
	}

	vl.DecayQuietVolumeSizes(10 * time.Second)

	vl.accessLock.RLock()
	defer vl.accessLock.RUnlock()
	if got := vl.sizeTracking[2].effectiveSize; got != 7000 {
		t.Errorf("the phantom-full volume was left at %d, want it decayed to 7000", got)
	}
}