mirror of
https://github.com/seaweedfs/seaweedfs.git
synced 2026-11-07 20:47:46 +01:00
* admin: re-send the master vacuum disable every poll so it survives leader changes
The admin disabled the master's scheduled vacuum only when a vacuum plugin
worker connected or disconnected. The master keeps that flag in memory and it
is not replicated through raft, so after a leader change or a leader restart
the new leader ran its own vacuum next to the plugin worker. When both picked
the same volume the master's commit failed ("compact swap aborted: missing
.cpd/.cpx") and it dropped that replica, leaving the volume under-replicated.
Send the desired state on every poll instead, and log on the master only
when the flag actually changes.
* admin: shorten the vacuum sync comments
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
---------
Co-authored-by: Kevin <kevin@leetify.com>
Co-authored-by: Chris Lu <chrislusf@users.noreply.github.com>
Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
161 lines
4.6 KiB
Go
161 lines
4.6 KiB
Go
package dash
|
|
|
|
import (
|
|
"errors"
|
|
"testing"
|
|
"time"
|
|
|
|
"github.com/seaweedfs/seaweedfs/weed/pb/plugin_pb"
|
|
|
|
adminplugin "github.com/seaweedfs/seaweedfs/weed/admin/plugin"
|
|
)
|
|
|
|
// TestVacuumMonitorStateTransitions verifies the state transition logic that
|
|
// drives the vacuum monitor goroutine: the plugin registry correctly reports
|
|
// whether a vacuum-capable worker is present or absent.
|
|
func TestVacuumMonitorStateTransitions(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
opts := adminplugin.Options{
|
|
DataDir: "",
|
|
}
|
|
p, err := adminplugin.New(opts)
|
|
if err != nil {
|
|
t.Fatalf("failed to create plugin: %v", err)
|
|
}
|
|
defer p.Shutdown()
|
|
|
|
// Initially no workers => no capable worker.
|
|
if p.HasCapableWorker("vacuum") {
|
|
t.Fatalf("expected no capable worker initially")
|
|
}
|
|
|
|
// Simulate state transition false -> true: worker connects.
|
|
p.WorkerConnectForTest(&plugin_pb.WorkerHello{
|
|
WorkerId: "vacuum-worker-1",
|
|
Capabilities: []*plugin_pb.JobTypeCapability{
|
|
{JobType: "vacuum", CanDetect: true, CanExecute: true},
|
|
},
|
|
})
|
|
|
|
if !p.HasCapableWorker("vacuum") {
|
|
t.Fatalf("expected capable worker after connect")
|
|
}
|
|
|
|
// Simulate state transition true -> false: worker disconnects.
|
|
p.WorkerDisconnectForTest("vacuum-worker-1")
|
|
|
|
// Give registry removal a moment (it's synchronous, but be safe).
|
|
time.Sleep(10 * time.Millisecond)
|
|
|
|
if p.HasCapableWorker("vacuum") {
|
|
t.Fatalf("expected no capable worker after disconnect")
|
|
}
|
|
}
|
|
|
|
// fakeToggler records disable/enable calls for testing.
|
|
type fakeToggler struct {
|
|
disableCalls int
|
|
enableCalls int
|
|
err error
|
|
}
|
|
|
|
func (f *fakeToggler) disableVacuum() error {
|
|
f.disableCalls++
|
|
return f.err
|
|
}
|
|
|
|
func (f *fakeToggler) enableVacuum() error {
|
|
f.enableCalls++
|
|
return f.err
|
|
}
|
|
|
|
func TestSyncVacuumState(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
t.Run("worker connects triggers disable", func(t *testing.T) {
|
|
tog := &fakeToggler{}
|
|
result, failed := syncVacuumState(true, false, tog, false)
|
|
if result != true || failed {
|
|
t.Error("expected true after disable, no failure")
|
|
}
|
|
if tog.disableCalls != 1 {
|
|
t.Errorf("expected 1 disable call, got %d", tog.disableCalls)
|
|
}
|
|
if tog.enableCalls != 0 {
|
|
t.Errorf("expected 0 enable calls, got %d", tog.enableCalls)
|
|
}
|
|
})
|
|
|
|
t.Run("worker disconnects triggers enable", func(t *testing.T) {
|
|
tog := &fakeToggler{}
|
|
result, failed := syncVacuumState(false, true, tog, false)
|
|
if result != false || failed {
|
|
t.Error("expected false after enable, no failure")
|
|
}
|
|
if tog.enableCalls != 1 {
|
|
t.Errorf("expected 1 enable call, got %d", tog.enableCalls)
|
|
}
|
|
if tog.disableCalls != 0 {
|
|
t.Errorf("expected 0 disable calls, got %d", tog.disableCalls)
|
|
}
|
|
})
|
|
|
|
// The master keeps the flag in memory only, so a new leader or a restarted
|
|
// master has lost it. The disable has to be re-sent while the worker stays
|
|
// connected, not only when it first connects.
|
|
t.Run("disable is re-sent every tick while the worker stays connected", func(t *testing.T) {
|
|
tog := &fakeToggler{}
|
|
active := false
|
|
for i := 0; i < 3; i++ {
|
|
var failed bool
|
|
active, failed = syncVacuumState(true, active, tog, false)
|
|
if !active || failed {
|
|
t.Fatalf("tick %d: expected active, no failure", i)
|
|
}
|
|
}
|
|
if tog.disableCalls != 3 {
|
|
t.Errorf("expected 3 disable calls, got %d", tog.disableCalls)
|
|
}
|
|
if tog.enableCalls != 0 {
|
|
t.Errorf("expected 0 enable calls, got %d", tog.enableCalls)
|
|
}
|
|
})
|
|
|
|
// Same for the enable side: a master that still holds an old plugin disable
|
|
// and becomes leader would otherwise never vacuum again.
|
|
t.Run("enable is re-sent every tick while no worker is connected", func(t *testing.T) {
|
|
tog := &fakeToggler{}
|
|
active := false
|
|
for i := 0; i < 3; i++ {
|
|
var failed bool
|
|
active, failed = syncVacuumState(false, active, tog, false)
|
|
if active || failed {
|
|
t.Fatalf("tick %d: expected inactive, no failure", i)
|
|
}
|
|
}
|
|
if tog.enableCalls != 3 {
|
|
t.Errorf("expected 3 enable calls, got %d", tog.enableCalls)
|
|
}
|
|
if tog.disableCalls != 0 {
|
|
t.Errorf("expected 0 disable calls, got %d", tog.disableCalls)
|
|
}
|
|
})
|
|
|
|
t.Run("failed call keeps the previous state and reports failure", func(t *testing.T) {
|
|
tog := &fakeToggler{err: errors.New("master unavailable")}
|
|
result, failed := syncVacuumState(true, false, tog, false)
|
|
if result != false || !failed {
|
|
t.Error("expected previous state false and a failure")
|
|
}
|
|
result, failed = syncVacuumState(true, true, tog, false)
|
|
if result != true || !failed {
|
|
t.Error("expected previous state true and a failure")
|
|
}
|
|
result, failed = syncVacuumState(false, true, tog, false)
|
|
if result != true || !failed {
|
|
t.Error("expected previous state true and a failure")
|
|
}
|
|
})
|
|
}
|