mirror of
https://github.com/seaweedfs/seaweedfs.git
synced 2026-10-06 14:31:57 +02:00
Chris Lu
7e809c9991
* admin: delete persisted state when scan cancels pending tasks Each detection cycle cancels every pending task of a type before re-detecting it, and the cancel path saved the cancelled task back to disk. Nothing ever removed those files, so -dataDir/tasks gained one orphaned .pb per candidate volume per scan cycle. Cancelled is terminal, so drop the file the same way CompleteTask does for completed/failed tasks. The cancelled entry stays in memory for the UI until the next purge. Refs #11595 * admin: delete persisted state when CancelTask cancels a pending task The manual cancel path only updated memory, leaving the pending .pb on disk where a restart would resurrect the cancelled task as pending and the file would linger until then. Delete it like the scan-cycle cancel path now does. * admin: count cancelled tasks toward task retention cleanup CleanupOldTasks and ConfigPersistence.CleanupCompletedTasks only filtered completed/failed tasks, so cancelled entries were exempt from retention in both memory and on disk. Treat all terminal states alike; nil CompletedAt entries also count and sort last, so they are pruned first. * admin: run task file retention in the periodic cleanup loop cleanupCompletedTasks had no callers, so the on-disk retention bound never ran during uptime. Invoke it from performCleanup alongside the in-memory CleanupOldTasks sweep. * admin: guard task state writes against stale saves and failed deletes saveTaskState runs after mq.mutex is released, so the task may have gone terminal in between; a delayed pending save could then recreate the file a cancel just deleted and resurrect the task on restart. Skip saving non-terminal snapshots once the live task is terminal or gone. If a cancel file removal fails, fall back to writing the cancelled snapshot so the file is terminal rather than pending. deleteTaskState now returns its error, and CancelTask captures task.Status while still holding the queue lock. * admin: serialize task file check+write against cancel deletes The saveTaskState guard still had a check-then-write window: a pending snapshot could pass the terminal check before a cancel deleted the file, then write it back after. A persistMu on the queue now covers the check+save and the cancel paths' delete (with its terminal-state fallback), so the two cannot interleave for the same task.
109 lines
3.0 KiB
Go
109 lines
3.0 KiB
Go
package dash
|
|
|
|
import (
|
|
"fmt"
|
|
"os"
|
|
"path/filepath"
|
|
"testing"
|
|
|
|
"github.com/seaweedfs/seaweedfs/weed/admin/maintenance"
|
|
)
|
|
|
|
func countTaskStateFiles(t *testing.T, dir string) int {
|
|
t.Helper()
|
|
entries, err := os.ReadDir(filepath.Join(dir, TasksSubdir))
|
|
if os.IsNotExist(err) {
|
|
return 0
|
|
}
|
|
if err != nil {
|
|
t.Fatalf("read tasks dir: %v", err)
|
|
}
|
|
count := 0
|
|
for _, entry := range entries {
|
|
if !entry.IsDir() && filepath.Ext(entry.Name()) == ".pb" {
|
|
count++
|
|
}
|
|
}
|
|
return count
|
|
}
|
|
|
|
// TestCancelledTaskFilesDoNotAccumulate reproduces issue #11595: every scan
|
|
// cycle cancels pending tasks of each detected type and re-detects them, so a
|
|
// cancelled task file per candidate volume accumulates on disk forever.
|
|
func TestCancelledTaskFilesDoNotAccumulate(t *testing.T) {
|
|
dir := t.TempDir()
|
|
cp := NewConfigPersistence(dir)
|
|
|
|
queue := maintenance.NewMaintenanceQueue(nil)
|
|
queue.SetPersistence(cp)
|
|
|
|
for cycle := 0; cycle < 3; cycle++ {
|
|
queue.AddTask(&maintenance.MaintenanceTask{
|
|
ID: fmt.Sprintf("ec_vol_%d_cycle_%d", cycle, cycle),
|
|
Type: "erasure_coding",
|
|
VolumeID: uint32(cycle + 1),
|
|
Server: "server1",
|
|
})
|
|
if cancelled := queue.CancelPendingTasksByType("erasure_coding"); cancelled != 1 {
|
|
t.Fatalf("cycle %d: cancelled %d tasks, want 1", cycle, cancelled)
|
|
}
|
|
}
|
|
|
|
if n := countTaskStateFiles(t, dir); n != 0 {
|
|
t.Errorf("%d task files on disk after %d cancel cycles, want 0", n, 3)
|
|
}
|
|
}
|
|
|
|
// TestManuallyCancelledTaskFileIsRemoved covers the CancelTask path used by
|
|
// the UI: a cancelled pending task must not leave its file behind, where a
|
|
// restart would resurrect it as pending.
|
|
func TestManuallyCancelledTaskFileIsRemoved(t *testing.T) {
|
|
dir := t.TempDir()
|
|
cp := NewConfigPersistence(dir)
|
|
|
|
manager := maintenance.NewMaintenanceManager(nil, nil, cp)
|
|
queue := manager.GetQueue()
|
|
queue.SetPersistence(cp)
|
|
|
|
queue.AddTask(&maintenance.MaintenanceTask{
|
|
ID: "manual_1",
|
|
Type: "vacuum",
|
|
VolumeID: 7,
|
|
Server: "server1",
|
|
})
|
|
if n := countTaskStateFiles(t, dir); n != 1 {
|
|
t.Fatalf("%d task files after AddTask, want 1", n)
|
|
}
|
|
|
|
if err := manager.CancelTask("manual_1"); err != nil {
|
|
t.Fatalf("CancelTask: %v", err)
|
|
}
|
|
if n := countTaskStateFiles(t, dir); n != 0 {
|
|
t.Errorf("%d task files on disk after CancelTask, want 0", n)
|
|
}
|
|
}
|
|
|
|
// TestCleanupCompletedTasksBoundsCancelledFiles checks retention covers
|
|
// cancelled files, e.g. ones written by older versions.
|
|
func TestCleanupCompletedTasksBoundsCancelledFiles(t *testing.T) {
|
|
dir := t.TempDir()
|
|
cp := NewConfigPersistence(dir)
|
|
|
|
for i := 0; i < MaxCompletedTasks+5; i++ {
|
|
if err := cp.SaveTaskState(&maintenance.MaintenanceTask{
|
|
ID: fmt.Sprintf("old_cancelled_%02d", i),
|
|
Type: "erasure_coding",
|
|
Status: maintenance.TaskStatusCancelled,
|
|
}); err != nil {
|
|
t.Fatalf("save task state: %v", err)
|
|
}
|
|
}
|
|
|
|
if err := cp.CleanupCompletedTasks(); err != nil {
|
|
t.Fatalf("CleanupCompletedTasks: %v", err)
|
|
}
|
|
if n := countTaskStateFiles(t, dir); n > MaxCompletedTasks {
|
|
t.Errorf("%d task files after cleanup, want at most %d", n, MaxCompletedTasks)
|
|
}
|
|
}
|