mirror of
https://github.com/seaweedfs/seaweedfs.git
synced 2026-11-17 17:26:06 +01:00
integration test. ec worker task
This commit is contained in:
1 parent
0d8e1f8a17
commit
30ad65bb5f
21 files changed
+6363
-486
No files matched your search
@@ -0,0 +1,324 @@
|
||||
package task
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/seaweedfs/seaweedfs/weed/wdclient"
|
||||
"github.com/seaweedfs/seaweedfs/weed/worker/types"
|
||||
)
|
||||
|
||||
// AdminConfig contains configuration for the admin server
|
||||
type AdminConfig struct {
|
||||
ScanInterval time.Duration
|
||||
WorkerTimeout time.Duration
|
||||
TaskTimeout time.Duration
|
||||
MaxRetries int
|
||||
ReconcileInterval time.Duration
|
||||
EnableFailureRecovery bool
|
||||
MaxConcurrentTasks int
|
||||
}
|
||||
|
||||
// AdminServer manages workers and tasks
|
||||
type AdminServer struct {
|
||||
config *AdminConfig
|
||||
masterClient *wdclient.MasterClient
|
||||
running bool
|
||||
mutex sync.RWMutex
|
||||
|
||||
// Task management
|
||||
tasks map[string]*types.Task
|
||||
taskQueue []*types.Task
|
||||
activeTasks map[string]*types.Task
|
||||
|
||||
// Worker management
|
||||
workers map[string]*types.Worker
|
||||
workerStatus map[string]*types.WorkerStatus
|
||||
|
||||
// Task history
|
||||
taskHistory []TaskHistoryEntry
|
||||
}
|
||||
|
||||
// TaskHistoryEntry represents a single task history entry
|
||||
type TaskHistoryEntry struct {
|
||||
TaskID string
|
||||
TaskType types.TaskType
|
||||
VolumeID uint32
|
||||
WorkerID string
|
||||
Status types.TaskStatus
|
||||
StartedAt time.Time
|
||||
CompletedAt time.Time
|
||||
Duration time.Duration
|
||||
ErrorMessage string
|
||||
}
|
||||
|
||||
// SystemStats represents system statistics
|
||||
type SystemStats struct {
|
||||
ActiveTasks int
|
||||
QueuedTasks int
|
||||
ActiveWorkers int
|
||||
TotalTasks int
|
||||
}
|
||||
|
||||
// NewAdminServer creates a new admin server
|
||||
func NewAdminServer(config *AdminConfig, masterClient *wdclient.MasterClient) *AdminServer {
|
||||
return &AdminServer{
|
||||
config: config,
|
||||
masterClient: masterClient,
|
||||
tasks: make(map[string]*types.Task),
|
||||
taskQueue: make([]*types.Task, 0),
|
||||
activeTasks: make(map[string]*types.Task),
|
||||
workers: make(map[string]*types.Worker),
|
||||
workerStatus: make(map[string]*types.WorkerStatus),
|
||||
taskHistory: make([]TaskHistoryEntry, 0),
|
||||
}
|
||||
}
|
||||
|
||||
// Start starts the admin server
|
||||
func (as *AdminServer) Start() error {
|
||||
as.mutex.Lock()
|
||||
defer as.mutex.Unlock()
|
||||
|
||||
if as.running {
|
||||
return fmt.Errorf("admin server is already running")
|
||||
}
|
||||
|
||||
as.running = true
|
||||
return nil
|
||||
}
|
||||
|
||||
// Stop stops the admin server
|
||||
func (as *AdminServer) Stop() error {
|
||||
as.mutex.Lock()
|
||||
defer as.mutex.Unlock()
|
||||
|
||||
as.running = false
|
||||
return nil
|
||||
}
|
||||
|
||||
// RegisterWorker registers a new worker
|
||||
func (as *AdminServer) RegisterWorker(worker *types.Worker) error {
|
||||
as.mutex.Lock()
|
||||
defer as.mutex.Unlock()
|
||||
|
||||
if !as.running {
|
||||
return fmt.Errorf("admin server is not running")
|
||||
}
|
||||
|
||||
as.workers[worker.ID] = worker
|
||||
as.workerStatus[worker.ID] = &types.WorkerStatus{
|
||||
Status: "active",
|
||||
CurrentLoad: 0,
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// QueueTask adds a new task to the task queue
|
||||
func (as *AdminServer) QueueTask(task *types.Task) error {
|
||||
as.mutex.Lock()
|
||||
defer as.mutex.Unlock()
|
||||
|
||||
if !as.running {
|
||||
return fmt.Errorf("admin server is not running")
|
||||
}
|
||||
|
||||
if task.ID == "" {
|
||||
task.ID = fmt.Sprintf("task-%d", time.Now().UnixNano())
|
||||
}
|
||||
|
||||
task.Status = types.TaskStatusPending
|
||||
task.CreatedAt = time.Now()
|
||||
|
||||
as.tasks[task.ID] = task
|
||||
as.taskQueue = append(as.taskQueue, task)
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// RequestTask requests a task for a worker
|
||||
func (as *AdminServer) RequestTask(workerID string, capabilities []types.TaskType) (*types.Task, error) {
|
||||
as.mutex.Lock()
|
||||
defer as.mutex.Unlock()
|
||||
|
||||
if !as.running {
|
||||
return nil, fmt.Errorf("admin server is not running")
|
||||
}
|
||||
|
||||
// Check if worker exists
|
||||
worker, exists := as.workers[workerID]
|
||||
if !exists {
|
||||
return nil, fmt.Errorf("worker %s not found", workerID)
|
||||
}
|
||||
|
||||
// Check if worker has capacity
|
||||
status := as.workerStatus[workerID]
|
||||
if status.CurrentLoad >= worker.MaxConcurrent {
|
||||
return nil, nil // No capacity
|
||||
}
|
||||
|
||||
// Find a suitable task
|
||||
for i, task := range as.taskQueue {
|
||||
if task.Status != types.TaskStatusPending {
|
||||
continue
|
||||
}
|
||||
|
||||
// Check if worker can handle this task type
|
||||
canHandle := false
|
||||
for _, capability := range capabilities {
|
||||
if task.Type == capability {
|
||||
canHandle = true
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
if canHandle {
|
||||
// Assign task to worker
|
||||
task.Status = types.TaskStatusInProgress
|
||||
task.WorkerID = workerID
|
||||
now := time.Now()
|
||||
task.StartedAt = &now
|
||||
|
||||
// Move task from queue to active tasks
|
||||
as.taskQueue = append(as.taskQueue[:i], as.taskQueue[i+1:]...)
|
||||
as.activeTasks[task.ID] = task
|
||||
|
||||
// Update worker load
|
||||
status.CurrentLoad++
|
||||
|
||||
return task, nil
|
||||
}
|
||||
}
|
||||
|
||||
return nil, nil // No suitable task found
|
||||
}
|
||||
|
||||
// UpdateTaskProgress updates task progress
|
||||
func (as *AdminServer) UpdateTaskProgress(taskID string, progress float64) error {
|
||||
as.mutex.Lock()
|
||||
defer as.mutex.Unlock()
|
||||
|
||||
task, exists := as.tasks[taskID]
|
||||
if !exists {
|
||||
return fmt.Errorf("task %s not found", taskID)
|
||||
}
|
||||
|
||||
task.Progress = progress
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// CompleteTask marks a task as completed
|
||||
func (as *AdminServer) CompleteTask(taskID string, success bool, errorMessage string) error {
|
||||
as.mutex.Lock()
|
||||
defer as.mutex.Unlock()
|
||||
|
||||
task, exists := as.tasks[taskID]
|
||||
if !exists {
|
||||
return fmt.Errorf("task %s not found", taskID)
|
||||
}
|
||||
|
||||
// Update task status
|
||||
if success {
|
||||
task.Status = types.TaskStatusCompleted
|
||||
} else {
|
||||
task.Status = types.TaskStatusFailed
|
||||
task.Error = errorMessage
|
||||
}
|
||||
|
||||
now := time.Now()
|
||||
task.CompletedAt = &now
|
||||
|
||||
// Remove from active tasks
|
||||
delete(as.activeTasks, taskID)
|
||||
|
||||
// Update worker load
|
||||
if task.WorkerID != "" {
|
||||
if status, exists := as.workerStatus[task.WorkerID]; exists {
|
||||
status.CurrentLoad--
|
||||
}
|
||||
}
|
||||
|
||||
// Add to history
|
||||
var duration time.Duration
|
||||
if task.StartedAt != nil {
|
||||
duration = now.Sub(*task.StartedAt)
|
||||
}
|
||||
|
||||
entry := TaskHistoryEntry{
|
||||
TaskID: task.ID,
|
||||
TaskType: task.Type,
|
||||
VolumeID: task.VolumeID,
|
||||
WorkerID: task.WorkerID,
|
||||
Status: task.Status,
|
||||
StartedAt: *task.StartedAt,
|
||||
CompletedAt: now,
|
||||
Duration: duration,
|
||||
ErrorMessage: errorMessage,
|
||||
}
|
||||
as.taskHistory = append(as.taskHistory, entry)
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// UpdateWorkerHeartbeat updates worker heartbeat
|
||||
func (as *AdminServer) UpdateWorkerHeartbeat(workerID string, status *types.WorkerStatus) error {
|
||||
as.mutex.Lock()
|
||||
defer as.mutex.Unlock()
|
||||
|
||||
worker, exists := as.workers[workerID]
|
||||
if !exists {
|
||||
return fmt.Errorf("worker %s not found", workerID)
|
||||
}
|
||||
|
||||
worker.LastHeartbeat = time.Now()
|
||||
as.workerStatus[workerID] = status
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// GetSystemStats returns system statistics
|
||||
func (as *AdminServer) GetSystemStats() *SystemStats {
|
||||
as.mutex.RLock()
|
||||
defer as.mutex.RUnlock()
|
||||
|
||||
activeWorkers := 0
|
||||
for _, status := range as.workerStatus {
|
||||
if status.Status == "active" {
|
||||
activeWorkers++
|
||||
}
|
||||
}
|
||||
|
||||
return &SystemStats{
|
||||
ActiveTasks: len(as.activeTasks),
|
||||
QueuedTasks: len(as.taskQueue),
|
||||
ActiveWorkers: activeWorkers,
|
||||
TotalTasks: len(as.tasks),
|
||||
}
|
||||
}
|
||||
|
||||
// GetQueuedTaskCount returns the number of queued tasks
|
||||
func (as *AdminServer) GetQueuedTaskCount() int {
|
||||
as.mutex.RLock()
|
||||
defer as.mutex.RUnlock()
|
||||
return len(as.taskQueue)
|
||||
}
|
||||
|
||||
// GetActiveTaskCount returns the number of active tasks
|
||||
func (as *AdminServer) GetActiveTaskCount() int {
|
||||
as.mutex.RLock()
|
||||
defer as.mutex.RUnlock()
|
||||
return len(as.activeTasks)
|
||||
}
|
||||
|
||||
// GetTaskHistory returns task history
|
||||
func (as *AdminServer) GetTaskHistory() []TaskHistoryEntry {
|
||||
as.mutex.RLock()
|
||||
defer as.mutex.RUnlock()
|
||||
|
||||
// Return a copy of the history
|
||||
history := make([]TaskHistoryEntry, len(as.taskHistory))
|
||||
copy(history, as.taskHistory)
|
||||
return history
|
||||
}
|
||||
@@ -0,0 +1,3 @@
|
||||
module task_minimal
|
||||
|
||||
go 1.24.1
|
||||
@@ -0,0 +1,233 @@
|
||||
package task
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/seaweedfs/seaweedfs/weed/worker/types"
|
||||
)
|
||||
|
||||
// TestSimpleIntegration tests basic admin-worker operational flow without complex dependencies
|
||||
func TestSimpleIntegration(t *testing.T) {
|
||||
t.Logf("Starting simple integration test")
|
||||
|
||||
// Step 1: Create a minimal admin server configuration
|
||||
config := &AdminConfig{
|
||||
ScanInterval: 10 * time.Second,
|
||||
WorkerTimeout: 30 * time.Second,
|
||||
TaskTimeout: 2 * time.Hour,
|
||||
MaxRetries: 3,
|
||||
ReconcileInterval: 5 * time.Minute,
|
||||
EnableFailureRecovery: true,
|
||||
MaxConcurrentTasks: 5,
|
||||
}
|
||||
|
||||
// Step 2: Create admin server with nil master client (for testing)
|
||||
adminServer := NewAdminServer(config, nil)
|
||||
|
||||
// Step 3: Start admin server
|
||||
err := adminServer.Start()
|
||||
if err != nil {
|
||||
t.Fatalf("Failed to start admin server: %v", err)
|
||||
}
|
||||
defer adminServer.Stop()
|
||||
|
||||
// Step 4: Test worker registration
|
||||
t.Logf("Testing worker registration")
|
||||
|
||||
worker := &types.Worker{
|
||||
ID: "test-worker-1",
|
||||
Address: "localhost:9001",
|
||||
Capabilities: []types.TaskType{types.TaskTypeVacuum},
|
||||
MaxConcurrent: 2,
|
||||
Status: "active",
|
||||
CurrentLoad: 0,
|
||||
LastHeartbeat: time.Now(),
|
||||
}
|
||||
|
||||
err = adminServer.RegisterWorker(worker)
|
||||
if err != nil {
|
||||
t.Fatalf("Failed to register worker: %v", err)
|
||||
}
|
||||
t.Logf("Successfully registered worker %s", worker.ID)
|
||||
|
||||
// Step 5: Test task queueing
|
||||
t.Logf("Testing task queueing")
|
||||
|
||||
task := &types.Task{
|
||||
ID: "test-task-1",
|
||||
Type: types.TaskTypeVacuum,
|
||||
VolumeID: 1001,
|
||||
Server: "localhost:8080",
|
||||
Status: types.TaskStatusPending,
|
||||
Priority: types.TaskPriorityNormal,
|
||||
Parameters: map[string]interface{}{
|
||||
"garbage_threshold": "0.3",
|
||||
},
|
||||
CreatedAt: time.Now(),
|
||||
}
|
||||
|
||||
err = adminServer.QueueTask(task)
|
||||
if err != nil {
|
||||
t.Fatalf("Failed to queue task: %v", err)
|
||||
}
|
||||
t.Logf("Successfully queued task %s", task.ID)
|
||||
|
||||
// Step 6: Test task request by worker
|
||||
t.Logf("Testing task request")
|
||||
|
||||
assignedTask, err := adminServer.RequestTask("test-worker-1", []types.TaskType{types.TaskTypeVacuum})
|
||||
if err != nil {
|
||||
t.Fatalf("Failed to request task: %v", err)
|
||||
}
|
||||
|
||||
if assignedTask != nil {
|
||||
t.Logf("Successfully assigned task %s to worker", assignedTask.ID)
|
||||
|
||||
// Step 7: Test task progress updates
|
||||
t.Logf("Testing task progress updates")
|
||||
|
||||
err = adminServer.UpdateTaskProgress(assignedTask.ID, 50.0)
|
||||
if err != nil {
|
||||
t.Errorf("Failed to update task progress: %v", err)
|
||||
}
|
||||
|
||||
err = adminServer.UpdateTaskProgress(assignedTask.ID, 100.0)
|
||||
if err != nil {
|
||||
t.Errorf("Failed to update task progress: %v", err)
|
||||
}
|
||||
|
||||
// Step 8: Test task completion
|
||||
t.Logf("Testing task completion")
|
||||
|
||||
err = adminServer.CompleteTask(assignedTask.ID, true, "")
|
||||
if err != nil {
|
||||
t.Errorf("Failed to complete task: %v", err)
|
||||
}
|
||||
t.Logf("Successfully completed task %s", assignedTask.ID)
|
||||
} else {
|
||||
t.Logf("No task was assigned (queue might be empty)")
|
||||
}
|
||||
|
||||
// Step 9: Test basic metrics
|
||||
t.Logf("Testing basic metrics")
|
||||
|
||||
stats := adminServer.GetSystemStats()
|
||||
if stats != nil {
|
||||
t.Logf("System stats: Active tasks=%d, Queued tasks=%d, Active workers=%d",
|
||||
stats.ActiveTasks, stats.QueuedTasks, stats.ActiveWorkers)
|
||||
}
|
||||
|
||||
queuedCount := adminServer.GetQueuedTaskCount()
|
||||
activeCount := adminServer.GetActiveTaskCount()
|
||||
t.Logf("Queue status: %d queued, %d active tasks", queuedCount, activeCount)
|
||||
|
||||
// Step 10: Test task history
|
||||
history := adminServer.GetTaskHistory()
|
||||
t.Logf("Task history contains %d entries", len(history))
|
||||
|
||||
t.Logf("Simple integration test completed successfully")
|
||||
}
|
||||
|
||||
// TestWorkerHeartbeat tests worker heartbeat functionality
|
||||
func TestWorkerHeartbeat(t *testing.T) {
|
||||
t.Logf("Testing worker heartbeat")
|
||||
|
||||
config := &AdminConfig{
|
||||
ScanInterval: 10 * time.Second,
|
||||
WorkerTimeout: 30 * time.Second,
|
||||
TaskTimeout: 2 * time.Hour,
|
||||
MaxRetries: 3,
|
||||
ReconcileInterval: 5 * time.Minute,
|
||||
EnableFailureRecovery: true,
|
||||
MaxConcurrentTasks: 5,
|
||||
}
|
||||
|
||||
adminServer := NewAdminServer(config, nil)
|
||||
err := adminServer.Start()
|
||||
if err != nil {
|
||||
t.Fatalf("Failed to start admin server: %v", err)
|
||||
}
|
||||
defer adminServer.Stop()
|
||||
|
||||
// Register a worker
|
||||
worker := &types.Worker{
|
||||
ID: "heartbeat-worker",
|
||||
Address: "localhost:9002",
|
||||
Capabilities: []types.TaskType{types.TaskTypeVacuum},
|
||||
MaxConcurrent: 1,
|
||||
Status: "active",
|
||||
CurrentLoad: 0,
|
||||
LastHeartbeat: time.Now(),
|
||||
}
|
||||
|
||||
err = adminServer.RegisterWorker(worker)
|
||||
if err != nil {
|
||||
t.Fatalf("Failed to register worker: %v", err)
|
||||
}
|
||||
|
||||
// Test heartbeat update
|
||||
status := &types.WorkerStatus{
|
||||
Status: "active",
|
||||
CurrentLoad: 0,
|
||||
}
|
||||
|
||||
err = adminServer.UpdateWorkerHeartbeat("heartbeat-worker", status)
|
||||
if err != nil {
|
||||
t.Errorf("Failed to update worker heartbeat: %v", err)
|
||||
}
|
||||
|
||||
t.Logf("Worker heartbeat test completed successfully")
|
||||
}
|
||||
|
||||
// TestTaskQueueOperations tests task queue operations
|
||||
func TestTaskQueueOperations(t *testing.T) {
|
||||
t.Logf("Testing task queue operations")
|
||||
|
||||
config := &AdminConfig{
|
||||
ScanInterval: 10 * time.Second,
|
||||
WorkerTimeout: 30 * time.Second,
|
||||
TaskTimeout: 2 * time.Hour,
|
||||
MaxRetries: 3,
|
||||
ReconcileInterval: 5 * time.Minute,
|
||||
EnableFailureRecovery: true,
|
||||
MaxConcurrentTasks: 5,
|
||||
}
|
||||
|
||||
adminServer := NewAdminServer(config, nil)
|
||||
err := adminServer.Start()
|
||||
if err != nil {
|
||||
t.Fatalf("Failed to start admin server: %v", err)
|
||||
}
|
||||
defer adminServer.Stop()
|
||||
|
||||
// Test queuing multiple tasks
|
||||
for i := 0; i < 3; i++ {
|
||||
task := &types.Task{
|
||||
ID: fmt.Sprintf("queue-test-task-%d", i),
|
||||
Type: types.TaskTypeVacuum,
|
||||
VolumeID: uint32(2000 + i),
|
||||
Server: "localhost:8080",
|
||||
Status: types.TaskStatusPending,
|
||||
Priority: types.TaskPriorityNormal,
|
||||
Parameters: map[string]interface{}{
|
||||
"garbage_threshold": "0.3",
|
||||
},
|
||||
CreatedAt: time.Now(),
|
||||
}
|
||||
|
||||
err = adminServer.QueueTask(task)
|
||||
if err != nil {
|
||||
t.Errorf("Failed to queue task %d: %v", i, err)
|
||||
}
|
||||
}
|
||||
|
||||
// Check queue size
|
||||
queuedCount := adminServer.GetQueuedTaskCount()
|
||||
if queuedCount != 3 {
|
||||
t.Errorf("Expected 3 queued tasks, got %d", queuedCount)
|
||||
}
|
||||
|
||||
t.Logf("Task queue operations test completed successfully")
|
||||
}
|
||||
Reference in new issue
Block a user