integration test. ec worker task

This commit is contained in:
chrislu committed 2025-07-24 07:50:03 -07:00
1 parent 0d8e1f8a17
commit 30ad65bb5f
21 files changed
+6363 -486

No files matched your search

+324
View File
@@ -0,0 +1,324 @@
package task
import (
"fmt"
"sync"
"time"
"github.com/seaweedfs/seaweedfs/weed/wdclient"
"github.com/seaweedfs/seaweedfs/weed/worker/types"
)
// AdminConfig contains configuration for the admin server
type AdminConfig struct {
ScanInterval time.Duration
WorkerTimeout time.Duration
TaskTimeout time.Duration
MaxRetries int
ReconcileInterval time.Duration
EnableFailureRecovery bool
MaxConcurrentTasks int
}
// AdminServer manages workers and tasks
type AdminServer struct {
config *AdminConfig
masterClient *wdclient.MasterClient
running bool
mutex sync.RWMutex
// Task management
tasks map[string]*types.Task
taskQueue []*types.Task
activeTasks map[string]*types.Task
// Worker management
workers map[string]*types.Worker
workerStatus map[string]*types.WorkerStatus
// Task history
taskHistory []TaskHistoryEntry
}
// TaskHistoryEntry represents a single task history entry
type TaskHistoryEntry struct {
TaskID string
TaskType types.TaskType
VolumeID uint32
WorkerID string
Status types.TaskStatus
StartedAt time.Time
CompletedAt time.Time
Duration time.Duration
ErrorMessage string
}
// SystemStats represents system statistics
type SystemStats struct {
ActiveTasks int
QueuedTasks int
ActiveWorkers int
TotalTasks int
}
// NewAdminServer creates a new admin server
func NewAdminServer(config *AdminConfig, masterClient *wdclient.MasterClient) *AdminServer {
return &AdminServer{
config: config,
masterClient: masterClient,
tasks: make(map[string]*types.Task),
taskQueue: make([]*types.Task, 0),
activeTasks: make(map[string]*types.Task),
workers: make(map[string]*types.Worker),
workerStatus: make(map[string]*types.WorkerStatus),
taskHistory: make([]TaskHistoryEntry, 0),
}
}
// Start starts the admin server
func (as *AdminServer) Start() error {
as.mutex.Lock()
defer as.mutex.Unlock()
if as.running {
return fmt.Errorf("admin server is already running")
}
as.running = true
return nil
}
// Stop stops the admin server
func (as *AdminServer) Stop() error {
as.mutex.Lock()
defer as.mutex.Unlock()
as.running = false
return nil
}
// RegisterWorker registers a new worker
func (as *AdminServer) RegisterWorker(worker *types.Worker) error {
as.mutex.Lock()
defer as.mutex.Unlock()
if !as.running {
return fmt.Errorf("admin server is not running")
}
as.workers[worker.ID] = worker
as.workerStatus[worker.ID] = &types.WorkerStatus{
Status: "active",
CurrentLoad: 0,
}
return nil
}
// QueueTask adds a new task to the task queue
func (as *AdminServer) QueueTask(task *types.Task) error {
as.mutex.Lock()
defer as.mutex.Unlock()
if !as.running {
return fmt.Errorf("admin server is not running")
}
if task.ID == "" {
task.ID = fmt.Sprintf("task-%d", time.Now().UnixNano())
}
task.Status = types.TaskStatusPending
task.CreatedAt = time.Now()
as.tasks[task.ID] = task
as.taskQueue = append(as.taskQueue, task)
return nil
}
// RequestTask requests a task for a worker
func (as *AdminServer) RequestTask(workerID string, capabilities []types.TaskType) (*types.Task, error) {
as.mutex.Lock()
defer as.mutex.Unlock()
if !as.running {
return nil, fmt.Errorf("admin server is not running")
}
// Check if worker exists
worker, exists := as.workers[workerID]
if !exists {
return nil, fmt.Errorf("worker %s not found", workerID)
}
// Check if worker has capacity
status := as.workerStatus[workerID]
if status.CurrentLoad >= worker.MaxConcurrent {
return nil, nil // No capacity
}
// Find a suitable task
for i, task := range as.taskQueue {
if task.Status != types.TaskStatusPending {
continue
}
// Check if worker can handle this task type
canHandle := false
for _, capability := range capabilities {
if task.Type == capability {
canHandle = true
break
}
}
if canHandle {
// Assign task to worker
task.Status = types.TaskStatusInProgress
task.WorkerID = workerID
now := time.Now()
task.StartedAt = &now
// Move task from queue to active tasks
as.taskQueue = append(as.taskQueue[:i], as.taskQueue[i+1:]...)
as.activeTasks[task.ID] = task
// Update worker load
status.CurrentLoad++
return task, nil
}
}
return nil, nil // No suitable task found
}
// UpdateTaskProgress updates task progress
func (as *AdminServer) UpdateTaskProgress(taskID string, progress float64) error {
as.mutex.Lock()
defer as.mutex.Unlock()
task, exists := as.tasks[taskID]
if !exists {
return fmt.Errorf("task %s not found", taskID)
}
task.Progress = progress
return nil
}
// CompleteTask marks a task as completed
func (as *AdminServer) CompleteTask(taskID string, success bool, errorMessage string) error {
as.mutex.Lock()
defer as.mutex.Unlock()
task, exists := as.tasks[taskID]
if !exists {
return fmt.Errorf("task %s not found", taskID)
}
// Update task status
if success {
task.Status = types.TaskStatusCompleted
} else {
task.Status = types.TaskStatusFailed
task.Error = errorMessage
}
now := time.Now()
task.CompletedAt = &now
// Remove from active tasks
delete(as.activeTasks, taskID)
// Update worker load
if task.WorkerID != "" {
if status, exists := as.workerStatus[task.WorkerID]; exists {
status.CurrentLoad--
}
}
// Add to history
var duration time.Duration
if task.StartedAt != nil {
duration = now.Sub(*task.StartedAt)
}
entry := TaskHistoryEntry{
TaskID: task.ID,
TaskType: task.Type,
VolumeID: task.VolumeID,
WorkerID: task.WorkerID,
Status: task.Status,
StartedAt: *task.StartedAt,
CompletedAt: now,
Duration: duration,
ErrorMessage: errorMessage,
}
as.taskHistory = append(as.taskHistory, entry)
return nil
}
// UpdateWorkerHeartbeat updates worker heartbeat
func (as *AdminServer) UpdateWorkerHeartbeat(workerID string, status *types.WorkerStatus) error {
as.mutex.Lock()
defer as.mutex.Unlock()
worker, exists := as.workers[workerID]
if !exists {
return fmt.Errorf("worker %s not found", workerID)
}
worker.LastHeartbeat = time.Now()
as.workerStatus[workerID] = status
return nil
}
// GetSystemStats returns system statistics
func (as *AdminServer) GetSystemStats() *SystemStats {
as.mutex.RLock()
defer as.mutex.RUnlock()
activeWorkers := 0
for _, status := range as.workerStatus {
if status.Status == "active" {
activeWorkers++
}
}
return &SystemStats{
ActiveTasks: len(as.activeTasks),
QueuedTasks: len(as.taskQueue),
ActiveWorkers: activeWorkers,
TotalTasks: len(as.tasks),
}
}
// GetQueuedTaskCount returns the number of queued tasks
func (as *AdminServer) GetQueuedTaskCount() int {
as.mutex.RLock()
defer as.mutex.RUnlock()
return len(as.taskQueue)
}
// GetActiveTaskCount returns the number of active tasks
func (as *AdminServer) GetActiveTaskCount() int {
as.mutex.RLock()
defer as.mutex.RUnlock()
return len(as.activeTasks)
}
// GetTaskHistory returns task history
func (as *AdminServer) GetTaskHistory() []TaskHistoryEntry {
as.mutex.RLock()
defer as.mutex.RUnlock()
// Return a copy of the history
history := make([]TaskHistoryEntry, len(as.taskHistory))
copy(history, as.taskHistory)
return history
}
+3
View File
@@ -0,0 +1,3 @@
module task_minimal
go 1.24.1
+233
View File
@@ -0,0 +1,233 @@
package task
import (
"fmt"
"testing"
"time"
"github.com/seaweedfs/seaweedfs/weed/worker/types"
)
// TestSimpleIntegration tests basic admin-worker operational flow without complex dependencies
func TestSimpleIntegration(t *testing.T) {
t.Logf("Starting simple integration test")
// Step 1: Create a minimal admin server configuration
config := &AdminConfig{
ScanInterval: 10 * time.Second,
WorkerTimeout: 30 * time.Second,
TaskTimeout: 2 * time.Hour,
MaxRetries: 3,
ReconcileInterval: 5 * time.Minute,
EnableFailureRecovery: true,
MaxConcurrentTasks: 5,
}
// Step 2: Create admin server with nil master client (for testing)
adminServer := NewAdminServer(config, nil)
// Step 3: Start admin server
err := adminServer.Start()
if err != nil {
t.Fatalf("Failed to start admin server: %v", err)
}
defer adminServer.Stop()
// Step 4: Test worker registration
t.Logf("Testing worker registration")
worker := &types.Worker{
ID: "test-worker-1",
Address: "localhost:9001",
Capabilities: []types.TaskType{types.TaskTypeVacuum},
MaxConcurrent: 2,
Status: "active",
CurrentLoad: 0,
LastHeartbeat: time.Now(),
}
err = adminServer.RegisterWorker(worker)
if err != nil {
t.Fatalf("Failed to register worker: %v", err)
}
t.Logf("Successfully registered worker %s", worker.ID)
// Step 5: Test task queueing
t.Logf("Testing task queueing")
task := &types.Task{
ID: "test-task-1",
Type: types.TaskTypeVacuum,
VolumeID: 1001,
Server: "localhost:8080",
Status: types.TaskStatusPending,
Priority: types.TaskPriorityNormal,
Parameters: map[string]interface{}{
"garbage_threshold": "0.3",
},
CreatedAt: time.Now(),
}
err = adminServer.QueueTask(task)
if err != nil {
t.Fatalf("Failed to queue task: %v", err)
}
t.Logf("Successfully queued task %s", task.ID)
// Step 6: Test task request by worker
t.Logf("Testing task request")
assignedTask, err := adminServer.RequestTask("test-worker-1", []types.TaskType{types.TaskTypeVacuum})
if err != nil {
t.Fatalf("Failed to request task: %v", err)
}
if assignedTask != nil {
t.Logf("Successfully assigned task %s to worker", assignedTask.ID)
// Step 7: Test task progress updates
t.Logf("Testing task progress updates")
err = adminServer.UpdateTaskProgress(assignedTask.ID, 50.0)
if err != nil {
t.Errorf("Failed to update task progress: %v", err)
}
err = adminServer.UpdateTaskProgress(assignedTask.ID, 100.0)
if err != nil {
t.Errorf("Failed to update task progress: %v", err)
}
// Step 8: Test task completion
t.Logf("Testing task completion")
err = adminServer.CompleteTask(assignedTask.ID, true, "")
if err != nil {
t.Errorf("Failed to complete task: %v", err)
}
t.Logf("Successfully completed task %s", assignedTask.ID)
} else {
t.Logf("No task was assigned (queue might be empty)")
}
// Step 9: Test basic metrics
t.Logf("Testing basic metrics")
stats := adminServer.GetSystemStats()
if stats != nil {
t.Logf("System stats: Active tasks=%d, Queued tasks=%d, Active workers=%d",
stats.ActiveTasks, stats.QueuedTasks, stats.ActiveWorkers)
}
queuedCount := adminServer.GetQueuedTaskCount()
activeCount := adminServer.GetActiveTaskCount()
t.Logf("Queue status: %d queued, %d active tasks", queuedCount, activeCount)
// Step 10: Test task history
history := adminServer.GetTaskHistory()
t.Logf("Task history contains %d entries", len(history))
t.Logf("Simple integration test completed successfully")
}
// TestWorkerHeartbeat tests worker heartbeat functionality
func TestWorkerHeartbeat(t *testing.T) {
t.Logf("Testing worker heartbeat")
config := &AdminConfig{
ScanInterval: 10 * time.Second,
WorkerTimeout: 30 * time.Second,
TaskTimeout: 2 * time.Hour,
MaxRetries: 3,
ReconcileInterval: 5 * time.Minute,
EnableFailureRecovery: true,
MaxConcurrentTasks: 5,
}
adminServer := NewAdminServer(config, nil)
err := adminServer.Start()
if err != nil {
t.Fatalf("Failed to start admin server: %v", err)
}
defer adminServer.Stop()
// Register a worker
worker := &types.Worker{
ID: "heartbeat-worker",
Address: "localhost:9002",
Capabilities: []types.TaskType{types.TaskTypeVacuum},
MaxConcurrent: 1,
Status: "active",
CurrentLoad: 0,
LastHeartbeat: time.Now(),
}
err = adminServer.RegisterWorker(worker)
if err != nil {
t.Fatalf("Failed to register worker: %v", err)
}
// Test heartbeat update
status := &types.WorkerStatus{
Status: "active",
CurrentLoad: 0,
}
err = adminServer.UpdateWorkerHeartbeat("heartbeat-worker", status)
if err != nil {
t.Errorf("Failed to update worker heartbeat: %v", err)
}
t.Logf("Worker heartbeat test completed successfully")
}
// TestTaskQueueOperations tests task queue operations
func TestTaskQueueOperations(t *testing.T) {
t.Logf("Testing task queue operations")
config := &AdminConfig{
ScanInterval: 10 * time.Second,
WorkerTimeout: 30 * time.Second,
TaskTimeout: 2 * time.Hour,
MaxRetries: 3,
ReconcileInterval: 5 * time.Minute,
EnableFailureRecovery: true,
MaxConcurrentTasks: 5,
}
adminServer := NewAdminServer(config, nil)
err := adminServer.Start()
if err != nil {
t.Fatalf("Failed to start admin server: %v", err)
}
defer adminServer.Stop()
// Test queuing multiple tasks
for i := 0; i < 3; i++ {
task := &types.Task{
ID: fmt.Sprintf("queue-test-task-%d", i),
Type: types.TaskTypeVacuum,
VolumeID: uint32(2000 + i),
Server: "localhost:8080",
Status: types.TaskStatusPending,
Priority: types.TaskPriorityNormal,
Parameters: map[string]interface{}{
"garbage_threshold": "0.3",
},
CreatedAt: time.Now(),
}
err = adminServer.QueueTask(task)
if err != nil {
t.Errorf("Failed to queue task %d: %v", i, err)
}
}
// Check queue size
queuedCount := adminServer.GetQueuedTaskCount()
if queuedCount != 3 {
t.Errorf("Expected 3 queued tasks, got %d", queuedCount)
}
t.Logf("Task queue operations test completed successfully")
}