admin: expose Prometheus metrics (#9652)

* admin: add -metricsPort flag to expose Prometheus metrics

The admin command had no metrics endpoint, so passing -metricsPort
(as the operator does for spec.admin.metricsPort) crashed the process
with "flag provided but not defined". Wire up -metricsPort/-metricsIp
and start the shared Prometheus metrics server, matching filer, master,
and volume.

* admin: emit maintenance task and worker fleet metrics

Add Prometheus metrics for the admin server's distinctive work: the
maintenance task queue and the worker fleet that executes it.

Task lifecycle: maintenance_tasks_by_status / _by_type gauges (snapshot
of the queue), maintenance_tasks_completed_total{type,outcome} counter
and maintenance_task_duration_seconds{type} histogram (recorded when a
task reaches a terminal state), and last/next scan timestamp gauges.

Worker fleet: workers_connected and worker_slots{used,max} gauges, plus
worker_events_total{event} counting register/unregister/stale removals.

Gauges are snapshotted by a background goroutine on the admin server;
counters and the histogram are recorded at their event sites.

* admin: read worker slot totals under lock, clear next-scan gauge when idle

GetWorkers returns live worker pointers; summing CurrentLoad/MaxConcurrent
outside the queue lock races with task assignment and completion. Add
GetWorkerSlotTotals to aggregate under the lock.

Also reset maintenance_next_scan_timestamp_seconds to 0 when the scanner
is not running, so it can't retain a stale value after a stop.
This commit is contained in:
Chris Lu authored and GitHub committed 2026-05-24 14:09:02 -07:00
1 parent 6fc212cedb
commit 25beb7ec48
6 files changed
+198 -7

No files matched your search

+52
View File
@@ -23,6 +23,7 @@ import (
"github.com/seaweedfs/seaweedfs/weed/pb/plugin_pb"
"github.com/seaweedfs/seaweedfs/weed/pb/schema_pb"
"github.com/seaweedfs/seaweedfs/weed/security"
stats_collect "github.com/seaweedfs/seaweedfs/weed/stats"
"github.com/seaweedfs/seaweedfs/weed/storage/erasure_coding"
"github.com/seaweedfs/seaweedfs/weed/storage/super_block"
"github.com/seaweedfs/seaweedfs/weed/util"
@@ -280,6 +281,8 @@ func NewAdminServer(masters string, templateFS http.FileSystem, dataDir string,
go server.monitorVacuumWorker(bgCtx)
}
go server.publishMaintenanceMetrics(bgCtx)
return server
}
@@ -364,6 +367,55 @@ func (s *AdminServer) monitorVacuumWorker(ctx context.Context) {
}
}
// publishMaintenanceMetrics periodically snapshots the maintenance queue and
// worker fleet into Prometheus gauges. Counters and durations are recorded at
// their event sites; these gauges reflect current state at scrape resolution.
func (s *AdminServer) publishMaintenanceMetrics(ctx context.Context) {
const interval = 15 * time.Second
ticker := time.NewTicker(interval)
defer ticker.Stop()
for {
select {
case <-ctx.Done():
return
case <-ticker.C:
s.collectMaintenanceMetrics()
}
}
}
func (s *AdminServer) collectMaintenanceMetrics() {
if s.maintenanceManager == nil {
return
}
stats := s.maintenanceManager.GetStats()
stats_collect.AdminMaintenanceTasksByStatus.Reset()
for status, count := range stats.TasksByStatus {
stats_collect.AdminMaintenanceTasksByStatus.WithLabelValues(string(status)).Set(float64(count))
}
stats_collect.AdminMaintenanceTasksByType.Reset()
for taskType, count := range stats.TasksByType {
stats_collect.AdminMaintenanceTasksByType.WithLabelValues(string(taskType)).Set(float64(count))
}
// NextScanTime is only meaningful while the scanner runs; GetStats computes
// it unconditionally, so clear the gauge when idle to avoid a stale value.
if s.maintenanceManager.IsRunning() && !stats.NextScanTime.IsZero() {
stats_collect.AdminMaintenanceNextScanTimestampSeconds.Set(float64(stats.NextScanTime.Unix()))
} else {
stats_collect.AdminMaintenanceNextScanTimestampSeconds.Set(0)
}
workers, usedSlots, maxSlots := s.maintenanceManager.GetWorkerSlotTotals()
stats_collect.AdminWorkersConnected.Set(float64(workers))
stats_collect.AdminWorkerSlots.WithLabelValues("used").Set(float64(usedSlots))
stats_collect.AdminWorkerSlots.WithLabelValues("max").Set(float64(maxSlots))
}
// loadTaskConfigurationsFromPersistence loads saved task configurations from protobuf files
func (s *AdminServer) loadTaskConfigurationsFromPersistence() {
if s.configPersistence == nil || !s.configPersistence.IsConfigured() {
+9 -6
View File
@@ -16,6 +16,7 @@ import (
"github.com/seaweedfs/seaweedfs/weed/pb/plugin_pb"
"github.com/seaweedfs/seaweedfs/weed/pb/worker_pb"
"github.com/seaweedfs/seaweedfs/weed/security"
stats_collect "github.com/seaweedfs/seaweedfs/weed/stats"
"github.com/seaweedfs/seaweedfs/weed/util"
"google.golang.org/grpc"
"google.golang.org/grpc/codes"
@@ -233,6 +234,7 @@ func (s *WorkerGrpcServer) WorkerStream(stream worker_pb.WorkerService_WorkerStr
}
s.connections[workerID] = conn
s.connMutex.Unlock()
stats_collect.AdminWorkerEventsTotal.WithLabelValues("registered").Inc()
// Register worker with maintenance manager
s.registerWorkerWithManager(conn)
@@ -265,11 +267,11 @@ func (s *WorkerGrpcServer) WorkerStream(stream worker_pb.WorkerService_WorkerStr
select {
case <-ctx.Done():
glog.Infof("Worker %s connection closed: %v", workerID, ctx.Err())
s.unregisterWorker(conn)
s.unregisterWorker(conn, "unregistered")
return nil
case <-connCtx.Done():
glog.Infof("Worker %s connection cancelled", workerID)
s.unregisterWorker(conn)
s.unregisterWorker(conn, "unregistered")
return nil
default:
}
@@ -285,7 +287,7 @@ func (s *WorkerGrpcServer) WorkerStream(stream worker_pb.WorkerService_WorkerStr
default:
glog.Errorf("Error receiving from worker %s: %v", workerID, err)
}
s.unregisterWorker(conn)
s.unregisterWorker(conn, "unregistered")
return err
}
@@ -338,7 +340,7 @@ func (s *WorkerGrpcServer) handleWorkerMessage(conn *WorkerConnection, msg *work
case *worker_pb.WorkerMessage_Shutdown:
glog.Infof("Worker %s shutting down: %s", workerID, m.Shutdown.Reason)
s.unregisterWorker(conn)
s.unregisterWorker(conn, "unregistered")
default:
glog.Warningf("Unknown message type from worker %s", workerID)
@@ -605,7 +607,7 @@ func (s *WorkerGrpcServer) safeCloseOutgoingChannel(conn *WorkerConnection, sour
}
// unregisterWorker removes a worker connection
func (s *WorkerGrpcServer) unregisterWorker(conn *WorkerConnection) {
func (s *WorkerGrpcServer) unregisterWorker(conn *WorkerConnection, event string) {
s.connMutex.Lock()
existingConn, exists := s.connections[conn.workerID]
if !exists {
@@ -624,6 +626,7 @@ func (s *WorkerGrpcServer) unregisterWorker(conn *WorkerConnection) {
// Remove from map first to prevent duplicate cleanup attempts
delete(s.connections, conn.workerID)
s.connMutex.Unlock()
stats_collect.AdminWorkerEventsTotal.WithLabelValues(event).Inc()
// Cancel context to signal goroutines to stop
conn.cancel()
@@ -665,7 +668,7 @@ func (s *WorkerGrpcServer) cleanupStaleConnections() {
for _, conn := range toRemove {
glog.Warningf("Cleaning up stale worker connection: %s", conn.workerID)
s.unregisterWorker(conn)
s.unregisterWorker(conn, "stale_removed")
}
}
@@ -8,6 +8,7 @@ import (
"github.com/seaweedfs/seaweedfs/weed/glog"
"github.com/seaweedfs/seaweedfs/weed/pb/worker_pb"
stats_collect "github.com/seaweedfs/seaweedfs/weed/stats"
"github.com/seaweedfs/seaweedfs/weed/worker/tasks/balance"
"github.com/seaweedfs/seaweedfs/weed/worker/tasks/erasure_coding"
"github.com/seaweedfs/seaweedfs/weed/worker/tasks/vacuum"
@@ -315,6 +316,7 @@ func (mm *MaintenanceManager) performScan() {
glog.Infof("Starting maintenance scan...")
results, err := mm.scanner.ScanForMaintenanceTasks()
stats_collect.AdminMaintenanceLastScanTimestampSeconds.SetToCurrentTime()
if err != nil {
// Handle scan error
mm.mutex.Lock()
@@ -518,6 +520,11 @@ func (mm *MaintenanceManager) GetWorkers() []*MaintenanceWorker {
return mm.queue.GetWorkers()
}
// GetWorkerSlotTotals returns worker count and aggregate used/max task slots.
func (mm *MaintenanceManager) GetWorkerSlotTotals() (workers, used, max int) {
return mm.queue.GetWorkerSlotTotals()
}
// TriggerScan manually triggers a maintenance scan
func (mm *MaintenanceManager) TriggerScan() error {
return mm.triggerScanInternal(true)
+30 -1
View File
@@ -8,6 +8,7 @@ import (
"time"
"github.com/seaweedfs/seaweedfs/weed/glog"
stats_collect "github.com/seaweedfs/seaweedfs/weed/stats"
)
// NewMaintenanceQueue creates a new maintenance queue
@@ -482,12 +483,14 @@ func (mq *MaintenanceQueue) CompleteTask(taskID string, error string) {
// Calculate task duration
var duration time.Duration
if task.StartedAt != nil {
hadStart := task.StartedAt != nil
if hadStart {
duration = completedTime.Sub(*task.StartedAt)
}
// Capture workerID before it may be cleared during retry
originalWorkerID := task.WorkerID
taskType := string(task.Type)
var taskToSave *MaintenanceTask
var logFn func()
@@ -577,6 +580,18 @@ func (mq *MaintenanceQueue) CompleteTask(taskID string, error string) {
}
mq.mutex.Unlock()
// Record terminal-state metrics. A retry leaves the task pending, so it
// is not counted as completed or failed here.
switch taskStatus {
case TaskStatusCompleted:
stats_collect.AdminMaintenanceTasksCompletedTotal.WithLabelValues(taskType, "completed").Inc()
case TaskStatusFailed:
stats_collect.AdminMaintenanceTasksCompletedTotal.WithLabelValues(taskType, "failed").Inc()
}
if hadStart && (taskStatus == TaskStatusCompleted || taskStatus == TaskStatusFailed) {
stats_collect.AdminMaintenanceTaskDurationSeconds.WithLabelValues(taskType).Observe(duration.Seconds())
}
// Only persist non-terminal tasks (retries). Completed/failed tasks stay
// in memory for the UI but are not written to disk — they would just
// accumulate and slow down future startups.
@@ -819,6 +834,20 @@ func (mq *MaintenanceQueue) GetWorkers() []*MaintenanceWorker {
return workers
}
// GetWorkerSlotTotals aggregates worker count and used/max task slots under the
// lock, so callers don't read live worker fields that task updates mutate.
func (mq *MaintenanceQueue) GetWorkerSlotTotals() (workers, used, max int) {
mq.mutex.RLock()
defer mq.mutex.RUnlock()
for _, worker := range mq.workers {
workers++
used += worker.CurrentLoad
max += worker.MaxConcurrent
}
return
}
// generateTaskID generates a unique ID for tasks
func generateTaskID() string {
const charset = "abcdefghijklmnopqrstuvwxyz0123456789"