From 12ce9ac76237a9ed0291287f1038ae08246c75b8 Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Mon, 14 Sep 2026 23:42:05 -0700 Subject: [PATCH] admin: key monitoring queries by metric name and scrape endpoint Sources are now metrics endpoints rather than component names, so cluster-wide aggregates match on any source and let the metric name identify the component. Detail pages iterate the scrape endpoints and only list one when it actually serves that component's series, which keeps a combined "weed server" from appearing as four separate nodes. The empty-state notice now tells operators to start servers with -metricsPort, since that is what creates the listener. --- weed/admin/dash/metrics_scraper.go | 19 ++++ weed/admin/dash/monitoring_data.go | 119 ++++++++++++------------ weed/admin/view/app/monitoring.templ | 8 +- weed/admin/view/app/monitoring_templ.go | 24 ++--- 4 files changed, 98 insertions(+), 72 deletions(-) diff --git a/weed/admin/dash/metrics_scraper.go b/weed/admin/dash/metrics_scraper.go index f3c37d689..2557bae77 100644 --- a/weed/admin/dash/metrics_scraper.go +++ b/weed/admin/dash/metrics_scraper.go @@ -169,6 +169,25 @@ type scrapeTarget struct { nodes []string } +// label names the target for the UI: the node that advertised the endpoint, or +// the endpoint address itself when several nodes share one listener. +func (t scrapeTarget) label() string { + if len(t.nodes) == 1 { + return t.nodes[0] + } + return t.address +} + +// anyNodeMatches reports whether any node behind this endpoint is flagged. +func (t scrapeTarget) anyNodeMatches(flagged map[string]bool) bool { + for _, node := range t.nodes { + if flagged[node] { + return true + } + } + return false +} + // scrapeTargets lists the distinct metrics endpoints advertised by the cluster. // Nodes started without -metricsPort advertise 0 and are skipped, so nothing is // scraped from a client-facing service port. diff --git a/weed/admin/dash/monitoring_data.go b/weed/admin/dash/monitoring_data.go index 015d9a80a..548d74d90 100644 --- a/weed/admin/dash/monitoring_data.go +++ b/weed/admin/dash/monitoring_data.go @@ -39,13 +39,13 @@ const ( mAdminWorkers = "SeaweedFS_admin_workers_connected" ) -// Scrape source components. +// Store sources. Scraped series are keyed by the metrics endpoint they came +// from, so cluster-wide aggregates match on srcAny and rely on the metric name +// to identify the component: one endpoint can serve master, volume, filer and +// S3 series at once when they share a process. const ( - srcMaster = "master" - srcVolume = "volume" - srcFiler = "filer" - srcS3 = "s3" - srcAdmin = "admin/local" + srcAdmin = "admin/local" + srcAny = "" ) type MonitoringData struct { @@ -186,25 +186,25 @@ func (s *AdminServer) fillOverview(d *MonitoringData) { o.UnderReplicatedVolumes = s.sum(leader, mMasterUnderReplicated) o.WritableVolumes = s.sum(leader, mMasterWritable) o.CrowdedVolumes = s.sum(leader, mMasterCrowded) - o.DiskUsagePct = s.diskUsagePct(srcVolume) + o.DiskUsagePct = s.diskUsagePct(srcAny) - o.VolumeReadRate = s.sumFiltered(srcVolume, mVolumeRequests+suffixRate, isReadRequest) - o.VolumeWriteRate = s.sumFiltered(srcVolume, mVolumeRequests+suffixRate, isWriteRequest) - o.FilerRequestRate = s.sum(srcFiler, mFilerRequests+suffixRate) - o.S3RequestRate = s.sum(srcS3, mS3Requests+suffixRate) + o.VolumeReadRate = s.sumFiltered(srcAny, mVolumeRequests+suffixRate, isReadRequest) + o.VolumeWriteRate = s.sumFiltered(srcAny, mVolumeRequests+suffixRate, isWriteRequest) + o.FilerRequestRate = s.sum(srcAny, mFilerRequests+suffixRate) + o.S3RequestRate = s.sum(srcAny, mS3Requests+suffixRate) - o.VolumeP50 = s.max(srcVolume, mVolumeLatency+suffixP50) - o.VolumeP95 = s.max(srcVolume, mVolumeLatency+suffixP95) - o.VolumeP99 = s.max(srcVolume, mVolumeLatency+suffixP99) - o.FilerP50 = s.max(srcFiler, mFilerLatency+suffixP50) - o.FilerP95 = s.max(srcFiler, mFilerLatency+suffixP95) - o.FilerP99 = s.max(srcFiler, mFilerLatency+suffixP99) + o.VolumeP50 = s.max(srcAny, mVolumeLatency+suffixP50) + o.VolumeP95 = s.max(srcAny, mVolumeLatency+suffixP95) + o.VolumeP99 = s.max(srcAny, mVolumeLatency+suffixP99) + o.FilerP50 = s.max(srcAny, mFilerLatency+suffixP50) + o.FilerP95 = s.max(srcAny, mFilerLatency+suffixP95) + o.FilerP99 = s.max(srcAny, mFilerLatency+suffixP99) - o.VolumeErrorRate = s.sumFiltered(srcVolume, mVolumeRequests+suffixRate, isErrorCode) - o.FilerErrorRate = s.sumFiltered(srcFiler, mFilerRequests+suffixRate, isErrorCode) - o.S3ErrorRate = s.sumFiltered(srcS3, mS3Requests+suffixRate, isErrorCode) - o.DiskErrors = s.sum(srcVolume, mVolumeDiskError) - o.Quarantined = s.sum(srcVolume, mVolumeQuarantine) + o.VolumeErrorRate = s.sumFiltered(srcAny, mVolumeRequests+suffixRate, isErrorCode) + o.FilerErrorRate = s.sumFiltered(srcAny, mFilerRequests+suffixRate, isErrorCode) + o.S3ErrorRate = s.sumFiltered(srcAny, mS3Requests+suffixRate, isErrorCode) + o.DiskErrors = s.sum(srcAny, mVolumeDiskError) + o.Quarantined = s.sum(srcAny, mVolumeQuarantine) o.QueueDepth = s.sumFiltered(srcAdmin, mAdminTasksByStatus, func(l map[string]string) bool { return l["status"] == "pending" || l["status"] == "assigned" || l["status"] == "in_progress" @@ -214,25 +214,27 @@ func (s *AdminServer) fillOverview(d *MonitoringData) { } func (s *AdminServer) fillVolumeServers(d *MonitoringData) { - for _, addr := range s.sourceAddresses(srcVolume) { - src := srcVolume + "/" + addr + for _, t := range s.scrapeTargets() { + src := t.source vs := MonitoringVolumeServer{ - Address: addr, + Address: t.label(), RequestRate: s.sum(src, mVolumeRequests+suffixRate), P99: s.max(src, mVolumeLatency+suffixP99), DiskUsagePct: s.diskUsagePct(src), ErrorRate: s.sumFiltered(src, mVolumeRequests+suffixRate, isErrorCode), } vs.HasData = len(vs.RequestRate) > 0 || len(vs.DiskUsagePct) > 0 - d.VolumeServers = append(d.VolumeServers, vs) + if vs.HasData { + d.VolumeServers = append(d.VolumeServers, vs) + } } } func (s *AdminServer) fillFilers(d *MonitoringData) { - for _, addr := range s.sourceAddresses(srcFiler) { - src := srcFiler + "/" + addr + for _, t := range s.scrapeTargets() { + src := t.source f := MonitoringFiler{ - Address: addr, + Address: t.label(), RequestRate: s.sum(src, mFilerRequests+suffixRate), P99: s.max(src, mFilerLatency+suffixP99), StoreP99: s.max(src, mFilerStoreLat+suffixP99), @@ -240,15 +242,17 @@ func (s *AdminServer) fillFilers(d *MonitoringData) { SyncLag: s.max(src, mFilerSyncLag), } f.HasData = len(f.RequestRate) > 0 || len(f.InFlight) > 0 - d.Filers = append(d.Filers, f) + if f.HasData { + d.Filers = append(d.Filers, f) + } } } func (s *AdminServer) fillS3(d *MonitoringData) { - for _, addr := range s.sourceAddresses(srcS3) { - src := srcS3 + "/" + addr + for _, t := range s.scrapeTargets() { + src := t.source n := MonitoringS3{ - Address: addr, + Address: t.label(), RequestRate: s.sum(src, mS3Requests+suffixRate), Errors4xx: s.sumFiltered(src, mS3Requests+suffixRate, func(l map[string]string) bool { return strings.HasPrefix(l["code"], "4") @@ -259,7 +263,9 @@ func (s *AdminServer) fillS3(d *MonitoringData) { P99: s.max(src, mS3Latency+suffixP99), } n.HasData = len(n.RequestRate) > 0 - d.S3 = append(d.S3, n) + if n.HasData { + d.S3 = append(d.S3, n) + } } } @@ -270,18 +276,20 @@ func (s *AdminServer) fillMasters(d *MonitoringData) { leaders[m.Address] = m.IsLeader } } - for _, addr := range s.sourceAddresses(srcMaster) { - src := srcMaster + "/" + addr + for _, t := range s.scrapeTargets() { + src := t.source m := MonitoringMaster{ - Address: addr, - IsLeader: leaders[addr], + Address: t.label(), + IsLeader: t.anyNodeMatches(leaders), HeartbeatRate: s.sum(src, mMasterHeartbeats+suffixRate), VolumeCreation: s.sum(src, mMasterVolumeCreation+suffixRate), LeaderChanges: s.sum(src, mMasterLeaderChanges+suffixRate), PlacementMiss: s.sum(src, mMasterPlacementMiss), } m.HasData = len(m.HeartbeatRate) > 0 || len(m.PlacementMiss) > 0 - d.Masters = append(d.Masters, m) + if m.HasData { + d.Masters = append(d.Masters, m) + } } } @@ -302,36 +310,31 @@ func (s *AdminServer) fillWorkers(d *MonitoringData) { } } -// leaderSource returns the store source for the current master leader, or -// srcMaster when the leader is unknown. Cluster-wide master gauges are only -// meaningful on the leader. +// leaderSource returns the metrics endpoint of the current master leader, or +// srcAny when the leader is unknown or advertises no metrics port. Cluster-wide +// master gauges are only maintained by the leader. func (s *AdminServer) leaderSource() string { md, err := s.GetClusterMasters() if err != nil || md == nil { - return srcMaster + return srcAny } + leader := "" for _, m := range md.Masters { if m.IsLeader { - return srcMaster + "/" + m.Address + leader = m.Address } } - return srcMaster -} - -// sourceAddresses lists the scraped server addresses for a component, sorted. -func (s *AdminServer) sourceAddresses(component string) []string { - seen := map[string]bool{} + if leader == "" { + return srcAny + } for _, t := range s.scrapeTargets() { - if strings.HasPrefix(t.source, component+"/") { - seen[strings.TrimPrefix(t.source, component+"/")] = true + for _, node := range t.nodes { + if node == leader { + return t.source + } } } - out := make([]string, 0, len(seen)) - for addr := range seen { - out = append(out, addr) - } - sort.Strings(out) - return out + return srcAny } func (s *AdminServer) sum(source, metric string) []Point { diff --git a/weed/admin/view/app/monitoring.templ b/weed/admin/view/app/monitoring.templ index 7275b4c56..99c0c6074 100644 --- a/weed/admin/view/app/monitoring.templ +++ b/weed/admin/view/app/monitoring.templ @@ -57,8 +57,12 @@ templ monitoringHeader(title string, icon string, lastUpdated string) { templ noMetricsNotice() { } diff --git a/weed/admin/view/app/monitoring_templ.go b/weed/admin/view/app/monitoring_templ.go index 1b7aef12c..2035d3335 100644 --- a/weed/admin/view/app/monitoring_templ.go +++ b/weed/admin/view/app/monitoring_templ.go @@ -328,7 +328,7 @@ func noMetricsNotice() templ.Component { templ_7745c5c3_Var21 = templ.NopComponent } ctx = templ.ClearChildren(ctx) - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 21, "
No metrics collected yet. The admin server scrapes each node's /metrics endpoint every 15 seconds; charts fill in as samples accumulate.
") + templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 21, "
No metrics collected. The admin server scrapes the Prometheus listener each server advertises, which is only started when that server runs with -metricsPort. Metrics are never served on a client-facing service port, so start the servers you want to monitor with -metricsPort=<port>. Charts then fill in as samples accumulate, one scrape every 15 seconds.
") if templ_7745c5c3_Err != nil { return templ_7745c5c3_Err } @@ -375,7 +375,7 @@ func Monitoring(data dash.MonitoringData) templ.Component { var templ_7745c5c3_Var23 string templ_7745c5c3_Var23, templ_7745c5c3_Err = templ.JoinStringErrs(fmt.Sprintf("%.0f", dash.LatestValue(data.Overview.UnderReplicatedVolumes))) if templ_7745c5c3_Err != nil { - return templ.Error{Err: templ_7745c5c3_Err, FileName: `weed/admin/view/app/monitoring.templ`, Line: 75, Col: 88} + return templ.Error{Err: templ_7745c5c3_Err, FileName: `weed/admin/view/app/monitoring.templ`, Line: 79, Col: 88} } _, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var23)) if templ_7745c5c3_Err != nil { @@ -545,7 +545,7 @@ func MonitoringVolumeServers(data dash.MonitoringData) templ.Component { var templ_7745c5c3_Var25 string templ_7745c5c3_Var25, templ_7745c5c3_Err = templ.JoinStringErrs(vs.Address) if templ_7745c5c3_Err != nil { - return templ.Error{Err: templ_7745c5c3_Err, FileName: `weed/admin/view/app/monitoring.templ`, Line: 174, Col: 24} + return templ.Error{Err: templ_7745c5c3_Err, FileName: `weed/admin/view/app/monitoring.templ`, Line: 178, Col: 24} } _, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var25)) if templ_7745c5c3_Err != nil { @@ -558,7 +558,7 @@ func MonitoringVolumeServers(data dash.MonitoringData) templ.Component { var templ_7745c5c3_Var26 string templ_7745c5c3_Var26, templ_7745c5c3_Err = templ.JoinStringErrs(dash.FormatChartValue(dash.LatestValue(vs.RequestRate), dash.UnitCount)) if templ_7745c5c3_Err != nil { - return templ.Error{Err: templ_7745c5c3_Err, FileName: `weed/admin/view/app/monitoring.templ`, Line: 175, Col: 85} + return templ.Error{Err: templ_7745c5c3_Err, FileName: `weed/admin/view/app/monitoring.templ`, Line: 179, Col: 85} } _, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var26)) if templ_7745c5c3_Err != nil { @@ -571,7 +571,7 @@ func MonitoringVolumeServers(data dash.MonitoringData) templ.Component { var templ_7745c5c3_Var27 string templ_7745c5c3_Var27, templ_7745c5c3_Err = templ.JoinStringErrs(dash.FormatChartValue(dash.LatestValue(vs.P99), dash.UnitSeconds)) if templ_7745c5c3_Err != nil { - return templ.Error{Err: templ_7745c5c3_Err, FileName: `weed/admin/view/app/monitoring.templ`, Line: 176, Col: 79} + return templ.Error{Err: templ_7745c5c3_Err, FileName: `weed/admin/view/app/monitoring.templ`, Line: 180, Col: 79} } _, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var27)) if templ_7745c5c3_Err != nil { @@ -584,7 +584,7 @@ func MonitoringVolumeServers(data dash.MonitoringData) templ.Component { var templ_7745c5c3_Var28 string templ_7745c5c3_Var28, templ_7745c5c3_Err = templ.JoinStringErrs(dash.FormatChartValue(dash.LatestValue(vs.DiskUsagePct), dash.UnitPercent)) if templ_7745c5c3_Err != nil { - return templ.Error{Err: templ_7745c5c3_Err, FileName: `weed/admin/view/app/monitoring.templ`, Line: 177, Col: 88} + return templ.Error{Err: templ_7745c5c3_Err, FileName: `weed/admin/view/app/monitoring.templ`, Line: 181, Col: 88} } _, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var28)) if templ_7745c5c3_Err != nil { @@ -602,7 +602,7 @@ func MonitoringVolumeServers(data dash.MonitoringData) templ.Component { var templ_7745c5c3_Var29 string templ_7745c5c3_Var29, templ_7745c5c3_Err = templ.JoinStringErrs(dash.FormatChartValue(dash.LatestValue(vs.ErrorRate), dash.UnitCount)) if templ_7745c5c3_Err != nil { - return templ.Error{Err: templ_7745c5c3_Err, FileName: `weed/admin/view/app/monitoring.templ`, Line: 181, Col: 82} + return templ.Error{Err: templ_7745c5c3_Err, FileName: `weed/admin/view/app/monitoring.templ`, Line: 185, Col: 82} } _, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var29)) if templ_7745c5c3_Err != nil { @@ -635,7 +635,7 @@ func MonitoringVolumeServers(data dash.MonitoringData) templ.Component { var templ_7745c5c3_Var30 string templ_7745c5c3_Var30, templ_7745c5c3_Err = templ.JoinStringErrs(vs.Address) if templ_7745c5c3_Err != nil { - return templ.Error{Err: templ_7745c5c3_Err, FileName: `weed/admin/view/app/monitoring.templ`, Line: 196, Col: 63} + return templ.Error{Err: templ_7745c5c3_Err, FileName: `weed/admin/view/app/monitoring.templ`, Line: 200, Col: 63} } _, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var30)) if templ_7745c5c3_Err != nil { @@ -717,7 +717,7 @@ func MonitoringFilers(data dash.MonitoringData) templ.Component { var templ_7745c5c3_Var32 string templ_7745c5c3_Var32, templ_7745c5c3_Err = templ.JoinStringErrs(f.Address) if templ_7745c5c3_Err != nil { - return templ.Error{Err: templ_7745c5c3_Err, FileName: `weed/admin/view/app/monitoring.templ`, Line: 222, Col: 62} + return templ.Error{Err: templ_7745c5c3_Err, FileName: `weed/admin/view/app/monitoring.templ`, Line: 226, Col: 62} } _, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var32)) if templ_7745c5c3_Err != nil { @@ -800,7 +800,7 @@ func MonitoringS3(data dash.MonitoringData) templ.Component { var templ_7745c5c3_Var34 string templ_7745c5c3_Var34, templ_7745c5c3_Err = templ.JoinStringErrs(n.Address) if templ_7745c5c3_Err != nil { - return templ.Error{Err: templ_7745c5c3_Err, FileName: `weed/admin/view/app/monitoring.templ`, Line: 249, Col: 62} + return templ.Error{Err: templ_7745c5c3_Err, FileName: `weed/admin/view/app/monitoring.templ`, Line: 253, Col: 62} } _, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var34)) if templ_7745c5c3_Err != nil { @@ -877,7 +877,7 @@ func MonitoringMasters(data dash.MonitoringData) templ.Component { var templ_7745c5c3_Var36 string templ_7745c5c3_Var36, templ_7745c5c3_Err = templ.JoinStringErrs(m.Address) if templ_7745c5c3_Err != nil { - return templ.Error{Err: templ_7745c5c3_Err, FileName: `weed/admin/view/app/monitoring.templ`, Line: 274, Col: 14} + return templ.Error{Err: templ_7745c5c3_Err, FileName: `weed/admin/view/app/monitoring.templ`, Line: 278, Col: 14} } _, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var36)) if templ_7745c5c3_Err != nil { @@ -1015,7 +1015,7 @@ func MonitoringWorkers(data dash.MonitoringData) templ.Component { var templ_7745c5c3_Var38 string templ_7745c5c3_Var38, templ_7745c5c3_Err = templ.JoinStringErrs(w.ID) if templ_7745c5c3_Err != nil { - return templ.Error{Err: templ_7745c5c3_Err, FileName: `weed/admin/view/app/monitoring.templ`, Line: 334, Col: 22} + return templ.Error{Err: templ_7745c5c3_Err, FileName: `weed/admin/view/app/monitoring.templ`, Line: 338, Col: 22} } _, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var38)) if templ_7745c5c3_Err != nil {