admin: key monitoring queries by metric name and scrape endpoint

Sources are now metrics endpoints rather than component names, so
cluster-wide aggregates match on any source and let the metric name
identify the component. Detail pages iterate the scrape endpoints and
only list one when it actually serves that component's series, which
keeps a combined "weed server" from appearing as four separate nodes.

The empty-state notice now tells operators to start servers with
-metricsPort, since that is what creates the listener.

weed/admin/dash/metrics_scraper.go

@@ -169,6 +169,25 @@ type scrapeTarget struct {
 	nodes   []string
 }
 
+// label names the target for the UI: the node that advertised the endpoint, or
+// the endpoint address itself when several nodes share one listener.
+func (t scrapeTarget) label() string {
+	if len(t.nodes) == 1 {
+		return t.nodes[0]
+	}
+	return t.address
+}
+
+// anyNodeMatches reports whether any node behind this endpoint is flagged.
+func (t scrapeTarget) anyNodeMatches(flagged map[string]bool) bool {
+	for _, node := range t.nodes {
+		if flagged[node] {
+			return true
+		}
+	}
+	return false
+}
+
 // scrapeTargets lists the distinct metrics endpoints advertised by the cluster.
 // Nodes started without -metricsPort advertise 0 and are skipped, so nothing is
 // scraped from a client-facing service port.

weed/admin/dash/monitoring_data.go

@@ -39,13 +39,13 @@ const (
 	mAdminWorkers       = "SeaweedFS_admin_workers_connected"
 )
 
-// Scrape source components.
+// Store sources. Scraped series are keyed by the metrics endpoint they came
+// from, so cluster-wide aggregates match on srcAny and rely on the metric name
+// to identify the component: one endpoint can serve master, volume, filer and
+// S3 series at once when they share a process.
 const (
-	srcMaster = "master"
-	srcVolume = "volume"
-	srcFiler  = "filer"
-	srcS3     = "s3"
-	srcAdmin  = "admin/local"
+	srcAdmin = "admin/local"
+	srcAny   = ""
 )
 
 type MonitoringData struct {
@@ -186,25 +186,25 @@ func (s *AdminServer) fillOverview(d *MonitoringData) {
 	o.UnderReplicatedVolumes = s.sum(leader, mMasterUnderReplicated)
 	o.WritableVolumes = s.sum(leader, mMasterWritable)
 	o.CrowdedVolumes = s.sum(leader, mMasterCrowded)
-	o.DiskUsagePct = s.diskUsagePct(srcVolume)
+	o.DiskUsagePct = s.diskUsagePct(srcAny)
 
-	o.VolumeReadRate = s.sumFiltered(srcVolume, mVolumeRequests+suffixRate, isReadRequest)
-	o.VolumeWriteRate = s.sumFiltered(srcVolume, mVolumeRequests+suffixRate, isWriteRequest)
-	o.FilerRequestRate = s.sum(srcFiler, mFilerRequests+suffixRate)
-	o.S3RequestRate = s.sum(srcS3, mS3Requests+suffixRate)
+	o.VolumeReadRate = s.sumFiltered(srcAny, mVolumeRequests+suffixRate, isReadRequest)
+	o.VolumeWriteRate = s.sumFiltered(srcAny, mVolumeRequests+suffixRate, isWriteRequest)
+	o.FilerRequestRate = s.sum(srcAny, mFilerRequests+suffixRate)
+	o.S3RequestRate = s.sum(srcAny, mS3Requests+suffixRate)
 
-	o.VolumeP50 = s.max(srcVolume, mVolumeLatency+suffixP50)
-	o.VolumeP95 = s.max(srcVolume, mVolumeLatency+suffixP95)
-	o.VolumeP99 = s.max(srcVolume, mVolumeLatency+suffixP99)
-	o.FilerP50 = s.max(srcFiler, mFilerLatency+suffixP50)
-	o.FilerP95 = s.max(srcFiler, mFilerLatency+suffixP95)
-	o.FilerP99 = s.max(srcFiler, mFilerLatency+suffixP99)
+	o.VolumeP50 = s.max(srcAny, mVolumeLatency+suffixP50)
+	o.VolumeP95 = s.max(srcAny, mVolumeLatency+suffixP95)
+	o.VolumeP99 = s.max(srcAny, mVolumeLatency+suffixP99)
+	o.FilerP50 = s.max(srcAny, mFilerLatency+suffixP50)
+	o.FilerP95 = s.max(srcAny, mFilerLatency+suffixP95)
+	o.FilerP99 = s.max(srcAny, mFilerLatency+suffixP99)
 
-	o.VolumeErrorRate = s.sumFiltered(srcVolume, mVolumeRequests+suffixRate, isErrorCode)
-	o.FilerErrorRate = s.sumFiltered(srcFiler, mFilerRequests+suffixRate, isErrorCode)
-	o.S3ErrorRate = s.sumFiltered(srcS3, mS3Requests+suffixRate, isErrorCode)
-	o.DiskErrors = s.sum(srcVolume, mVolumeDiskError)
-	o.Quarantined = s.sum(srcVolume, mVolumeQuarantine)
+	o.VolumeErrorRate = s.sumFiltered(srcAny, mVolumeRequests+suffixRate, isErrorCode)
+	o.FilerErrorRate = s.sumFiltered(srcAny, mFilerRequests+suffixRate, isErrorCode)
+	o.S3ErrorRate = s.sumFiltered(srcAny, mS3Requests+suffixRate, isErrorCode)
+	o.DiskErrors = s.sum(srcAny, mVolumeDiskError)
+	o.Quarantined = s.sum(srcAny, mVolumeQuarantine)
 
 	o.QueueDepth = s.sumFiltered(srcAdmin, mAdminTasksByStatus, func(l map[string]string) bool {
 		return l["status"] == "pending" || l["status"] == "assigned" || l["status"] == "in_progress"
@@ -214,25 +214,27 @@ func (s *AdminServer) fillOverview(d *MonitoringData) {
 }
 
 func (s *AdminServer) fillVolumeServers(d *MonitoringData) {
-	for _, addr := range s.sourceAddresses(srcVolume) {
-		src := srcVolume + "/" + addr
+	for _, t := range s.scrapeTargets() {
+		src := t.source
 		vs := MonitoringVolumeServer{
-			Address:      addr,
+			Address:      t.label(),
 			RequestRate:  s.sum(src, mVolumeRequests+suffixRate),
 			P99:          s.max(src, mVolumeLatency+suffixP99),
 			DiskUsagePct: s.diskUsagePct(src),
 			ErrorRate:    s.sumFiltered(src, mVolumeRequests+suffixRate, isErrorCode),
 		}
 		vs.HasData = len(vs.RequestRate) > 0 || len(vs.DiskUsagePct) > 0
-		d.VolumeServers = append(d.VolumeServers, vs)
+		if vs.HasData {
+			d.VolumeServers = append(d.VolumeServers, vs)
+		}
 	}
 }
 
 func (s *AdminServer) fillFilers(d *MonitoringData) {
-	for _, addr := range s.sourceAddresses(srcFiler) {
-		src := srcFiler + "/" + addr
+	for _, t := range s.scrapeTargets() {
+		src := t.source
 		f := MonitoringFiler{
-			Address:     addr,
+			Address:     t.label(),
 			RequestRate: s.sum(src, mFilerRequests+suffixRate),
 			P99:         s.max(src, mFilerLatency+suffixP99),
 			StoreP99:    s.max(src, mFilerStoreLat+suffixP99),
@@ -240,15 +242,17 @@ func (s *AdminServer) fillFilers(d *MonitoringData) {
 			SyncLag:     s.max(src, mFilerSyncLag),
 		}
 		f.HasData = len(f.RequestRate) > 0 || len(f.InFlight) > 0
-		d.Filers = append(d.Filers, f)
+		if f.HasData {
+			d.Filers = append(d.Filers, f)
+		}
 	}
 }
 
 func (s *AdminServer) fillS3(d *MonitoringData) {
-	for _, addr := range s.sourceAddresses(srcS3) {
-		src := srcS3 + "/" + addr
+	for _, t := range s.scrapeTargets() {
+		src := t.source
 		n := MonitoringS3{
-			Address:     addr,
+			Address:     t.label(),
 			RequestRate: s.sum(src, mS3Requests+suffixRate),
 			Errors4xx: s.sumFiltered(src, mS3Requests+suffixRate, func(l map[string]string) bool {
 				return strings.HasPrefix(l["code"], "4")
@@ -259,7 +263,9 @@ func (s *AdminServer) fillS3(d *MonitoringData) {
 			P99: s.max(src, mS3Latency+suffixP99),
 		}
 		n.HasData = len(n.RequestRate) > 0
-		d.S3 = append(d.S3, n)
+		if n.HasData {
+			d.S3 = append(d.S3, n)
+		}
 	}
 }
 
@@ -270,18 +276,20 @@ func (s *AdminServer) fillMasters(d *MonitoringData) {
 			leaders[m.Address] = m.IsLeader
 		}
 	}
-	for _, addr := range s.sourceAddresses(srcMaster) {
-		src := srcMaster + "/" + addr
+	for _, t := range s.scrapeTargets() {
+		src := t.source
 		m := MonitoringMaster{
-			Address:        addr,
-			IsLeader:       leaders[addr],
+			Address:        t.label(),
+			IsLeader:       t.anyNodeMatches(leaders),
 			HeartbeatRate:  s.sum(src, mMasterHeartbeats+suffixRate),
 			VolumeCreation: s.sum(src, mMasterVolumeCreation+suffixRate),
 			LeaderChanges:  s.sum(src, mMasterLeaderChanges+suffixRate),
 			PlacementMiss:  s.sum(src, mMasterPlacementMiss),
 		}
 		m.HasData = len(m.HeartbeatRate) > 0 || len(m.PlacementMiss) > 0
-		d.Masters = append(d.Masters, m)
+		if m.HasData {
+			d.Masters = append(d.Masters, m)
+		}
 	}
 }
 
@@ -302,36 +310,31 @@ func (s *AdminServer) fillWorkers(d *MonitoringData) {
 	}
 }
 
-// leaderSource returns the store source for the current master leader, or
-// srcMaster when the leader is unknown. Cluster-wide master gauges are only
-// meaningful on the leader.
+// leaderSource returns the metrics endpoint of the current master leader, or
+// srcAny when the leader is unknown or advertises no metrics port. Cluster-wide
+// master gauges are only maintained by the leader.
 func (s *AdminServer) leaderSource() string {
 	md, err := s.GetClusterMasters()
 	if err != nil || md == nil {
-		return srcMaster
+		return srcAny
 	}
+	leader := ""
 	for _, m := range md.Masters {
 		if m.IsLeader {
-			return srcMaster + "/" + m.Address
+			leader = m.Address
 		}
 	}
-	return srcMaster
-}
-
-// sourceAddresses lists the scraped server addresses for a component, sorted.
-func (s *AdminServer) sourceAddresses(component string) []string {
-	seen := map[string]bool{}
+	if leader == "" {
+		return srcAny
+	}
 	for _, t := range s.scrapeTargets() {
-		if strings.HasPrefix(t.source, component+"/") {
-			seen[strings.TrimPrefix(t.source, component+"/")] = true
+		for _, node := range t.nodes {
+			if node == leader {
+				return t.source
+			}
 		}
 	}
-	out := make([]string, 0, len(seen))
-	for addr := range seen {
-		out = append(out, addr)
-	}
-	sort.Strings(out)
-	return out
+	return srcAny
 }
 
 func (s *AdminServer) sum(source, metric string) []Point {

weed/admin/view/app/monitoring.templ

@@ -57,8 +57,12 @@ templ monitoringHeader(title string, icon string, lastUpdated string) {
 templ noMetricsNotice() {
 	<div class="alert alert-info" role="alert">
 		<i class="fas fa-info-circle me-2"></i>
-		No metrics collected yet. The admin server scrapes each node's <code>/metrics</code>
-		endpoint every 15 seconds; charts fill in as samples accumulate.
+		<strong>No metrics collected.</strong>
+		The admin server scrapes the Prometheus listener each server advertises, which
+		is only started when that server runs with <code>-metricsPort</code>. Metrics are
+		never served on a client-facing service port, so start the servers you want to
+		monitor with <code>-metricsPort=&lt;port&gt;</code>. Charts then fill in as samples
+		accumulate, one scrape every 15 seconds.
 	</div>
 }
 

weed/admin/view/app/monitoring_templ.go

@@ -328,7 +328,7 @@ func noMetricsNotice() templ.Component {
 			templ_7745c5c3_Var21 = templ.NopComponent
 		}
 		ctx = templ.ClearChildren(ctx)
-		templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 21, "<div class=\"alert alert-info\" role=\"alert\"><i class=\"fas fa-info-circle me-2\"></i> No metrics collected yet. The admin server scrapes each node's <code>/metrics</code> endpoint every 15 seconds; charts fill in as samples accumulate.</div>")
+		templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 21, "<div class=\"alert alert-info\" role=\"alert\"><i class=\"fas fa-info-circle me-2\"></i> <strong>No metrics collected.</strong> The admin server scrapes the Prometheus listener each server advertises, which is only started when that server runs with <code>-metricsPort</code>. Metrics are never served on a client-facing service port, so start the servers you want to monitor with <code>-metricsPort=&lt;port&gt;</code>. Charts then fill in as samples accumulate, one scrape every 15 seconds.</div>")
 		if templ_7745c5c3_Err != nil {
 			return templ_7745c5c3_Err
 		}
@@ -375,7 +375,7 @@ func Monitoring(data dash.MonitoringData) templ.Component {
 			var templ_7745c5c3_Var23 string
 			templ_7745c5c3_Var23, templ_7745c5c3_Err = templ.JoinStringErrs(fmt.Sprintf("%.0f", dash.LatestValue(data.Overview.UnderReplicatedVolumes)))
 			if templ_7745c5c3_Err != nil {
-				return templ.Error{Err: templ_7745c5c3_Err, FileName: `weed/admin/view/app/monitoring.templ`, Line: 75, Col: 88}
+				return templ.Error{Err: templ_7745c5c3_Err, FileName: `weed/admin/view/app/monitoring.templ`, Line: 79, Col: 88}
 			}
 			_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var23))
 			if templ_7745c5c3_Err != nil {
@@ -545,7 +545,7 @@ func MonitoringVolumeServers(data dash.MonitoringData) templ.Component {
 			var templ_7745c5c3_Var25 string
 			templ_7745c5c3_Var25, templ_7745c5c3_Err = templ.JoinStringErrs(vs.Address)
 			if templ_7745c5c3_Err != nil {
-				return templ.Error{Err: templ_7745c5c3_Err, FileName: `weed/admin/view/app/monitoring.templ`, Line: 174, Col: 24}
+				return templ.Error{Err: templ_7745c5c3_Err, FileName: `weed/admin/view/app/monitoring.templ`, Line: 178, Col: 24}
 			}
 			_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var25))
 			if templ_7745c5c3_Err != nil {
@@ -558,7 +558,7 @@ func MonitoringVolumeServers(data dash.MonitoringData) templ.Component {
 			var templ_7745c5c3_Var26 string
 			templ_7745c5c3_Var26, templ_7745c5c3_Err = templ.JoinStringErrs(dash.FormatChartValue(dash.LatestValue(vs.RequestRate), dash.UnitCount))
 			if templ_7745c5c3_Err != nil {
-				return templ.Error{Err: templ_7745c5c3_Err, FileName: `weed/admin/view/app/monitoring.templ`, Line: 175, Col: 85}
+				return templ.Error{Err: templ_7745c5c3_Err, FileName: `weed/admin/view/app/monitoring.templ`, Line: 179, Col: 85}
 			}
 			_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var26))
 			if templ_7745c5c3_Err != nil {
@@ -571,7 +571,7 @@ func MonitoringVolumeServers(data dash.MonitoringData) templ.Component {
 			var templ_7745c5c3_Var27 string
 			templ_7745c5c3_Var27, templ_7745c5c3_Err = templ.JoinStringErrs(dash.FormatChartValue(dash.LatestValue(vs.P99), dash.UnitSeconds))
 			if templ_7745c5c3_Err != nil {
-				return templ.Error{Err: templ_7745c5c3_Err, FileName: `weed/admin/view/app/monitoring.templ`, Line: 176, Col: 79}
+				return templ.Error{Err: templ_7745c5c3_Err, FileName: `weed/admin/view/app/monitoring.templ`, Line: 180, Col: 79}
 			}
 			_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var27))
 			if templ_7745c5c3_Err != nil {
@@ -584,7 +584,7 @@ func MonitoringVolumeServers(data dash.MonitoringData) templ.Component {
 			var templ_7745c5c3_Var28 string
 			templ_7745c5c3_Var28, templ_7745c5c3_Err = templ.JoinStringErrs(dash.FormatChartValue(dash.LatestValue(vs.DiskUsagePct), dash.UnitPercent))
 			if templ_7745c5c3_Err != nil {
-				return templ.Error{Err: templ_7745c5c3_Err, FileName: `weed/admin/view/app/monitoring.templ`, Line: 177, Col: 88}
+				return templ.Error{Err: templ_7745c5c3_Err, FileName: `weed/admin/view/app/monitoring.templ`, Line: 181, Col: 88}
 			}
 			_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var28))
 			if templ_7745c5c3_Err != nil {
@@ -602,7 +602,7 @@ func MonitoringVolumeServers(data dash.MonitoringData) templ.Component {
 				var templ_7745c5c3_Var29 string
 				templ_7745c5c3_Var29, templ_7745c5c3_Err = templ.JoinStringErrs(dash.FormatChartValue(dash.LatestValue(vs.ErrorRate), dash.UnitCount))
 				if templ_7745c5c3_Err != nil {
-					return templ.Error{Err: templ_7745c5c3_Err, FileName: `weed/admin/view/app/monitoring.templ`, Line: 181, Col: 82}
+					return templ.Error{Err: templ_7745c5c3_Err, FileName: `weed/admin/view/app/monitoring.templ`, Line: 185, Col: 82}
 				}
 				_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var29))
 				if templ_7745c5c3_Err != nil {
@@ -635,7 +635,7 @@ func MonitoringVolumeServers(data dash.MonitoringData) templ.Component {
 			var templ_7745c5c3_Var30 string
 			templ_7745c5c3_Var30, templ_7745c5c3_Err = templ.JoinStringErrs(vs.Address)
 			if templ_7745c5c3_Err != nil {
-				return templ.Error{Err: templ_7745c5c3_Err, FileName: `weed/admin/view/app/monitoring.templ`, Line: 196, Col: 63}
+				return templ.Error{Err: templ_7745c5c3_Err, FileName: `weed/admin/view/app/monitoring.templ`, Line: 200, Col: 63}
 			}
 			_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var30))
 			if templ_7745c5c3_Err != nil {
@@ -717,7 +717,7 @@ func MonitoringFilers(data dash.MonitoringData) templ.Component {
 			var templ_7745c5c3_Var32 string
 			templ_7745c5c3_Var32, templ_7745c5c3_Err = templ.JoinStringErrs(f.Address)
 			if templ_7745c5c3_Err != nil {
-				return templ.Error{Err: templ_7745c5c3_Err, FileName: `weed/admin/view/app/monitoring.templ`, Line: 222, Col: 62}
+				return templ.Error{Err: templ_7745c5c3_Err, FileName: `weed/admin/view/app/monitoring.templ`, Line: 226, Col: 62}
 			}
 			_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var32))
 			if templ_7745c5c3_Err != nil {
@@ -800,7 +800,7 @@ func MonitoringS3(data dash.MonitoringData) templ.Component {
 			var templ_7745c5c3_Var34 string
 			templ_7745c5c3_Var34, templ_7745c5c3_Err = templ.JoinStringErrs(n.Address)
 			if templ_7745c5c3_Err != nil {
-				return templ.Error{Err: templ_7745c5c3_Err, FileName: `weed/admin/view/app/monitoring.templ`, Line: 249, Col: 62}
+				return templ.Error{Err: templ_7745c5c3_Err, FileName: `weed/admin/view/app/monitoring.templ`, Line: 253, Col: 62}
 			}
 			_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var34))
 			if templ_7745c5c3_Err != nil {
@@ -877,7 +877,7 @@ func MonitoringMasters(data dash.MonitoringData) templ.Component {
 			var templ_7745c5c3_Var36 string
 			templ_7745c5c3_Var36, templ_7745c5c3_Err = templ.JoinStringErrs(m.Address)
 			if templ_7745c5c3_Err != nil {
-				return templ.Error{Err: templ_7745c5c3_Err, FileName: `weed/admin/view/app/monitoring.templ`, Line: 274, Col: 14}
+				return templ.Error{Err: templ_7745c5c3_Err, FileName: `weed/admin/view/app/monitoring.templ`, Line: 278, Col: 14}
 			}
 			_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var36))
 			if templ_7745c5c3_Err != nil {
@@ -1015,7 +1015,7 @@ func MonitoringWorkers(data dash.MonitoringData) templ.Component {
 				var templ_7745c5c3_Var38 string
 				templ_7745c5c3_Var38, templ_7745c5c3_Err = templ.JoinStringErrs(w.ID)
 				if templ_7745c5c3_Err != nil {
-					return templ.Error{Err: templ_7745c5c3_Err, FileName: `weed/admin/view/app/monitoring.templ`, Line: 334, Col: 22}
+					return templ.Error{Err: templ_7745c5c3_Err, FileName: `weed/admin/view/app/monitoring.templ`, Line: 338, Col: 22}
 				}
 				_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var38))
 				if templ_7745c5c3_Err != nil {