diff --git a/weed/admin/dash/metrics_scraper.go b/weed/admin/dash/metrics_scraper.go index f3c37d689..2557bae77 100644 --- a/weed/admin/dash/metrics_scraper.go +++ b/weed/admin/dash/metrics_scraper.go @@ -169,6 +169,25 @@ type scrapeTarget struct { nodes []string } +// label names the target for the UI: the node that advertised the endpoint, or +// the endpoint address itself when several nodes share one listener. +func (t scrapeTarget) label() string { + if len(t.nodes) == 1 { + return t.nodes[0] + } + return t.address +} + +// anyNodeMatches reports whether any node behind this endpoint is flagged. +func (t scrapeTarget) anyNodeMatches(flagged map[string]bool) bool { + for _, node := range t.nodes { + if flagged[node] { + return true + } + } + return false +} + // scrapeTargets lists the distinct metrics endpoints advertised by the cluster. // Nodes started without -metricsPort advertise 0 and are skipped, so nothing is // scraped from a client-facing service port. diff --git a/weed/admin/dash/monitoring_data.go b/weed/admin/dash/monitoring_data.go index 015d9a80a..548d74d90 100644 --- a/weed/admin/dash/monitoring_data.go +++ b/weed/admin/dash/monitoring_data.go @@ -39,13 +39,13 @@ const ( mAdminWorkers = "SeaweedFS_admin_workers_connected" ) -// Scrape source components. +// Store sources. Scraped series are keyed by the metrics endpoint they came +// from, so cluster-wide aggregates match on srcAny and rely on the metric name +// to identify the component: one endpoint can serve master, volume, filer and +// S3 series at once when they share a process. const ( - srcMaster = "master" - srcVolume = "volume" - srcFiler = "filer" - srcS3 = "s3" - srcAdmin = "admin/local" + srcAdmin = "admin/local" + srcAny = "" ) type MonitoringData struct { @@ -186,25 +186,25 @@ func (s *AdminServer) fillOverview(d *MonitoringData) { o.UnderReplicatedVolumes = s.sum(leader, mMasterUnderReplicated) o.WritableVolumes = s.sum(leader, mMasterWritable) o.CrowdedVolumes = s.sum(leader, mMasterCrowded) - o.DiskUsagePct = s.diskUsagePct(srcVolume) + o.DiskUsagePct = s.diskUsagePct(srcAny) - o.VolumeReadRate = s.sumFiltered(srcVolume, mVolumeRequests+suffixRate, isReadRequest) - o.VolumeWriteRate = s.sumFiltered(srcVolume, mVolumeRequests+suffixRate, isWriteRequest) - o.FilerRequestRate = s.sum(srcFiler, mFilerRequests+suffixRate) - o.S3RequestRate = s.sum(srcS3, mS3Requests+suffixRate) + o.VolumeReadRate = s.sumFiltered(srcAny, mVolumeRequests+suffixRate, isReadRequest) + o.VolumeWriteRate = s.sumFiltered(srcAny, mVolumeRequests+suffixRate, isWriteRequest) + o.FilerRequestRate = s.sum(srcAny, mFilerRequests+suffixRate) + o.S3RequestRate = s.sum(srcAny, mS3Requests+suffixRate) - o.VolumeP50 = s.max(srcVolume, mVolumeLatency+suffixP50) - o.VolumeP95 = s.max(srcVolume, mVolumeLatency+suffixP95) - o.VolumeP99 = s.max(srcVolume, mVolumeLatency+suffixP99) - o.FilerP50 = s.max(srcFiler, mFilerLatency+suffixP50) - o.FilerP95 = s.max(srcFiler, mFilerLatency+suffixP95) - o.FilerP99 = s.max(srcFiler, mFilerLatency+suffixP99) + o.VolumeP50 = s.max(srcAny, mVolumeLatency+suffixP50) + o.VolumeP95 = s.max(srcAny, mVolumeLatency+suffixP95) + o.VolumeP99 = s.max(srcAny, mVolumeLatency+suffixP99) + o.FilerP50 = s.max(srcAny, mFilerLatency+suffixP50) + o.FilerP95 = s.max(srcAny, mFilerLatency+suffixP95) + o.FilerP99 = s.max(srcAny, mFilerLatency+suffixP99) - o.VolumeErrorRate = s.sumFiltered(srcVolume, mVolumeRequests+suffixRate, isErrorCode) - o.FilerErrorRate = s.sumFiltered(srcFiler, mFilerRequests+suffixRate, isErrorCode) - o.S3ErrorRate = s.sumFiltered(srcS3, mS3Requests+suffixRate, isErrorCode) - o.DiskErrors = s.sum(srcVolume, mVolumeDiskError) - o.Quarantined = s.sum(srcVolume, mVolumeQuarantine) + o.VolumeErrorRate = s.sumFiltered(srcAny, mVolumeRequests+suffixRate, isErrorCode) + o.FilerErrorRate = s.sumFiltered(srcAny, mFilerRequests+suffixRate, isErrorCode) + o.S3ErrorRate = s.sumFiltered(srcAny, mS3Requests+suffixRate, isErrorCode) + o.DiskErrors = s.sum(srcAny, mVolumeDiskError) + o.Quarantined = s.sum(srcAny, mVolumeQuarantine) o.QueueDepth = s.sumFiltered(srcAdmin, mAdminTasksByStatus, func(l map[string]string) bool { return l["status"] == "pending" || l["status"] == "assigned" || l["status"] == "in_progress" @@ -214,25 +214,27 @@ func (s *AdminServer) fillOverview(d *MonitoringData) { } func (s *AdminServer) fillVolumeServers(d *MonitoringData) { - for _, addr := range s.sourceAddresses(srcVolume) { - src := srcVolume + "/" + addr + for _, t := range s.scrapeTargets() { + src := t.source vs := MonitoringVolumeServer{ - Address: addr, + Address: t.label(), RequestRate: s.sum(src, mVolumeRequests+suffixRate), P99: s.max(src, mVolumeLatency+suffixP99), DiskUsagePct: s.diskUsagePct(src), ErrorRate: s.sumFiltered(src, mVolumeRequests+suffixRate, isErrorCode), } vs.HasData = len(vs.RequestRate) > 0 || len(vs.DiskUsagePct) > 0 - d.VolumeServers = append(d.VolumeServers, vs) + if vs.HasData { + d.VolumeServers = append(d.VolumeServers, vs) + } } } func (s *AdminServer) fillFilers(d *MonitoringData) { - for _, addr := range s.sourceAddresses(srcFiler) { - src := srcFiler + "/" + addr + for _, t := range s.scrapeTargets() { + src := t.source f := MonitoringFiler{ - Address: addr, + Address: t.label(), RequestRate: s.sum(src, mFilerRequests+suffixRate), P99: s.max(src, mFilerLatency+suffixP99), StoreP99: s.max(src, mFilerStoreLat+suffixP99), @@ -240,15 +242,17 @@ func (s *AdminServer) fillFilers(d *MonitoringData) { SyncLag: s.max(src, mFilerSyncLag), } f.HasData = len(f.RequestRate) > 0 || len(f.InFlight) > 0 - d.Filers = append(d.Filers, f) + if f.HasData { + d.Filers = append(d.Filers, f) + } } } func (s *AdminServer) fillS3(d *MonitoringData) { - for _, addr := range s.sourceAddresses(srcS3) { - src := srcS3 + "/" + addr + for _, t := range s.scrapeTargets() { + src := t.source n := MonitoringS3{ - Address: addr, + Address: t.label(), RequestRate: s.sum(src, mS3Requests+suffixRate), Errors4xx: s.sumFiltered(src, mS3Requests+suffixRate, func(l map[string]string) bool { return strings.HasPrefix(l["code"], "4") @@ -259,7 +263,9 @@ func (s *AdminServer) fillS3(d *MonitoringData) { P99: s.max(src, mS3Latency+suffixP99), } n.HasData = len(n.RequestRate) > 0 - d.S3 = append(d.S3, n) + if n.HasData { + d.S3 = append(d.S3, n) + } } } @@ -270,18 +276,20 @@ func (s *AdminServer) fillMasters(d *MonitoringData) { leaders[m.Address] = m.IsLeader } } - for _, addr := range s.sourceAddresses(srcMaster) { - src := srcMaster + "/" + addr + for _, t := range s.scrapeTargets() { + src := t.source m := MonitoringMaster{ - Address: addr, - IsLeader: leaders[addr], + Address: t.label(), + IsLeader: t.anyNodeMatches(leaders), HeartbeatRate: s.sum(src, mMasterHeartbeats+suffixRate), VolumeCreation: s.sum(src, mMasterVolumeCreation+suffixRate), LeaderChanges: s.sum(src, mMasterLeaderChanges+suffixRate), PlacementMiss: s.sum(src, mMasterPlacementMiss), } m.HasData = len(m.HeartbeatRate) > 0 || len(m.PlacementMiss) > 0 - d.Masters = append(d.Masters, m) + if m.HasData { + d.Masters = append(d.Masters, m) + } } } @@ -302,36 +310,31 @@ func (s *AdminServer) fillWorkers(d *MonitoringData) { } } -// leaderSource returns the store source for the current master leader, or -// srcMaster when the leader is unknown. Cluster-wide master gauges are only -// meaningful on the leader. +// leaderSource returns the metrics endpoint of the current master leader, or +// srcAny when the leader is unknown or advertises no metrics port. Cluster-wide +// master gauges are only maintained by the leader. func (s *AdminServer) leaderSource() string { md, err := s.GetClusterMasters() if err != nil || md == nil { - return srcMaster + return srcAny } + leader := "" for _, m := range md.Masters { if m.IsLeader { - return srcMaster + "/" + m.Address + leader = m.Address } } - return srcMaster -} - -// sourceAddresses lists the scraped server addresses for a component, sorted. -func (s *AdminServer) sourceAddresses(component string) []string { - seen := map[string]bool{} + if leader == "" { + return srcAny + } for _, t := range s.scrapeTargets() { - if strings.HasPrefix(t.source, component+"/") { - seen[strings.TrimPrefix(t.source, component+"/")] = true + for _, node := range t.nodes { + if node == leader { + return t.source + } } } - out := make([]string, 0, len(seen)) - for addr := range seen { - out = append(out, addr) - } - sort.Strings(out) - return out + return srcAny } func (s *AdminServer) sum(source, metric string) []Point { diff --git a/weed/admin/view/app/monitoring.templ b/weed/admin/view/app/monitoring.templ index 7275b4c56..99c0c6074 100644 --- a/weed/admin/view/app/monitoring.templ +++ b/weed/admin/view/app/monitoring.templ @@ -57,8 +57,12 @@ templ monitoringHeader(title string, icon string, lastUpdated string) { templ noMetricsNotice() { } diff --git a/weed/admin/view/app/monitoring_templ.go b/weed/admin/view/app/monitoring_templ.go index 1b7aef12c..2035d3335 100644 --- a/weed/admin/view/app/monitoring_templ.go +++ b/weed/admin/view/app/monitoring_templ.go @@ -328,7 +328,7 @@ func noMetricsNotice() templ.Component { templ_7745c5c3_Var21 = templ.NopComponent } ctx = templ.ClearChildren(ctx) - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 21, "
No metrics collected yet. The admin server scrapes each node's /metrics endpoint every 15 seconds; charts fill in as samples accumulate.
") + templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 21, "
No metrics collected. The admin server scrapes the Prometheus listener each server advertises, which is only started when that server runs with -metricsPort. Metrics are never served on a client-facing service port, so start the servers you want to monitor with -metricsPort=<port>. Charts then fill in as samples accumulate, one scrape every 15 seconds.
") if templ_7745c5c3_Err != nil { return templ_7745c5c3_Err } @@ -375,7 +375,7 @@ func Monitoring(data dash.MonitoringData) templ.Component { var templ_7745c5c3_Var23 string templ_7745c5c3_Var23, templ_7745c5c3_Err = templ.JoinStringErrs(fmt.Sprintf("%.0f", dash.LatestValue(data.Overview.UnderReplicatedVolumes))) if templ_7745c5c3_Err != nil { - return templ.Error{Err: templ_7745c5c3_Err, FileName: `weed/admin/view/app/monitoring.templ`, Line: 75, Col: 88} + return templ.Error{Err: templ_7745c5c3_Err, FileName: `weed/admin/view/app/monitoring.templ`, Line: 79, Col: 88} } _, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var23)) if templ_7745c5c3_Err != nil { @@ -545,7 +545,7 @@ func MonitoringVolumeServers(data dash.MonitoringData) templ.Component { var templ_7745c5c3_Var25 string templ_7745c5c3_Var25, templ_7745c5c3_Err = templ.JoinStringErrs(vs.Address) if templ_7745c5c3_Err != nil { - return templ.Error{Err: templ_7745c5c3_Err, FileName: `weed/admin/view/app/monitoring.templ`, Line: 174, Col: 24} + return templ.Error{Err: templ_7745c5c3_Err, FileName: `weed/admin/view/app/monitoring.templ`, Line: 178, Col: 24} } _, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var25)) if templ_7745c5c3_Err != nil { @@ -558,7 +558,7 @@ func MonitoringVolumeServers(data dash.MonitoringData) templ.Component { var templ_7745c5c3_Var26 string templ_7745c5c3_Var26, templ_7745c5c3_Err = templ.JoinStringErrs(dash.FormatChartValue(dash.LatestValue(vs.RequestRate), dash.UnitCount)) if templ_7745c5c3_Err != nil { - return templ.Error{Err: templ_7745c5c3_Err, FileName: `weed/admin/view/app/monitoring.templ`, Line: 175, Col: 85} + return templ.Error{Err: templ_7745c5c3_Err, FileName: `weed/admin/view/app/monitoring.templ`, Line: 179, Col: 85} } _, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var26)) if templ_7745c5c3_Err != nil { @@ -571,7 +571,7 @@ func MonitoringVolumeServers(data dash.MonitoringData) templ.Component { var templ_7745c5c3_Var27 string templ_7745c5c3_Var27, templ_7745c5c3_Err = templ.JoinStringErrs(dash.FormatChartValue(dash.LatestValue(vs.P99), dash.UnitSeconds)) if templ_7745c5c3_Err != nil { - return templ.Error{Err: templ_7745c5c3_Err, FileName: `weed/admin/view/app/monitoring.templ`, Line: 176, Col: 79} + return templ.Error{Err: templ_7745c5c3_Err, FileName: `weed/admin/view/app/monitoring.templ`, Line: 180, Col: 79} } _, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var27)) if templ_7745c5c3_Err != nil { @@ -584,7 +584,7 @@ func MonitoringVolumeServers(data dash.MonitoringData) templ.Component { var templ_7745c5c3_Var28 string templ_7745c5c3_Var28, templ_7745c5c3_Err = templ.JoinStringErrs(dash.FormatChartValue(dash.LatestValue(vs.DiskUsagePct), dash.UnitPercent)) if templ_7745c5c3_Err != nil { - return templ.Error{Err: templ_7745c5c3_Err, FileName: `weed/admin/view/app/monitoring.templ`, Line: 177, Col: 88} + return templ.Error{Err: templ_7745c5c3_Err, FileName: `weed/admin/view/app/monitoring.templ`, Line: 181, Col: 88} } _, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var28)) if templ_7745c5c3_Err != nil { @@ -602,7 +602,7 @@ func MonitoringVolumeServers(data dash.MonitoringData) templ.Component { var templ_7745c5c3_Var29 string templ_7745c5c3_Var29, templ_7745c5c3_Err = templ.JoinStringErrs(dash.FormatChartValue(dash.LatestValue(vs.ErrorRate), dash.UnitCount)) if templ_7745c5c3_Err != nil { - return templ.Error{Err: templ_7745c5c3_Err, FileName: `weed/admin/view/app/monitoring.templ`, Line: 181, Col: 82} + return templ.Error{Err: templ_7745c5c3_Err, FileName: `weed/admin/view/app/monitoring.templ`, Line: 185, Col: 82} } _, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var29)) if templ_7745c5c3_Err != nil { @@ -635,7 +635,7 @@ func MonitoringVolumeServers(data dash.MonitoringData) templ.Component { var templ_7745c5c3_Var30 string templ_7745c5c3_Var30, templ_7745c5c3_Err = templ.JoinStringErrs(vs.Address) if templ_7745c5c3_Err != nil { - return templ.Error{Err: templ_7745c5c3_Err, FileName: `weed/admin/view/app/monitoring.templ`, Line: 196, Col: 63} + return templ.Error{Err: templ_7745c5c3_Err, FileName: `weed/admin/view/app/monitoring.templ`, Line: 200, Col: 63} } _, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var30)) if templ_7745c5c3_Err != nil { @@ -717,7 +717,7 @@ func MonitoringFilers(data dash.MonitoringData) templ.Component { var templ_7745c5c3_Var32 string templ_7745c5c3_Var32, templ_7745c5c3_Err = templ.JoinStringErrs(f.Address) if templ_7745c5c3_Err != nil { - return templ.Error{Err: templ_7745c5c3_Err, FileName: `weed/admin/view/app/monitoring.templ`, Line: 222, Col: 62} + return templ.Error{Err: templ_7745c5c3_Err, FileName: `weed/admin/view/app/monitoring.templ`, Line: 226, Col: 62} } _, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var32)) if templ_7745c5c3_Err != nil { @@ -800,7 +800,7 @@ func MonitoringS3(data dash.MonitoringData) templ.Component { var templ_7745c5c3_Var34 string templ_7745c5c3_Var34, templ_7745c5c3_Err = templ.JoinStringErrs(n.Address) if templ_7745c5c3_Err != nil { - return templ.Error{Err: templ_7745c5c3_Err, FileName: `weed/admin/view/app/monitoring.templ`, Line: 249, Col: 62} + return templ.Error{Err: templ_7745c5c3_Err, FileName: `weed/admin/view/app/monitoring.templ`, Line: 253, Col: 62} } _, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var34)) if templ_7745c5c3_Err != nil { @@ -877,7 +877,7 @@ func MonitoringMasters(data dash.MonitoringData) templ.Component { var templ_7745c5c3_Var36 string templ_7745c5c3_Var36, templ_7745c5c3_Err = templ.JoinStringErrs(m.Address) if templ_7745c5c3_Err != nil { - return templ.Error{Err: templ_7745c5c3_Err, FileName: `weed/admin/view/app/monitoring.templ`, Line: 274, Col: 14} + return templ.Error{Err: templ_7745c5c3_Err, FileName: `weed/admin/view/app/monitoring.templ`, Line: 278, Col: 14} } _, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var36)) if templ_7745c5c3_Err != nil { @@ -1015,7 +1015,7 @@ func MonitoringWorkers(data dash.MonitoringData) templ.Component { var templ_7745c5c3_Var38 string templ_7745c5c3_Var38, templ_7745c5c3_Err = templ.JoinStringErrs(w.ID) if templ_7745c5c3_Err != nil { - return templ.Error{Err: templ_7745c5c3_Err, FileName: `weed/admin/view/app/monitoring.templ`, Line: 334, Col: 22} + return templ.Error{Err: templ_7745c5c3_Err, FileName: `weed/admin/view/app/monitoring.templ`, Line: 338, Col: 22} } _, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var38)) if templ_7745c5c3_Err != nil {