diff --git a/weed/admin/dash/metrics_scraper.go b/weed/admin/dash/metrics_scraper.go index f50ff1852..0ba8e587c 100644 --- a/weed/admin/dash/metrics_scraper.go +++ b/weed/admin/dash/metrics_scraper.go @@ -12,6 +12,7 @@ import ( "github.com/prometheus/common/expfmt" "github.com/seaweedfs/seaweedfs/weed/glog" + stats_collect "github.com/seaweedfs/seaweedfs/weed/stats" util_http "github.com/seaweedfs/seaweedfs/weed/util/http" ) @@ -62,31 +63,55 @@ func parsePrometheusText(r io.Reader) ([]scrapedMetric, error) { for _, l := range m.Label { labels[l.GetName()] = l.GetValue() } - var v float64 - switch { - case m.Gauge != nil: - v = m.Gauge.GetValue() - case m.Counter != nil: - v = m.Counter.GetValue() - case m.Untyped != nil: - v = m.Untyped.GetValue() - case m.Histogram != nil: - v = m.Histogram.GetSampleSum() - case m.Summary != nil: - v = m.Summary.GetSampleSum() - } - out = append(out, scrapedMetric{name: fam.GetName(), labels: labels, value: v}) + out = append(out, scrapedMetric{name: fam.GetName(), labels: labels, value: metricValue(m)}) } } return out, nil } +func metricValue(m *dto.Metric) float64 { + switch { + case m.Gauge != nil: + return m.Gauge.GetValue() + case m.Counter != nil: + return m.Counter.GetValue() + case m.Untyped != nil: + return m.Untyped.GetValue() + case m.Histogram != nil: + return m.Histogram.GetSampleSum() + case m.Summary != nil: + return m.Summary.GetSampleSum() + } + return 0 +} + +// gatherLocalMetrics records the admin's own registry (maintenance tasks, +// worker slots) without a network round trip. +func (s *AdminServer) gatherLocalMetrics(now time.Time) { + families, err := stats_collect.Gather.Gather() + if err != nil { + glog.V(1).Infof("gather admin metrics: %v", err) + return + } + for _, fam := range families { + for _, m := range fam.Metric { + labels := map[string]string{} + for _, l := range m.Label { + labels[l.GetName()] = l.GetValue() + } + s.metricsStore.recordLabeled("admin/local", fam.GetName(), labels, metricValue(m), now) + } + } +} + func (s *AdminServer) scrapeAllServers(ctx context.Context) { + now := time.Now() + s.gatherLocalMetrics(now) + targets := s.scrapeTargets() if len(targets) == 0 { return } - now := time.Now() type result struct { source string metrics []scrapedMetric diff --git a/weed/admin/dash/metrics_store.go b/weed/admin/dash/metrics_store.go index eabbc3413..bf1a7c6af 100644 --- a/weed/admin/dash/metrics_store.go +++ b/weed/admin/dash/metrics_store.go @@ -1,6 +1,7 @@ package dash import ( + "strings" "sync" "time" ) @@ -39,8 +40,8 @@ func (s *metricsSeries) snapshot() []metricsSample { } type metricsStore struct { - mu sync.Mutex - series map[string]*metricsSeries + mu sync.Mutex + series map[string]*metricsSeries } func newMetricsStore() *metricsStore { @@ -93,6 +94,36 @@ func (s *metricsStore) getLabeled(source, name string, labels map[string]string) return ser.snapshot() } +// match returns every series whose source has the given prefix and whose +// metric name matches exactly. +func (s *metricsStore) match(sourcePrefix, name string) []*metricsSeries { + s.mu.Lock() + defer s.mu.Unlock() + var out []*metricsSeries + for k, ser := range s.series { + if !strings.HasPrefix(k, sourcePrefix) { + continue + } + rest := k[len(sourcePrefix):] + if !strings.HasPrefix(rest, "/") { + continue + } + rest = rest[1:] + // rest is either "/[/]" or "[/]". + if rest == name || strings.HasPrefix(rest, name+"/") { + out = append(out, ser) + continue + } + if i := strings.Index(rest, "/"); i >= 0 { + tail := rest[i+1:] + if tail == name || strings.HasPrefix(tail, name+"/") { + out = append(out, ser) + } + } + } + return out +} + func labelKey(labels map[string]string) string { if len(labels) == 0 { return ""