admin: gather the admin's own registry into the metrics store

The admin's maintenance and worker metrics live in the local
stats.Gather registry, so record them directly under the admin/local
source instead of scraping over HTTP. Adds metricsStore.match for
prefix/metric lookups.

weed/admin/dash/metrics_scraper.go

@@ -12,6 +12,7 @@ import (
 	"github.com/prometheus/common/expfmt"
 
 	"github.com/seaweedfs/seaweedfs/weed/glog"
+	stats_collect "github.com/seaweedfs/seaweedfs/weed/stats"
 	util_http "github.com/seaweedfs/seaweedfs/weed/util/http"
 )
 
@@ -62,31 +63,55 @@ func parsePrometheusText(r io.Reader) ([]scrapedMetric, error) {
 			for _, l := range m.Label {
 				labels[l.GetName()] = l.GetValue()
 			}
-			var v float64
-			switch {
-			case m.Gauge != nil:
-				v = m.Gauge.GetValue()
-			case m.Counter != nil:
-				v = m.Counter.GetValue()
-			case m.Untyped != nil:
-				v = m.Untyped.GetValue()
-			case m.Histogram != nil:
-				v = m.Histogram.GetSampleSum()
-			case m.Summary != nil:
-				v = m.Summary.GetSampleSum()
-			}
-			out = append(out, scrapedMetric{name: fam.GetName(), labels: labels, value: v})
+			out = append(out, scrapedMetric{name: fam.GetName(), labels: labels, value: metricValue(m)})
 		}
 	}
 	return out, nil
 }
 
+func metricValue(m *dto.Metric) float64 {
+	switch {
+	case m.Gauge != nil:
+		return m.Gauge.GetValue()
+	case m.Counter != nil:
+		return m.Counter.GetValue()
+	case m.Untyped != nil:
+		return m.Untyped.GetValue()
+	case m.Histogram != nil:
+		return m.Histogram.GetSampleSum()
+	case m.Summary != nil:
+		return m.Summary.GetSampleSum()
+	}
+	return 0
+}
+
+// gatherLocalMetrics records the admin's own registry (maintenance tasks,
+// worker slots) without a network round trip.
+func (s *AdminServer) gatherLocalMetrics(now time.Time) {
+	families, err := stats_collect.Gather.Gather()
+	if err != nil {
+		glog.V(1).Infof("gather admin metrics: %v", err)
+		return
+	}
+	for _, fam := range families {
+		for _, m := range fam.Metric {
+			labels := map[string]string{}
+			for _, l := range m.Label {
+				labels[l.GetName()] = l.GetValue()
+			}
+			s.metricsStore.recordLabeled("admin/local", fam.GetName(), labels, metricValue(m), now)
+		}
+	}
+}
+
 func (s *AdminServer) scrapeAllServers(ctx context.Context) {
+	now := time.Now()
+	s.gatherLocalMetrics(now)
+
 	targets := s.scrapeTargets()
 	if len(targets) == 0 {
 		return
 	}
-	now := time.Now()
 	type result struct {
 		source  string
 		metrics []scrapedMetric

weed/admin/dash/metrics_store.go

@@ -1,6 +1,7 @@
 package dash
 
 import (
+	"strings"
 	"sync"
 	"time"
 )
@@ -39,8 +40,8 @@ func (s *metricsSeries) snapshot() []metricsSample {
 }
 
 type metricsStore struct {
-	mu      sync.Mutex
-	series  map[string]*metricsSeries
+	mu     sync.Mutex
+	series map[string]*metricsSeries
 }
 
 func newMetricsStore() *metricsStore {
@@ -93,6 +94,36 @@ func (s *metricsStore) getLabeled(source, name string, labels map[string]string)
 	return ser.snapshot()
 }
 
+// match returns every series whose source has the given prefix and whose
+// metric name matches exactly.
+func (s *metricsStore) match(sourcePrefix, name string) []*metricsSeries {
+	s.mu.Lock()
+	defer s.mu.Unlock()
+	var out []*metricsSeries
+	for k, ser := range s.series {
+		if !strings.HasPrefix(k, sourcePrefix) {
+			continue
+		}
+		rest := k[len(sourcePrefix):]
+		if !strings.HasPrefix(rest, "/") {
+			continue
+		}
+		rest = rest[1:]
+		// rest is either "<addr>/<metric>[/<labels>]" or "<metric>[/<labels>]".
+		if rest == name || strings.HasPrefix(rest, name+"/") {
+			out = append(out, ser)
+			continue
+		}
+		if i := strings.Index(rest, "/"); i >= 0 {
+			tail := rest[i+1:]
+			if tail == name || strings.HasPrefix(tail, name+"/") {
+				out = append(out, ser)
+			}
+		}
+	}
+	return out
+}
+
 func labelKey(labels map[string]string) string {
 	if len(labels) == 0 {
 		return ""