mirror of
https://github.com/seaweedfs/seaweedfs.git
synced 2026-10-17 19:48:35 +02:00
Chris Lu
3e9154def2
* telemetry: persist server state across restarts The telemetry server kept the instance map and Prometheus gauges only in process memory, so every deploy or restart reset all collected metrics until clusters re-reported over the next 24h. Snapshot the instance map to a JSON state file (atomic tmp+rename) on a debounced interval and on SIGTERM, and restore it on startup, preserving received_at so the cleanup and active-cluster windows stay correct. Defaults to data/telemetry-state.json, which the deployed systemd unit's WorkingDirectory already provides; -state-file='' disables. * telemetry: keep instances for 90 days by default With state now persisted across restarts, a longer retention default is meaningful; raise -max-age from 30 to 90 days so per-cluster data survives long enough for quarterly views.
76 lines
1.9 KiB
Go
76 lines
1.9 KiB
Go
package storage
|
|
|
|
import (
|
|
"path/filepath"
|
|
"testing"
|
|
"time"
|
|
|
|
"github.com/seaweedfs/seaweedfs/telemetry/proto"
|
|
)
|
|
|
|
func TestStateRoundTrip(t *testing.T) {
|
|
path := filepath.Join(t.TempDir(), "data", "telemetry-state.json")
|
|
|
|
// promauto registers on the global registry, so the whole test uses a
|
|
// single storage instance.
|
|
s := NewPrometheusStorage()
|
|
|
|
if err := s.SaveStateIfDirty(path); err != nil {
|
|
t.Fatalf("clean save: %v", err)
|
|
}
|
|
if _, err := s.LoadState(path); err != nil {
|
|
t.Fatalf("load with no state file: %v", err)
|
|
}
|
|
|
|
report := &proto.TelemetryData{
|
|
TopologyId: "test-cluster-1",
|
|
Version: "4.40",
|
|
Os: "linux/amd64",
|
|
VolumeServerCount: 5,
|
|
TotalDiskBytes: 123456789,
|
|
TotalVolumeCount: 42,
|
|
FilerCount: 2,
|
|
BrokerCount: 1,
|
|
Timestamp: time.Now().Unix(),
|
|
}
|
|
if err := s.StoreTelemetry(report); err != nil {
|
|
t.Fatalf("store: %v", err)
|
|
}
|
|
receivedAt := s.instances[report.TopologyId].ReceivedAt
|
|
|
|
if err := s.SaveStateIfDirty(path); err != nil {
|
|
t.Fatalf("save: %v", err)
|
|
}
|
|
if s.dirty {
|
|
t.Fatal("dirty flag not cleared after save")
|
|
}
|
|
|
|
// Simulate a restart: wipe the in-memory map, then restore.
|
|
s.instances = make(map[string]*telemetryData)
|
|
n, err := s.LoadState(path)
|
|
if err != nil {
|
|
t.Fatalf("load: %v", err)
|
|
}
|
|
if n != 1 {
|
|
t.Fatalf("loaded %d instances, want 1", n)
|
|
}
|
|
|
|
got, ok := s.instances[report.TopologyId]
|
|
if !ok {
|
|
t.Fatal("instance missing after load")
|
|
}
|
|
if !got.ReceivedAt.Equal(receivedAt) {
|
|
t.Errorf("ReceivedAt not preserved: got %v, want %v", got.ReceivedAt, receivedAt)
|
|
}
|
|
if got.TelemetryData.TotalDiskBytes != report.TotalDiskBytes ||
|
|
got.TelemetryData.Version != report.Version ||
|
|
got.TelemetryData.VolumeServerCount != report.VolumeServerCount {
|
|
t.Errorf("fields not preserved: got %+v", got.TelemetryData)
|
|
}
|
|
|
|
// Load must not mark state dirty.
|
|
if s.dirty {
|
|
t.Error("dirty flag set after load")
|
|
}
|
|
}
|