package weed_server import ( "encoding/json" "net/http" "net/http/httptest" "os" "path/filepath" "runtime" "strings" "testing" engine "github.com/seaweedfs/seaweedfs/sw-block/engine/replication" "github.com/seaweedfs/seaweedfs/weed/storage/blockvol" "github.com/seaweedfs/seaweedfs/weed/storage/blockvol/blockapi" ) // --- Health-state derivation tests --- func TestHealthState_Healthy_PrimaryWithReplicas(t *testing.T) { entry := &BlockVolumeEntry{ Role: blockvol.RoleToWire(blockvol.RolePrimary), ReplicaFactor: 2, Replicas: []ReplicaInfo{ {Server: "vs2:9333", Role: blockvol.RoleToWire(blockvol.RoleReplica)}, }, } state := deriveHealthState(entry) if state != HealthStateHealthy { t.Fatalf("expected %q, got %q", HealthStateHealthy, state) } } func TestHealthState_Unsafe_NotPrimary(t *testing.T) { entry := &BlockVolumeEntry{ Role: blockvol.RoleToWire(blockvol.RoleReplica), ReplicaFactor: 2, } state := deriveHealthState(entry) if state != HealthStateUnsafe { t.Fatalf("expected %q, got %q", HealthStateUnsafe, state) } } func TestHealthState_Unsafe_StrictBelowRequired(t *testing.T) { entry := &BlockVolumeEntry{ Role: blockvol.RoleToWire(blockvol.RolePrimary), ReplicaFactor: 2, DurabilityMode: "sync_all", Replicas: nil, // 0 replicas, sync_all requires 1 } state := deriveHealthState(entry) if state != HealthStateUnsafe { t.Fatalf("expected %q for sync_all with 0 replicas, got %q", HealthStateUnsafe, state) } } func TestHealthState_Rebuilding(t *testing.T) { entry := &BlockVolumeEntry{ Role: blockvol.RoleToWire(blockvol.RolePrimary), ReplicaFactor: 2, Replicas: []ReplicaInfo{ {Server: "vs2:9333", Role: blockvol.RoleToWire(blockvol.RoleRebuilding)}, }, } state := deriveHealthState(entry) if state != HealthStateRebuilding { t.Fatalf("expected %q, got %q", HealthStateRebuilding, state) } } func TestHealthState_Degraded_BelowDesired(t *testing.T) { entry := &BlockVolumeEntry{ Role: blockvol.RoleToWire(blockvol.RolePrimary), ReplicaFactor: 3, Replicas: []ReplicaInfo{ {Server: "vs2:9333", Role: blockvol.RoleToWire(blockvol.RoleReplica)}, }, // RF=3 needs 2 replicas, only has 1 } state := deriveHealthState(entry) if state != HealthStateDegraded { t.Fatalf("expected %q, got %q", HealthStateDegraded, state) } } func TestHealthState_Degraded_FlagSet(t *testing.T) { entry := &BlockVolumeEntry{ Role: blockvol.RoleToWire(blockvol.RolePrimary), ReplicaFactor: 2, Replicas: []ReplicaInfo{{Server: "vs2:9333", Role: blockvol.RoleToWire(blockvol.RoleReplica)}}, ReplicaDegraded: true, } state := deriveHealthState(entry) if state != HealthStateDegraded { t.Fatalf("expected %q, got %q", HealthStateDegraded, state) } } func TestHealthState_WithLiveness_PrimaryDead(t *testing.T) { entry := &BlockVolumeEntry{ Role: blockvol.RoleToWire(blockvol.RolePrimary), ReplicaFactor: 2, Replicas: []ReplicaInfo{{Server: "vs2:9333", Role: blockvol.RoleToWire(blockvol.RoleReplica)}}, } state := deriveHealthStateWithLiveness(entry, false) if state != HealthStateUnsafe { t.Fatalf("expected %q when primary dead, got %q", HealthStateUnsafe, state) } } func TestHealthState_BestEffort_ZeroReplicas_NotUnsafe(t *testing.T) { // best_effort with RF=2 but 0 replicas: degraded, not unsafe entry := &BlockVolumeEntry{ Role: blockvol.RoleToWire(blockvol.RolePrimary), ReplicaFactor: 2, DurabilityMode: "best_effort", Replicas: nil, } state := deriveHealthState(entry) if state != HealthStateDegraded { t.Fatalf("expected %q for best_effort with 0 replicas, got %q", HealthStateDegraded, state) } } func TestHealthState_RF1_NoReplicas_Healthy(t *testing.T) { entry := &BlockVolumeEntry{ Role: blockvol.RoleToWire(blockvol.RolePrimary), ReplicaFactor: 1, Replicas: nil, } state := deriveHealthState(entry) if state != HealthStateHealthy { t.Fatalf("expected %q for RF=1, got %q", HealthStateHealthy, state) } } // --- Cluster health summary test --- func TestClusterHealthSummary(t *testing.T) { r := NewBlockVolumeRegistry() r.MarkBlockCapable("vs1:9333") r.MarkBlockCapable("vs2:9333") // Register a healthy volume r.Register(&BlockVolumeEntry{ Name: "healthy-vol", VolumeServer: "vs1:9333", Path: "/data/healthy.blk", Role: blockvol.RoleToWire(blockvol.RolePrimary), ReplicaFactor: 2, ReplicaReady: true, Replicas: []ReplicaInfo{{Server: "vs2:9333", Role: blockvol.RoleToWire(blockvol.RoleReplica), Ready: true}}, Status: StatusActive, }) // Register a degraded volume (missing replica) r.Register(&BlockVolumeEntry{ Name: "degraded-vol", VolumeServer: "vs1:9333", Path: "/data/degraded.blk", Role: blockvol.RoleToWire(blockvol.RolePrimary), ReplicaFactor: 2, Replicas: nil, // missing replica Status: StatusActive, }) summary := r.ComputeClusterHealthSummary() if summary.Healthy != 1 { t.Fatalf("expected 1 healthy, got %d", summary.Healthy) } if summary.Degraded != 1 { t.Fatalf("expected 1 degraded, got %d", summary.Degraded) } if summary.Unsafe != 0 { t.Fatalf("expected 0 unsafe, got %d", summary.Unsafe) } } // --- Status handler test --- func TestBlockStatusHandler_IncludesHealthCounts(t *testing.T) { ms := qaPlanMaster(t) // Create a volume to have some state r := ms.blockRegistry r.Register(&BlockVolumeEntry{ Name: "status-test", VolumeServer: "vs1:9333", Path: "/data/status.blk", Role: blockvol.RoleToWire(blockvol.RolePrimary), ReplicaFactor: 2, ReplicaReady: true, Replicas: []ReplicaInfo{{Server: "vs2:9333", Role: blockvol.RoleToWire(blockvol.RoleReplica), Ready: true}}, Status: StatusActive, }) req := httptest.NewRequest(http.MethodGet, "/block/status", nil) w := httptest.NewRecorder() ms.blockStatusHandler(w, req) if w.Code != http.StatusOK { t.Fatalf("expected 200, got %d", w.Code) } var resp blockapi.BlockStatusResponse if err := json.NewDecoder(w.Body).Decode(&resp); err != nil { t.Fatalf("decode: %v", err) } if resp.VolumeCount != 1 { t.Fatalf("expected volume_count=1, got %d", resp.VolumeCount) } if resp.HealthyCount != 1 { t.Fatalf("expected healthy_count=1, got %d", resp.HealthyCount) } if resp.NvmeCapableServers < 0 { t.Fatal("nvme_capable_servers should be >= 0") } } func TestBlockStatusHandler_ReflectsCoreInfluencedConsumeCounts(t *testing.T) { ms := qaPlanMaster(t) // Ready path: replica heartbeat publishes receiver addrs from the // core-influenced ready path, then registry consumes it into a healthy volume. readyBS := newTestBlockServiceDirect(t) readyPath := createTestVolDirect(t, readyBS, "status-ready") errs := readyBS.ApplyAssignments([]blockvol.BlockVolumeAssignment{{ Path: readyPath, Epoch: 1, Role: blockvol.RoleToWire(blockvol.RoleReplica), LeaseTtlMs: 30000, ReplicaDataAddr: "127.0.0.1:0", ReplicaCtrlAddr: "127.0.0.1:0", }}) if len(errs) != 1 || errs[0] != nil { t.Fatalf("ready apply errs=%v", errs) } readyBS.replMu.Lock() readyState := readyBS.replStates[readyPath] if readyState == nil { readyBS.replMu.Unlock() t.Fatal("missing ready repl state") } readyState.publishHealthy = false readyBS.replMu.Unlock() readyHB := findHeartbeatMsg(readyBS.CollectBlockVolumeHeartbeat(), readyPath) if readyHB == nil { t.Fatal("ready heartbeat missing") } if err := ms.blockRegistry.Register(&BlockVolumeEntry{ Name: "status-ready", VolumeServer: "vs1:9333", Path: "/blocks/status-ready-primary.blk", Epoch: 1, Status: StatusActive, Role: blockvol.RoleToWire(blockvol.RolePrimary), ReplicaFactor: 2, Replicas: []ReplicaInfo{{ Server: "vs2:9333", Path: readyPath, }}, }); err != nil { t.Fatalf("register ready: %v", err) } ms.blockRegistry.mu.Lock() ms.blockRegistry.addToServer("vs2:9333", "status-ready") ms.blockRegistry.mu.Unlock() ms.blockRegistry.UpdateFullHeartbeat("vs2:9333", blockvol.InfoMessagesToProto([]blockvol.BlockVolumeInfoMessage{*readyHB}), "") // Degraded path: primary heartbeat publishes degraded bit from the // core-influenced degraded path, then registry consumes it into a degraded volume. degradedBS := newTestBlockServiceDirect(t) degradedPath := createTestVolDirect(t, degradedBS, "status-degraded") errs = degradedBS.ApplyAssignments([]blockvol.BlockVolumeAssignment{{ Path: degradedPath, Epoch: 1, Role: blockvol.RoleToWire(blockvol.RolePrimary), LeaseTtlMs: 30000, ReplicaServerID: "vs1:9333", ReplicaDataAddr: "10.0.0.3:4260", ReplicaCtrlAddr: "10.0.0.3:4261", }}) if len(errs) != 1 || errs[0] != nil { t.Fatalf("degraded apply errs=%v", errs) } degradedBS.applyCoreEvent(engine.BarrierRejected{ID: degradedPath, Reason: "barrier_timeout"}) degradedHB := findHeartbeatMsg(degradedBS.CollectBlockVolumeHeartbeat(), degradedPath) if degradedHB == nil { t.Fatal("degraded heartbeat missing") } if err := ms.blockRegistry.Register(&BlockVolumeEntry{ Name: "status-degraded", VolumeServer: "vs3:9333", Path: degradedPath, Epoch: 1, Status: StatusActive, Role: blockvol.RoleToWire(blockvol.RolePrimary), ReplicaFactor: 2, Replicas: []ReplicaInfo{{ Server: "vs1:9333", Path: "/blocks/status-degraded-replica.blk", Ready: true, }}, }); err != nil { t.Fatalf("register degraded: %v", err) } ms.blockRegistry.UpdateFullHeartbeat("vs3:9333", blockvol.InfoMessagesToProto([]blockvol.BlockVolumeInfoMessage{*degradedHB}), "") req := httptest.NewRequest(http.MethodGet, "/block/status", nil) w := httptest.NewRecorder() ms.blockStatusHandler(w, req) if w.Code != http.StatusOK { t.Fatalf("expected 200, got %d", w.Code) } var resp blockapi.BlockStatusResponse if err := json.NewDecoder(w.Body).Decode(&resp); err != nil { t.Fatalf("decode: %v", err) } if resp.VolumeCount != 2 { t.Fatalf("expected volume_count=2, got %d", resp.VolumeCount) } if resp.HealthyCount != 1 { t.Fatalf("expected healthy_count=1, got %d", resp.HealthyCount) } if resp.DegradedCount != 1 { t.Fatalf("expected degraded_count=1, got %d", resp.DegradedCount) } if resp.RebuildingCount != 0 { t.Fatalf("expected rebuilding_count=0, got %d", resp.RebuildingCount) } if resp.UnsafeCount != 0 { t.Fatalf("expected unsafe_count=0, got %d", resp.UnsafeCount) } } // --- VolumeInfo includes health_state --- func TestEntryToVolumeInfo_IncludesHealthState(t *testing.T) { entry := &BlockVolumeEntry{ Name: "health-vol", VolumeServer: "vs1:9333", Role: blockvol.RoleToWire(blockvol.RolePrimary), ReplicaFactor: 2, Replicas: []ReplicaInfo{{Server: "vs2:9333", Role: blockvol.RoleToWire(blockvol.RoleReplica)}}, Status: StatusActive, } info := entryToVolumeInfo(entry, true) // primary alive if info.HealthState != HealthStateHealthy { t.Fatalf("expected %q, got %q", HealthStateHealthy, info.HealthState) } } func TestEntryToVolumeInfo_ReflectsCoreInfluencedReadyConsume(t *testing.T) { bs := newTestBlockServiceDirect(t) path := createTestVolDirect(t, bs, "vol-info-ready") errs := bs.ApplyAssignments([]blockvol.BlockVolumeAssignment{ { Path: path, Epoch: 1, Role: blockvol.RoleToWire(blockvol.RoleReplica), LeaseTtlMs: 30000, ReplicaDataAddr: "127.0.0.1:0", ReplicaCtrlAddr: "127.0.0.1:0", }, }) if len(errs) != 1 || errs[0] != nil { t.Fatalf("apply assignment errs=%v", errs) } bs.replMu.Lock() state := bs.replStates[path] if state == nil { bs.replMu.Unlock() t.Fatal("missing repl state") } state.publishHealthy = false bs.replMu.Unlock() hb := findHeartbeatMsg(bs.CollectBlockVolumeHeartbeat(), path) if hb == nil { t.Fatal("heartbeat volume missing") } r := NewBlockVolumeRegistry() r.MarkBlockCapable("primary-server:8080") if err := r.Register(&BlockVolumeEntry{ Name: "vol-info-ready", VolumeServer: "primary-server:8080", Path: "/blocks/vol-info-ready-primary.blk", Status: StatusActive, Role: blockvol.RoleToWire(blockvol.RolePrimary), ReplicaFactor: 2, Replicas: []ReplicaInfo{{ Server: "replica-server:8080", Path: path, }}, }); err != nil { t.Fatalf("register: %v", err) } r.UpdateFullHeartbeat("replica-server:8080", blockvol.InfoMessagesToProto([]blockvol.BlockVolumeInfoMessage{*hb}), "") entry, _ := r.Lookup("vol-info-ready") info := entryToVolumeInfo(&entry, true) if !info.ReplicaReady { t.Fatalf("expected outward ReplicaReady=true, info=%+v", info) } if info.ReplicaDegraded { t.Fatalf("expected outward ReplicaDegraded=false, info=%+v", info) } if info.VolumeMode != "publish_healthy" { t.Fatalf("expected outward VolumeMode=publish_healthy, got %q", info.VolumeMode) } if info.VolumeModeReason != "" { t.Fatalf("expected empty outward VolumeModeReason for publish_healthy, got %q", info.VolumeModeReason) } if info.HealthState != HealthStateHealthy { t.Fatalf("expected outward HealthState=%q, got %q", HealthStateHealthy, info.HealthState) } } func TestEntryToVolumeInfo_ReflectsCoreInfluencedDegradedConsume(t *testing.T) { bs := newTestBlockServiceDirect(t) path := createTestVolDirect(t, bs, "vol-info-degraded") errs := bs.ApplyAssignments([]blockvol.BlockVolumeAssignment{ { Path: path, Epoch: 1, Role: blockvol.RoleToWire(blockvol.RolePrimary), LeaseTtlMs: 30000, ReplicaServerID: "replica-server:8080", ReplicaDataAddr: "10.0.0.2:4260", ReplicaCtrlAddr: "10.0.0.2:4261", }, }) if len(errs) != 1 || errs[0] != nil { t.Fatalf("apply assignment errs=%v", errs) } bs.applyCoreEvent(engine.BarrierRejected{ID: path, Reason: "barrier_timeout"}) hb := findHeartbeatMsg(bs.CollectBlockVolumeHeartbeat(), path) if hb == nil { t.Fatal("heartbeat volume missing") } r := NewBlockVolumeRegistry() r.MarkBlockCapable("primary-server:8080") if err := r.Register(&BlockVolumeEntry{ Name: "vol-info-degraded", VolumeServer: "primary-server:8080", Path: path, Status: StatusActive, Role: blockvol.RoleToWire(blockvol.RolePrimary), ReplicaFactor: 2, Replicas: []ReplicaInfo{{ Server: "replica-server:8080", Path: "/blocks/vol-info-degraded-replica.blk", Ready: true, }}, }); err != nil { t.Fatalf("register: %v", err) } r.UpdateFullHeartbeat("primary-server:8080", blockvol.InfoMessagesToProto([]blockvol.BlockVolumeInfoMessage{*hb}), "") entry, _ := r.Lookup("vol-info-degraded") info := entryToVolumeInfo(&entry, true) if !info.ReplicaDegraded { t.Fatalf("expected outward ReplicaDegraded=true, info=%+v", info) } if info.VolumeMode != "degraded" { t.Fatalf("expected outward VolumeMode=degraded, got %q", info.VolumeMode) } if info.VolumeModeReason != "barrier_timeout" { t.Fatalf("expected outward VolumeModeReason=barrier_timeout, got %q", info.VolumeModeReason) } if info.HealthState != HealthStateDegraded { t.Fatalf("expected outward HealthState=%q, got %q", HealthStateDegraded, info.HealthState) } } func TestEntryToVolumeInfo_PrimaryDead_Unsafe(t *testing.T) { entry := &BlockVolumeEntry{ Name: "dead-primary-vol", VolumeServer: "vs1:9333", Role: blockvol.RoleToWire(blockvol.RolePrimary), ReplicaFactor: 2, Replicas: []ReplicaInfo{{Server: "vs2:9333", Role: blockvol.RoleToWire(blockvol.RoleReplica)}}, Status: StatusActive, } info := entryToVolumeInfo(entry, false) // primary NOT alive if info.HealthState != HealthStateUnsafe { t.Fatalf("expected %q when primary dead, got %q", HealthStateUnsafe, info.HealthState) } } // --- Alert rules parse validity --- func testdataDir() string { _, filename, _, _ := runtime.Caller(0) return filepath.Join(filepath.Dir(filename), "..", "storage", "blockvol", "monitoring") } func TestAlertRules_FileExists(t *testing.T) { path := filepath.Join(testdataDir(), "alerts", "block-alerts.yaml") data, err := os.ReadFile(path) if err != nil { t.Fatalf("alert rules file not found: %v", err) } if len(data) == 0 { t.Fatal("alert rules file is empty") } // Basic structure check: must contain "groups:" and "alert:" and known metric names content := string(data) for _, required := range []string{"groups:", "alert:", "seaweedfs_blockvol_"} { if !strings.Contains(content, required) { t.Fatalf("alert rules missing required content: %q", required) } } } func TestAlertRules_ReferencesRealMetrics(t *testing.T) { path := filepath.Join(testdataDir(), "alerts", "block-alerts.yaml") data, err := os.ReadFile(path) if err != nil { t.Skipf("alert rules file not found: %v", err) } content := string(data) // All referenced metrics must be real exported metric names knownMetrics := []string{ "seaweedfs_blockvol_wal_used_fraction", "seaweedfs_blockvol_replica_failed_barriers_total", "seaweedfs_blockvol_wal_pressure_state", "seaweedfs_blockvol_flusher_errors_total", "seaweedfs_blockvol_scrub_errors_total", } for _, m := range knownMetrics { if !strings.Contains(content, m) { t.Fatalf("alert rules should reference metric %q", m) } } } // --- Dashboard asset presence --- func TestDashboard_FileExists(t *testing.T) { path := filepath.Join(testdataDir(), "dashboards", "block-overview.json") data, err := os.ReadFile(path) if err != nil { t.Fatalf("dashboard file not found: %v", err) } if len(data) == 0 { t.Fatal("dashboard file is empty") } // Must be valid JSON var dash map[string]interface{} if err := json.Unmarshal(data, &dash); err != nil { t.Fatalf("dashboard is not valid JSON: %v", err) } // Must have panels panels, ok := dash["panels"] if !ok { t.Fatal("dashboard missing 'panels' key") } panelList, ok := panels.([]interface{}) if !ok || len(panelList) == 0 { t.Fatal("dashboard has no panels") } } func TestDashboard_QueriesReferenceRealMetrics(t *testing.T) { path := filepath.Join(testdataDir(), "dashboards", "block-overview.json") data, err := os.ReadFile(path) if err != nil { t.Skipf("dashboard file not found: %v", err) } content := string(data) requiredMetrics := []string{ "seaweedfs_blockvol_write_ops_total", "seaweedfs_blockvol_read_ops_total", "seaweedfs_blockvol_wal_used_fraction", "seaweedfs_blockvol_wal_pressure_state", "seaweedfs_blockvol_flusher_bytes_total", "seaweedfs_blockvol_flusher_errors_total", } for _, m := range requiredMetrics { if !strings.Contains(content, m) { t.Fatalf("dashboard should reference metric %q", m) } } }