mirror of
https://github.com/seaweedfs/seaweedfs.git
synced 2026-09-20 13:30:46 +02:00
pingqiuandClaude Opus 4.6
ceb68cc66b
Fix three tester findings on T5:
1. RF2 with missing replicas now reports "degraded" instead of
"no_replicas". Only RF=1 with no replicas returns "no_replicas".
Missing replica in an RF2 set is a degraded cluster state.
2. TransportDegraded signal now incorporated: if master-observed
transport is degraded, ClusterReplicationMode is at least
"degraded" regardless of individual replica health.
3. API surface exposure: EngineProjectionMode and
ClusterReplicationMode now appear on blockapi.VolumeInfo and are
populated in entryToVolumeInfo(). Operators can consume both
through GET /block/volume/{name} with distinct JSON field names.
12 tests: keepup, catching_up, stale degraded, LSN gap needs_rebuild,
rebuilding role, RF1 no_replicas, RF2 missing degraded, transport
degraded, distinctness, heartbeat update, worst dominates, API
surface distinct naming.
Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
318 lines
12 KiB
Go
318 lines
12 KiB
Go
package weed_server
|
|
|
|
import (
|
|
"testing"
|
|
"time"
|
|
|
|
"github.com/seaweedfs/seaweedfs/weed/pb/master_pb"
|
|
"github.com/seaweedfs/seaweedfs/weed/storage/blockvol"
|
|
)
|
|
|
|
func TestT5_AllReplicasHealthy_Keepup(t *testing.T) {
|
|
r := NewBlockVolumeRegistry()
|
|
if err := r.Register(&BlockVolumeEntry{
|
|
Name: "vol-crm-keepup", VolumeServer: "primary:8080",
|
|
Path: "/data/vol-crm-keepup.blk", Status: StatusActive,
|
|
Role: blockvol.RoleToWire(blockvol.RolePrimary), ReplicaFactor: 2,
|
|
WALHeadLSN: 100,
|
|
Replicas: []ReplicaInfo{{
|
|
Server: "replica:8080", Path: "/data/vol-crm-keepup.blk",
|
|
HealthScore: 1.0, WALHeadLSN: 100, Ready: true,
|
|
Role: blockvol.RoleToWire(blockvol.RoleReplica), LastHeartbeat: time.Now(),
|
|
}},
|
|
}); err != nil {
|
|
t.Fatalf("register: %v", err)
|
|
}
|
|
|
|
entry, _ := r.Lookup("vol-crm-keepup")
|
|
if entry.ClusterReplicationMode != "keepup" {
|
|
t.Fatalf("ClusterReplicationMode=%q, want %q", entry.ClusterReplicationMode, "keepup")
|
|
}
|
|
}
|
|
|
|
func TestT5_ReplicaBehind_CatchingUp(t *testing.T) {
|
|
r := NewBlockVolumeRegistry()
|
|
if err := r.Register(&BlockVolumeEntry{
|
|
Name: "vol-crm-catch", VolumeServer: "primary:8080",
|
|
Path: "/data/vol-crm-catch.blk", Status: StatusActive,
|
|
Role: blockvol.RoleToWire(blockvol.RolePrimary), ReplicaFactor: 2,
|
|
WALHeadLSN: 100,
|
|
Replicas: []ReplicaInfo{{
|
|
Server: "replica:8080", Path: "/data/vol-crm-catch.blk",
|
|
HealthScore: 1.0, WALHeadLSN: 90, Ready: true,
|
|
Role: blockvol.RoleToWire(blockvol.RoleReplica), LastHeartbeat: time.Now(),
|
|
}},
|
|
}); err != nil {
|
|
t.Fatalf("register: %v", err)
|
|
}
|
|
|
|
entry, _ := r.Lookup("vol-crm-catch")
|
|
if entry.ClusterReplicationMode != "catching_up" {
|
|
t.Fatalf("ClusterReplicationMode=%q, want %q", entry.ClusterReplicationMode, "catching_up")
|
|
}
|
|
}
|
|
|
|
func TestT5_StaleHeartbeat_Degraded(t *testing.T) {
|
|
r := NewBlockVolumeRegistry()
|
|
if err := r.Register(&BlockVolumeEntry{
|
|
Name: "vol-crm-degrade", VolumeServer: "primary:8080",
|
|
Path: "/data/vol-crm-degrade.blk", Status: StatusActive,
|
|
Role: blockvol.RoleToWire(blockvol.RolePrimary), ReplicaFactor: 2,
|
|
WALHeadLSN: 100,
|
|
Replicas: []ReplicaInfo{{
|
|
Server: "replica:8080", Path: "/data/vol-crm-degrade.blk",
|
|
HealthScore: 1.0, WALHeadLSN: 100, Ready: true,
|
|
Role: blockvol.RoleToWire(blockvol.RoleReplica),
|
|
LastHeartbeat: time.Now().Add(-120 * time.Second), // stale
|
|
}},
|
|
}); err != nil {
|
|
t.Fatalf("register: %v", err)
|
|
}
|
|
|
|
entry, _ := r.Lookup("vol-crm-degrade")
|
|
if entry.ClusterReplicationMode != "degraded" {
|
|
t.Fatalf("ClusterReplicationMode=%q, want %q", entry.ClusterReplicationMode, "degraded")
|
|
}
|
|
}
|
|
|
|
func TestT5_UnrecoverableGap_NeedsRebuild(t *testing.T) {
|
|
r := NewBlockVolumeRegistry()
|
|
if err := r.Register(&BlockVolumeEntry{
|
|
Name: "vol-crm-rebuild", VolumeServer: "primary:8080",
|
|
Path: "/data/vol-crm-rebuild.blk", Status: StatusActive,
|
|
Role: blockvol.RoleToWire(blockvol.RolePrimary), ReplicaFactor: 2,
|
|
WALHeadLSN: 5000,
|
|
Replicas: []ReplicaInfo{{
|
|
Server: "replica:8080", Path: "/data/vol-crm-rebuild.blk",
|
|
HealthScore: 1.0, WALHeadLSN: 100, Ready: true, // lag > 1000
|
|
Role: blockvol.RoleToWire(blockvol.RoleReplica), LastHeartbeat: time.Now(),
|
|
}},
|
|
}); err != nil {
|
|
t.Fatalf("register: %v", err)
|
|
}
|
|
|
|
entry, _ := r.Lookup("vol-crm-rebuild")
|
|
if entry.ClusterReplicationMode != "needs_rebuild" {
|
|
t.Fatalf("ClusterReplicationMode=%q, want %q", entry.ClusterReplicationMode, "needs_rebuild")
|
|
}
|
|
}
|
|
|
|
func TestT5_RebuildingRole_NeedsRebuild(t *testing.T) {
|
|
r := NewBlockVolumeRegistry()
|
|
if err := r.Register(&BlockVolumeEntry{
|
|
Name: "vol-crm-rebuilding", VolumeServer: "primary:8080",
|
|
Path: "/data/vol-crm-rebuilding.blk", Status: StatusActive,
|
|
Role: blockvol.RoleToWire(blockvol.RolePrimary), ReplicaFactor: 2,
|
|
WALHeadLSN: 100,
|
|
Replicas: []ReplicaInfo{{
|
|
Server: "replica:8080", Path: "/data/vol-crm-rebuilding.blk",
|
|
HealthScore: 1.0, WALHeadLSN: 100, Ready: true,
|
|
Role: blockvol.RoleToWire(blockvol.RoleRebuilding), LastHeartbeat: time.Now(),
|
|
}},
|
|
}); err != nil {
|
|
t.Fatalf("register: %v", err)
|
|
}
|
|
|
|
entry, _ := r.Lookup("vol-crm-rebuilding")
|
|
if entry.ClusterReplicationMode != "needs_rebuild" {
|
|
t.Fatalf("ClusterReplicationMode=%q, want %q", entry.ClusterReplicationMode, "needs_rebuild")
|
|
}
|
|
}
|
|
|
|
func TestT5_RF1_NoReplicas(t *testing.T) {
|
|
r := NewBlockVolumeRegistry()
|
|
if err := r.Register(&BlockVolumeEntry{
|
|
Name: "vol-crm-rf1", VolumeServer: "primary:8080",
|
|
Path: "/data/vol-crm-rf1.blk", Status: StatusActive,
|
|
Role: blockvol.RoleToWire(blockvol.RolePrimary), ReplicaFactor: 1,
|
|
}); err != nil {
|
|
t.Fatalf("register: %v", err)
|
|
}
|
|
|
|
entry, _ := r.Lookup("vol-crm-rf1")
|
|
if entry.ClusterReplicationMode != "no_replicas" {
|
|
t.Fatalf("ClusterReplicationMode=%q, want %q for RF=1", entry.ClusterReplicationMode, "no_replicas")
|
|
}
|
|
}
|
|
|
|
func TestT5_RF2_MissingReplica_Degraded(t *testing.T) {
|
|
r := NewBlockVolumeRegistry()
|
|
if err := r.Register(&BlockVolumeEntry{
|
|
Name: "vol-crm-missing", VolumeServer: "primary:8080",
|
|
Path: "/data/vol-crm-missing.blk", Status: StatusActive,
|
|
Role: blockvol.RoleToWire(blockvol.RolePrimary), ReplicaFactor: 2,
|
|
// RF=2 but no replicas registered → degraded, not "no_replicas"
|
|
}); err != nil {
|
|
t.Fatalf("register: %v", err)
|
|
}
|
|
|
|
entry, _ := r.Lookup("vol-crm-missing")
|
|
if entry.ClusterReplicationMode != "degraded" {
|
|
t.Fatalf("ClusterReplicationMode=%q, want %q for RF=2 with missing replica", entry.ClusterReplicationMode, "degraded")
|
|
}
|
|
}
|
|
|
|
func TestT5_TransportDegraded_Degraded(t *testing.T) {
|
|
r := NewBlockVolumeRegistry()
|
|
if err := r.Register(&BlockVolumeEntry{
|
|
Name: "vol-crm-transport", VolumeServer: "primary:8080",
|
|
Path: "/data/vol-crm-transport.blk", Status: StatusActive,
|
|
Role: blockvol.RoleToWire(blockvol.RolePrimary), ReplicaFactor: 2,
|
|
WALHeadLSN: 100, TransportDegraded: true,
|
|
Replicas: []ReplicaInfo{{
|
|
Server: "replica:8080", Path: "/data/vol-crm-transport.blk",
|
|
HealthScore: 1.0, WALHeadLSN: 100, Ready: true,
|
|
Role: blockvol.RoleToWire(blockvol.RoleReplica), LastHeartbeat: time.Now(),
|
|
}},
|
|
}); err != nil {
|
|
t.Fatalf("register: %v", err)
|
|
}
|
|
|
|
entry, _ := r.Lookup("vol-crm-transport")
|
|
if entry.ClusterReplicationMode != "degraded" {
|
|
t.Fatalf("ClusterReplicationMode=%q, want %q for transport-degraded", entry.ClusterReplicationMode, "degraded")
|
|
}
|
|
}
|
|
|
|
func TestT5_ClusterReplicationModeDistinctFromEngineProjectionMode(t *testing.T) {
|
|
r := NewBlockVolumeRegistry()
|
|
if err := r.Register(&BlockVolumeEntry{
|
|
Name: "vol-crm-distinct", VolumeServer: "primary:8080",
|
|
Path: "/data/vol-crm-distinct.blk", Status: StatusActive,
|
|
Role: blockvol.RoleToWire(blockvol.RolePrimary), ReplicaFactor: 2,
|
|
WALHeadLSN: 100,
|
|
// EngineProjectionMode says "publish_healthy" (VS-local).
|
|
EngineProjectionMode: "publish_healthy",
|
|
HasEngineProjectionMode: true,
|
|
Replicas: []ReplicaInfo{{
|
|
Server: "replica:8080", Path: "/data/vol-crm-distinct.blk",
|
|
HealthScore: 1.0, WALHeadLSN: 50, Ready: true, // behind → catching_up
|
|
Role: blockvol.RoleToWire(blockvol.RoleReplica), LastHeartbeat: time.Now(),
|
|
}},
|
|
}); err != nil {
|
|
t.Fatalf("register: %v", err)
|
|
}
|
|
|
|
entry, _ := r.Lookup("vol-crm-distinct")
|
|
// EngineProjectionMode is VS-local: "publish_healthy"
|
|
if entry.EngineProjectionMode != "publish_healthy" {
|
|
t.Fatalf("EngineProjectionMode=%q, want %q", entry.EngineProjectionMode, "publish_healthy")
|
|
}
|
|
// ClusterReplicationMode is master-computed: "catching_up" because replica is behind.
|
|
if entry.ClusterReplicationMode != "catching_up" {
|
|
t.Fatalf("ClusterReplicationMode=%q, want %q", entry.ClusterReplicationMode, "catching_up")
|
|
}
|
|
// They must be different — proving the two are independent.
|
|
if entry.EngineProjectionMode == entry.ClusterReplicationMode {
|
|
t.Fatal("EngineProjectionMode and ClusterReplicationMode should differ in this scenario")
|
|
}
|
|
}
|
|
|
|
func TestT5_HeartbeatUpdatesClusterReplicationMode(t *testing.T) {
|
|
r := NewBlockVolumeRegistry()
|
|
r.MarkBlockCapable("primary:8080")
|
|
r.MarkBlockCapable("replica:8080")
|
|
if err := r.Register(&BlockVolumeEntry{
|
|
Name: "vol-crm-hb", VolumeServer: "primary:8080",
|
|
Path: "/data/vol-crm-hb.blk", Status: StatusActive,
|
|
Role: blockvol.RoleToWire(blockvol.RolePrimary), ReplicaFactor: 2,
|
|
WALHeadLSN: 100,
|
|
Replicas: []ReplicaInfo{{
|
|
Server: "replica:8080", Path: "/data/vol-crm-hb-replica.blk",
|
|
HealthScore: 1.0, WALHeadLSN: 100, Ready: true,
|
|
Role: blockvol.RoleToWire(blockvol.RoleReplica), LastHeartbeat: time.Now(),
|
|
}},
|
|
}); err != nil {
|
|
t.Fatalf("register: %v", err)
|
|
}
|
|
|
|
// Initial state: keepup.
|
|
entry, _ := r.Lookup("vol-crm-hb")
|
|
if entry.ClusterReplicationMode != "keepup" {
|
|
t.Fatalf("initial ClusterReplicationMode=%q, want keepup", entry.ClusterReplicationMode)
|
|
}
|
|
|
|
// Primary heartbeat with higher WALHeadLSN → replica now behind.
|
|
r.UpdateFullHeartbeat("primary:8080", []*master_pb.BlockVolumeInfoMessage{{
|
|
Path: "/data/vol-crm-hb.blk",
|
|
Role: blockvol.RoleToWire(blockvol.RolePrimary),
|
|
WalHeadLsn: 200,
|
|
}}, "")
|
|
|
|
entry, _ = r.Lookup("vol-crm-hb")
|
|
if entry.ClusterReplicationMode != "catching_up" {
|
|
t.Fatalf("after primary advance: ClusterReplicationMode=%q, want catching_up", entry.ClusterReplicationMode)
|
|
}
|
|
}
|
|
|
|
func TestT5_WorstReplicaDominates(t *testing.T) {
|
|
r := NewBlockVolumeRegistry()
|
|
if err := r.Register(&BlockVolumeEntry{
|
|
Name: "vol-crm-worst", VolumeServer: "primary:8080",
|
|
Path: "/data/vol-crm-worst.blk", Status: StatusActive,
|
|
Role: blockvol.RoleToWire(blockvol.RolePrimary), ReplicaFactor: 3,
|
|
WALHeadLSN: 100,
|
|
Replicas: []ReplicaInfo{
|
|
{
|
|
Server: "replica1:8080", Path: "/data/vol-crm-worst.blk",
|
|
HealthScore: 1.0, WALHeadLSN: 100, Ready: true, // keepup
|
|
Role: blockvol.RoleToWire(blockvol.RoleReplica), LastHeartbeat: time.Now(),
|
|
},
|
|
{
|
|
Server: "replica2:8080", Path: "/data/vol-crm-worst.blk",
|
|
HealthScore: 1.0, WALHeadLSN: 100, Ready: true,
|
|
Role: blockvol.RoleToWire(blockvol.RoleRebuilding), LastHeartbeat: time.Now(), // needs_rebuild
|
|
},
|
|
},
|
|
}); err != nil {
|
|
t.Fatalf("register: %v", err)
|
|
}
|
|
|
|
entry, _ := r.Lookup("vol-crm-worst")
|
|
// One replica is keepup, one is needs_rebuild → worst dominates.
|
|
if entry.ClusterReplicationMode != "needs_rebuild" {
|
|
t.Fatalf("ClusterReplicationMode=%q, want needs_rebuild (worst dominates)", entry.ClusterReplicationMode)
|
|
}
|
|
}
|
|
|
|
func TestT5_APISurface_DistinctNaming(t *testing.T) {
|
|
r := NewBlockVolumeRegistry()
|
|
r.MarkBlockCapable("primary:8080")
|
|
if err := r.Register(&BlockVolumeEntry{
|
|
Name: "vol-crm-api", VolumeServer: "primary:8080",
|
|
Path: "/data/vol-crm-api.blk", Status: StatusActive,
|
|
Role: blockvol.RoleToWire(blockvol.RolePrimary), ReplicaFactor: 2,
|
|
WALHeadLSN: 100,
|
|
EngineProjectionMode: "publish_healthy",
|
|
HasEngineProjectionMode: true,
|
|
Replicas: []ReplicaInfo{{
|
|
Server: "replica:8080", Path: "/data/vol-crm-api.blk",
|
|
HealthScore: 1.0, WALHeadLSN: 80, Ready: true,
|
|
Role: blockvol.RoleToWire(blockvol.RoleReplica), LastHeartbeat: time.Now(),
|
|
}},
|
|
}); err != nil {
|
|
t.Fatalf("register: %v", err)
|
|
}
|
|
|
|
entry, _ := r.Lookup("vol-crm-api")
|
|
info := entryToVolumeInfo(&entry, true)
|
|
|
|
// All three mode fields must be present and distinct.
|
|
if info.VolumeMode == "" {
|
|
t.Fatal("VolumeMode missing from API surface")
|
|
}
|
|
if info.EngineProjectionMode == "" {
|
|
t.Fatal("EngineProjectionMode missing from API surface")
|
|
}
|
|
if info.ClusterReplicationMode == "" {
|
|
t.Fatal("ClusterReplicationMode missing from API surface")
|
|
}
|
|
// EngineProjectionMode is VS-local (publish_healthy).
|
|
// ClusterReplicationMode is master-computed (catching_up because replica behind).
|
|
// They must differ in this scenario.
|
|
if info.EngineProjectionMode == info.ClusterReplicationMode {
|
|
t.Fatalf("EngineProjectionMode=%q should differ from ClusterReplicationMode=%q on API surface",
|
|
info.EngineProjectionMode, info.ClusterReplicationMode)
|
|
}
|
|
}
|