mirror of
https://github.com/seaweedfs/seaweedfs.git
synced 2026-10-19 12:34:29 +02:00
Introduce weed shell command ec.check.replication. (#10328)
* Introduce weed shell command `ec.check.replication`.
This command performs a quick check of EC volume shard replication, reporting
volumes whose shards are over- or under-replicated. Each volume is checked
against its own data+parity ratio, obtained via
erasure_coding.EcShardsVolume{Data,Parity}Shards, so builds that derive the EC
ratio per volume report custom ratios correctly.
The name follows the shell's convention (cluster.check, volume.check.disk); the
closest normal-volume counterpart is volume.fix.replication.
* shell: ec.check.replication reports mixed under+over-replication in both lists
This commit is contained in:
Chris Lu authored and GitHub committed
1 parent
cd7c8ae784
commit
10cdaf3818
3 files changed
+589
No files matched your search
@@ -0,0 +1,289 @@
|
||||
package shell
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"fmt"
|
||||
"reflect"
|
||||
"testing"
|
||||
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb/master_pb"
|
||||
)
|
||||
|
||||
// TestEcCheckReplication exercises the ec.check.replication classification and reporting over a
|
||||
// fake topology. EC volume 1 is fully replicated, EC volume 2 is
|
||||
// under-replicated, and EC volume 3 is over-replicated.
|
||||
func TestEcCheckReplication(t *testing.T) {
|
||||
nodes := []*master_pb.DataNodeInfo{
|
||||
{
|
||||
Address: "localhost:1111",
|
||||
DiskInfos: map[string]*master_pb.DiskInfo{
|
||||
"d1": {
|
||||
EcShardInfos: []*master_pb.VolumeEcShardInformationMessage{
|
||||
{
|
||||
Id: 1,
|
||||
EcIndexBits: 0b0000000011111,
|
||||
},
|
||||
{
|
||||
Id: 2,
|
||||
EcIndexBits: 0b0000000011111,
|
||||
},
|
||||
{
|
||||
Id: 3,
|
||||
EcIndexBits: 0b11111111111111,
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
Address: "localhost:2222",
|
||||
DiskInfos: map[string]*master_pb.DiskInfo{
|
||||
"d2": {
|
||||
EcShardInfos: []*master_pb.VolumeEcShardInformationMessage{
|
||||
{
|
||||
Id: 1,
|
||||
EcIndexBits: 0b11111111100000,
|
||||
},
|
||||
{
|
||||
Id: 2,
|
||||
EcIndexBits: 0b11111000000000,
|
||||
},
|
||||
{
|
||||
Id: 3,
|
||||
EcIndexBits: 0b00000000001111,
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
testCases := []struct {
|
||||
name string
|
||||
showDetails bool
|
||||
volumeIDMap map[uint32]bool
|
||||
want string
|
||||
wantErr error
|
||||
}{
|
||||
{
|
||||
name: "default",
|
||||
showDetails: false,
|
||||
want: `Found 1/3 under-replicated EC volumes: [2]
|
||||
Found 1/3 over-replicated EC volumes: [3]
|
||||
`,
|
||||
},
|
||||
{
|
||||
name: "single healthy volume",
|
||||
showDetails: false,
|
||||
volumeIDMap: map[uint32]bool{1: true},
|
||||
want: `EC volumes are healthy.
|
||||
`,
|
||||
},
|
||||
{
|
||||
name: "filtered by volume ID",
|
||||
showDetails: false,
|
||||
volumeIDMap: map[uint32]bool{1: true, 2: true, 5: true},
|
||||
want: `Found 1/2 under-replicated EC volumes: [2]
|
||||
`,
|
||||
},
|
||||
{
|
||||
name: "no valid volume IDs",
|
||||
showDetails: false,
|
||||
volumeIDMap: map[uint32]bool{5: true, 6: true},
|
||||
want: "",
|
||||
wantErr: fmt.Errorf("no EC volumes found"),
|
||||
},
|
||||
{
|
||||
name: "with details",
|
||||
showDetails: true,
|
||||
want: `Found 1/3 under-replicated EC volumes: [2]
|
||||
Found 1/3 over-replicated EC volumes: [3]
|
||||
|
||||
Shards map for under-replicated EC volume 2 (10/14 shards):
|
||||
00 => [localhost:1111]
|
||||
01 => [localhost:1111]
|
||||
02 => [localhost:1111]
|
||||
03 => [localhost:1111]
|
||||
04 => [localhost:1111]
|
||||
05 is missing
|
||||
06 is missing
|
||||
07 is missing
|
||||
08 is missing
|
||||
09 => [localhost:2222]
|
||||
10 (parity) => [localhost:2222]
|
||||
11 (parity) => [localhost:2222]
|
||||
12 (parity) => [localhost:2222]
|
||||
13 (parity) => [localhost:2222]
|
||||
|
||||
Shards map for over-replicated EC volume 3 (18/14 shards):
|
||||
00 => [localhost:1111 localhost:2222]
|
||||
01 => [localhost:1111 localhost:2222]
|
||||
02 => [localhost:1111 localhost:2222]
|
||||
03 => [localhost:1111 localhost:2222]
|
||||
04 => [localhost:1111]
|
||||
05 => [localhost:1111]
|
||||
06 => [localhost:1111]
|
||||
07 => [localhost:1111]
|
||||
08 => [localhost:1111]
|
||||
09 => [localhost:1111]
|
||||
10 (parity) => [localhost:1111]
|
||||
11 (parity) => [localhost:1111]
|
||||
12 (parity) => [localhost:1111]
|
||||
13 (parity) => [localhost:1111]
|
||||
`,
|
||||
},
|
||||
}
|
||||
|
||||
for _, tc := range testCases {
|
||||
var buf bytes.Buffer
|
||||
|
||||
runner := &ecCheckReplicationRunner{
|
||||
writer: &buf,
|
||||
dataNodes: nodes,
|
||||
volumeIDMap: tc.volumeIDMap,
|
||||
}
|
||||
gotErr := runner.checkEcVolumes(tc.showDetails)
|
||||
got := buf.String()
|
||||
|
||||
if got != tc.want {
|
||||
t.Errorf("%s: got %v, want %v", tc.name, got, tc.want)
|
||||
}
|
||||
if !reflect.DeepEqual(gotErr, tc.wantErr) {
|
||||
t.Errorf("%s: got error %v, want %v", tc.name, gotErr, tc.wantErr)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// TestEcCheckReplicationNoVolumes checks the empty-topology behavior: an unfiltered
|
||||
// run over a cluster with no EC volumes is healthy (informational message, no
|
||||
// error), while a run filtered to volume IDs that are not EC volumes errors.
|
||||
func TestEcCheckReplicationNoVolumes(t *testing.T) {
|
||||
// a data node reporting a normal volume but no EC shards
|
||||
nodes := []*master_pb.DataNodeInfo{
|
||||
{
|
||||
Address: "localhost:1111",
|
||||
DiskInfos: map[string]*master_pb.DiskInfo{"d1": {}},
|
||||
},
|
||||
}
|
||||
|
||||
t.Run("unfiltered", func(t *testing.T) {
|
||||
var buf bytes.Buffer
|
||||
runner := &ecCheckReplicationRunner{writer: &buf, dataNodes: nodes}
|
||||
if err := runner.checkEcVolumes(false); err != nil {
|
||||
t.Errorf("got error %v, want nil", err)
|
||||
}
|
||||
if got, want := buf.String(), "No EC volumes found.\n"; got != want {
|
||||
t.Errorf("got %q, want %q", got, want)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("filtered no match", func(t *testing.T) {
|
||||
var buf bytes.Buffer
|
||||
runner := &ecCheckReplicationRunner{writer: &buf, dataNodes: nodes, volumeIDMap: map[uint32]bool{5: true}}
|
||||
if err := runner.checkEcVolumes(false); err == nil {
|
||||
t.Errorf("got nil error, want %q", "no EC volumes found")
|
||||
}
|
||||
if got := buf.String(); got != "" {
|
||||
t.Errorf("got output %q, want none", got)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
// TestEcCheckReplicationUnexpectedShard verifies that a shard id beyond a volume's
|
||||
// expected data+parity range is flagged as over-replication and surfaced in the
|
||||
// details map, rather than being silently ignored. Volume 20 is a healthy 10+4
|
||||
// (shards 0..13 on nodeA) with a stray shard 14 on nodeB.
|
||||
func TestEcCheckReplicationUnexpectedShard(t *testing.T) {
|
||||
nodes := []*master_pb.DataNodeInfo{
|
||||
{
|
||||
Address: "nodeA",
|
||||
DiskInfos: map[string]*master_pb.DiskInfo{
|
||||
"d1": {
|
||||
EcShardInfos: []*master_pb.VolumeEcShardInformationMessage{
|
||||
{Id: 20, EcIndexBits: 0b11111111111111}, // shards 0..13
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
Address: "nodeB",
|
||||
DiskInfos: map[string]*master_pb.DiskInfo{
|
||||
"d2": {
|
||||
EcShardInfos: []*master_pb.VolumeEcShardInformationMessage{
|
||||
{Id: 20, EcIndexBits: 0b100000000000000}, // stray shard 14
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
want := `Found 1/1 over-replicated EC volumes: [20]
|
||||
|
||||
Shards map for over-replicated EC volume 20 (15/14 shards):
|
||||
00 => [nodeA]
|
||||
01 => [nodeA]
|
||||
02 => [nodeA]
|
||||
03 => [nodeA]
|
||||
04 => [nodeA]
|
||||
05 => [nodeA]
|
||||
06 => [nodeA]
|
||||
07 => [nodeA]
|
||||
08 => [nodeA]
|
||||
09 => [nodeA]
|
||||
10 (parity) => [nodeA]
|
||||
11 (parity) => [nodeA]
|
||||
12 (parity) => [nodeA]
|
||||
13 (parity) => [nodeA]
|
||||
14 (unexpected) => [nodeB]
|
||||
`
|
||||
|
||||
var buf bytes.Buffer
|
||||
runner := &ecCheckReplicationRunner{writer: &buf, dataNodes: nodes}
|
||||
if err := runner.checkEcVolumes(true); err != nil {
|
||||
t.Fatalf("unexpected error: %v", err)
|
||||
}
|
||||
if got := buf.String(); got != want {
|
||||
t.Errorf("unexpected shard:\n got: %q\nwant: %q", got, want)
|
||||
}
|
||||
}
|
||||
|
||||
// TestEcCheckReplicationMixedAnomalies verifies that a volume with both a missing
|
||||
// shard and a duplicated shard is reported in both lists. Volume 30 has shard 5
|
||||
// missing everywhere (under-replicated) and shard 0 on two nodes (over-replicated).
|
||||
func TestEcCheckReplicationMixedAnomalies(t *testing.T) {
|
||||
nodes := []*master_pb.DataNodeInfo{
|
||||
{
|
||||
Address: "nodeA",
|
||||
DiskInfos: map[string]*master_pb.DiskInfo{
|
||||
"d1": {
|
||||
EcShardInfos: []*master_pb.VolumeEcShardInformationMessage{
|
||||
{Id: 30, EcIndexBits: 0b11111111011111}, // shards 0..13 except 5
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
Address: "nodeB",
|
||||
DiskInfos: map[string]*master_pb.DiskInfo{
|
||||
"d2": {
|
||||
EcShardInfos: []*master_pb.VolumeEcShardInformationMessage{
|
||||
{Id: 30, EcIndexBits: 0b00000000000001}, // duplicate of shard 0
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
want := `Found 1/1 under-replicated EC volumes: [30]
|
||||
Found 1/1 over-replicated EC volumes: [30]
|
||||
`
|
||||
|
||||
var buf bytes.Buffer
|
||||
runner := &ecCheckReplicationRunner{writer: &buf, dataNodes: nodes}
|
||||
if err := runner.checkEcVolumes(false); err != nil {
|
||||
t.Fatalf("unexpected error: %v", err)
|
||||
}
|
||||
if got := buf.String(); got != want {
|
||||
t.Errorf("mixed anomalies:\n got: %q\nwant: %q", got, want)
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user