mirror of
https://github.com/seaweedfs/seaweedfs.git
synced 2026-11-12 23:09:33 +01:00
Chris LuandClaude Opus 5.5
a3b68aa37c
* shell: confirm trim survivors still serve the volume before deleting volume.fix.replication compared the doomed replica's index with every survivor and then deleted it unconditionally. A survivor that went away while the indexes were read still let the delete through, leaving fewer copies than the check had counted. Re-read each survivor's VolumeStatus after the index checks and skip the trim if any no longer has the volume. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * shell: test a trim whose survivor goes away after its index read Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> --------- Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
810 lines
24 KiB
Go
810 lines
24 KiB
Go
package shell
|
|
|
|
import (
|
|
"context"
|
|
"io"
|
|
"sync/atomic"
|
|
"testing"
|
|
|
|
"github.com/seaweedfs/seaweedfs/weed/pb/master_pb"
|
|
"github.com/seaweedfs/seaweedfs/weed/pb/volume_server_pb"
|
|
"github.com/seaweedfs/seaweedfs/weed/storage/needle_map"
|
|
"github.com/seaweedfs/seaweedfs/weed/storage/super_block"
|
|
"github.com/seaweedfs/seaweedfs/weed/storage/types"
|
|
"google.golang.org/grpc/codes"
|
|
"google.golang.org/grpc/status"
|
|
)
|
|
|
|
type testcase struct {
|
|
name string
|
|
replication string
|
|
replicas []*VolumeReplica
|
|
possibleLocation location
|
|
expected bool
|
|
}
|
|
|
|
func TestSatisfyReplicaPlacementComplicated(t *testing.T) {
|
|
|
|
var tests = []testcase{
|
|
{
|
|
name: "test 100 negative",
|
|
replication: "100",
|
|
replicas: []*VolumeReplica{
|
|
{
|
|
location: &location{"dc1", "r1", &master_pb.DataNodeInfo{Id: "dn1"}},
|
|
},
|
|
},
|
|
possibleLocation: location{"dc1", "r2", &master_pb.DataNodeInfo{Id: "dn2"}},
|
|
expected: false,
|
|
},
|
|
{
|
|
name: "test 100 positive",
|
|
replication: "100",
|
|
replicas: []*VolumeReplica{
|
|
{
|
|
location: &location{"dc1", "r1", &master_pb.DataNodeInfo{Id: "dn1"}},
|
|
},
|
|
},
|
|
possibleLocation: location{"dc2", "r2", &master_pb.DataNodeInfo{Id: "dn2"}},
|
|
expected: true,
|
|
},
|
|
{
|
|
name: "test 022 positive",
|
|
replication: "022",
|
|
replicas: []*VolumeReplica{
|
|
{
|
|
location: &location{"dc1", "r1", &master_pb.DataNodeInfo{Id: "dn1"}},
|
|
},
|
|
{
|
|
location: &location{"dc1", "r2", &master_pb.DataNodeInfo{Id: "dn2"}},
|
|
},
|
|
{
|
|
location: &location{"dc1", "r3", &master_pb.DataNodeInfo{Id: "dn3"}},
|
|
},
|
|
},
|
|
possibleLocation: location{"dc1", "r1", &master_pb.DataNodeInfo{Id: "dn4"}},
|
|
expected: true,
|
|
},
|
|
{
|
|
name: "test 022 negative",
|
|
replication: "022",
|
|
replicas: []*VolumeReplica{
|
|
{
|
|
location: &location{"dc1", "r1", &master_pb.DataNodeInfo{Id: "dn1"}},
|
|
},
|
|
{
|
|
location: &location{"dc1", "r2", &master_pb.DataNodeInfo{Id: "dn2"}},
|
|
},
|
|
{
|
|
location: &location{"dc1", "r3", &master_pb.DataNodeInfo{Id: "dn3"}},
|
|
},
|
|
},
|
|
possibleLocation: location{"dc1", "r4", &master_pb.DataNodeInfo{Id: "dn4"}},
|
|
expected: false,
|
|
},
|
|
{
|
|
name: "test 210 moved from 200 positive",
|
|
replication: "210",
|
|
replicas: []*VolumeReplica{
|
|
{
|
|
location: &location{"dc1", "r1", &master_pb.DataNodeInfo{Id: "dn1"}},
|
|
},
|
|
{
|
|
location: &location{"dc2", "r2", &master_pb.DataNodeInfo{Id: "dn2"}},
|
|
},
|
|
{
|
|
location: &location{"dc3", "r3", &master_pb.DataNodeInfo{Id: "dn3"}},
|
|
},
|
|
},
|
|
possibleLocation: location{"dc1", "r4", &master_pb.DataNodeInfo{Id: "dn4"}},
|
|
expected: true,
|
|
},
|
|
{
|
|
name: "test 210 moved from 200 negative extra dc",
|
|
replication: "210",
|
|
replicas: []*VolumeReplica{
|
|
{
|
|
location: &location{"dc1", "r1", &master_pb.DataNodeInfo{Id: "dn1"}},
|
|
},
|
|
{
|
|
location: &location{"dc2", "r2", &master_pb.DataNodeInfo{Id: "dn2"}},
|
|
},
|
|
{
|
|
location: &location{"dc3", "r3", &master_pb.DataNodeInfo{Id: "dn3"}},
|
|
},
|
|
},
|
|
possibleLocation: location{"dc4", "r4", &master_pb.DataNodeInfo{Id: "dn4"}},
|
|
expected: false,
|
|
},
|
|
{
|
|
name: "test 210 moved from 200 negative extra data node",
|
|
replication: "210",
|
|
replicas: []*VolumeReplica{
|
|
{
|
|
location: &location{"dc1", "r1", &master_pb.DataNodeInfo{Id: "dn1"}},
|
|
},
|
|
{
|
|
location: &location{"dc2", "r2", &master_pb.DataNodeInfo{Id: "dn2"}},
|
|
},
|
|
{
|
|
location: &location{"dc3", "r3", &master_pb.DataNodeInfo{Id: "dn3"}},
|
|
},
|
|
},
|
|
possibleLocation: location{"dc1", "r1", &master_pb.DataNodeInfo{Id: "dn4"}},
|
|
expected: false,
|
|
},
|
|
}
|
|
|
|
runTests(tests, t)
|
|
|
|
}
|
|
|
|
func TestSatisfyReplicaPlacement01x(t *testing.T) {
|
|
|
|
var tests = []testcase{
|
|
{
|
|
name: "test 011 same existing rack",
|
|
replication: "011",
|
|
replicas: []*VolumeReplica{
|
|
{
|
|
location: &location{"dc1", "r1", &master_pb.DataNodeInfo{Id: "dn1"}},
|
|
},
|
|
{
|
|
location: &location{"dc1", "r1", &master_pb.DataNodeInfo{Id: "dn2"}},
|
|
},
|
|
},
|
|
possibleLocation: location{"dc1", "r2", &master_pb.DataNodeInfo{Id: "dn3"}},
|
|
expected: true,
|
|
},
|
|
{
|
|
name: "test 011 negative",
|
|
replication: "011",
|
|
replicas: []*VolumeReplica{
|
|
{
|
|
location: &location{"dc1", "r1", &master_pb.DataNodeInfo{Id: "dn1"}},
|
|
},
|
|
{
|
|
location: &location{"dc1", "r1", &master_pb.DataNodeInfo{Id: "dn2"}},
|
|
},
|
|
},
|
|
possibleLocation: location{"dc1", "r1", &master_pb.DataNodeInfo{Id: "dn3"}},
|
|
expected: false,
|
|
},
|
|
{
|
|
name: "test 011 different existing racks",
|
|
replication: "011",
|
|
replicas: []*VolumeReplica{
|
|
{
|
|
location: &location{"dc1", "r1", &master_pb.DataNodeInfo{Id: "dn1"}},
|
|
},
|
|
{
|
|
location: &location{"dc1", "r2", &master_pb.DataNodeInfo{Id: "dn2"}},
|
|
},
|
|
},
|
|
possibleLocation: location{"dc1", "r2", &master_pb.DataNodeInfo{Id: "dn3"}},
|
|
expected: true,
|
|
},
|
|
{
|
|
name: "test 011 different existing racks negative",
|
|
replication: "011",
|
|
replicas: []*VolumeReplica{
|
|
{
|
|
location: &location{"dc1", "r1", &master_pb.DataNodeInfo{Id: "dn1"}},
|
|
},
|
|
{
|
|
location: &location{"dc1", "r2", &master_pb.DataNodeInfo{Id: "dn2"}},
|
|
},
|
|
},
|
|
possibleLocation: location{"dc1", "r3", &master_pb.DataNodeInfo{Id: "dn3"}},
|
|
expected: false,
|
|
},
|
|
}
|
|
|
|
runTests(tests, t)
|
|
|
|
}
|
|
|
|
func TestSatisfyReplicaPlacement00x(t *testing.T) {
|
|
|
|
var tests = []testcase{
|
|
{
|
|
name: "test 001",
|
|
replication: "001",
|
|
replicas: []*VolumeReplica{
|
|
{
|
|
location: &location{"dc1", "r1", &master_pb.DataNodeInfo{Id: "dn1"}},
|
|
},
|
|
},
|
|
possibleLocation: location{"dc1", "r1", &master_pb.DataNodeInfo{Id: "dn2"}},
|
|
expected: true,
|
|
},
|
|
{
|
|
name: "test 002 positive",
|
|
replication: "002",
|
|
replicas: []*VolumeReplica{
|
|
{
|
|
location: &location{"dc1", "r1", &master_pb.DataNodeInfo{Id: "dn1"}},
|
|
},
|
|
{
|
|
location: &location{"dc1", "r1", &master_pb.DataNodeInfo{Id: "dn2"}},
|
|
},
|
|
},
|
|
possibleLocation: location{"dc1", "r1", &master_pb.DataNodeInfo{Id: "dn3"}},
|
|
expected: true,
|
|
},
|
|
{
|
|
name: "test 002 negative, repeat the same node",
|
|
replication: "002",
|
|
replicas: []*VolumeReplica{
|
|
{
|
|
location: &location{"dc1", "r1", &master_pb.DataNodeInfo{Id: "dn1"}},
|
|
},
|
|
{
|
|
location: &location{"dc1", "r1", &master_pb.DataNodeInfo{Id: "dn2"}},
|
|
},
|
|
},
|
|
possibleLocation: location{"dc1", "r1", &master_pb.DataNodeInfo{Id: "dn2"}},
|
|
expected: false,
|
|
},
|
|
{
|
|
name: "test 002 negative, enough node already",
|
|
replication: "002",
|
|
replicas: []*VolumeReplica{
|
|
{
|
|
location: &location{"dc1", "r1", &master_pb.DataNodeInfo{Id: "dn1"}},
|
|
},
|
|
{
|
|
location: &location{"dc1", "r1", &master_pb.DataNodeInfo{Id: "dn2"}},
|
|
},
|
|
{
|
|
location: &location{"dc1", "r1", &master_pb.DataNodeInfo{Id: "dn3"}},
|
|
},
|
|
},
|
|
possibleLocation: location{"dc1", "r1", &master_pb.DataNodeInfo{Id: "dn4"}},
|
|
expected: false,
|
|
},
|
|
}
|
|
|
|
runTests(tests, t)
|
|
|
|
}
|
|
|
|
func TestSatisfyReplicaPlacement100(t *testing.T) {
|
|
|
|
var tests = []testcase{
|
|
{
|
|
name: "test 100",
|
|
replication: "100",
|
|
replicas: []*VolumeReplica{
|
|
{
|
|
location: &location{"dc1", "r1", &master_pb.DataNodeInfo{Id: "dn1"}},
|
|
},
|
|
{
|
|
location: &location{"dc1", "r2", &master_pb.DataNodeInfo{Id: "dn2"}},
|
|
},
|
|
},
|
|
possibleLocation: location{"dc2", "r3", &master_pb.DataNodeInfo{Id: "dn3"}},
|
|
expected: true,
|
|
},
|
|
}
|
|
|
|
runTests(tests, t)
|
|
|
|
}
|
|
|
|
func runTests(tests []testcase, t *testing.T) {
|
|
for _, tt := range tests {
|
|
replicaPlacement, _ := super_block.NewReplicaPlacementFromString(tt.replication)
|
|
println("replication:", tt.replication, "expected", tt.expected, "name:", tt.name)
|
|
if satisfyReplicaPlacement(replicaPlacement, tt.replicas, tt.possibleLocation) != tt.expected {
|
|
t.Errorf("%s: expect %v add %v to %s %+v",
|
|
tt.name, tt.expected, tt.possibleLocation, tt.replication, tt.replicas)
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestMisplacedChecking(t *testing.T) {
|
|
|
|
var tests = []testcase{
|
|
{
|
|
name: "test 001",
|
|
replication: "001",
|
|
replicas: []*VolumeReplica{
|
|
{
|
|
location: &location{"dc1", "r1", &master_pb.DataNodeInfo{Id: "dn1"}},
|
|
},
|
|
{
|
|
location: &location{"dc1", "r2", &master_pb.DataNodeInfo{Id: "dn2"}},
|
|
},
|
|
},
|
|
expected: true,
|
|
},
|
|
{
|
|
name: "test 010",
|
|
replication: "010",
|
|
replicas: []*VolumeReplica{
|
|
{
|
|
location: &location{"dc1", "r1", &master_pb.DataNodeInfo{Id: "dn1"}},
|
|
},
|
|
{
|
|
location: &location{"dc1", "r2", &master_pb.DataNodeInfo{Id: "dn2"}},
|
|
},
|
|
},
|
|
expected: false,
|
|
},
|
|
{
|
|
name: "test 011",
|
|
replication: "011",
|
|
replicas: []*VolumeReplica{
|
|
{
|
|
location: &location{"dc1", "r1", &master_pb.DataNodeInfo{Id: "dn1"}},
|
|
},
|
|
{
|
|
location: &location{"dc1", "r1", &master_pb.DataNodeInfo{Id: "dn2"}},
|
|
},
|
|
{
|
|
location: &location{"dc1", "r2", &master_pb.DataNodeInfo{Id: "dn3"}},
|
|
},
|
|
},
|
|
expected: false,
|
|
},
|
|
{
|
|
name: "test 110",
|
|
replication: "110",
|
|
replicas: []*VolumeReplica{
|
|
{
|
|
location: &location{"dc1", "r1", &master_pb.DataNodeInfo{Id: "dn1"}},
|
|
},
|
|
{
|
|
location: &location{"dc1", "r1", &master_pb.DataNodeInfo{Id: "dn2"}},
|
|
},
|
|
{
|
|
location: &location{"dc2", "r2", &master_pb.DataNodeInfo{Id: "dn3"}},
|
|
},
|
|
},
|
|
expected: true,
|
|
},
|
|
{
|
|
name: "test 100",
|
|
replication: "100",
|
|
replicas: []*VolumeReplica{
|
|
{
|
|
location: &location{"dc1", "r1", &master_pb.DataNodeInfo{Id: "dn1"}},
|
|
},
|
|
{
|
|
location: &location{"dc1", "r2", &master_pb.DataNodeInfo{Id: "dn2"}},
|
|
},
|
|
},
|
|
expected: true,
|
|
},
|
|
}
|
|
|
|
for _, tt := range tests {
|
|
replicaPlacement, _ := super_block.NewReplicaPlacementFromString(tt.replication)
|
|
println("replication:", tt.replication, "expected", tt.expected, "name:", tt.name)
|
|
if isMisplaced(tt.replicas, replicaPlacement) != tt.expected {
|
|
t.Errorf("%s: expect %v %v %+v",
|
|
tt.name, tt.expected, tt.replication, tt.replicas)
|
|
}
|
|
}
|
|
|
|
}
|
|
|
|
func TestPickingMisplacedVolumeToDelete(t *testing.T) {
|
|
|
|
var tests = []testcase{
|
|
{
|
|
name: "test 001",
|
|
replication: "001",
|
|
replicas: []*VolumeReplica{
|
|
{
|
|
location: &location{"dc1", "r1", &master_pb.DataNodeInfo{Id: "dn1"}},
|
|
info: &master_pb.VolumeInformationMessage{
|
|
Size: 100,
|
|
},
|
|
},
|
|
{
|
|
location: &location{"dc1", "r2", &master_pb.DataNodeInfo{Id: "dn2"}},
|
|
info: &master_pb.VolumeInformationMessage{
|
|
Size: 99,
|
|
},
|
|
},
|
|
},
|
|
possibleLocation: location{"dc1", "r2", &master_pb.DataNodeInfo{Id: "dn2"}},
|
|
},
|
|
{
|
|
name: "test 100",
|
|
replication: "100",
|
|
replicas: []*VolumeReplica{
|
|
{
|
|
location: &location{"dc1", "r1", &master_pb.DataNodeInfo{Id: "dn1"}},
|
|
info: &master_pb.VolumeInformationMessage{
|
|
Size: 100,
|
|
},
|
|
},
|
|
{
|
|
location: &location{"dc1", "r2", &master_pb.DataNodeInfo{Id: "dn2"}},
|
|
info: &master_pb.VolumeInformationMessage{
|
|
Size: 99,
|
|
},
|
|
},
|
|
},
|
|
possibleLocation: location{"dc1", "r2", &master_pb.DataNodeInfo{Id: "dn2"}},
|
|
},
|
|
}
|
|
|
|
for _, tt := range tests {
|
|
replicaPlacement, _ := super_block.NewReplicaPlacementFromString(tt.replication)
|
|
println("replication:", tt.replication, "name:", tt.name)
|
|
if x := pickOneMisplacedVolume(tt.replicas, replicaPlacement); x.location.dataNode.Id != tt.possibleLocation.dataNode.Id {
|
|
t.Errorf("%s: picked %+v for replication %v",
|
|
tt.name, x.location.dataNode.Id, tt.replication)
|
|
} else {
|
|
t.Logf("%s: picked %+v %v",
|
|
tt.name, x.location.dataNode.Id, tt.replication)
|
|
}
|
|
}
|
|
|
|
}
|
|
|
|
func TestPickOneReplicaToDeleteSkipsReadOnlySurvivor(t *testing.T) {
|
|
replicaPlacement, _ := super_block.NewReplicaPlacementFromString("001")
|
|
|
|
// One writable replica and one read-only replica. The trim must never
|
|
// delete the only writable replica while the survivor is read-only:
|
|
// VolumeStatus alone cannot prove the read-only .dat is readable, so the
|
|
// trim is refused.
|
|
t.Run("refuses to delete the only writable replica while a read-only survivor remains", func(t *testing.T) {
|
|
replicas := []*VolumeReplica{
|
|
{
|
|
location: &location{"dc1", "r1", &master_pb.DataNodeInfo{Id: "dn-writable"}},
|
|
info: &master_pb.VolumeInformationMessage{Size: 90, ReadOnly: false},
|
|
},
|
|
{
|
|
location: &location{"dc1", "r2", &master_pb.DataNodeInfo{Id: "dn-readonly"}},
|
|
info: &master_pb.VolumeInformationMessage{Size: 100, ReadOnly: true},
|
|
},
|
|
}
|
|
if got := pickOneReplicaToDelete(replicas, replicaPlacement); got != nil {
|
|
t.Fatalf("expected no replica to delete, got %s", got.location.dataNode.Id)
|
|
}
|
|
})
|
|
|
|
// All survivors read-only: never trim.
|
|
t.Run("refuses when all replicas are read-only", func(t *testing.T) {
|
|
replicas := []*VolumeReplica{
|
|
{
|
|
location: &location{"dc1", "r1", &master_pb.DataNodeInfo{Id: "dn1"}},
|
|
info: &master_pb.VolumeInformationMessage{Size: 100, ReadOnly: true},
|
|
},
|
|
{
|
|
location: &location{"dc1", "r2", &master_pb.DataNodeInfo{Id: "dn2"}},
|
|
info: &master_pb.VolumeInformationMessage{Size: 100, ReadOnly: true},
|
|
},
|
|
}
|
|
if got := pickOneReplicaToDelete(replicas, replicaPlacement); got != nil {
|
|
t.Fatalf("expected no replica to delete, got %s", got.location.dataNode.Id)
|
|
}
|
|
})
|
|
|
|
// Two healthy writable replicas plus a read-only one: trim the smallest
|
|
// writable, never the read-only one.
|
|
t.Run("trims the smallest writable replica and never the read-only one", func(t *testing.T) {
|
|
replicas := []*VolumeReplica{
|
|
{
|
|
location: &location{"dc1", "r1", &master_pb.DataNodeInfo{Id: "dn-big"}},
|
|
info: &master_pb.VolumeInformationMessage{Size: 100, ReadOnly: false},
|
|
},
|
|
{
|
|
location: &location{"dc1", "r2", &master_pb.DataNodeInfo{Id: "dn-small"}},
|
|
info: &master_pb.VolumeInformationMessage{Size: 90, ReadOnly: false},
|
|
},
|
|
{
|
|
location: &location{"dc1", "r3", &master_pb.DataNodeInfo{Id: "dn-readonly"}},
|
|
info: &master_pb.VolumeInformationMessage{Size: 80, ReadOnly: true},
|
|
},
|
|
}
|
|
got := pickOneReplicaToDelete(replicas, replicaPlacement)
|
|
if got == nil {
|
|
t.Fatalf("expected a writable replica to delete, got nil")
|
|
}
|
|
if got.location.dataNode.Id != "dn-small" {
|
|
t.Fatalf("expected to delete smallest writable dn-small, got %s", got.location.dataNode.Id)
|
|
}
|
|
})
|
|
}
|
|
|
|
// The over-replication writable-survivor guard must NOT leak into the misplaced
|
|
// relocation path: a misplaced volume whose replicas are all read-only (e.g. a
|
|
// full volume) must still be relocated, picking the smallest replica to delete
|
|
// and recreate at a correct placement.
|
|
func TestPickOneMisplacedVolumeRelocatesReadOnlyReplicas(t *testing.T) {
|
|
replicaPlacement, _ := super_block.NewReplicaPlacementFromString("001")
|
|
replicas := []*VolumeReplica{
|
|
{
|
|
location: &location{"dc1", "r1", &master_pb.DataNodeInfo{Id: "dn1"}},
|
|
info: &master_pb.VolumeInformationMessage{Size: 100, ReadOnly: true},
|
|
},
|
|
{
|
|
location: &location{"dc1", "r2", &master_pb.DataNodeInfo{Id: "dn2"}},
|
|
info: &master_pb.VolumeInformationMessage{Size: 99, ReadOnly: true},
|
|
},
|
|
}
|
|
got := pickOneMisplacedVolume(replicas, replicaPlacement)
|
|
if got == nil {
|
|
t.Fatal("misplaced read-only volume must still be relocated, got nil")
|
|
}
|
|
if got.location.dataNode.Id != "dn2" {
|
|
t.Fatalf("expected to relocate smallest replica dn2, got %s", got.location.dataNode.Id)
|
|
}
|
|
}
|
|
|
|
// The over-replication trim must not strip the only replica in a required
|
|
// failure domain: with "100" and replicas in dc1 + two in dc2, deleting the
|
|
// smallest (dc1) leaves both survivors in dc2 and violates placement, so the
|
|
// trim must instead delete a dc2 writable replica.
|
|
func TestPickOneReplicaToDeletePreservesPlacement(t *testing.T) {
|
|
replicaPlacement, _ := super_block.NewReplicaPlacementFromString("100")
|
|
replicas := []*VolumeReplica{
|
|
{
|
|
location: &location{"dc1", "r1", &master_pb.DataNodeInfo{Id: "dc1-writable"}},
|
|
info: &master_pb.VolumeInformationMessage{Size: 90, ReadOnly: false},
|
|
},
|
|
{
|
|
location: &location{"dc2", "r2", &master_pb.DataNodeInfo{Id: "dc2-readonly"}},
|
|
info: &master_pb.VolumeInformationMessage{Size: 80, ReadOnly: true},
|
|
},
|
|
{
|
|
location: &location{"dc2", "r3", &master_pb.DataNodeInfo{Id: "dc2-writable"}},
|
|
info: &master_pb.VolumeInformationMessage{Size: 100, ReadOnly: false},
|
|
},
|
|
}
|
|
got := pickOneReplicaToDelete(replicas, replicaPlacement)
|
|
if got == nil {
|
|
t.Fatal("expected a replica to delete")
|
|
}
|
|
if got.location.dataNode.Id != "dc2-writable" {
|
|
t.Fatalf("expected to delete dc2-writable to preserve placement, got %s", got.location.dataNode.Id)
|
|
}
|
|
}
|
|
|
|
func TestSatisfyReplicaCurrentLocation(t *testing.T) {
|
|
|
|
var tests = []testcase{
|
|
{
|
|
name: "test 001",
|
|
replication: "001",
|
|
replicas: []*VolumeReplica{
|
|
{
|
|
location: &location{"dc1", "r1", &master_pb.DataNodeInfo{Id: "dn1"}},
|
|
},
|
|
{
|
|
location: &location{"dc1", "r2", &master_pb.DataNodeInfo{Id: "dn2"}},
|
|
},
|
|
},
|
|
expected: false,
|
|
},
|
|
{
|
|
name: "test 010",
|
|
replication: "010",
|
|
replicas: []*VolumeReplica{
|
|
{
|
|
location: &location{"dc1", "r1", &master_pb.DataNodeInfo{Id: "dn1"}},
|
|
},
|
|
{
|
|
location: &location{"dc1", "r2", &master_pb.DataNodeInfo{Id: "dn2"}},
|
|
},
|
|
},
|
|
expected: true,
|
|
},
|
|
{
|
|
name: "test 011",
|
|
replication: "011",
|
|
replicas: []*VolumeReplica{
|
|
{
|
|
location: &location{"dc1", "r1", &master_pb.DataNodeInfo{Id: "dn1"}},
|
|
},
|
|
{
|
|
location: &location{"dc1", "r1", &master_pb.DataNodeInfo{Id: "dn2"}},
|
|
},
|
|
{
|
|
location: &location{"dc1", "r2", &master_pb.DataNodeInfo{Id: "dn3"}},
|
|
},
|
|
},
|
|
expected: true,
|
|
},
|
|
{
|
|
name: "test 110",
|
|
replication: "110",
|
|
replicas: []*VolumeReplica{
|
|
{
|
|
location: &location{"dc1", "r1", &master_pb.DataNodeInfo{Id: "dn1"}},
|
|
},
|
|
{
|
|
location: &location{"dc1", "r1", &master_pb.DataNodeInfo{Id: "dn2"}},
|
|
},
|
|
{
|
|
location: &location{"dc2", "r2", &master_pb.DataNodeInfo{Id: "dn3"}},
|
|
},
|
|
},
|
|
expected: true,
|
|
},
|
|
{
|
|
name: "test 100",
|
|
replication: "100",
|
|
replicas: []*VolumeReplica{
|
|
{
|
|
location: &location{"dc1", "r1", &master_pb.DataNodeInfo{Id: "dn1"}},
|
|
},
|
|
{
|
|
location: &location{"dc1", "r2", &master_pb.DataNodeInfo{Id: "dn2"}},
|
|
},
|
|
},
|
|
expected: false,
|
|
},
|
|
}
|
|
|
|
for _, tt := range tests {
|
|
t.Run(tt.name, func(t *testing.T) {
|
|
replicaPlacement, _ := super_block.NewReplicaPlacementFromString(tt.replication)
|
|
if satisfyReplicaCurrentLocation(replicaPlacement, tt.replicas) != tt.expected {
|
|
t.Errorf("%s: expect %v %v %+v",
|
|
tt.name, tt.expected, tt.replication, tt.replicas)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestClassifyReplicaSet(t *testing.T) {
|
|
|
|
var tests = []struct {
|
|
name string
|
|
replication string
|
|
replicas []*VolumeReplica
|
|
expected replicaFix
|
|
}{
|
|
{
|
|
// The full replica count is present but two copies crowd one rack.
|
|
// Deleting a misplaced replica first would drop the volume below its
|
|
// intended durability, so add a well-placed replica first.
|
|
name: "110 misplaced without surplus adds before trimming",
|
|
replication: "110",
|
|
replicas: []*VolumeReplica{
|
|
{location: &location{"dc1", "r1", &master_pb.DataNodeInfo{Id: "dn1"}}},
|
|
{location: &location{"dc1", "r1", &master_pb.DataNodeInfo{Id: "dn2"}}},
|
|
{location: &location{"dc2", "r2", &master_pb.DataNodeInfo{Id: "dn3"}}},
|
|
},
|
|
expected: replicaFixAddOneBeforeTrim,
|
|
},
|
|
{
|
|
// Same misplaced set once the well-placed replica registered: now
|
|
// there is a surplus, and the misplaced replica can be trimmed.
|
|
name: "110 misplaced with surplus trims",
|
|
replication: "110",
|
|
replicas: []*VolumeReplica{
|
|
{location: &location{"dc1", "r1", &master_pb.DataNodeInfo{Id: "dn1"}}},
|
|
{location: &location{"dc1", "r1", &master_pb.DataNodeInfo{Id: "dn2"}}},
|
|
{location: &location{"dc1", "r3", &master_pb.DataNodeInfo{Id: "dn4"}}},
|
|
{location: &location{"dc2", "r2", &master_pb.DataNodeInfo{Id: "dn3"}}},
|
|
},
|
|
expected: replicaFixTrimMisplaced,
|
|
},
|
|
{
|
|
name: "010 missing a replica adds",
|
|
replication: "010",
|
|
replicas: []*VolumeReplica{
|
|
{location: &location{"dc1", "r1", &master_pb.DataNodeInfo{Id: "dn1"}}},
|
|
},
|
|
expected: replicaFixAddOne,
|
|
},
|
|
{
|
|
// Simultaneously over-provisioned in count and lacking rack spread:
|
|
// the missing failure domain is filled first, trims come later.
|
|
name: "010 surplus count but missing rack spread adds",
|
|
replication: "010",
|
|
replicas: []*VolumeReplica{
|
|
{location: &location{"dc1", "r1", &master_pb.DataNodeInfo{Id: "dn1"}}},
|
|
{location: &location{"dc1", "r1", &master_pb.DataNodeInfo{Id: "dn2"}}},
|
|
{location: &location{"dc1", "r1", &master_pb.DataNodeInfo{Id: "dn3"}}},
|
|
},
|
|
expected: replicaFixAddOne,
|
|
},
|
|
{
|
|
name: "010 well placed needs nothing",
|
|
replication: "010",
|
|
replicas: []*VolumeReplica{
|
|
{location: &location{"dc1", "r1", &master_pb.DataNodeInfo{Id: "dn1"}}},
|
|
{location: &location{"dc1", "r2", &master_pb.DataNodeInfo{Id: "dn2"}}},
|
|
},
|
|
expected: replicaFixNothing,
|
|
},
|
|
{
|
|
name: "000 duplicate copy trims",
|
|
replication: "000",
|
|
replicas: []*VolumeReplica{
|
|
{location: &location{"dc1", "r1", &master_pb.DataNodeInfo{Id: "dn1"}}},
|
|
{location: &location{"dc1", "r2", &master_pb.DataNodeInfo{Id: "dn2"}}},
|
|
},
|
|
expected: replicaFixTrimMisplaced,
|
|
},
|
|
}
|
|
|
|
for _, tt := range tests {
|
|
t.Run(tt.name, func(t *testing.T) {
|
|
replicaPlacement, _ := super_block.NewReplicaPlacementFromString(tt.replication)
|
|
if got := classifyReplicaSet(replicaPlacement, tt.replicas); got != tt.expected {
|
|
t.Errorf("%s: expect %v got %v for %v %+v",
|
|
tt.name, tt.expected, got, tt.replication, tt.replicas)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
type trimTestVolumeServer struct {
|
|
volume_server_pb.UnimplementedVolumeServerServer
|
|
afterIndex func()
|
|
gone atomic.Bool
|
|
deletes atomic.Int32
|
|
}
|
|
|
|
func (s *trimTestVolumeServer) CopyFile(_ *volume_server_pb.CopyFileRequest, stream volume_server_pb.VolumeServer_CopyFileServer) error {
|
|
if s.gone.Load() {
|
|
return status.Error(codes.NotFound, "volume not found")
|
|
}
|
|
if err := stream.Send(&volume_server_pb.CopyFileResponse{FileContent: needle_map.ToBytes(1, types.ToOffset(8), 4)}); err != nil {
|
|
return err
|
|
}
|
|
if s.afterIndex != nil {
|
|
s.afterIndex()
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func (s *trimTestVolumeServer) VolumeStatus(context.Context, *volume_server_pb.VolumeStatusRequest) (*volume_server_pb.VolumeStatusResponse, error) {
|
|
if s.gone.Load() {
|
|
return nil, status.Error(codes.NotFound, "volume not found")
|
|
}
|
|
return &volume_server_pb.VolumeStatusResponse{}, nil
|
|
}
|
|
|
|
func (s *trimTestVolumeServer) VolumeDelete(context.Context, *volume_server_pb.VolumeDeleteRequest) (*volume_server_pb.VolumeDeleteResponse, error) {
|
|
s.deletes.Add(1)
|
|
return &volume_server_pb.VolumeDeleteResponse{}, nil
|
|
}
|
|
|
|
func TestDeleteOneVolumeRechecksSurvivorsBeforeDelete(t *testing.T) {
|
|
for _, tc := range []struct {
|
|
name string
|
|
survivorGoes bool
|
|
wantDeleted int
|
|
}{
|
|
{name: "survivor stays", wantDeleted: 1},
|
|
{name: "survivor gone after its index read", survivorGoes: true, wantDeleted: 0},
|
|
} {
|
|
t.Run(tc.name, func(t *testing.T) {
|
|
doomed, survivor := &trimTestVolumeServer{}, &trimTestVolumeServer{}
|
|
if tc.survivorGoes {
|
|
survivor.afterIndex = func() { survivor.gone.Store(true) }
|
|
}
|
|
commandEnv, _ := configureReplicationTestEnv()
|
|
replicas := []*VolumeReplica{
|
|
{location: &location{dataNode: startConfigureReplicationTestVolumeServer(t, doomed, 1)}, info: &master_pb.VolumeInformationMessage{Id: 1}},
|
|
{location: &location{dataNode: startConfigureReplicationTestVolumeServer(t, survivor, 1)}, info: &master_pb.VolumeInformationMessage{Id: 1}},
|
|
}
|
|
pickDoomed := func([]*VolumeReplica, *super_block.ReplicaPlacement) *VolumeReplica { return replicas[0] }
|
|
|
|
c := &commandVolumeFixReplication{}
|
|
deleted, err := c.deleteOneVolume(commandEnv, io.Discard, true, true, []uint32{1}, map[uint32][]*VolumeReplica{1: replicas}, pickDoomed)
|
|
if err != nil {
|
|
t.Fatalf("deleteOneVolume: %v", err)
|
|
}
|
|
if tc.survivorGoes && !survivor.gone.Load() {
|
|
t.Fatal("the survivor never served its index")
|
|
}
|
|
if deleted != tc.wantDeleted || int(doomed.deletes.Load()) != tc.wantDeleted {
|
|
t.Fatalf("deleted=%d VolumeDelete calls=%d, want %d", deleted, doomed.deletes.Load(), tc.wantDeleted)
|
|
}
|
|
})
|
|
}
|
|
}
|