mirror of
https://github.com/seaweedfs/seaweedfs.git
synced 2026-09-13 18:10:49 +02:00
Chris Lu
5ec813b4f1
* topology: follow a volume that moved between a server's disks The heartbeat diff asked only whether a volume id was reported anywhere on the node, so a volume that moved to a disk of another type stayed on the disk it left as well. The master then held two copies of it forever: the volume count was overstated, and GetVolumesById returned whichever disk the map iterated first, so lookups could hand back the disk the volume had already left. Track which disk types the heartbeat named each volume on, and treat a volume named on another disk as absent from this one. Disk types are interned to an index because a server reports a handful of them across hundreds of thousands of volumes. A volume named on two disks at once is a stale twin rather than a move, and is still kept on both -- dropping one would tell the master a replica vanished. Only a volume named twice on one disk type is unrepresentable, so that is now what marks the node, rather than any repeat of an id. * master: do not tell clients a moved volume left the node A volume moved between a node's disks is removed from one and added to the other, so it lands in both lists of the same heartbeat. Clients apply additions before deletions, so the removal wins and they end up with no location for a volume that never went anywhere. Skip removals for volumes the node still holds, as the ec shard paths already do, and update the topology before judging the delta removals so an unmount that really did happen is still reported. * trim the comments on this change to the parts that are not evident * master: judge a volume removal on normal replicas alone HasVolumesById answers for ec shards as well, so a replica encoded into ec shards looked like it was still on the node and clients were never told the normal location had gone. They hold normal and ec locations separately and prefer the normal one from the same generation, so that location would have gone on shadowing the shards.
113 lines
3.7 KiB
Go
113 lines
3.7 KiB
Go
package topology
|
|
|
|
import (
|
|
"testing"
|
|
|
|
"github.com/seaweedfs/seaweedfs/weed/pb/master_pb"
|
|
"github.com/seaweedfs/seaweedfs/weed/storage/needle"
|
|
)
|
|
|
|
func diskMoveNode(t *testing.T) (*Topology, *DataNode) {
|
|
t.Helper()
|
|
topo := NewTopology("move", nil, 32*1024*1024*1024, 5, false)
|
|
dn := topo.GetOrCreateDataCenter("dc1").GetOrCreateRack("rack1").
|
|
GetOrCreateDataNode("127.0.0.1", 8080, 18080, "", "", map[string]uint32{"": 100, "ssd": 100})
|
|
return topo, dn
|
|
}
|
|
|
|
func diskMoveVolume(id uint32, diskType string, diskId uint32) *master_pb.VolumeInformationMessage {
|
|
return &master_pb.VolumeInformationMessage{
|
|
Id: id, Size: 1024, Collection: "c", Version: 3, DiskType: diskType, DiskId: diskId,
|
|
}
|
|
}
|
|
|
|
func heldCopies(dn *DataNode) int {
|
|
total := 0
|
|
for _, c := range dn.Children() {
|
|
total += c.(*Disk).VolumeCount()
|
|
}
|
|
return total
|
|
}
|
|
|
|
func TestVolumeMovedBetweenDisks(t *testing.T) {
|
|
for _, tc := range []struct {
|
|
name string
|
|
from, to string
|
|
}{
|
|
{"SameDiskType", "", ""},
|
|
{"DifferentDiskType", "", "ssd"},
|
|
{"BackAgain", "ssd", ""},
|
|
} {
|
|
t.Run(tc.name, func(t *testing.T) {
|
|
topo, dn := diskMoveNode(t)
|
|
topo.SyncDataNodeRegistration(
|
|
[]*master_pb.VolumeInformationMessage{diskMoveVolume(1, tc.from, 0)}, dn)
|
|
topo.SyncDataNodeRegistration(
|
|
[]*master_pb.VolumeInformationMessage{diskMoveVolume(1, tc.to, 1)}, dn)
|
|
|
|
if got := heldCopies(dn); got != 1 {
|
|
t.Errorf("master holds %d copies of a volume that moved, want 1", got)
|
|
}
|
|
stored, err := dn.GetVolumesById(needle.VolumeId(1))
|
|
if err != nil {
|
|
t.Fatalf("moved volume is no longer on the node: %v", err)
|
|
}
|
|
if stored.DiskType != tc.to || stored.DiskId != 1 {
|
|
t.Errorf("master has the volume on disk %q/%d, server reports %q/1",
|
|
stored.DiskType, stored.DiskId, tc.to)
|
|
}
|
|
if !dn.HasConsistentVolumeIndex() {
|
|
t.Error("the move left the lookup index disagreeing with the disks")
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
// A stale twin is two reports, not a move: dropping one would tell the master a
|
|
// replica vanished.
|
|
func TestVolumeReportedOnTwoDiskTypesIsKept(t *testing.T) {
|
|
topo, dn := diskMoveNode(t)
|
|
topo.SyncDataNodeRegistration([]*master_pb.VolumeInformationMessage{
|
|
diskMoveVolume(1, "", 0), diskMoveVolume(1, "ssd", 1),
|
|
}, dn)
|
|
|
|
if got := heldCopies(dn); got != 2 {
|
|
t.Errorf("master holds %d copies of a volume reported on two disks, want 2", got)
|
|
}
|
|
if dn.HasDuplicateVolumeIds() {
|
|
t.Error("a volume on two disk types is representable, so it should not disable digest comparison")
|
|
}
|
|
|
|
topo.SyncDataNodeRegistration([]*master_pb.VolumeInformationMessage{diskMoveVolume(1, "ssd", 1)}, dn)
|
|
if got := heldCopies(dn); got != 1 {
|
|
t.Errorf("master holds %d copies after the twin was unmounted, want 1", got)
|
|
}
|
|
}
|
|
|
|
func TestVolumeReportedTwiceOnOneDiskTypeIsFlagged(t *testing.T) {
|
|
topo, dn := diskMoveNode(t)
|
|
first := diskMoveVolume(1, "ssd", 0)
|
|
second := diskMoveVolume(1, "ssd", 1)
|
|
topo.SyncDataNodeRegistration([]*master_pb.VolumeInformationMessage{first, second}, dn)
|
|
|
|
if !dn.HasDuplicateVolumeIds() {
|
|
t.Error("a volume reported twice on one disk type went unflagged")
|
|
}
|
|
}
|
|
|
|
func TestVolumeDigestSurvivesADiskMove(t *testing.T) {
|
|
topo, dn := diskMoveNode(t)
|
|
topo.SyncDataNodeRegistration([]*master_pb.VolumeInformationMessage{diskMoveVolume(1, "", 0)}, dn)
|
|
|
|
moved := diskMoveVolume(1, "ssd", 1)
|
|
topo.SyncDataNodeRegistration([]*master_pb.VolumeInformationMessage{moved}, dn)
|
|
|
|
reference, referenceNode := diskMoveNode(t)
|
|
reference.SyncDataNodeRegistration([]*master_pb.VolumeInformationMessage{moved}, referenceNode)
|
|
|
|
if dn.VolumeDigest() != referenceNode.VolumeDigest() {
|
|
t.Errorf("after a disk move the digest is %d, a server holding only the moved volume reports %d",
|
|
dn.VolumeDigest(), referenceNode.VolumeDigest())
|
|
}
|
|
}
|