Files
seaweedfs/weed/topology/topology_test.go
T
Chris LuandClaude Opus 5.5 5c910ad546 topology: list collections without nesting the collection map read lock (#12495)
* topology: list collections without nesting the collection map read lock

Items() already takes the read lock and copies the values. Holding it
around the call made ListCollections take the read lock twice, so a
DeleteCollection waiting for the write lock in between blocked the
second read lock and wedged the master.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

* topology: test ListCollections against a concurrent collection delete

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

---------

Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-08 15:05:41 -08:00

977 lines
35 KiB
Go

package topology
import (
"reflect"
"testing"
"time"
"github.com/prometheus/client_golang/prometheus/testutil"
"github.com/seaweedfs/seaweedfs/weed/pb"
"github.com/seaweedfs/seaweedfs/weed/pb/master_pb"
"github.com/seaweedfs/seaweedfs/weed/sequence"
"github.com/seaweedfs/seaweedfs/weed/stats"
"github.com/seaweedfs/seaweedfs/weed/storage"
"github.com/seaweedfs/seaweedfs/weed/storage/erasure_coding"
"github.com/seaweedfs/seaweedfs/weed/storage/needle"
"github.com/seaweedfs/seaweedfs/weed/storage/super_block"
"github.com/seaweedfs/seaweedfs/weed/storage/types"
)
func TestRemoveDataCenter(t *testing.T) {
topo := setup(topologyLayout)
topo.UnlinkChildNode(NodeId("dc2"))
if topo.diskUsages.usages[types.HardDriveType].activeVolumeCount != 15 {
t.Fail()
}
topo.UnlinkChildNode(NodeId("dc3"))
if topo.diskUsages.usages[types.HardDriveType].activeVolumeCount != 12 {
t.Fail()
}
}
func TestHandlingVolumeServerHeartbeat(t *testing.T) {
topo := NewTopology("weedfs", sequence.NewMemorySequencer(), 32*1024, 5, false)
dc := topo.GetOrCreateDataCenter("dc1")
rack := dc.GetOrCreateRack("rack1")
maxVolumeCounts := make(map[string]uint32)
maxVolumeCounts[""] = 25
maxVolumeCounts["ssd"] = 12
dn := rack.GetOrCreateDataNode("127.0.0.1", 34534, 0, "127.0.0.1", "", maxVolumeCounts)
{
volumeCount := 7
var volumeMessages []*master_pb.VolumeInformationMessage
for k := 1; k <= volumeCount; k++ {
volumeMessage := &master_pb.VolumeInformationMessage{
Id: uint32(k),
Size: uint64(25432),
Collection: "",
FileCount: uint64(2343),
DeleteCount: uint64(345),
DeletedByteCount: 34524,
ReadOnly: false,
ReplicaPlacement: uint32(0),
Version: uint32(needle.GetCurrentVersion()),
Ttl: 0,
}
volumeMessages = append(volumeMessages, volumeMessage)
}
for k := 1; k <= volumeCount; k++ {
volumeMessage := &master_pb.VolumeInformationMessage{
Id: uint32(volumeCount + k),
Size: uint64(25432),
Collection: "",
FileCount: uint64(2343),
DeleteCount: uint64(345),
DeletedByteCount: 34524,
ReadOnly: false,
ReplicaPlacement: uint32(0),
Version: uint32(needle.GetCurrentVersion()),
Ttl: 0,
DiskType: "ssd",
}
volumeMessages = append(volumeMessages, volumeMessage)
}
topo.SyncDataNodeRegistration(volumeMessages, dn)
usageCounts := topo.diskUsages.usages[types.HardDriveType]
assert(t, "activeVolumeCount1", int(usageCounts.activeVolumeCount), volumeCount)
assert(t, "volumeCount", int(usageCounts.volumeCount), volumeCount)
assert(t, "ssdVolumeCount", int(topo.diskUsages.usages[types.SsdType].volumeCount), volumeCount)
}
{
volumeCount := 7 - 1
var volumeMessages []*master_pb.VolumeInformationMessage
for k := 1; k <= volumeCount; k++ {
volumeMessage := &master_pb.VolumeInformationMessage{
Id: uint32(k),
Size: uint64(30000),
Collection: "",
FileCount: uint64(2343),
DeleteCount: uint64(345),
DeletedByteCount: 345240,
ReadOnly: false,
ReplicaPlacement: uint32(0),
Version: uint32(needle.GetCurrentVersion()),
Ttl: 0,
}
volumeMessages = append(volumeMessages, volumeMessage)
}
topo.SyncDataNodeRegistration(volumeMessages, dn)
//rp, _ := storage.NewReplicaPlacementFromString("000")
//layout := topo.GetVolumeLayout("", rp, needle.EMPTY_TTL)
//assert(t, "writables", len(layout.writables), volumeCount)
usageCounts := topo.diskUsages.usages[types.HardDriveType]
assert(t, "activeVolumeCount1", int(usageCounts.activeVolumeCount), volumeCount)
assert(t, "volumeCount", int(usageCounts.volumeCount), volumeCount)
}
{
volumeCount := 6
newVolumeShortMessage := &master_pb.VolumeShortInformationMessage{
Id: uint32(3),
Collection: "",
ReplicaPlacement: uint32(0),
Version: uint32(needle.GetCurrentVersion()),
Ttl: 0,
}
topo.IncrementalSyncDataNodeRegistration(
[]*master_pb.VolumeShortInformationMessage{newVolumeShortMessage},
nil,
dn)
rp, _ := super_block.NewReplicaPlacementFromString("000")
layout := topo.GetVolumeLayout("", rp, needle.EMPTY_TTL, types.HardDriveType)
assert(t, "writables after repeated add", len(layout.writables), volumeCount)
usageCounts := topo.diskUsages.usages[types.HardDriveType]
assert(t, "activeVolumeCount1", int(usageCounts.activeVolumeCount), volumeCount)
assert(t, "volumeCount", int(usageCounts.volumeCount), volumeCount)
topo.IncrementalSyncDataNodeRegistration(
nil,
[]*master_pb.VolumeShortInformationMessage{newVolumeShortMessage},
dn)
assert(t, "writables after deletion", len(layout.writables), volumeCount-1)
assert(t, "activeVolumeCount1", int(usageCounts.activeVolumeCount), volumeCount-1)
assert(t, "volumeCount", int(usageCounts.volumeCount), volumeCount-1)
topo.IncrementalSyncDataNodeRegistration(
[]*master_pb.VolumeShortInformationMessage{newVolumeShortMessage},
nil,
dn)
for vid := range layout.vid2location {
println("after add volume id", vid)
}
for _, vid := range layout.writables {
println("after add writable volume id", vid)
}
assert(t, "writables after add back", len(layout.writables), volumeCount)
}
topo.UnRegisterDataNode(dn)
usageCounts := topo.diskUsages.usages[types.HardDriveType]
assert(t, "activeVolumeCount2", int(usageCounts.activeVolumeCount), 0)
}
func TestIncrementalSyncReplacesVolumeReadOnlyState(t *testing.T) {
for _, tc := range []struct {
name string
fromReadOnly bool
toReadOnly bool
}{
{name: "writable to read-only", toReadOnly: true},
{name: "read-only to writable", fromReadOnly: true},
} {
t.Run(tc.name, func(t *testing.T) {
topo := NewTopology("weedfs", sequence.NewMemorySequencer(), 32*1024, 5, false)
dn := topo.GetOrCreateDataCenter("dc1").GetOrCreateRack("rack1").
GetOrCreateDataNode("127.0.0.1", 34534, 0, "127.0.0.1", "", map[string]uint32{"": 25})
volume := func(readOnly bool) *master_pb.VolumeShortInformationMessage {
return &master_pb.VolumeShortInformationMessage{
Id: 1, Collection: "c", Version: uint32(needle.GetCurrentVersion()), ReadOnly: readOnly,
}
}
oldVolume := volume(tc.fromReadOnly)
topo.IncrementalSyncDataNodeRegistration([]*master_pb.VolumeShortInformationMessage{oldVolume}, nil, dn)
topo.IncrementalSyncDataNodeRegistration(
[]*master_pb.VolumeShortInformationMessage{volume(tc.toReadOnly)},
[]*master_pb.VolumeShortInformationMessage{oldVolume}, dn)
locations := topo.Lookup("c", needle.VolumeId(1))
if len(locations) != 1 || locations[0] != dn {
t.Fatalf("lookup locations = %v, want only %v", locations, dn)
}
stored, err := dn.GetVolumesById(needle.VolumeId(1))
if err != nil || stored.ReadOnly != tc.toReadOnly {
t.Fatalf("stored volume = %+v, err = %v, want read-only %t", stored, err, tc.toReadOnly)
}
rp, _ := super_block.NewReplicaPlacementFromString("000")
active, _ := topo.GetVolumeLayout("c", rp, needle.EMPTY_TTL, types.HardDriveType).GetWritableVolumeCount()
want := 0
if !tc.toReadOnly {
want = 1
}
if active != want {
t.Fatalf("writable count = %d, want %d", active, want)
}
if !dn.HasConsistentVolumeIndex() {
t.Fatal("replacement left the held and servable volume indexes inconsistent")
}
})
}
}
func TestIncrementalSyncRegistersMovedVolumeBeforeRemoval(t *testing.T) {
topo := NewTopology("weedfs", sequence.NewMemorySequencer(), 32*1024, 5, false)
dn := topo.GetOrCreateDataCenter("dc1").GetOrCreateRack("rack1").
GetOrCreateDataNode("127.0.0.1", 34534, 0, "127.0.0.1", "", map[string]uint32{"": 25, "ssd": 25})
oldVolume := &master_pb.VolumeShortInformationMessage{
Id: 1, Collection: "c", Version: uint32(needle.GetCurrentVersion()),
}
newVolume := &master_pb.VolumeShortInformationMessage{
Id: 1, Collection: "c", Version: uint32(needle.GetCurrentVersion()), DiskType: "ssd",
}
topo.IncrementalSyncDataNodeRegistration([]*master_pb.VolumeShortInformationMessage{oldVolume}, nil, dn)
rp, _ := super_block.NewReplicaPlacementFromString("000")
oldLayout := topo.GetVolumeLayout("c", rp, needle.EMPTY_TTL, types.HardDriveType)
newLayout := topo.GetVolumeLayout("c", rp, needle.EMPTY_TTL, types.SsdType)
oldLayout.accessLock.Lock()
done := make(chan struct{})
go func() {
topo.IncrementalSyncDataNodeRegistration(
[]*master_pb.VolumeShortInformationMessage{newVolume},
[]*master_pb.VolumeShortInformationMessage{oldVolume}, dn)
close(done)
}()
deadline := time.NewTimer(time.Second)
defer deadline.Stop()
ticker := time.NewTicker(time.Millisecond)
defer ticker.Stop()
movedBeforeRemoval := false
for !movedBeforeRemoval {
select {
case <-deadline.C:
oldLayout.accessLock.Unlock()
<-done
t.Fatal("destination layout was not registered before source removal")
case <-ticker.C:
movedBeforeRemoval = len(newLayout.Lookup(needle.VolumeId(1))) == 1
}
}
oldLayout.accessLock.Unlock()
<-done
locations := topo.Lookup("c", needle.VolumeId(1))
if len(locations) != 1 || locations[0] != dn {
t.Fatalf("lookup locations = %v, want only %v", locations, dn)
}
}
func TestDataNodeToDataNodeInfo_IncludeEmptyDiskFromUsage(t *testing.T) {
dn := NewDataNode("node-1")
dn.Ip = "127.0.0.1"
dn.Port = 18080
dn.GrpcPort = 28080
// Simulate a node that has slot counters but no mounted volumes yet.
usage := dn.diskUsages.getOrCreateDisk(types.HardDriveType)
usage.maxVolumeCount = 8
info := dn.ToDataNodeInfo(VolumeFilter{})
diskInfo, found := info.DiskInfos[""]
if !found {
t.Fatalf("expected default disk entry for empty node")
}
if diskInfo.MaxVolumeCount != 8 {
t.Fatalf("unexpected max volume count: got=%d want=8", diskInfo.MaxVolumeCount)
}
if len(diskInfo.VolumeInfos) != 0 {
t.Fatalf("expected no volumes for empty disk, got=%d", len(diskInfo.VolumeInfos))
}
}
func assert(t *testing.T, message string, actual, expected int) {
if actual != expected {
t.Fatalf("unexpected %s: %d, expected: %d", message, actual, expected)
}
}
func TestAddRemoveVolume(t *testing.T) {
topo := NewTopology("weedfs", sequence.NewMemorySequencer(), 32*1024, 5, false)
dc := topo.GetOrCreateDataCenter("dc1")
rack := dc.GetOrCreateRack("rack1")
maxVolumeCounts := make(map[string]uint32)
maxVolumeCounts[""] = 25
maxVolumeCounts["ssd"] = 12
dn := rack.GetOrCreateDataNode("127.0.0.1", 34534, 0, "127.0.0.1", "", maxVolumeCounts)
v := storage.VolumeInfo{
Id: needle.VolumeId(1),
Size: 100,
Collection: "xcollection",
DiskType: "ssd",
FileCount: 123,
DeleteCount: 23,
DeletedByteCount: 45,
ReadOnly: false,
Version: needle.GetCurrentVersion(),
ReplicaPlacement: &super_block.ReplicaPlacement{},
Ttl: needle.EMPTY_TTL,
}
dn.UpdateVolumes([]storage.VolumeInfo{v})
topo.RegisterVolumeLayout(v, dn)
topo.RegisterVolumeLayout(v, dn)
if _, hasCollection := topo.FindCollection(v.Collection); !hasCollection {
t.Errorf("collection %v should exist", v.Collection)
}
topo.UnRegisterVolumeLayout(v, dn)
if _, hasCollection := topo.FindCollection(v.Collection); hasCollection {
t.Errorf("collection %v should not exist", v.Collection)
}
}
func TestUnRegisterVolumeLayoutClearsReplicaPlacementMismatchMetric(t *testing.T) {
stats.MasterReplicaPlacementMismatch.Reset()
t.Cleanup(stats.MasterReplicaPlacementMismatch.Reset)
topo := NewTopology("weedfs", sequence.NewMemorySequencer(), 32*1024, 5, false)
dc := topo.GetOrCreateDataCenter("dc1")
rack := dc.GetOrCreateRack("rack1")
maxVolumeCounts := map[string]uint32{"": 25}
dn := rack.GetOrCreateDataNode("127.0.0.1", 34534, 0, "127.0.0.1", "", maxVolumeCounts)
rp, err := super_block.NewReplicaPlacementFromString("001")
if err != nil {
t.Fatalf("NewReplicaPlacementFromString: %v", err)
}
v := storage.VolumeInfo{
Id: needle.VolumeId(42),
Size: 100,
Collection: "metrics-test",
ReplicaPlacement: rp,
Ttl: needle.EMPTY_TTL,
}
dn.UpdateVolumes([]storage.VolumeInfo{v})
topo.RegisterVolumeLayout(v, dn)
stats.MasterReplicaPlacementMismatch.WithLabelValues(v.Collection, v.Id.String()).Set(1)
if n := testutil.CollectAndCount(stats.MasterReplicaPlacementMismatch); n != 1 {
t.Fatalf("expected 1 replica_placement_mismatch series, got %d", n)
}
topo.UnRegisterVolumeLayout(v, dn)
if n := testutil.CollectAndCount(stats.MasterReplicaPlacementMismatch); n != 0 {
t.Errorf("%d replica_placement_mismatch series left after volume left topology", n)
}
}
func TestUnRegisterVolumeLayoutKeepsReplicaPlacementMismatchMetricWhilePlacementsRemain(t *testing.T) {
stats.MasterReplicaPlacementMismatch.Reset()
t.Cleanup(stats.MasterReplicaPlacementMismatch.Reset)
topo := NewTopology("weedfs", sequence.NewMemorySequencer(), 32*1024, 5, false)
dc := topo.GetOrCreateDataCenter("dc1")
rack := dc.GetOrCreateRack("rack1")
maxVolumeCounts := map[string]uint32{"": 25}
dn1 := rack.GetOrCreateDataNode("127.0.0.1", 34534, 0, "127.0.0.1", "", maxVolumeCounts)
dn2 := rack.GetOrCreateDataNode("127.0.0.1", 34535, 0, "127.0.0.1", "", maxVolumeCounts)
rp, err := super_block.NewReplicaPlacementFromString("001")
if err != nil {
t.Fatalf("NewReplicaPlacementFromString: %v", err)
}
v := storage.VolumeInfo{
Id: needle.VolumeId(42),
Size: 100,
Collection: "metrics-test",
ReplicaPlacement: rp,
Ttl: needle.EMPTY_TTL,
}
dn1.UpdateVolumes([]storage.VolumeInfo{v})
dn2.UpdateVolumes([]storage.VolumeInfo{v})
topo.RegisterVolumeLayout(v, dn1)
topo.RegisterVolumeLayout(v, dn2)
stats.MasterReplicaPlacementMismatch.WithLabelValues(v.Collection, v.Id.String()).Set(1)
if n := testutil.CollectAndCount(stats.MasterReplicaPlacementMismatch); n != 1 {
t.Fatalf("expected 1 replica_placement_mismatch series, got %d", n)
}
topo.UnRegisterVolumeLayout(v, dn1)
if n := testutil.CollectAndCount(stats.MasterReplicaPlacementMismatch); n != 1 {
t.Errorf("expected series to remain while %s still holds the volume, got %d", dn2.Id(), n)
}
if got := len(topo.Lookup(v.Collection, v.Id)); got != 1 {
t.Fatalf("expected 1 remaining placement, got %d", got)
}
topo.UnRegisterVolumeLayout(v, dn2)
if n := testutil.CollectAndCount(stats.MasterReplicaPlacementMismatch); n != 0 {
t.Errorf("%d replica_placement_mismatch series left after last placement left", n)
}
}
func TestVolumeReadOnlyStatusChange(t *testing.T) {
topo := NewTopology("weedfs", sequence.NewMemorySequencer(), 32*1024, 5, false)
dc := topo.GetOrCreateDataCenter("dc1")
rack := dc.GetOrCreateRack("rack1")
maxVolumeCounts := make(map[string]uint32)
maxVolumeCounts[""] = 25
dn := rack.GetOrCreateDataNode("127.0.0.1", 34534, 0, "127.0.0.1", "", maxVolumeCounts)
// Create a writable volume
v := storage.VolumeInfo{
Id: needle.VolumeId(1),
Size: 100,
Collection: "",
DiskType: "",
FileCount: 10,
DeleteCount: 0,
DeletedByteCount: 0,
ReadOnly: false, // Initially writable
Version: needle.GetCurrentVersion(),
ReplicaPlacement: &super_block.ReplicaPlacement{},
Ttl: needle.EMPTY_TTL,
}
dn.UpdateVolumes([]storage.VolumeInfo{v})
topo.RegisterVolumeLayout(v, dn)
// Check initial active count (should be 1 since volume is writable)
usageCounts := topo.diskUsages.usages[types.HardDriveType]
assert(t, "initial activeVolumeCount", int(usageCounts.activeVolumeCount), 1)
assert(t, "initial remoteVolumeCount", int(usageCounts.remoteVolumeCount), 0)
// Change volume to read-only
v.ReadOnly = true
dn.UpdateVolumes([]storage.VolumeInfo{v})
// Check active count after marking read-only (should be 0)
usageCounts = topo.diskUsages.usages[types.HardDriveType]
assert(t, "activeVolumeCount after read-only", int(usageCounts.activeVolumeCount), 0)
// Change volume back to writable
v.ReadOnly = false
dn.UpdateVolumes([]storage.VolumeInfo{v})
// Check active count after marking writable again (should be 1)
usageCounts = topo.diskUsages.usages[types.HardDriveType]
assert(t, "activeVolumeCount after writable again", int(usageCounts.activeVolumeCount), 1)
}
func TestDeletingReadOnlyVolumeKeepsActiveCount(t *testing.T) {
topo := NewTopology("weedfs", sequence.NewMemorySequencer(), 32*1024, 5, false)
dn := topo.GetOrCreateDataCenter("dc1").GetOrCreateRack("rack1").
GetOrCreateDataNode("127.0.0.1", 34534, 0, "127.0.0.1", "", map[string]uint32{"": 25})
v := storage.VolumeInfo{
Id: needle.VolumeId(1),
Version: needle.GetCurrentVersion(),
ReplicaPlacement: &super_block.ReplicaPlacement{},
Ttl: needle.EMPTY_TTL,
}
dn.UpdateVolumes([]storage.VolumeInfo{v})
topo.RegisterVolumeLayout(v, dn)
v.ReadOnly = true
dn.UpdateVolumes([]storage.VolumeInfo{v})
assert(t, "activeVolumeCount after read-only", int(topo.diskUsages.usages[types.HardDriveType].activeVolumeCount), 0)
// The server's delete report does not say the volume was read-only.
topo.IncrementalSyncDataNodeRegistration(nil, []*master_pb.VolumeShortInformationMessage{
{Id: 1, Version: uint32(needle.GetCurrentVersion())},
}, dn)
for name, usage := range map[string]*DiskUsageCounts{
"topology": topo.diskUsages.usages[types.HardDriveType],
"node": dn.diskUsages.usages[types.HardDriveType],
} {
assert(t, name+" volumeCount after delete", int(usage.volumeCount), 0)
assert(t, name+" activeVolumeCount after delete", int(usage.activeVolumeCount), 0)
}
}
func TestVolumeReadOnlyAndRemoteStatusChange(t *testing.T) {
topo := NewTopology("weedfs", sequence.NewMemorySequencer(), 32*1024, 5, false)
dc := topo.GetOrCreateDataCenter("dc1")
rack := dc.GetOrCreateRack("rack1")
maxVolumeCounts := make(map[string]uint32)
maxVolumeCounts[""] = 25
dn := rack.GetOrCreateDataNode("127.0.0.1", 34534, 0, "127.0.0.1", "", maxVolumeCounts)
// Create a writable, local volume
v := storage.VolumeInfo{
Id: needle.VolumeId(1),
Size: 100,
Collection: "",
DiskType: "",
FileCount: 10,
DeleteCount: 0,
DeletedByteCount: 0,
ReadOnly: false, // Initially writable
RemoteStorageName: "", // Initially local
Version: needle.GetCurrentVersion(),
ReplicaPlacement: &super_block.ReplicaPlacement{},
Ttl: needle.EMPTY_TTL,
}
dn.UpdateVolumes([]storage.VolumeInfo{v})
topo.RegisterVolumeLayout(v, dn)
// Check initial counts
usageCounts := topo.diskUsages.usages[types.HardDriveType]
assert(t, "initial activeVolumeCount", int(usageCounts.activeVolumeCount), 1)
assert(t, "initial remoteVolumeCount", int(usageCounts.remoteVolumeCount), 0)
// Simultaneously change to read-only AND remote
v.ReadOnly = true
v.RemoteStorageName = "s3"
v.RemoteStorageName = "s3.default"
dn.UpdateVolumes([]storage.VolumeInfo{v})
// Check counts after both changes
usageCounts = topo.diskUsages.usages[types.HardDriveType]
assert(t, "activeVolumeCount after read-only+remote", int(usageCounts.activeVolumeCount), 0)
assert(t, "remoteVolumeCount after read-only+remote", int(usageCounts.remoteVolumeCount), 1)
// Change back to writable but keep remote
v.ReadOnly = false
dn.UpdateVolumes([]storage.VolumeInfo{v})
// Check counts - should be writable (active=1) and still remote
usageCounts = topo.diskUsages.usages[types.HardDriveType]
assert(t, "activeVolumeCount after writable+remote", int(usageCounts.activeVolumeCount), 1)
assert(t, "remoteVolumeCount after writable+remote", int(usageCounts.remoteVolumeCount), 1)
// Change back to local AND read-only simultaneously
v.ReadOnly = true
v.RemoteStorageName = ""
v.RemoteStorageName = ""
dn.UpdateVolumes([]storage.VolumeInfo{v})
// Check final counts
usageCounts = topo.diskUsages.usages[types.HardDriveType]
assert(t, "final activeVolumeCount", int(usageCounts.activeVolumeCount), 0)
assert(t, "final remoteVolumeCount", int(usageCounts.remoteVolumeCount), 0)
}
func TestListCollections(t *testing.T) {
rp, _ := super_block.NewReplicaPlacementFromString("002")
topo := NewTopology("weedfs", sequence.NewMemorySequencer(), 32*1024, 5, false)
dc := topo.GetOrCreateDataCenter("dc1")
rack := dc.GetOrCreateRack("rack1")
dn := rack.GetOrCreateDataNode("127.0.0.1", 34534, 0, "127.0.0.1", "", nil)
topo.RegisterVolumeLayout(storage.VolumeInfo{
Id: needle.VolumeId(1111),
ReplicaPlacement: rp,
}, dn)
topo.RegisterVolumeLayout(storage.VolumeInfo{
Id: needle.VolumeId(2222),
ReplicaPlacement: rp,
Collection: "vol_collection_a",
}, dn)
topo.RegisterVolumeLayout(storage.VolumeInfo{
Id: needle.VolumeId(3333),
ReplicaPlacement: rp,
Collection: "vol_collection_b",
}, dn)
topo.RegisterEcShards(&erasure_coding.EcVolumeInfo{
VolumeId: needle.VolumeId(4444),
Collection: "ec_collection_a",
ShardsInfo: erasure_coding.NewShardsInfo(),
}, dn)
topo.RegisterEcShards(&erasure_coding.EcVolumeInfo{
VolumeId: needle.VolumeId(5555),
Collection: "ec_collection_b",
ShardsInfo: erasure_coding.NewShardsInfo(),
}, dn)
testCases := []struct {
name string
includeNormalVolumes bool
includeEcVolumes bool
want []string
}{
{
name: "no volume types selected",
includeNormalVolumes: false,
includeEcVolumes: false,
want: nil,
}, {
name: "normal volumes",
includeNormalVolumes: true,
includeEcVolumes: false,
want: []string{"", "vol_collection_a", "vol_collection_b"},
}, {
name: "EC volumes",
includeNormalVolumes: false,
includeEcVolumes: true,
want: []string{"ec_collection_a", "ec_collection_b"},
}, {
name: "normal + EC volumes",
includeNormalVolumes: true,
includeEcVolumes: true,
want: []string{"", "ec_collection_a", "ec_collection_b", "vol_collection_a", "vol_collection_b"},
},
}
for _, tc := range testCases {
t.Run(tc.name, func(t *testing.T) {
got := topo.ListCollections(tc.includeNormalVolumes, tc.includeEcVolumes)
if !reflect.DeepEqual(got, tc.want) {
t.Errorf("got %v, want %v", got, tc.want)
}
})
}
}
// A collection delete that queues for the write lock while ListCollections
// reads the collections must not wedge both.
func TestListCollectionsWithConcurrentCollectionDelete(t *testing.T) {
rp, _ := super_block.NewReplicaPlacementFromString("000")
topo := NewTopology("weedfs", sequence.NewMemorySequencer(), 32*1024, 5, false)
stop := make(chan struct{})
defer close(stop)
go func() {
for {
select {
case <-stop:
return
default:
}
topo.GetVolumeLayout("churn", rp, needle.EMPTY_TTL, types.HardDriveType)
topo.DeleteCollection("churn")
}
}()
listed := make(chan struct{})
go func() {
defer close(listed)
for i := 0; i < 100000; i++ {
topo.ListCollections(true, false)
}
}()
select {
case <-listed:
case <-time.After(10 * time.Second):
t.Fatal("ListCollections deadlocked with a concurrent DeleteCollection")
}
}
func TestDataNodeIdBasedIdentification(t *testing.T) {
topo := NewTopology("weedfs", sequence.NewMemorySequencer(), 32*1024, 5, false)
dc := topo.GetOrCreateDataCenter("dc1")
rack := dc.GetOrCreateRack("rack1")
maxVolumeCounts := make(map[string]uint32)
maxVolumeCounts[""] = 10
// Test 1: Create a DataNode with explicit id
dn1 := rack.GetOrCreateDataNode("10.0.0.1", 8080, 18080, "10.0.0.1:8080", "node-1", maxVolumeCounts)
if string(dn1.Id()) != "node-1" {
t.Errorf("expected node id 'node-1', got '%s'", dn1.Id())
}
if dn1.Ip != "10.0.0.1" {
t.Errorf("expected ip '10.0.0.1', got '%s'", dn1.Ip)
}
// Test 2: Same id with different IP should return the same DataNode (K8s pod reschedule scenario)
dn2 := rack.GetOrCreateDataNode("10.0.0.2", 8080, 18080, "10.0.0.2:8080", "node-1", maxVolumeCounts)
if dn1 != dn2 {
t.Errorf("expected same DataNode for same id, got different nodes")
}
// IP should be updated to the new value
if dn2.Ip != "10.0.0.2" {
t.Errorf("expected ip to be updated to '10.0.0.2', got '%s'", dn2.Ip)
}
if dn2.PublicUrl != "10.0.0.2:8080" {
t.Errorf("expected publicUrl to be updated to '10.0.0.2:8080', got '%s'", dn2.PublicUrl)
}
// Test 3: Different id should create a new DataNode
dn3 := rack.GetOrCreateDataNode("10.0.0.3", 8080, 18080, "10.0.0.3:8080", "node-2", maxVolumeCounts)
if string(dn3.Id()) != "node-2" {
t.Errorf("expected node id 'node-2', got '%s'", dn3.Id())
}
if dn1 == dn3 {
t.Errorf("expected different DataNode for different id")
}
// Test 4: Empty id should fall back to ip:port (backward compatibility)
dn4 := rack.GetOrCreateDataNode("10.0.0.4", 8080, 18080, "10.0.0.4:8080", "", maxVolumeCounts)
if string(dn4.Id()) != "10.0.0.4:8080" {
t.Errorf("expected node id '10.0.0.4:8080' for empty id, got '%s'", dn4.Id())
}
// Test 5: Same ip:port with empty id should return the same DataNode
dn5 := rack.GetOrCreateDataNode("10.0.0.4", 8080, 18080, "10.0.0.4:8080", "", maxVolumeCounts)
if dn4 != dn5 {
t.Errorf("expected same DataNode for same ip:port with empty id")
}
// Verify we have 3 unique DataNodes total:
// - node-1 (dn1/dn2 share the same id)
// - node-2 (dn3)
// - 10.0.0.4:8080 (dn4/dn5 share the same ip:port)
children := rack.Children()
if len(children) != 3 {
t.Errorf("expected 3 DataNodes, got %d", len(children))
}
// Test 6: Transition from ip:port to explicit id
// First, the node exists with ip:port as id (dn4/dn5)
// Now the same volume server starts sending an explicit id
dn6 := rack.GetOrCreateDataNode("10.0.0.4", 8080, 18080, "10.0.0.4:8080", "node-4-explicit", maxVolumeCounts)
// Should return the same DataNode instance
if dn6 != dn4 {
t.Errorf("expected same DataNode instance during transition")
}
// But the id should now be updated to the explicit id
if string(dn6.Id()) != "node-4-explicit" {
t.Errorf("expected node id to transition to 'node-4-explicit', got '%s'", dn6.Id())
}
// The node should be re-keyed in the children map
if rack.FindDataNodeById("node-4-explicit") != dn6 {
t.Errorf("expected to find DataNode by new explicit id")
}
// Old ip:port key should no longer work
if rack.FindDataNodeById("10.0.0.4:8080") != nil {
t.Errorf("expected old ip:port id to be removed from children map")
}
// Still 3 unique DataNodes (node-1, node-2, node-4-explicit)
children = rack.Children()
if len(children) != 3 {
t.Errorf("expected 3 DataNodes after transition, got %d", len(children))
}
// Test 7: Prevent incorrect transition when a new node reuses ip:port of a node with explicit id
// Scenario: node-1 runs at 10.0.0.1:8080, dies, new node-99 starts at same ip:port
// The transition should NOT happen because node-1 already has an explicit id
dn7 := rack.GetOrCreateDataNode("10.0.0.1", 8080, 18080, "10.0.0.1:8080", "node-99", maxVolumeCounts)
// Should create a NEW DataNode, not reuse node-1
if dn7 == dn1 {
t.Errorf("expected new DataNode for node-99, got reused node-1")
}
if string(dn7.Id()) != "node-99" {
t.Errorf("expected node id 'node-99', got '%s'", dn7.Id())
}
// node-1 should still exist with its original id
if rack.FindDataNodeById("node-1") == nil {
t.Errorf("node-1 should still exist")
}
// Now we have 4 DataNodes
children = rack.Children()
if len(children) != 4 {
t.Errorf("expected 4 DataNodes, got %d", len(children))
}
}
func TestLookupDataNodeByAddress(t *testing.T) {
topo := NewTopology("weedfs", sequence.NewMemorySequencer(), 32*1024, 5, false)
dc := topo.GetOrCreateDataCenter("dc1")
rack := dc.GetOrCreateRack("rack1")
maxVolumeCounts := map[string]uint32{"": 10}
// Brand-new registration must be discoverable by both the http and
// grpc forms of the address.
dn := rack.GetOrCreateDataNode("10.1.2.3", 8080, 18080, "10.1.2.3:8080", "n1", maxVolumeCounts)
if got := topo.LookupDataNodeByAddress(pb.ServerAddress("10.1.2.3:8080")); got != dn {
t.Fatalf("lookup by http address: got %v, want %v", got, dn)
}
if got := topo.LookupDataNodeByAddress(pb.ServerAddress("10.1.2.3:8080.18080")); got != dn {
t.Fatalf("lookup by grpc-suffix address: got %v, want %v", got, dn)
}
// Unknown addresses must miss.
if got := topo.LookupDataNodeByAddress(pb.ServerAddress("127.0.0.1:1")); got != nil {
t.Fatalf("unknown address must not be found, got %v", got)
}
// Heartbeat from a moved pod (same id, new ip) updates the index in
// place: the old address is dropped and the new one resolves.
dnMoved := rack.GetOrCreateDataNode("10.9.9.9", 8080, 18080, "10.9.9.9:8080", "n1", maxVolumeCounts)
if dnMoved != dn {
t.Fatalf("expected same node instance after move, got different")
}
if got := topo.LookupDataNodeByAddress(pb.ServerAddress("10.1.2.3:8080")); got != nil {
t.Fatalf("old address must be unregistered after move, got %v", got)
}
if got := topo.LookupDataNodeByAddress(pb.ServerAddress("10.9.9.9:8080")); got != dn {
t.Fatalf("new address lookup: got %v, want %v", got, dn)
}
// UnRegisterDataNode evicts the index entry.
topo.UnRegisterDataNode(dn)
if got := topo.LookupDataNodeByAddress(pb.ServerAddress("10.9.9.9:8080")); got != nil {
t.Fatalf("address must be unregistered after UnRegisterDataNode, got %v", got)
}
}
// TestSyncDataNodeRegistrationReRegistersMissingVolume reproduces the divergence
// where a volume present on a data node (shown by volume.list / admin UI) is
// missing from the lookup index, which surfaces as "volume id not found" on
// LookupVolume. SetVolumeUnavailable (used by
// UnRegisterDataNode on a disconnect) drops the volume from the index, and the
// reconnecting full heartbeat used to skip it because it was no longer "new" to
// the disk map. The full heartbeat now self-heals.
func TestSyncDataNodeRegistrationReRegistersMissingVolume(t *testing.T) {
topo := NewTopology("weedfs", sequence.NewMemorySequencer(), 32*1024, 5, false)
dc := topo.GetOrCreateDataCenter("dc1")
rack := dc.GetOrCreateRack("rack1")
dn := rack.GetOrCreateDataNode("127.0.0.1", 34534, 0, "127.0.0.1", "", map[string]uint32{"": 25})
vid := needle.VolumeId(18994)
volumeMessage := &master_pb.VolumeInformationMessage{
Id: uint32(vid),
Size: 100,
Collection: "drr",
ReplicaPlacement: uint32(0),
Version: uint32(needle.GetCurrentVersion()),
Ttl: 0,
}
// Initial full heartbeat registers the volume in the lookup index.
topo.SyncDataNodeRegistration([]*master_pb.VolumeInformationMessage{volumeMessage}, dn)
if got := topo.Lookup("", vid); len(got) != 1 {
t.Fatalf("after registration: lookup %d got %v, want 1 location", vid, got)
}
// Drop the volume from the index the way UnRegisterDataNode does, but leave
// it in the data node's disk map (the reconnecting heartbeat did not report
// it as new).
rp, _ := super_block.NewReplicaPlacementFromString("000")
vl := topo.GetVolumeLayout("drr", rp, needle.EMPTY_TTL, types.HardDriveType)
vl.SetVolumeUnavailable(dn, vid)
// The empty entry must be removed, otherwise Lookup returns a non-nil empty
// list that still reads as "not found".
if got := topo.Lookup("", vid); got != nil {
t.Fatalf("after SetVolumeUnavailable: expected lookup miss, got %v", got)
}
if _, err := dn.GetVolumesById(vid); err != nil {
t.Fatalf("volume %d should still be in the data node disk map: %v", vid, err)
}
// The next full heartbeat re-registers the volume even though it is not new.
topo.SyncDataNodeRegistration([]*master_pb.VolumeInformationMessage{volumeMessage}, dn)
if got := topo.Lookup("", vid); len(got) != 1 {
t.Fatalf("after self-heal: lookup %d got %v, want 1 location", vid, got)
}
}
// TestSetVolumeAvailableRepairsMissingVolume covers the vacuum-commit variant of
// the same divergence. A disconnect during a long vacuum can drop a
// single-replica volume from the lookup index while it stays on the node; the
// commit then calls SetVolumeAvailable on it. That used to dereference a nil
// location and panic; it now re-creates the entry and repairs the split.
func TestSetVolumeAvailableRepairsMissingVolume(t *testing.T) {
topo := NewTopology("weedfs", sequence.NewMemorySequencer(), 32*1024, 5, false)
dc := topo.GetOrCreateDataCenter("dc1")
rack := dc.GetOrCreateRack("rack1")
dn := rack.GetOrCreateDataNode("127.0.0.1", 34534, 0, "127.0.0.1", "", map[string]uint32{"": 25})
vid := needle.VolumeId(2640)
volumeMessage := &master_pb.VolumeInformationMessage{
Id: uint32(vid),
Size: 100,
Collection: "drr",
ReplicaPlacement: uint32(0),
Version: uint32(needle.GetCurrentVersion()),
Ttl: 0,
}
topo.SyncDataNodeRegistration([]*master_pb.VolumeInformationMessage{volumeMessage}, dn)
rp, _ := super_block.NewReplicaPlacementFromString("000")
vl := topo.GetVolumeLayout("drr", rp, needle.EMPTY_TTL, types.HardDriveType)
// Disconnect drops the volume from the index but leaves it on the node.
vl.SetVolumeUnavailable(dn, vid)
if got := topo.Lookup("", vid); got != nil {
t.Fatalf("after SetVolumeUnavailable: expected lookup miss, got %v", got)
}
if _, err := dn.GetVolumesById(vid); err != nil {
t.Fatalf("volume %d should still be in the data node disk map: %v", vid, err)
}
// The vacuum commit re-marks the volume available; it must re-register it.
vl.SetVolumeAvailable(dn, vid, 100, false, false, false)
if got := topo.Lookup("", vid); len(got) != 1 {
t.Fatalf("after SetVolumeAvailable: lookup %d got %v, want 1 location", vid, got)
}
// Size tracking must be seeded too, or assigns go uncounted until the next
// heartbeat and the volume can overfill.
vl.accessLock.RLock()
_, tracked := vl.sizeTracking[vid]
vl.accessLock.RUnlock()
if !tracked {
t.Fatalf("after SetVolumeAvailable: size tracking for %d not seeded", vid)
}
}
// A read-only change that arrives in the regular heartbeat must reach the
// layout's per-replica flag, which the vacuum sweep consults. Registration
// and volume.mark already set it; the heartbeat path did not.
func TestFullHeartbeatUpdatesLayoutReadOnlyFlag(t *testing.T) {
topo := NewTopology("weedfs", sequence.NewMemorySequencer(), 32*1024, 5, false)
dn := topo.GetOrCreateDataCenter("dc1").GetOrCreateRack("rack1").
GetOrCreateDataNode("127.0.0.1", 34534, 0, "127.0.0.1", "", map[string]uint32{"": 25})
report := func(readOnly bool) []*master_pb.VolumeInformationMessage {
return []*master_pb.VolumeInformationMessage{{
Id: 1, Collection: "c", Size: 1 << 20, Version: uint32(needle.GetCurrentVersion()), ReadOnly: readOnly,
}}
}
rp, _ := super_block.NewReplicaPlacementFromString("000")
flag := func() bool {
vl := topo.GetVolumeLayout("c", rp, needle.EMPTY_TTL, types.HardDriveType)
vl.accessLock.RLock()
defer vl.accessLock.RUnlock()
return vl.vid2location[needle.VolumeId(1)].AnyReadOnly()
}
// the full list, which a server sends first and whenever the digests disagree
topo.SyncDataNodeRegistration(report(false), dn)
if flag() {
t.Fatal("volume registered writable is flagged read-only")
}
topo.SyncDataNodeRegistration(report(true), dn)
if !flag() {
t.Fatal("full heartbeat turned the volume read-only but the layout flag did not follow")
}
topo.SyncDataNodeRegistration(report(false), dn)
if flag() {
t.Fatal("full heartbeat turned the volume writable again but the layout flag stayed read-only")
}
// the changed-volumes delta, which is what a running server normally sends
topo.ApplyVolumeChanges(report(true), dn)
if !flag() {
t.Fatal("delta heartbeat turned the volume read-only but the layout flag did not follow")
}
topo.ApplyVolumeChanges(report(false), dn)
if flag() {
t.Fatal("delta heartbeat turned the volume writable again but the layout flag stayed read-only")
}
}