mirror of
https://github.com/seaweedfs/seaweedfs.git
synced 2026-11-16 16:56:04 +01:00
* topology: list collections without nesting the collection map read lock Items() already takes the read lock and copies the values. Holding it around the call made ListCollections take the read lock twice, so a DeleteCollection waiting for the write lock in between blocked the second read lock and wedged the master. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * topology: test ListCollections against a concurrent collection delete Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> --------- Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
977 lines
35 KiB
Go
977 lines
35 KiB
Go
package topology
|
|
|
|
import (
|
|
"reflect"
|
|
"testing"
|
|
"time"
|
|
|
|
"github.com/prometheus/client_golang/prometheus/testutil"
|
|
"github.com/seaweedfs/seaweedfs/weed/pb"
|
|
"github.com/seaweedfs/seaweedfs/weed/pb/master_pb"
|
|
"github.com/seaweedfs/seaweedfs/weed/sequence"
|
|
"github.com/seaweedfs/seaweedfs/weed/stats"
|
|
"github.com/seaweedfs/seaweedfs/weed/storage"
|
|
"github.com/seaweedfs/seaweedfs/weed/storage/erasure_coding"
|
|
"github.com/seaweedfs/seaweedfs/weed/storage/needle"
|
|
"github.com/seaweedfs/seaweedfs/weed/storage/super_block"
|
|
"github.com/seaweedfs/seaweedfs/weed/storage/types"
|
|
)
|
|
|
|
func TestRemoveDataCenter(t *testing.T) {
|
|
topo := setup(topologyLayout)
|
|
topo.UnlinkChildNode(NodeId("dc2"))
|
|
if topo.diskUsages.usages[types.HardDriveType].activeVolumeCount != 15 {
|
|
t.Fail()
|
|
}
|
|
topo.UnlinkChildNode(NodeId("dc3"))
|
|
if topo.diskUsages.usages[types.HardDriveType].activeVolumeCount != 12 {
|
|
t.Fail()
|
|
}
|
|
}
|
|
|
|
func TestHandlingVolumeServerHeartbeat(t *testing.T) {
|
|
topo := NewTopology("weedfs", sequence.NewMemorySequencer(), 32*1024, 5, false)
|
|
|
|
dc := topo.GetOrCreateDataCenter("dc1")
|
|
rack := dc.GetOrCreateRack("rack1")
|
|
maxVolumeCounts := make(map[string]uint32)
|
|
maxVolumeCounts[""] = 25
|
|
maxVolumeCounts["ssd"] = 12
|
|
dn := rack.GetOrCreateDataNode("127.0.0.1", 34534, 0, "127.0.0.1", "", maxVolumeCounts)
|
|
|
|
{
|
|
volumeCount := 7
|
|
var volumeMessages []*master_pb.VolumeInformationMessage
|
|
for k := 1; k <= volumeCount; k++ {
|
|
volumeMessage := &master_pb.VolumeInformationMessage{
|
|
Id: uint32(k),
|
|
Size: uint64(25432),
|
|
Collection: "",
|
|
FileCount: uint64(2343),
|
|
DeleteCount: uint64(345),
|
|
DeletedByteCount: 34524,
|
|
ReadOnly: false,
|
|
ReplicaPlacement: uint32(0),
|
|
Version: uint32(needle.GetCurrentVersion()),
|
|
Ttl: 0,
|
|
}
|
|
volumeMessages = append(volumeMessages, volumeMessage)
|
|
}
|
|
|
|
for k := 1; k <= volumeCount; k++ {
|
|
volumeMessage := &master_pb.VolumeInformationMessage{
|
|
Id: uint32(volumeCount + k),
|
|
Size: uint64(25432),
|
|
Collection: "",
|
|
FileCount: uint64(2343),
|
|
DeleteCount: uint64(345),
|
|
DeletedByteCount: 34524,
|
|
ReadOnly: false,
|
|
ReplicaPlacement: uint32(0),
|
|
Version: uint32(needle.GetCurrentVersion()),
|
|
Ttl: 0,
|
|
DiskType: "ssd",
|
|
}
|
|
volumeMessages = append(volumeMessages, volumeMessage)
|
|
}
|
|
|
|
topo.SyncDataNodeRegistration(volumeMessages, dn)
|
|
|
|
usageCounts := topo.diskUsages.usages[types.HardDriveType]
|
|
|
|
assert(t, "activeVolumeCount1", int(usageCounts.activeVolumeCount), volumeCount)
|
|
assert(t, "volumeCount", int(usageCounts.volumeCount), volumeCount)
|
|
assert(t, "ssdVolumeCount", int(topo.diskUsages.usages[types.SsdType].volumeCount), volumeCount)
|
|
}
|
|
|
|
{
|
|
volumeCount := 7 - 1
|
|
var volumeMessages []*master_pb.VolumeInformationMessage
|
|
for k := 1; k <= volumeCount; k++ {
|
|
volumeMessage := &master_pb.VolumeInformationMessage{
|
|
Id: uint32(k),
|
|
Size: uint64(30000),
|
|
Collection: "",
|
|
FileCount: uint64(2343),
|
|
DeleteCount: uint64(345),
|
|
DeletedByteCount: 345240,
|
|
ReadOnly: false,
|
|
ReplicaPlacement: uint32(0),
|
|
Version: uint32(needle.GetCurrentVersion()),
|
|
Ttl: 0,
|
|
}
|
|
volumeMessages = append(volumeMessages, volumeMessage)
|
|
}
|
|
topo.SyncDataNodeRegistration(volumeMessages, dn)
|
|
|
|
//rp, _ := storage.NewReplicaPlacementFromString("000")
|
|
//layout := topo.GetVolumeLayout("", rp, needle.EMPTY_TTL)
|
|
//assert(t, "writables", len(layout.writables), volumeCount)
|
|
|
|
usageCounts := topo.diskUsages.usages[types.HardDriveType]
|
|
|
|
assert(t, "activeVolumeCount1", int(usageCounts.activeVolumeCount), volumeCount)
|
|
assert(t, "volumeCount", int(usageCounts.volumeCount), volumeCount)
|
|
}
|
|
|
|
{
|
|
volumeCount := 6
|
|
newVolumeShortMessage := &master_pb.VolumeShortInformationMessage{
|
|
Id: uint32(3),
|
|
Collection: "",
|
|
ReplicaPlacement: uint32(0),
|
|
Version: uint32(needle.GetCurrentVersion()),
|
|
Ttl: 0,
|
|
}
|
|
topo.IncrementalSyncDataNodeRegistration(
|
|
[]*master_pb.VolumeShortInformationMessage{newVolumeShortMessage},
|
|
nil,
|
|
dn)
|
|
rp, _ := super_block.NewReplicaPlacementFromString("000")
|
|
layout := topo.GetVolumeLayout("", rp, needle.EMPTY_TTL, types.HardDriveType)
|
|
assert(t, "writables after repeated add", len(layout.writables), volumeCount)
|
|
|
|
usageCounts := topo.diskUsages.usages[types.HardDriveType]
|
|
|
|
assert(t, "activeVolumeCount1", int(usageCounts.activeVolumeCount), volumeCount)
|
|
assert(t, "volumeCount", int(usageCounts.volumeCount), volumeCount)
|
|
|
|
topo.IncrementalSyncDataNodeRegistration(
|
|
nil,
|
|
[]*master_pb.VolumeShortInformationMessage{newVolumeShortMessage},
|
|
dn)
|
|
assert(t, "writables after deletion", len(layout.writables), volumeCount-1)
|
|
assert(t, "activeVolumeCount1", int(usageCounts.activeVolumeCount), volumeCount-1)
|
|
assert(t, "volumeCount", int(usageCounts.volumeCount), volumeCount-1)
|
|
|
|
topo.IncrementalSyncDataNodeRegistration(
|
|
[]*master_pb.VolumeShortInformationMessage{newVolumeShortMessage},
|
|
nil,
|
|
dn)
|
|
|
|
for vid := range layout.vid2location {
|
|
println("after add volume id", vid)
|
|
}
|
|
for _, vid := range layout.writables {
|
|
println("after add writable volume id", vid)
|
|
}
|
|
|
|
assert(t, "writables after add back", len(layout.writables), volumeCount)
|
|
|
|
}
|
|
|
|
topo.UnRegisterDataNode(dn)
|
|
|
|
usageCounts := topo.diskUsages.usages[types.HardDriveType]
|
|
|
|
assert(t, "activeVolumeCount2", int(usageCounts.activeVolumeCount), 0)
|
|
|
|
}
|
|
|
|
func TestIncrementalSyncReplacesVolumeReadOnlyState(t *testing.T) {
|
|
for _, tc := range []struct {
|
|
name string
|
|
fromReadOnly bool
|
|
toReadOnly bool
|
|
}{
|
|
{name: "writable to read-only", toReadOnly: true},
|
|
{name: "read-only to writable", fromReadOnly: true},
|
|
} {
|
|
t.Run(tc.name, func(t *testing.T) {
|
|
topo := NewTopology("weedfs", sequence.NewMemorySequencer(), 32*1024, 5, false)
|
|
dn := topo.GetOrCreateDataCenter("dc1").GetOrCreateRack("rack1").
|
|
GetOrCreateDataNode("127.0.0.1", 34534, 0, "127.0.0.1", "", map[string]uint32{"": 25})
|
|
volume := func(readOnly bool) *master_pb.VolumeShortInformationMessage {
|
|
return &master_pb.VolumeShortInformationMessage{
|
|
Id: 1, Collection: "c", Version: uint32(needle.GetCurrentVersion()), ReadOnly: readOnly,
|
|
}
|
|
}
|
|
|
|
oldVolume := volume(tc.fromReadOnly)
|
|
topo.IncrementalSyncDataNodeRegistration([]*master_pb.VolumeShortInformationMessage{oldVolume}, nil, dn)
|
|
topo.IncrementalSyncDataNodeRegistration(
|
|
[]*master_pb.VolumeShortInformationMessage{volume(tc.toReadOnly)},
|
|
[]*master_pb.VolumeShortInformationMessage{oldVolume}, dn)
|
|
|
|
locations := topo.Lookup("c", needle.VolumeId(1))
|
|
if len(locations) != 1 || locations[0] != dn {
|
|
t.Fatalf("lookup locations = %v, want only %v", locations, dn)
|
|
}
|
|
stored, err := dn.GetVolumesById(needle.VolumeId(1))
|
|
if err != nil || stored.ReadOnly != tc.toReadOnly {
|
|
t.Fatalf("stored volume = %+v, err = %v, want read-only %t", stored, err, tc.toReadOnly)
|
|
}
|
|
rp, _ := super_block.NewReplicaPlacementFromString("000")
|
|
active, _ := topo.GetVolumeLayout("c", rp, needle.EMPTY_TTL, types.HardDriveType).GetWritableVolumeCount()
|
|
want := 0
|
|
if !tc.toReadOnly {
|
|
want = 1
|
|
}
|
|
if active != want {
|
|
t.Fatalf("writable count = %d, want %d", active, want)
|
|
}
|
|
if !dn.HasConsistentVolumeIndex() {
|
|
t.Fatal("replacement left the held and servable volume indexes inconsistent")
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestIncrementalSyncRegistersMovedVolumeBeforeRemoval(t *testing.T) {
|
|
topo := NewTopology("weedfs", sequence.NewMemorySequencer(), 32*1024, 5, false)
|
|
dn := topo.GetOrCreateDataCenter("dc1").GetOrCreateRack("rack1").
|
|
GetOrCreateDataNode("127.0.0.1", 34534, 0, "127.0.0.1", "", map[string]uint32{"": 25, "ssd": 25})
|
|
oldVolume := &master_pb.VolumeShortInformationMessage{
|
|
Id: 1, Collection: "c", Version: uint32(needle.GetCurrentVersion()),
|
|
}
|
|
newVolume := &master_pb.VolumeShortInformationMessage{
|
|
Id: 1, Collection: "c", Version: uint32(needle.GetCurrentVersion()), DiskType: "ssd",
|
|
}
|
|
topo.IncrementalSyncDataNodeRegistration([]*master_pb.VolumeShortInformationMessage{oldVolume}, nil, dn)
|
|
|
|
rp, _ := super_block.NewReplicaPlacementFromString("000")
|
|
oldLayout := topo.GetVolumeLayout("c", rp, needle.EMPTY_TTL, types.HardDriveType)
|
|
newLayout := topo.GetVolumeLayout("c", rp, needle.EMPTY_TTL, types.SsdType)
|
|
oldLayout.accessLock.Lock()
|
|
done := make(chan struct{})
|
|
go func() {
|
|
topo.IncrementalSyncDataNodeRegistration(
|
|
[]*master_pb.VolumeShortInformationMessage{newVolume},
|
|
[]*master_pb.VolumeShortInformationMessage{oldVolume}, dn)
|
|
close(done)
|
|
}()
|
|
|
|
deadline := time.NewTimer(time.Second)
|
|
defer deadline.Stop()
|
|
ticker := time.NewTicker(time.Millisecond)
|
|
defer ticker.Stop()
|
|
movedBeforeRemoval := false
|
|
for !movedBeforeRemoval {
|
|
select {
|
|
case <-deadline.C:
|
|
oldLayout.accessLock.Unlock()
|
|
<-done
|
|
t.Fatal("destination layout was not registered before source removal")
|
|
case <-ticker.C:
|
|
movedBeforeRemoval = len(newLayout.Lookup(needle.VolumeId(1))) == 1
|
|
}
|
|
}
|
|
oldLayout.accessLock.Unlock()
|
|
<-done
|
|
|
|
locations := topo.Lookup("c", needle.VolumeId(1))
|
|
if len(locations) != 1 || locations[0] != dn {
|
|
t.Fatalf("lookup locations = %v, want only %v", locations, dn)
|
|
}
|
|
}
|
|
|
|
func TestDataNodeToDataNodeInfo_IncludeEmptyDiskFromUsage(t *testing.T) {
|
|
dn := NewDataNode("node-1")
|
|
dn.Ip = "127.0.0.1"
|
|
dn.Port = 18080
|
|
dn.GrpcPort = 28080
|
|
|
|
// Simulate a node that has slot counters but no mounted volumes yet.
|
|
usage := dn.diskUsages.getOrCreateDisk(types.HardDriveType)
|
|
usage.maxVolumeCount = 8
|
|
|
|
info := dn.ToDataNodeInfo(VolumeFilter{})
|
|
diskInfo, found := info.DiskInfos[""]
|
|
if !found {
|
|
t.Fatalf("expected default disk entry for empty node")
|
|
}
|
|
if diskInfo.MaxVolumeCount != 8 {
|
|
t.Fatalf("unexpected max volume count: got=%d want=8", diskInfo.MaxVolumeCount)
|
|
}
|
|
if len(diskInfo.VolumeInfos) != 0 {
|
|
t.Fatalf("expected no volumes for empty disk, got=%d", len(diskInfo.VolumeInfos))
|
|
}
|
|
}
|
|
|
|
func assert(t *testing.T, message string, actual, expected int) {
|
|
if actual != expected {
|
|
t.Fatalf("unexpected %s: %d, expected: %d", message, actual, expected)
|
|
}
|
|
}
|
|
|
|
func TestAddRemoveVolume(t *testing.T) {
|
|
|
|
topo := NewTopology("weedfs", sequence.NewMemorySequencer(), 32*1024, 5, false)
|
|
|
|
dc := topo.GetOrCreateDataCenter("dc1")
|
|
rack := dc.GetOrCreateRack("rack1")
|
|
maxVolumeCounts := make(map[string]uint32)
|
|
maxVolumeCounts[""] = 25
|
|
maxVolumeCounts["ssd"] = 12
|
|
dn := rack.GetOrCreateDataNode("127.0.0.1", 34534, 0, "127.0.0.1", "", maxVolumeCounts)
|
|
|
|
v := storage.VolumeInfo{
|
|
Id: needle.VolumeId(1),
|
|
Size: 100,
|
|
Collection: "xcollection",
|
|
DiskType: "ssd",
|
|
FileCount: 123,
|
|
DeleteCount: 23,
|
|
DeletedByteCount: 45,
|
|
ReadOnly: false,
|
|
Version: needle.GetCurrentVersion(),
|
|
ReplicaPlacement: &super_block.ReplicaPlacement{},
|
|
Ttl: needle.EMPTY_TTL,
|
|
}
|
|
|
|
dn.UpdateVolumes([]storage.VolumeInfo{v})
|
|
topo.RegisterVolumeLayout(v, dn)
|
|
topo.RegisterVolumeLayout(v, dn)
|
|
|
|
if _, hasCollection := topo.FindCollection(v.Collection); !hasCollection {
|
|
t.Errorf("collection %v should exist", v.Collection)
|
|
}
|
|
|
|
topo.UnRegisterVolumeLayout(v, dn)
|
|
|
|
if _, hasCollection := topo.FindCollection(v.Collection); hasCollection {
|
|
t.Errorf("collection %v should not exist", v.Collection)
|
|
}
|
|
}
|
|
|
|
func TestUnRegisterVolumeLayoutClearsReplicaPlacementMismatchMetric(t *testing.T) {
|
|
stats.MasterReplicaPlacementMismatch.Reset()
|
|
t.Cleanup(stats.MasterReplicaPlacementMismatch.Reset)
|
|
|
|
topo := NewTopology("weedfs", sequence.NewMemorySequencer(), 32*1024, 5, false)
|
|
|
|
dc := topo.GetOrCreateDataCenter("dc1")
|
|
rack := dc.GetOrCreateRack("rack1")
|
|
maxVolumeCounts := map[string]uint32{"": 25}
|
|
dn := rack.GetOrCreateDataNode("127.0.0.1", 34534, 0, "127.0.0.1", "", maxVolumeCounts)
|
|
|
|
rp, err := super_block.NewReplicaPlacementFromString("001")
|
|
if err != nil {
|
|
t.Fatalf("NewReplicaPlacementFromString: %v", err)
|
|
}
|
|
v := storage.VolumeInfo{
|
|
Id: needle.VolumeId(42),
|
|
Size: 100,
|
|
Collection: "metrics-test",
|
|
ReplicaPlacement: rp,
|
|
Ttl: needle.EMPTY_TTL,
|
|
}
|
|
|
|
dn.UpdateVolumes([]storage.VolumeInfo{v})
|
|
topo.RegisterVolumeLayout(v, dn)
|
|
|
|
stats.MasterReplicaPlacementMismatch.WithLabelValues(v.Collection, v.Id.String()).Set(1)
|
|
if n := testutil.CollectAndCount(stats.MasterReplicaPlacementMismatch); n != 1 {
|
|
t.Fatalf("expected 1 replica_placement_mismatch series, got %d", n)
|
|
}
|
|
|
|
topo.UnRegisterVolumeLayout(v, dn)
|
|
|
|
if n := testutil.CollectAndCount(stats.MasterReplicaPlacementMismatch); n != 0 {
|
|
t.Errorf("%d replica_placement_mismatch series left after volume left topology", n)
|
|
}
|
|
}
|
|
|
|
func TestUnRegisterVolumeLayoutKeepsReplicaPlacementMismatchMetricWhilePlacementsRemain(t *testing.T) {
|
|
stats.MasterReplicaPlacementMismatch.Reset()
|
|
t.Cleanup(stats.MasterReplicaPlacementMismatch.Reset)
|
|
|
|
topo := NewTopology("weedfs", sequence.NewMemorySequencer(), 32*1024, 5, false)
|
|
|
|
dc := topo.GetOrCreateDataCenter("dc1")
|
|
rack := dc.GetOrCreateRack("rack1")
|
|
maxVolumeCounts := map[string]uint32{"": 25}
|
|
dn1 := rack.GetOrCreateDataNode("127.0.0.1", 34534, 0, "127.0.0.1", "", maxVolumeCounts)
|
|
dn2 := rack.GetOrCreateDataNode("127.0.0.1", 34535, 0, "127.0.0.1", "", maxVolumeCounts)
|
|
|
|
rp, err := super_block.NewReplicaPlacementFromString("001")
|
|
if err != nil {
|
|
t.Fatalf("NewReplicaPlacementFromString: %v", err)
|
|
}
|
|
v := storage.VolumeInfo{
|
|
Id: needle.VolumeId(42),
|
|
Size: 100,
|
|
Collection: "metrics-test",
|
|
ReplicaPlacement: rp,
|
|
Ttl: needle.EMPTY_TTL,
|
|
}
|
|
|
|
dn1.UpdateVolumes([]storage.VolumeInfo{v})
|
|
dn2.UpdateVolumes([]storage.VolumeInfo{v})
|
|
topo.RegisterVolumeLayout(v, dn1)
|
|
topo.RegisterVolumeLayout(v, dn2)
|
|
|
|
stats.MasterReplicaPlacementMismatch.WithLabelValues(v.Collection, v.Id.String()).Set(1)
|
|
if n := testutil.CollectAndCount(stats.MasterReplicaPlacementMismatch); n != 1 {
|
|
t.Fatalf("expected 1 replica_placement_mismatch series, got %d", n)
|
|
}
|
|
|
|
topo.UnRegisterVolumeLayout(v, dn1)
|
|
|
|
if n := testutil.CollectAndCount(stats.MasterReplicaPlacementMismatch); n != 1 {
|
|
t.Errorf("expected series to remain while %s still holds the volume, got %d", dn2.Id(), n)
|
|
}
|
|
if got := len(topo.Lookup(v.Collection, v.Id)); got != 1 {
|
|
t.Fatalf("expected 1 remaining placement, got %d", got)
|
|
}
|
|
|
|
topo.UnRegisterVolumeLayout(v, dn2)
|
|
|
|
if n := testutil.CollectAndCount(stats.MasterReplicaPlacementMismatch); n != 0 {
|
|
t.Errorf("%d replica_placement_mismatch series left after last placement left", n)
|
|
}
|
|
}
|
|
|
|
func TestVolumeReadOnlyStatusChange(t *testing.T) {
|
|
topo := NewTopology("weedfs", sequence.NewMemorySequencer(), 32*1024, 5, false)
|
|
|
|
dc := topo.GetOrCreateDataCenter("dc1")
|
|
rack := dc.GetOrCreateRack("rack1")
|
|
maxVolumeCounts := make(map[string]uint32)
|
|
maxVolumeCounts[""] = 25
|
|
dn := rack.GetOrCreateDataNode("127.0.0.1", 34534, 0, "127.0.0.1", "", maxVolumeCounts)
|
|
|
|
// Create a writable volume
|
|
v := storage.VolumeInfo{
|
|
Id: needle.VolumeId(1),
|
|
Size: 100,
|
|
Collection: "",
|
|
DiskType: "",
|
|
FileCount: 10,
|
|
DeleteCount: 0,
|
|
DeletedByteCount: 0,
|
|
ReadOnly: false, // Initially writable
|
|
Version: needle.GetCurrentVersion(),
|
|
ReplicaPlacement: &super_block.ReplicaPlacement{},
|
|
Ttl: needle.EMPTY_TTL,
|
|
}
|
|
|
|
dn.UpdateVolumes([]storage.VolumeInfo{v})
|
|
topo.RegisterVolumeLayout(v, dn)
|
|
|
|
// Check initial active count (should be 1 since volume is writable)
|
|
usageCounts := topo.diskUsages.usages[types.HardDriveType]
|
|
assert(t, "initial activeVolumeCount", int(usageCounts.activeVolumeCount), 1)
|
|
assert(t, "initial remoteVolumeCount", int(usageCounts.remoteVolumeCount), 0)
|
|
|
|
// Change volume to read-only
|
|
v.ReadOnly = true
|
|
dn.UpdateVolumes([]storage.VolumeInfo{v})
|
|
|
|
// Check active count after marking read-only (should be 0)
|
|
usageCounts = topo.diskUsages.usages[types.HardDriveType]
|
|
assert(t, "activeVolumeCount after read-only", int(usageCounts.activeVolumeCount), 0)
|
|
|
|
// Change volume back to writable
|
|
v.ReadOnly = false
|
|
dn.UpdateVolumes([]storage.VolumeInfo{v})
|
|
|
|
// Check active count after marking writable again (should be 1)
|
|
usageCounts = topo.diskUsages.usages[types.HardDriveType]
|
|
assert(t, "activeVolumeCount after writable again", int(usageCounts.activeVolumeCount), 1)
|
|
}
|
|
|
|
func TestDeletingReadOnlyVolumeKeepsActiveCount(t *testing.T) {
|
|
topo := NewTopology("weedfs", sequence.NewMemorySequencer(), 32*1024, 5, false)
|
|
dn := topo.GetOrCreateDataCenter("dc1").GetOrCreateRack("rack1").
|
|
GetOrCreateDataNode("127.0.0.1", 34534, 0, "127.0.0.1", "", map[string]uint32{"": 25})
|
|
|
|
v := storage.VolumeInfo{
|
|
Id: needle.VolumeId(1),
|
|
Version: needle.GetCurrentVersion(),
|
|
ReplicaPlacement: &super_block.ReplicaPlacement{},
|
|
Ttl: needle.EMPTY_TTL,
|
|
}
|
|
dn.UpdateVolumes([]storage.VolumeInfo{v})
|
|
topo.RegisterVolumeLayout(v, dn)
|
|
v.ReadOnly = true
|
|
dn.UpdateVolumes([]storage.VolumeInfo{v})
|
|
assert(t, "activeVolumeCount after read-only", int(topo.diskUsages.usages[types.HardDriveType].activeVolumeCount), 0)
|
|
|
|
// The server's delete report does not say the volume was read-only.
|
|
topo.IncrementalSyncDataNodeRegistration(nil, []*master_pb.VolumeShortInformationMessage{
|
|
{Id: 1, Version: uint32(needle.GetCurrentVersion())},
|
|
}, dn)
|
|
|
|
for name, usage := range map[string]*DiskUsageCounts{
|
|
"topology": topo.diskUsages.usages[types.HardDriveType],
|
|
"node": dn.diskUsages.usages[types.HardDriveType],
|
|
} {
|
|
assert(t, name+" volumeCount after delete", int(usage.volumeCount), 0)
|
|
assert(t, name+" activeVolumeCount after delete", int(usage.activeVolumeCount), 0)
|
|
}
|
|
}
|
|
|
|
func TestVolumeReadOnlyAndRemoteStatusChange(t *testing.T) {
|
|
topo := NewTopology("weedfs", sequence.NewMemorySequencer(), 32*1024, 5, false)
|
|
|
|
dc := topo.GetOrCreateDataCenter("dc1")
|
|
rack := dc.GetOrCreateRack("rack1")
|
|
maxVolumeCounts := make(map[string]uint32)
|
|
maxVolumeCounts[""] = 25
|
|
dn := rack.GetOrCreateDataNode("127.0.0.1", 34534, 0, "127.0.0.1", "", maxVolumeCounts)
|
|
|
|
// Create a writable, local volume
|
|
v := storage.VolumeInfo{
|
|
Id: needle.VolumeId(1),
|
|
Size: 100,
|
|
Collection: "",
|
|
DiskType: "",
|
|
FileCount: 10,
|
|
DeleteCount: 0,
|
|
DeletedByteCount: 0,
|
|
ReadOnly: false, // Initially writable
|
|
RemoteStorageName: "", // Initially local
|
|
Version: needle.GetCurrentVersion(),
|
|
ReplicaPlacement: &super_block.ReplicaPlacement{},
|
|
Ttl: needle.EMPTY_TTL,
|
|
}
|
|
|
|
dn.UpdateVolumes([]storage.VolumeInfo{v})
|
|
topo.RegisterVolumeLayout(v, dn)
|
|
|
|
// Check initial counts
|
|
usageCounts := topo.diskUsages.usages[types.HardDriveType]
|
|
assert(t, "initial activeVolumeCount", int(usageCounts.activeVolumeCount), 1)
|
|
assert(t, "initial remoteVolumeCount", int(usageCounts.remoteVolumeCount), 0)
|
|
|
|
// Simultaneously change to read-only AND remote
|
|
v.ReadOnly = true
|
|
v.RemoteStorageName = "s3"
|
|
v.RemoteStorageName = "s3.default"
|
|
dn.UpdateVolumes([]storage.VolumeInfo{v})
|
|
|
|
// Check counts after both changes
|
|
usageCounts = topo.diskUsages.usages[types.HardDriveType]
|
|
assert(t, "activeVolumeCount after read-only+remote", int(usageCounts.activeVolumeCount), 0)
|
|
assert(t, "remoteVolumeCount after read-only+remote", int(usageCounts.remoteVolumeCount), 1)
|
|
|
|
// Change back to writable but keep remote
|
|
v.ReadOnly = false
|
|
dn.UpdateVolumes([]storage.VolumeInfo{v})
|
|
|
|
// Check counts - should be writable (active=1) and still remote
|
|
usageCounts = topo.diskUsages.usages[types.HardDriveType]
|
|
assert(t, "activeVolumeCount after writable+remote", int(usageCounts.activeVolumeCount), 1)
|
|
assert(t, "remoteVolumeCount after writable+remote", int(usageCounts.remoteVolumeCount), 1)
|
|
|
|
// Change back to local AND read-only simultaneously
|
|
v.ReadOnly = true
|
|
v.RemoteStorageName = ""
|
|
v.RemoteStorageName = ""
|
|
dn.UpdateVolumes([]storage.VolumeInfo{v})
|
|
|
|
// Check final counts
|
|
usageCounts = topo.diskUsages.usages[types.HardDriveType]
|
|
assert(t, "final activeVolumeCount", int(usageCounts.activeVolumeCount), 0)
|
|
assert(t, "final remoteVolumeCount", int(usageCounts.remoteVolumeCount), 0)
|
|
}
|
|
|
|
func TestListCollections(t *testing.T) {
|
|
rp, _ := super_block.NewReplicaPlacementFromString("002")
|
|
|
|
topo := NewTopology("weedfs", sequence.NewMemorySequencer(), 32*1024, 5, false)
|
|
dc := topo.GetOrCreateDataCenter("dc1")
|
|
rack := dc.GetOrCreateRack("rack1")
|
|
dn := rack.GetOrCreateDataNode("127.0.0.1", 34534, 0, "127.0.0.1", "", nil)
|
|
|
|
topo.RegisterVolumeLayout(storage.VolumeInfo{
|
|
Id: needle.VolumeId(1111),
|
|
ReplicaPlacement: rp,
|
|
}, dn)
|
|
topo.RegisterVolumeLayout(storage.VolumeInfo{
|
|
Id: needle.VolumeId(2222),
|
|
ReplicaPlacement: rp,
|
|
Collection: "vol_collection_a",
|
|
}, dn)
|
|
topo.RegisterVolumeLayout(storage.VolumeInfo{
|
|
Id: needle.VolumeId(3333),
|
|
ReplicaPlacement: rp,
|
|
Collection: "vol_collection_b",
|
|
}, dn)
|
|
|
|
topo.RegisterEcShards(&erasure_coding.EcVolumeInfo{
|
|
VolumeId: needle.VolumeId(4444),
|
|
Collection: "ec_collection_a",
|
|
ShardsInfo: erasure_coding.NewShardsInfo(),
|
|
}, dn)
|
|
topo.RegisterEcShards(&erasure_coding.EcVolumeInfo{
|
|
VolumeId: needle.VolumeId(5555),
|
|
Collection: "ec_collection_b",
|
|
ShardsInfo: erasure_coding.NewShardsInfo(),
|
|
}, dn)
|
|
|
|
testCases := []struct {
|
|
name string
|
|
includeNormalVolumes bool
|
|
includeEcVolumes bool
|
|
want []string
|
|
}{
|
|
{
|
|
name: "no volume types selected",
|
|
includeNormalVolumes: false,
|
|
includeEcVolumes: false,
|
|
want: nil,
|
|
}, {
|
|
name: "normal volumes",
|
|
includeNormalVolumes: true,
|
|
includeEcVolumes: false,
|
|
want: []string{"", "vol_collection_a", "vol_collection_b"},
|
|
}, {
|
|
name: "EC volumes",
|
|
includeNormalVolumes: false,
|
|
includeEcVolumes: true,
|
|
want: []string{"ec_collection_a", "ec_collection_b"},
|
|
}, {
|
|
name: "normal + EC volumes",
|
|
includeNormalVolumes: true,
|
|
includeEcVolumes: true,
|
|
want: []string{"", "ec_collection_a", "ec_collection_b", "vol_collection_a", "vol_collection_b"},
|
|
},
|
|
}
|
|
|
|
for _, tc := range testCases {
|
|
t.Run(tc.name, func(t *testing.T) {
|
|
got := topo.ListCollections(tc.includeNormalVolumes, tc.includeEcVolumes)
|
|
|
|
if !reflect.DeepEqual(got, tc.want) {
|
|
t.Errorf("got %v, want %v", got, tc.want)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
// A collection delete that queues for the write lock while ListCollections
|
|
// reads the collections must not wedge both.
|
|
func TestListCollectionsWithConcurrentCollectionDelete(t *testing.T) {
|
|
rp, _ := super_block.NewReplicaPlacementFromString("000")
|
|
topo := NewTopology("weedfs", sequence.NewMemorySequencer(), 32*1024, 5, false)
|
|
|
|
stop := make(chan struct{})
|
|
defer close(stop)
|
|
go func() {
|
|
for {
|
|
select {
|
|
case <-stop:
|
|
return
|
|
default:
|
|
}
|
|
topo.GetVolumeLayout("churn", rp, needle.EMPTY_TTL, types.HardDriveType)
|
|
topo.DeleteCollection("churn")
|
|
}
|
|
}()
|
|
|
|
listed := make(chan struct{})
|
|
go func() {
|
|
defer close(listed)
|
|
for i := 0; i < 100000; i++ {
|
|
topo.ListCollections(true, false)
|
|
}
|
|
}()
|
|
select {
|
|
case <-listed:
|
|
case <-time.After(10 * time.Second):
|
|
t.Fatal("ListCollections deadlocked with a concurrent DeleteCollection")
|
|
}
|
|
}
|
|
|
|
func TestDataNodeIdBasedIdentification(t *testing.T) {
|
|
topo := NewTopology("weedfs", sequence.NewMemorySequencer(), 32*1024, 5, false)
|
|
dc := topo.GetOrCreateDataCenter("dc1")
|
|
rack := dc.GetOrCreateRack("rack1")
|
|
|
|
maxVolumeCounts := make(map[string]uint32)
|
|
maxVolumeCounts[""] = 10
|
|
|
|
// Test 1: Create a DataNode with explicit id
|
|
dn1 := rack.GetOrCreateDataNode("10.0.0.1", 8080, 18080, "10.0.0.1:8080", "node-1", maxVolumeCounts)
|
|
if string(dn1.Id()) != "node-1" {
|
|
t.Errorf("expected node id 'node-1', got '%s'", dn1.Id())
|
|
}
|
|
if dn1.Ip != "10.0.0.1" {
|
|
t.Errorf("expected ip '10.0.0.1', got '%s'", dn1.Ip)
|
|
}
|
|
|
|
// Test 2: Same id with different IP should return the same DataNode (K8s pod reschedule scenario)
|
|
dn2 := rack.GetOrCreateDataNode("10.0.0.2", 8080, 18080, "10.0.0.2:8080", "node-1", maxVolumeCounts)
|
|
if dn1 != dn2 {
|
|
t.Errorf("expected same DataNode for same id, got different nodes")
|
|
}
|
|
// IP should be updated to the new value
|
|
if dn2.Ip != "10.0.0.2" {
|
|
t.Errorf("expected ip to be updated to '10.0.0.2', got '%s'", dn2.Ip)
|
|
}
|
|
if dn2.PublicUrl != "10.0.0.2:8080" {
|
|
t.Errorf("expected publicUrl to be updated to '10.0.0.2:8080', got '%s'", dn2.PublicUrl)
|
|
}
|
|
|
|
// Test 3: Different id should create a new DataNode
|
|
dn3 := rack.GetOrCreateDataNode("10.0.0.3", 8080, 18080, "10.0.0.3:8080", "node-2", maxVolumeCounts)
|
|
if string(dn3.Id()) != "node-2" {
|
|
t.Errorf("expected node id 'node-2', got '%s'", dn3.Id())
|
|
}
|
|
if dn1 == dn3 {
|
|
t.Errorf("expected different DataNode for different id")
|
|
}
|
|
|
|
// Test 4: Empty id should fall back to ip:port (backward compatibility)
|
|
dn4 := rack.GetOrCreateDataNode("10.0.0.4", 8080, 18080, "10.0.0.4:8080", "", maxVolumeCounts)
|
|
if string(dn4.Id()) != "10.0.0.4:8080" {
|
|
t.Errorf("expected node id '10.0.0.4:8080' for empty id, got '%s'", dn4.Id())
|
|
}
|
|
|
|
// Test 5: Same ip:port with empty id should return the same DataNode
|
|
dn5 := rack.GetOrCreateDataNode("10.0.0.4", 8080, 18080, "10.0.0.4:8080", "", maxVolumeCounts)
|
|
if dn4 != dn5 {
|
|
t.Errorf("expected same DataNode for same ip:port with empty id")
|
|
}
|
|
|
|
// Verify we have 3 unique DataNodes total:
|
|
// - node-1 (dn1/dn2 share the same id)
|
|
// - node-2 (dn3)
|
|
// - 10.0.0.4:8080 (dn4/dn5 share the same ip:port)
|
|
children := rack.Children()
|
|
if len(children) != 3 {
|
|
t.Errorf("expected 3 DataNodes, got %d", len(children))
|
|
}
|
|
|
|
// Test 6: Transition from ip:port to explicit id
|
|
// First, the node exists with ip:port as id (dn4/dn5)
|
|
// Now the same volume server starts sending an explicit id
|
|
dn6 := rack.GetOrCreateDataNode("10.0.0.4", 8080, 18080, "10.0.0.4:8080", "node-4-explicit", maxVolumeCounts)
|
|
// Should return the same DataNode instance
|
|
if dn6 != dn4 {
|
|
t.Errorf("expected same DataNode instance during transition")
|
|
}
|
|
// But the id should now be updated to the explicit id
|
|
if string(dn6.Id()) != "node-4-explicit" {
|
|
t.Errorf("expected node id to transition to 'node-4-explicit', got '%s'", dn6.Id())
|
|
}
|
|
// The node should be re-keyed in the children map
|
|
if rack.FindDataNodeById("node-4-explicit") != dn6 {
|
|
t.Errorf("expected to find DataNode by new explicit id")
|
|
}
|
|
// Old ip:port key should no longer work
|
|
if rack.FindDataNodeById("10.0.0.4:8080") != nil {
|
|
t.Errorf("expected old ip:port id to be removed from children map")
|
|
}
|
|
|
|
// Still 3 unique DataNodes (node-1, node-2, node-4-explicit)
|
|
children = rack.Children()
|
|
if len(children) != 3 {
|
|
t.Errorf("expected 3 DataNodes after transition, got %d", len(children))
|
|
}
|
|
|
|
// Test 7: Prevent incorrect transition when a new node reuses ip:port of a node with explicit id
|
|
// Scenario: node-1 runs at 10.0.0.1:8080, dies, new node-99 starts at same ip:port
|
|
// The transition should NOT happen because node-1 already has an explicit id
|
|
dn7 := rack.GetOrCreateDataNode("10.0.0.1", 8080, 18080, "10.0.0.1:8080", "node-99", maxVolumeCounts)
|
|
// Should create a NEW DataNode, not reuse node-1
|
|
if dn7 == dn1 {
|
|
t.Errorf("expected new DataNode for node-99, got reused node-1")
|
|
}
|
|
if string(dn7.Id()) != "node-99" {
|
|
t.Errorf("expected node id 'node-99', got '%s'", dn7.Id())
|
|
}
|
|
// node-1 should still exist with its original id
|
|
if rack.FindDataNodeById("node-1") == nil {
|
|
t.Errorf("node-1 should still exist")
|
|
}
|
|
// Now we have 4 DataNodes
|
|
children = rack.Children()
|
|
if len(children) != 4 {
|
|
t.Errorf("expected 4 DataNodes, got %d", len(children))
|
|
}
|
|
}
|
|
|
|
func TestLookupDataNodeByAddress(t *testing.T) {
|
|
topo := NewTopology("weedfs", sequence.NewMemorySequencer(), 32*1024, 5, false)
|
|
dc := topo.GetOrCreateDataCenter("dc1")
|
|
rack := dc.GetOrCreateRack("rack1")
|
|
|
|
maxVolumeCounts := map[string]uint32{"": 10}
|
|
|
|
// Brand-new registration must be discoverable by both the http and
|
|
// grpc forms of the address.
|
|
dn := rack.GetOrCreateDataNode("10.1.2.3", 8080, 18080, "10.1.2.3:8080", "n1", maxVolumeCounts)
|
|
if got := topo.LookupDataNodeByAddress(pb.ServerAddress("10.1.2.3:8080")); got != dn {
|
|
t.Fatalf("lookup by http address: got %v, want %v", got, dn)
|
|
}
|
|
if got := topo.LookupDataNodeByAddress(pb.ServerAddress("10.1.2.3:8080.18080")); got != dn {
|
|
t.Fatalf("lookup by grpc-suffix address: got %v, want %v", got, dn)
|
|
}
|
|
|
|
// Unknown addresses must miss.
|
|
if got := topo.LookupDataNodeByAddress(pb.ServerAddress("127.0.0.1:1")); got != nil {
|
|
t.Fatalf("unknown address must not be found, got %v", got)
|
|
}
|
|
|
|
// Heartbeat from a moved pod (same id, new ip) updates the index in
|
|
// place: the old address is dropped and the new one resolves.
|
|
dnMoved := rack.GetOrCreateDataNode("10.9.9.9", 8080, 18080, "10.9.9.9:8080", "n1", maxVolumeCounts)
|
|
if dnMoved != dn {
|
|
t.Fatalf("expected same node instance after move, got different")
|
|
}
|
|
if got := topo.LookupDataNodeByAddress(pb.ServerAddress("10.1.2.3:8080")); got != nil {
|
|
t.Fatalf("old address must be unregistered after move, got %v", got)
|
|
}
|
|
if got := topo.LookupDataNodeByAddress(pb.ServerAddress("10.9.9.9:8080")); got != dn {
|
|
t.Fatalf("new address lookup: got %v, want %v", got, dn)
|
|
}
|
|
|
|
// UnRegisterDataNode evicts the index entry.
|
|
topo.UnRegisterDataNode(dn)
|
|
if got := topo.LookupDataNodeByAddress(pb.ServerAddress("10.9.9.9:8080")); got != nil {
|
|
t.Fatalf("address must be unregistered after UnRegisterDataNode, got %v", got)
|
|
}
|
|
}
|
|
|
|
// TestSyncDataNodeRegistrationReRegistersMissingVolume reproduces the divergence
|
|
// where a volume present on a data node (shown by volume.list / admin UI) is
|
|
// missing from the lookup index, which surfaces as "volume id not found" on
|
|
// LookupVolume. SetVolumeUnavailable (used by
|
|
// UnRegisterDataNode on a disconnect) drops the volume from the index, and the
|
|
// reconnecting full heartbeat used to skip it because it was no longer "new" to
|
|
// the disk map. The full heartbeat now self-heals.
|
|
func TestSyncDataNodeRegistrationReRegistersMissingVolume(t *testing.T) {
|
|
topo := NewTopology("weedfs", sequence.NewMemorySequencer(), 32*1024, 5, false)
|
|
|
|
dc := topo.GetOrCreateDataCenter("dc1")
|
|
rack := dc.GetOrCreateRack("rack1")
|
|
dn := rack.GetOrCreateDataNode("127.0.0.1", 34534, 0, "127.0.0.1", "", map[string]uint32{"": 25})
|
|
|
|
vid := needle.VolumeId(18994)
|
|
volumeMessage := &master_pb.VolumeInformationMessage{
|
|
Id: uint32(vid),
|
|
Size: 100,
|
|
Collection: "drr",
|
|
ReplicaPlacement: uint32(0),
|
|
Version: uint32(needle.GetCurrentVersion()),
|
|
Ttl: 0,
|
|
}
|
|
|
|
// Initial full heartbeat registers the volume in the lookup index.
|
|
topo.SyncDataNodeRegistration([]*master_pb.VolumeInformationMessage{volumeMessage}, dn)
|
|
if got := topo.Lookup("", vid); len(got) != 1 {
|
|
t.Fatalf("after registration: lookup %d got %v, want 1 location", vid, got)
|
|
}
|
|
|
|
// Drop the volume from the index the way UnRegisterDataNode does, but leave
|
|
// it in the data node's disk map (the reconnecting heartbeat did not report
|
|
// it as new).
|
|
rp, _ := super_block.NewReplicaPlacementFromString("000")
|
|
vl := topo.GetVolumeLayout("drr", rp, needle.EMPTY_TTL, types.HardDriveType)
|
|
vl.SetVolumeUnavailable(dn, vid)
|
|
|
|
// The empty entry must be removed, otherwise Lookup returns a non-nil empty
|
|
// list that still reads as "not found".
|
|
if got := topo.Lookup("", vid); got != nil {
|
|
t.Fatalf("after SetVolumeUnavailable: expected lookup miss, got %v", got)
|
|
}
|
|
if _, err := dn.GetVolumesById(vid); err != nil {
|
|
t.Fatalf("volume %d should still be in the data node disk map: %v", vid, err)
|
|
}
|
|
|
|
// The next full heartbeat re-registers the volume even though it is not new.
|
|
topo.SyncDataNodeRegistration([]*master_pb.VolumeInformationMessage{volumeMessage}, dn)
|
|
if got := topo.Lookup("", vid); len(got) != 1 {
|
|
t.Fatalf("after self-heal: lookup %d got %v, want 1 location", vid, got)
|
|
}
|
|
}
|
|
|
|
// TestSetVolumeAvailableRepairsMissingVolume covers the vacuum-commit variant of
|
|
// the same divergence. A disconnect during a long vacuum can drop a
|
|
// single-replica volume from the lookup index while it stays on the node; the
|
|
// commit then calls SetVolumeAvailable on it. That used to dereference a nil
|
|
// location and panic; it now re-creates the entry and repairs the split.
|
|
func TestSetVolumeAvailableRepairsMissingVolume(t *testing.T) {
|
|
topo := NewTopology("weedfs", sequence.NewMemorySequencer(), 32*1024, 5, false)
|
|
|
|
dc := topo.GetOrCreateDataCenter("dc1")
|
|
rack := dc.GetOrCreateRack("rack1")
|
|
dn := rack.GetOrCreateDataNode("127.0.0.1", 34534, 0, "127.0.0.1", "", map[string]uint32{"": 25})
|
|
|
|
vid := needle.VolumeId(2640)
|
|
volumeMessage := &master_pb.VolumeInformationMessage{
|
|
Id: uint32(vid),
|
|
Size: 100,
|
|
Collection: "drr",
|
|
ReplicaPlacement: uint32(0),
|
|
Version: uint32(needle.GetCurrentVersion()),
|
|
Ttl: 0,
|
|
}
|
|
|
|
topo.SyncDataNodeRegistration([]*master_pb.VolumeInformationMessage{volumeMessage}, dn)
|
|
|
|
rp, _ := super_block.NewReplicaPlacementFromString("000")
|
|
vl := topo.GetVolumeLayout("drr", rp, needle.EMPTY_TTL, types.HardDriveType)
|
|
|
|
// Disconnect drops the volume from the index but leaves it on the node.
|
|
vl.SetVolumeUnavailable(dn, vid)
|
|
if got := topo.Lookup("", vid); got != nil {
|
|
t.Fatalf("after SetVolumeUnavailable: expected lookup miss, got %v", got)
|
|
}
|
|
if _, err := dn.GetVolumesById(vid); err != nil {
|
|
t.Fatalf("volume %d should still be in the data node disk map: %v", vid, err)
|
|
}
|
|
|
|
// The vacuum commit re-marks the volume available; it must re-register it.
|
|
vl.SetVolumeAvailable(dn, vid, 100, false, false, false)
|
|
if got := topo.Lookup("", vid); len(got) != 1 {
|
|
t.Fatalf("after SetVolumeAvailable: lookup %d got %v, want 1 location", vid, got)
|
|
}
|
|
// Size tracking must be seeded too, or assigns go uncounted until the next
|
|
// heartbeat and the volume can overfill.
|
|
vl.accessLock.RLock()
|
|
_, tracked := vl.sizeTracking[vid]
|
|
vl.accessLock.RUnlock()
|
|
if !tracked {
|
|
t.Fatalf("after SetVolumeAvailable: size tracking for %d not seeded", vid)
|
|
}
|
|
}
|
|
|
|
// A read-only change that arrives in the regular heartbeat must reach the
|
|
// layout's per-replica flag, which the vacuum sweep consults. Registration
|
|
// and volume.mark already set it; the heartbeat path did not.
|
|
func TestFullHeartbeatUpdatesLayoutReadOnlyFlag(t *testing.T) {
|
|
topo := NewTopology("weedfs", sequence.NewMemorySequencer(), 32*1024, 5, false)
|
|
dn := topo.GetOrCreateDataCenter("dc1").GetOrCreateRack("rack1").
|
|
GetOrCreateDataNode("127.0.0.1", 34534, 0, "127.0.0.1", "", map[string]uint32{"": 25})
|
|
report := func(readOnly bool) []*master_pb.VolumeInformationMessage {
|
|
return []*master_pb.VolumeInformationMessage{{
|
|
Id: 1, Collection: "c", Size: 1 << 20, Version: uint32(needle.GetCurrentVersion()), ReadOnly: readOnly,
|
|
}}
|
|
}
|
|
rp, _ := super_block.NewReplicaPlacementFromString("000")
|
|
flag := func() bool {
|
|
vl := topo.GetVolumeLayout("c", rp, needle.EMPTY_TTL, types.HardDriveType)
|
|
vl.accessLock.RLock()
|
|
defer vl.accessLock.RUnlock()
|
|
return vl.vid2location[needle.VolumeId(1)].AnyReadOnly()
|
|
}
|
|
|
|
// the full list, which a server sends first and whenever the digests disagree
|
|
topo.SyncDataNodeRegistration(report(false), dn)
|
|
if flag() {
|
|
t.Fatal("volume registered writable is flagged read-only")
|
|
}
|
|
topo.SyncDataNodeRegistration(report(true), dn)
|
|
if !flag() {
|
|
t.Fatal("full heartbeat turned the volume read-only but the layout flag did not follow")
|
|
}
|
|
topo.SyncDataNodeRegistration(report(false), dn)
|
|
if flag() {
|
|
t.Fatal("full heartbeat turned the volume writable again but the layout flag stayed read-only")
|
|
}
|
|
|
|
// the changed-volumes delta, which is what a running server normally sends
|
|
topo.ApplyVolumeChanges(report(true), dn)
|
|
if !flag() {
|
|
t.Fatal("delta heartbeat turned the volume read-only but the layout flag did not follow")
|
|
}
|
|
topo.ApplyVolumeChanges(report(false), dn)
|
|
if flag() {
|
|
t.Fatal("delta heartbeat turned the volume writable again but the layout flag stayed read-only")
|
|
}
|
|
}
|