seaweedfs/weed/topology/topology_test.go

package topology

import (
	"reflect"
	"testing"
	"time"

	"github.com/prometheus/client_golang/prometheus/testutil"
	"github.com/seaweedfs/seaweedfs/weed/pb"
	"github.com/seaweedfs/seaweedfs/weed/pb/master_pb"
	"github.com/seaweedfs/seaweedfs/weed/sequence"
	"github.com/seaweedfs/seaweedfs/weed/stats"
	"github.com/seaweedfs/seaweedfs/weed/storage"
	"github.com/seaweedfs/seaweedfs/weed/storage/erasure_coding"
	"github.com/seaweedfs/seaweedfs/weed/storage/needle"
	"github.com/seaweedfs/seaweedfs/weed/storage/super_block"
	"github.com/seaweedfs/seaweedfs/weed/storage/types"
)

func TestRemoveDataCenter(t *testing.T) {
	topo := setup(topologyLayout)
	topo.UnlinkChildNode(NodeId("dc2"))
	if topo.diskUsages.usages[types.HardDriveType].activeVolumeCount != 15 {
		t.Fail()
	}
	topo.UnlinkChildNode(NodeId("dc3"))
	if topo.diskUsages.usages[types.HardDriveType].activeVolumeCount != 12 {
		t.Fail()
	}
}

func TestHandlingVolumeServerHeartbeat(t *testing.T) {
	topo := NewTopology("weedfs", sequence.NewMemorySequencer(), 32*1024, 5, false)

	dc := topo.GetOrCreateDataCenter("dc1")
	rack := dc.GetOrCreateRack("rack1")
	maxVolumeCounts := make(map[string]uint32)
	maxVolumeCounts[""] = 25
	maxVolumeCounts["ssd"] = 12
	dn := rack.GetOrCreateDataNode("127.0.0.1", 34534, 0, "127.0.0.1", "", maxVolumeCounts)

	{
		volumeCount := 7
		var volumeMessages []*master_pb.VolumeInformationMessage
		for k := 1; k <= volumeCount; k++ {
			volumeMessage := &master_pb.VolumeInformationMessage{
				Id:               uint32(k),
				Size:             uint64(25432),
				Collection:       "",
				FileCount:        uint64(2343),
				DeleteCount:      uint64(345),
				DeletedByteCount: 34524,
				ReadOnly:         false,
				ReplicaPlacement: uint32(0),
				Version:          uint32(needle.GetCurrentVersion()),
				Ttl:              0,
			}
			volumeMessages = append(volumeMessages, volumeMessage)
		}

		for k := 1; k <= volumeCount; k++ {
			volumeMessage := &master_pb.VolumeInformationMessage{
				Id:               uint32(volumeCount + k),
				Size:             uint64(25432),
				Collection:       "",
				FileCount:        uint64(2343),
				DeleteCount:      uint64(345),
				DeletedByteCount: 34524,
				ReadOnly:         false,
				ReplicaPlacement: uint32(0),
				Version:          uint32(needle.GetCurrentVersion()),
				Ttl:              0,
				DiskType:         "ssd",
			}
			volumeMessages = append(volumeMessages, volumeMessage)
		}

		topo.SyncDataNodeRegistration(volumeMessages, dn)

		usageCounts := topo.diskUsages.usages[types.HardDriveType]

		assert(t, "activeVolumeCount1", int(usageCounts.activeVolumeCount), volumeCount)
		assert(t, "volumeCount", int(usageCounts.volumeCount), volumeCount)
		assert(t, "ssdVolumeCount", int(topo.diskUsages.usages[types.SsdType].volumeCount), volumeCount)
	}

	{
		volumeCount := 7 - 1
		var volumeMessages []*master_pb.VolumeInformationMessage
		for k := 1; k <= volumeCount; k++ {
			volumeMessage := &master_pb.VolumeInformationMessage{
				Id:               uint32(k),
				Size:             uint64(30000),
				Collection:       "",
				FileCount:        uint64(2343),
				DeleteCount:      uint64(345),
				DeletedByteCount: 345240,
				ReadOnly:         false,
				ReplicaPlacement: uint32(0),
				Version:          uint32(needle.GetCurrentVersion()),
				Ttl:              0,
			}
			volumeMessages = append(volumeMessages, volumeMessage)
		}
		topo.SyncDataNodeRegistration(volumeMessages, dn)

		//rp, _ := storage.NewReplicaPlacementFromString("000")
		//layout := topo.GetVolumeLayout("", rp, needle.EMPTY_TTL)
		//assert(t, "writables", len(layout.writables), volumeCount)

		usageCounts := topo.diskUsages.usages[types.HardDriveType]

		assert(t, "activeVolumeCount1", int(usageCounts.activeVolumeCount), volumeCount)
		assert(t, "volumeCount", int(usageCounts.volumeCount), volumeCount)
	}

	{
		volumeCount := 6
		newVolumeShortMessage := &master_pb.VolumeShortInformationMessage{
			Id:               uint32(3),
			Collection:       "",
			ReplicaPlacement: uint32(0),
			Version:          uint32(needle.GetCurrentVersion()),
			Ttl:              0,
		}
		topo.IncrementalSyncDataNodeRegistration(
			[]*master_pb.VolumeShortInformationMessage{newVolumeShortMessage},
			nil,
			dn)
		rp, _ := super_block.NewReplicaPlacementFromString("000")
		layout := topo.GetVolumeLayout("", rp, needle.EMPTY_TTL, types.HardDriveType)
		assert(t, "writables after repeated add", len(layout.writables), volumeCount)

		usageCounts := topo.diskUsages.usages[types.HardDriveType]

		assert(t, "activeVolumeCount1", int(usageCounts.activeVolumeCount), volumeCount)
		assert(t, "volumeCount", int(usageCounts.volumeCount), volumeCount)

		topo.IncrementalSyncDataNodeRegistration(
			nil,
			[]*master_pb.VolumeShortInformationMessage{newVolumeShortMessage},
			dn)
		assert(t, "writables after deletion", len(layout.writables), volumeCount-1)
		assert(t, "activeVolumeCount1", int(usageCounts.activeVolumeCount), volumeCount-1)
		assert(t, "volumeCount", int(usageCounts.volumeCount), volumeCount-1)

		topo.IncrementalSyncDataNodeRegistration(
			[]*master_pb.VolumeShortInformationMessage{newVolumeShortMessage},
			nil,
			dn)

		for vid := range layout.vid2location {
			println("after add volume id", vid)
		}
		for _, vid := range layout.writables {
			println("after add writable volume id", vid)
		}

		assert(t, "writables after add back", len(layout.writables), volumeCount)

	}

	topo.UnRegisterDataNode(dn)

	usageCounts := topo.diskUsages.usages[types.HardDriveType]

	assert(t, "activeVolumeCount2", int(usageCounts.activeVolumeCount), 0)

}

func TestIncrementalSyncReplacesVolumeReadOnlyState(t *testing.T) {
	for _, tc := range []struct {
		name         string
		fromReadOnly bool
		toReadOnly   bool
	}{
		{name: "writable to read-only", toReadOnly: true},
		{name: "read-only to writable", fromReadOnly: true},
	} {
		t.Run(tc.name, func(t *testing.T) {
			topo := NewTopology("weedfs", sequence.NewMemorySequencer(), 32*1024, 5, false)
			dn := topo.GetOrCreateDataCenter("dc1").GetOrCreateRack("rack1").
				GetOrCreateDataNode("127.0.0.1", 34534, 0, "127.0.0.1", "", map[string]uint32{"": 25})
			volume := func(readOnly bool) *master_pb.VolumeShortInformationMessage {
				return &master_pb.VolumeShortInformationMessage{
					Id: 1, Collection: "c", Version: uint32(needle.GetCurrentVersion()), ReadOnly: readOnly,
				}
			}

			oldVolume := volume(tc.fromReadOnly)
			topo.IncrementalSyncDataNodeRegistration([]*master_pb.VolumeShortInformationMessage{oldVolume}, nil, dn)
			topo.IncrementalSyncDataNodeRegistration(
				[]*master_pb.VolumeShortInformationMessage{volume(tc.toReadOnly)},
				[]*master_pb.VolumeShortInformationMessage{oldVolume}, dn)

			locations := topo.Lookup("c", needle.VolumeId(1))
			if len(locations) != 1 || locations[0] != dn {
				t.Fatalf("lookup locations = %v, want only %v", locations, dn)
			}
			stored, err := dn.GetVolumesById(needle.VolumeId(1))
			if err != nil || stored.ReadOnly != tc.toReadOnly {
				t.Fatalf("stored volume = %+v, err = %v, want read-only %t", stored, err, tc.toReadOnly)
			}
			rp, _ := super_block.NewReplicaPlacementFromString("000")
			active, _ := topo.GetVolumeLayout("c", rp, needle.EMPTY_TTL, types.HardDriveType).GetWritableVolumeCount()
			want := 0
			if !tc.toReadOnly {
				want = 1
			}
			if active != want {
				t.Fatalf("writable count = %d, want %d", active, want)
			}
			if !dn.HasConsistentVolumeIndex() {
				t.Fatal("replacement left the held and servable volume indexes inconsistent")
			}
		})
	}
}

func TestIncrementalSyncRegistersMovedVolumeBeforeRemoval(t *testing.T) {
	topo := NewTopology("weedfs", sequence.NewMemorySequencer(), 32*1024, 5, false)
	dn := topo.GetOrCreateDataCenter("dc1").GetOrCreateRack("rack1").
		GetOrCreateDataNode("127.0.0.1", 34534, 0, "127.0.0.1", "", map[string]uint32{"": 25, "ssd": 25})
	oldVolume := &master_pb.VolumeShortInformationMessage{
		Id: 1, Collection: "c", Version: uint32(needle.GetCurrentVersion()),
	}
	newVolume := &master_pb.VolumeShortInformationMessage{
		Id: 1, Collection: "c", Version: uint32(needle.GetCurrentVersion()), DiskType: "ssd",
	}
	topo.IncrementalSyncDataNodeRegistration([]*master_pb.VolumeShortInformationMessage{oldVolume}, nil, dn)

	rp, _ := super_block.NewReplicaPlacementFromString("000")
	oldLayout := topo.GetVolumeLayout("c", rp, needle.EMPTY_TTL, types.HardDriveType)
	newLayout := topo.GetVolumeLayout("c", rp, needle.EMPTY_TTL, types.SsdType)
	oldLayout.accessLock.Lock()
	done := make(chan struct{})
	go func() {
		topo.IncrementalSyncDataNodeRegistration(
			[]*master_pb.VolumeShortInformationMessage{newVolume},
			[]*master_pb.VolumeShortInformationMessage{oldVolume}, dn)
		close(done)
	}()

	deadline := time.NewTimer(time.Second)
	defer deadline.Stop()
	ticker := time.NewTicker(time.Millisecond)
	defer ticker.Stop()
	movedBeforeRemoval := false
	for !movedBeforeRemoval {
		select {
		case <-deadline.C:
			oldLayout.accessLock.Unlock()
			<-done
			t.Fatal("destination layout was not registered before source removal")
		case <-ticker.C:
			movedBeforeRemoval = len(newLayout.Lookup(needle.VolumeId(1))) == 1
		}
	}
	oldLayout.accessLock.Unlock()
	<-done

	locations := topo.Lookup("c", needle.VolumeId(1))
	if len(locations) != 1 || locations[0] != dn {
		t.Fatalf("lookup locations = %v, want only %v", locations, dn)
	}
}

func TestDataNodeToDataNodeInfo_IncludeEmptyDiskFromUsage(t *testing.T) {
	dn := NewDataNode("node-1")
	dn.Ip = "127.0.0.1"
	dn.Port = 18080
	dn.GrpcPort = 28080

	// Simulate a node that has slot counters but no mounted volumes yet.
	usage := dn.diskUsages.getOrCreateDisk(types.HardDriveType)
	usage.maxVolumeCount = 8

	info := dn.ToDataNodeInfo(VolumeFilter{})
	diskInfo, found := info.DiskInfos[""]
	if !found {
		t.Fatalf("expected default disk entry for empty node")
	}
	if diskInfo.MaxVolumeCount != 8 {
		t.Fatalf("unexpected max volume count: got=%d want=8", diskInfo.MaxVolumeCount)
	}
	if len(diskInfo.VolumeInfos) != 0 {
		t.Fatalf("expected no volumes for empty disk, got=%d", len(diskInfo.VolumeInfos))
	}
}

func assert(t *testing.T, message string, actual, expected int) {
	if actual != expected {
		t.Fatalf("unexpected %s: %d, expected: %d", message, actual, expected)
	}
}

func TestAddRemoveVolume(t *testing.T) {

	topo := NewTopology("weedfs", sequence.NewMemorySequencer(), 32*1024, 5, false)

	dc := topo.GetOrCreateDataCenter("dc1")
	rack := dc.GetOrCreateRack("rack1")
	maxVolumeCounts := make(map[string]uint32)
	maxVolumeCounts[""] = 25
	maxVolumeCounts["ssd"] = 12
	dn := rack.GetOrCreateDataNode("127.0.0.1", 34534, 0, "127.0.0.1", "", maxVolumeCounts)

	v := storage.VolumeInfo{
		Id:               needle.VolumeId(1),
		Size:             100,
		Collection:       "xcollection",
		DiskType:         "ssd",
		FileCount:        123,
		DeleteCount:      23,
		DeletedByteCount: 45,
		ReadOnly:         false,
		Version:          needle.GetCurrentVersion(),
		ReplicaPlacement: &super_block.ReplicaPlacement{},
		Ttl:              needle.EMPTY_TTL,
	}

	dn.UpdateVolumes([]storage.VolumeInfo{v})
	topo.RegisterVolumeLayout(v, dn)
	topo.RegisterVolumeLayout(v, dn)

	if _, hasCollection := topo.FindCollection(v.Collection); !hasCollection {
		t.Errorf("collection %v should exist", v.Collection)
	}

	topo.UnRegisterVolumeLayout(v, dn)

	if _, hasCollection := topo.FindCollection(v.Collection); hasCollection {
		t.Errorf("collection %v should not exist", v.Collection)
	}
}

func TestUnRegisterVolumeLayoutClearsReplicaPlacementMismatchMetric(t *testing.T) {
	stats.MasterReplicaPlacementMismatch.Reset()
	t.Cleanup(stats.MasterReplicaPlacementMismatch.Reset)

	topo := NewTopology("weedfs", sequence.NewMemorySequencer(), 32*1024, 5, false)

	dc := topo.GetOrCreateDataCenter("dc1")
	rack := dc.GetOrCreateRack("rack1")
	maxVolumeCounts := map[string]uint32{"": 25}
	dn := rack.GetOrCreateDataNode("127.0.0.1", 34534, 0, "127.0.0.1", "", maxVolumeCounts)

	rp, err := super_block.NewReplicaPlacementFromString("001")
	if err != nil {
		t.Fatalf("NewReplicaPlacementFromString: %v", err)
	}
	v := storage.VolumeInfo{
		Id:               needle.VolumeId(42),
		Size:             100,
		Collection:       "metrics-test",
		ReplicaPlacement: rp,
		Ttl:              needle.EMPTY_TTL,
	}

	dn.UpdateVolumes([]storage.VolumeInfo{v})
	topo.RegisterVolumeLayout(v, dn)

	stats.MasterReplicaPlacementMismatch.WithLabelValues(v.Collection, v.Id.String()).Set(1)
	if n := testutil.CollectAndCount(stats.MasterReplicaPlacementMismatch); n != 1 {
		t.Fatalf("expected 1 replica_placement_mismatch series, got %d", n)
	}

	topo.UnRegisterVolumeLayout(v, dn)

	if n := testutil.CollectAndCount(stats.MasterReplicaPlacementMismatch); n != 0 {
		t.Errorf("%d replica_placement_mismatch series left after volume left topology", n)
	}
}

func TestUnRegisterVolumeLayoutKeepsReplicaPlacementMismatchMetricWhilePlacementsRemain(t *testing.T) {
	stats.MasterReplicaPlacementMismatch.Reset()
	t.Cleanup(stats.MasterReplicaPlacementMismatch.Reset)

	topo := NewTopology("weedfs", sequence.NewMemorySequencer(), 32*1024, 5, false)

	dc := topo.GetOrCreateDataCenter("dc1")
	rack := dc.GetOrCreateRack("rack1")
	maxVolumeCounts := map[string]uint32{"": 25}
	dn1 := rack.GetOrCreateDataNode("127.0.0.1", 34534, 0, "127.0.0.1", "", maxVolumeCounts)
	dn2 := rack.GetOrCreateDataNode("127.0.0.1", 34535, 0, "127.0.0.1", "", maxVolumeCounts)

	rp, err := super_block.NewReplicaPlacementFromString("001")
	if err != nil {
		t.Fatalf("NewReplicaPlacementFromString: %v", err)
	}
	v := storage.VolumeInfo{
		Id:               needle.VolumeId(42),
		Size:             100,
		Collection:       "metrics-test",
		ReplicaPlacement: rp,
		Ttl:              needle.EMPTY_TTL,
	}

	dn1.UpdateVolumes([]storage.VolumeInfo{v})
	dn2.UpdateVolumes([]storage.VolumeInfo{v})
	topo.RegisterVolumeLayout(v, dn1)
	topo.RegisterVolumeLayout(v, dn2)

	stats.MasterReplicaPlacementMismatch.WithLabelValues(v.Collection, v.Id.String()).Set(1)
	if n := testutil.CollectAndCount(stats.MasterReplicaPlacementMismatch); n != 1 {
		t.Fatalf("expected 1 replica_placement_mismatch series, got %d", n)
	}

	topo.UnRegisterVolumeLayout(v, dn1)

	if n := testutil.CollectAndCount(stats.MasterReplicaPlacementMismatch); n != 1 {
		t.Errorf("expected series to remain while %s still holds the volume, got %d", dn2.Id(), n)
	}
	if got := len(topo.Lookup(v.Collection, v.Id)); got != 1 {
		t.Fatalf("expected 1 remaining placement, got %d", got)
	}

	topo.UnRegisterVolumeLayout(v, dn2)

	if n := testutil.CollectAndCount(stats.MasterReplicaPlacementMismatch); n != 0 {
		t.Errorf("%d replica_placement_mismatch series left after last placement left", n)
	}
}

func TestVolumeReadOnlyStatusChange(t *testing.T) {
	topo := NewTopology("weedfs", sequence.NewMemorySequencer(), 32*1024, 5, false)

	dc := topo.GetOrCreateDataCenter("dc1")
	rack := dc.GetOrCreateRack("rack1")
	maxVolumeCounts := make(map[string]uint32)
	maxVolumeCounts[""] = 25
	dn := rack.GetOrCreateDataNode("127.0.0.1", 34534, 0, "127.0.0.1", "", maxVolumeCounts)

	// Create a writable volume
	v := storage.VolumeInfo{
		Id:               needle.VolumeId(1),
		Size:             100,
		Collection:       "",
		DiskType:         "",
		FileCount:        10,
		DeleteCount:      0,
		DeletedByteCount: 0,
		ReadOnly:         false, // Initially writable
		Version:          needle.GetCurrentVersion(),
		ReplicaPlacement: &super_block.ReplicaPlacement{},
		Ttl:              needle.EMPTY_TTL,
	}

	dn.UpdateVolumes([]storage.VolumeInfo{v})
	topo.RegisterVolumeLayout(v, dn)

	// Check initial active count (should be 1 since volume is writable)
	usageCounts := topo.diskUsages.usages[types.HardDriveType]
	assert(t, "initial activeVolumeCount", int(usageCounts.activeVolumeCount), 1)
	assert(t, "initial remoteVolumeCount", int(usageCounts.remoteVolumeCount), 0)

	// Change volume to read-only
	v.ReadOnly = true
	dn.UpdateVolumes([]storage.VolumeInfo{v})

	// Check active count after marking read-only (should be 0)
	usageCounts = topo.diskUsages.usages[types.HardDriveType]
	assert(t, "activeVolumeCount after read-only", int(usageCounts.activeVolumeCount), 0)

	// Change volume back to writable
	v.ReadOnly = false
	dn.UpdateVolumes([]storage.VolumeInfo{v})

	// Check active count after marking writable again (should be 1)
	usageCounts = topo.diskUsages.usages[types.HardDriveType]
	assert(t, "activeVolumeCount after writable again", int(usageCounts.activeVolumeCount), 1)
}

func TestVolumeReadOnlyAndRemoteStatusChange(t *testing.T) {
	topo := NewTopology("weedfs", sequence.NewMemorySequencer(), 32*1024, 5, false)

	dc := topo.GetOrCreateDataCenter("dc1")
	rack := dc.GetOrCreateRack("rack1")
	maxVolumeCounts := make(map[string]uint32)
	maxVolumeCounts[""] = 25
	dn := rack.GetOrCreateDataNode("127.0.0.1", 34534, 0, "127.0.0.1", "", maxVolumeCounts)

	// Create a writable, local volume
	v := storage.VolumeInfo{
		Id:                needle.VolumeId(1),
		Size:              100,
		Collection:        "",
		DiskType:          "",
		FileCount:         10,
		DeleteCount:       0,
		DeletedByteCount:  0,
		ReadOnly:          false, // Initially writable
		RemoteStorageName: "",    // Initially local
		Version:           needle.GetCurrentVersion(),
		ReplicaPlacement:  &super_block.ReplicaPlacement{},
		Ttl:               needle.EMPTY_TTL,
	}

	dn.UpdateVolumes([]storage.VolumeInfo{v})
	topo.RegisterVolumeLayout(v, dn)

	// Check initial counts
	usageCounts := topo.diskUsages.usages[types.HardDriveType]
	assert(t, "initial activeVolumeCount", int(usageCounts.activeVolumeCount), 1)
	assert(t, "initial remoteVolumeCount", int(usageCounts.remoteVolumeCount), 0)

	// Simultaneously change to read-only AND remote
	v.ReadOnly = true
	v.RemoteStorageName = "s3"
	v.RemoteStorageName = "s3.default"
	dn.UpdateVolumes([]storage.VolumeInfo{v})

	// Check counts after both changes
	usageCounts = topo.diskUsages.usages[types.HardDriveType]
	assert(t, "activeVolumeCount after read-only+remote", int(usageCounts.activeVolumeCount), 0)
	assert(t, "remoteVolumeCount after read-only+remote", int(usageCounts.remoteVolumeCount), 1)

	// Change back to writable but keep remote
	v.ReadOnly = false
	dn.UpdateVolumes([]storage.VolumeInfo{v})

	// Check counts - should be writable (active=1) and still remote
	usageCounts = topo.diskUsages.usages[types.HardDriveType]
	assert(t, "activeVolumeCount after writable+remote", int(usageCounts.activeVolumeCount), 1)
	assert(t, "remoteVolumeCount after writable+remote", int(usageCounts.remoteVolumeCount), 1)

	// Change back to local AND read-only simultaneously
	v.ReadOnly = true
	v.RemoteStorageName = ""
	v.RemoteStorageName = ""
	dn.UpdateVolumes([]storage.VolumeInfo{v})

	// Check final counts
	usageCounts = topo.diskUsages.usages[types.HardDriveType]
	assert(t, "final activeVolumeCount", int(usageCounts.activeVolumeCount), 0)
	assert(t, "final remoteVolumeCount", int(usageCounts.remoteVolumeCount), 0)
}

func TestListCollections(t *testing.T) {
	rp, _ := super_block.NewReplicaPlacementFromString("002")

	topo := NewTopology("weedfs", sequence.NewMemorySequencer(), 32*1024, 5, false)
	dc := topo.GetOrCreateDataCenter("dc1")
	rack := dc.GetOrCreateRack("rack1")
	dn := rack.GetOrCreateDataNode("127.0.0.1", 34534, 0, "127.0.0.1", "", nil)

	topo.RegisterVolumeLayout(storage.VolumeInfo{
		Id:               needle.VolumeId(1111),
		ReplicaPlacement: rp,
	}, dn)
	topo.RegisterVolumeLayout(storage.VolumeInfo{
		Id:               needle.VolumeId(2222),
		ReplicaPlacement: rp,
		Collection:       "vol_collection_a",
	}, dn)
	topo.RegisterVolumeLayout(storage.VolumeInfo{
		Id:               needle.VolumeId(3333),
		ReplicaPlacement: rp,
		Collection:       "vol_collection_b",
	}, dn)

	topo.RegisterEcShards(&erasure_coding.EcVolumeInfo{
		VolumeId:   needle.VolumeId(4444),
		Collection: "ec_collection_a",
		ShardsInfo: erasure_coding.NewShardsInfo(),
	}, dn)
	topo.RegisterEcShards(&erasure_coding.EcVolumeInfo{
		VolumeId:   needle.VolumeId(5555),
		Collection: "ec_collection_b",
		ShardsInfo: erasure_coding.NewShardsInfo(),
	}, dn)

	testCases := []struct {
		name                 string
		includeNormalVolumes bool
		includeEcVolumes     bool
		want                 []string
	}{
		{
			name:                 "no volume types selected",
			includeNormalVolumes: false,
			includeEcVolumes:     false,
			want:                 nil,
		}, {
			name:                 "normal volumes",
			includeNormalVolumes: true,
			includeEcVolumes:     false,
			want:                 []string{"", "vol_collection_a", "vol_collection_b"},
		}, {
			name:                 "EC volumes",
			includeNormalVolumes: false,
			includeEcVolumes:     true,
			want:                 []string{"ec_collection_a", "ec_collection_b"},
		}, {
			name:                 "normal + EC volumes",
			includeNormalVolumes: true,
			includeEcVolumes:     true,
			want:                 []string{"", "ec_collection_a", "ec_collection_b", "vol_collection_a", "vol_collection_b"},
		},
	}

	for _, tc := range testCases {
		t.Run(tc.name, func(t *testing.T) {
			got := topo.ListCollections(tc.includeNormalVolumes, tc.includeEcVolumes)

			if !reflect.DeepEqual(got, tc.want) {
				t.Errorf("got %v, want %v", got, tc.want)
			}
		})
	}
}

func TestDataNodeIdBasedIdentification(t *testing.T) {
	topo := NewTopology("weedfs", sequence.NewMemorySequencer(), 32*1024, 5, false)
	dc := topo.GetOrCreateDataCenter("dc1")
	rack := dc.GetOrCreateRack("rack1")

	maxVolumeCounts := make(map[string]uint32)
	maxVolumeCounts[""] = 10

	// Test 1: Create a DataNode with explicit id
	dn1 := rack.GetOrCreateDataNode("10.0.0.1", 8080, 18080, "10.0.0.1:8080", "node-1", maxVolumeCounts)
	if string(dn1.Id()) != "node-1" {
		t.Errorf("expected node id 'node-1', got '%s'", dn1.Id())
	}
	if dn1.Ip != "10.0.0.1" {
		t.Errorf("expected ip '10.0.0.1', got '%s'", dn1.Ip)
	}

	// Test 2: Same id with different IP should return the same DataNode (K8s pod reschedule scenario)
	dn2 := rack.GetOrCreateDataNode("10.0.0.2", 8080, 18080, "10.0.0.2:8080", "node-1", maxVolumeCounts)
	if dn1 != dn2 {
		t.Errorf("expected same DataNode for same id, got different nodes")
	}
	// IP should be updated to the new value
	if dn2.Ip != "10.0.0.2" {
		t.Errorf("expected ip to be updated to '10.0.0.2', got '%s'", dn2.Ip)
	}
	if dn2.PublicUrl != "10.0.0.2:8080" {
		t.Errorf("expected publicUrl to be updated to '10.0.0.2:8080', got '%s'", dn2.PublicUrl)
	}

	// Test 3: Different id should create a new DataNode
	dn3 := rack.GetOrCreateDataNode("10.0.0.3", 8080, 18080, "10.0.0.3:8080", "node-2", maxVolumeCounts)
	if string(dn3.Id()) != "node-2" {
		t.Errorf("expected node id 'node-2', got '%s'", dn3.Id())
	}
	if dn1 == dn3 {
		t.Errorf("expected different DataNode for different id")
	}

	// Test 4: Empty id should fall back to ip:port (backward compatibility)
	dn4 := rack.GetOrCreateDataNode("10.0.0.4", 8080, 18080, "10.0.0.4:8080", "", maxVolumeCounts)
	if string(dn4.Id()) != "10.0.0.4:8080" {
		t.Errorf("expected node id '10.0.0.4:8080' for empty id, got '%s'", dn4.Id())
	}

	// Test 5: Same ip:port with empty id should return the same DataNode
	dn5 := rack.GetOrCreateDataNode("10.0.0.4", 8080, 18080, "10.0.0.4:8080", "", maxVolumeCounts)
	if dn4 != dn5 {
		t.Errorf("expected same DataNode for same ip:port with empty id")
	}

	// Verify we have 3 unique DataNodes total:
	// - node-1 (dn1/dn2 share the same id)
	// - node-2 (dn3)
	// - 10.0.0.4:8080 (dn4/dn5 share the same ip:port)
	children := rack.Children()
	if len(children) != 3 {
		t.Errorf("expected 3 DataNodes, got %d", len(children))
	}

	// Test 6: Transition from ip:port to explicit id
	// First, the node exists with ip:port as id (dn4/dn5)
	// Now the same volume server starts sending an explicit id
	dn6 := rack.GetOrCreateDataNode("10.0.0.4", 8080, 18080, "10.0.0.4:8080", "node-4-explicit", maxVolumeCounts)
	// Should return the same DataNode instance
	if dn6 != dn4 {
		t.Errorf("expected same DataNode instance during transition")
	}
	// But the id should now be updated to the explicit id
	if string(dn6.Id()) != "node-4-explicit" {
		t.Errorf("expected node id to transition to 'node-4-explicit', got '%s'", dn6.Id())
	}
	// The node should be re-keyed in the children map
	if rack.FindDataNodeById("node-4-explicit") != dn6 {
		t.Errorf("expected to find DataNode by new explicit id")
	}
	// Old ip:port key should no longer work
	if rack.FindDataNodeById("10.0.0.4:8080") != nil {
		t.Errorf("expected old ip:port id to be removed from children map")
	}

	// Still 3 unique DataNodes (node-1, node-2, node-4-explicit)
	children = rack.Children()
	if len(children) != 3 {
		t.Errorf("expected 3 DataNodes after transition, got %d", len(children))
	}

	// Test 7: Prevent incorrect transition when a new node reuses ip:port of a node with explicit id
	// Scenario: node-1 runs at 10.0.0.1:8080, dies, new node-99 starts at same ip:port
	// The transition should NOT happen because node-1 already has an explicit id
	dn7 := rack.GetOrCreateDataNode("10.0.0.1", 8080, 18080, "10.0.0.1:8080", "node-99", maxVolumeCounts)
	// Should create a NEW DataNode, not reuse node-1
	if dn7 == dn1 {
		t.Errorf("expected new DataNode for node-99, got reused node-1")
	}
	if string(dn7.Id()) != "node-99" {
		t.Errorf("expected node id 'node-99', got '%s'", dn7.Id())
	}
	// node-1 should still exist with its original id
	if rack.FindDataNodeById("node-1") == nil {
		t.Errorf("node-1 should still exist")
	}
	// Now we have 4 DataNodes
	children = rack.Children()
	if len(children) != 4 {
		t.Errorf("expected 4 DataNodes, got %d", len(children))
	}
}

func TestLookupDataNodeByAddress(t *testing.T) {
	topo := NewTopology("weedfs", sequence.NewMemorySequencer(), 32*1024, 5, false)
	dc := topo.GetOrCreateDataCenter("dc1")
	rack := dc.GetOrCreateRack("rack1")

	maxVolumeCounts := map[string]uint32{"": 10}

	// Brand-new registration must be discoverable by both the http and
	// grpc forms of the address.
	dn := rack.GetOrCreateDataNode("10.1.2.3", 8080, 18080, "10.1.2.3:8080", "n1", maxVolumeCounts)
	if got := topo.LookupDataNodeByAddress(pb.ServerAddress("10.1.2.3:8080")); got != dn {
		t.Fatalf("lookup by http address: got %v, want %v", got, dn)
	}
	if got := topo.LookupDataNodeByAddress(pb.ServerAddress("10.1.2.3:8080.18080")); got != dn {
		t.Fatalf("lookup by grpc-suffix address: got %v, want %v", got, dn)
	}

	// Unknown addresses must miss.
	if got := topo.LookupDataNodeByAddress(pb.ServerAddress("127.0.0.1:1")); got != nil {
		t.Fatalf("unknown address must not be found, got %v", got)
	}

	// Heartbeat from a moved pod (same id, new ip) updates the index in
	// place: the old address is dropped and the new one resolves.
	dnMoved := rack.GetOrCreateDataNode("10.9.9.9", 8080, 18080, "10.9.9.9:8080", "n1", maxVolumeCounts)
	if dnMoved != dn {
		t.Fatalf("expected same node instance after move, got different")
	}
	if got := topo.LookupDataNodeByAddress(pb.ServerAddress("10.1.2.3:8080")); got != nil {
		t.Fatalf("old address must be unregistered after move, got %v", got)
	}
	if got := topo.LookupDataNodeByAddress(pb.ServerAddress("10.9.9.9:8080")); got != dn {
		t.Fatalf("new address lookup: got %v, want %v", got, dn)
	}

	// UnRegisterDataNode evicts the index entry.
	topo.UnRegisterDataNode(dn)
	if got := topo.LookupDataNodeByAddress(pb.ServerAddress("10.9.9.9:8080")); got != nil {
		t.Fatalf("address must be unregistered after UnRegisterDataNode, got %v", got)
	}
}

// TestSyncDataNodeRegistrationReRegistersMissingVolume reproduces the divergence
// where a volume present on a data node (shown by volume.list / admin UI) is
// missing from the lookup index, which surfaces as "volume id not found" on
// LookupVolume. SetVolumeUnavailable (used by
// UnRegisterDataNode on a disconnect) drops the volume from the index, and the
// reconnecting full heartbeat used to skip it because it was no longer "new" to
// the disk map. The full heartbeat now self-heals.
func TestSyncDataNodeRegistrationReRegistersMissingVolume(t *testing.T) {
	topo := NewTopology("weedfs", sequence.NewMemorySequencer(), 32*1024, 5, false)

	dc := topo.GetOrCreateDataCenter("dc1")
	rack := dc.GetOrCreateRack("rack1")
	dn := rack.GetOrCreateDataNode("127.0.0.1", 34534, 0, "127.0.0.1", "", map[string]uint32{"": 25})

	vid := needle.VolumeId(18994)
	volumeMessage := &master_pb.VolumeInformationMessage{
		Id:               uint32(vid),
		Size:             100,
		Collection:       "drr",
		ReplicaPlacement: uint32(0),
		Version:          uint32(needle.GetCurrentVersion()),
		Ttl:              0,
	}

	// Initial full heartbeat registers the volume in the lookup index.
	topo.SyncDataNodeRegistration([]*master_pb.VolumeInformationMessage{volumeMessage}, dn)
	if got := topo.Lookup("", vid); len(got) != 1 {
		t.Fatalf("after registration: lookup %d got %v, want 1 location", vid, got)
	}

	// Drop the volume from the index the way UnRegisterDataNode does, but leave
	// it in the data node's disk map (the reconnecting heartbeat did not report
	// it as new).
	rp, _ := super_block.NewReplicaPlacementFromString("000")
	vl := topo.GetVolumeLayout("drr", rp, needle.EMPTY_TTL, types.HardDriveType)
	vl.SetVolumeUnavailable(dn, vid)

	// The empty entry must be removed, otherwise Lookup returns a non-nil empty
	// list that still reads as "not found".
	if got := topo.Lookup("", vid); got != nil {
		t.Fatalf("after SetVolumeUnavailable: expected lookup miss, got %v", got)
	}
	if _, err := dn.GetVolumesById(vid); err != nil {
		t.Fatalf("volume %d should still be in the data node disk map: %v", vid, err)
	}

	// The next full heartbeat re-registers the volume even though it is not new.
	topo.SyncDataNodeRegistration([]*master_pb.VolumeInformationMessage{volumeMessage}, dn)
	if got := topo.Lookup("", vid); len(got) != 1 {
		t.Fatalf("after self-heal: lookup %d got %v, want 1 location", vid, got)
	}
}

// TestSetVolumeAvailableRepairsMissingVolume covers the vacuum-commit variant of
// the same divergence. A disconnect during a long vacuum can drop a
// single-replica volume from the lookup index while it stays on the node; the
// commit then calls SetVolumeAvailable on it. That used to dereference a nil
// location and panic; it now re-creates the entry and repairs the split.
func TestSetVolumeAvailableRepairsMissingVolume(t *testing.T) {
	topo := NewTopology("weedfs", sequence.NewMemorySequencer(), 32*1024, 5, false)

	dc := topo.GetOrCreateDataCenter("dc1")
	rack := dc.GetOrCreateRack("rack1")
	dn := rack.GetOrCreateDataNode("127.0.0.1", 34534, 0, "127.0.0.1", "", map[string]uint32{"": 25})

	vid := needle.VolumeId(2640)
	volumeMessage := &master_pb.VolumeInformationMessage{
		Id:               uint32(vid),
		Size:             100,
		Collection:       "drr",
		ReplicaPlacement: uint32(0),
		Version:          uint32(needle.GetCurrentVersion()),
		Ttl:              0,
	}

	topo.SyncDataNodeRegistration([]*master_pb.VolumeInformationMessage{volumeMessage}, dn)

	rp, _ := super_block.NewReplicaPlacementFromString("000")
	vl := topo.GetVolumeLayout("drr", rp, needle.EMPTY_TTL, types.HardDriveType)

	// Disconnect drops the volume from the index but leaves it on the node.
	vl.SetVolumeUnavailable(dn, vid)
	if got := topo.Lookup("", vid); got != nil {
		t.Fatalf("after SetVolumeUnavailable: expected lookup miss, got %v", got)
	}
	if _, err := dn.GetVolumesById(vid); err != nil {
		t.Fatalf("volume %d should still be in the data node disk map: %v", vid, err)
	}

	// The vacuum commit re-marks the volume available; it must re-register it.
	vl.SetVolumeAvailable(dn, vid, false, false, false)
	if got := topo.Lookup("", vid); len(got) != 1 {
		t.Fatalf("after SetVolumeAvailable: lookup %d got %v, want 1 location", vid, got)
	}
	// Size tracking must be seeded too, or assigns go uncounted until the next
	// heartbeat and the volume can overfill.
	vl.accessLock.RLock()
	_, tracked := vl.sizeTracking[vid]
	vl.accessLock.RUnlock()
	if !tracked {
		t.Fatalf("after SetVolumeAvailable: size tracking for %d not seeded", vid)
	}
}

// A read-only change that arrives in the regular heartbeat must reach the
// layout's per-replica flag, which the vacuum sweep consults. Registration
// and volume.mark already set it; the heartbeat path did not.
func TestFullHeartbeatUpdatesLayoutReadOnlyFlag(t *testing.T) {
	topo := NewTopology("weedfs", sequence.NewMemorySequencer(), 32*1024, 5, false)
	dn := topo.GetOrCreateDataCenter("dc1").GetOrCreateRack("rack1").
		GetOrCreateDataNode("127.0.0.1", 34534, 0, "127.0.0.1", "", map[string]uint32{"": 25})
	report := func(readOnly bool) []*master_pb.VolumeInformationMessage {
		return []*master_pb.VolumeInformationMessage{{
			Id: 1, Collection: "c", Size: 1 << 20, Version: uint32(needle.GetCurrentVersion()), ReadOnly: readOnly,
		}}
	}
	rp, _ := super_block.NewReplicaPlacementFromString("000")
	flag := func() bool {
		vl := topo.GetVolumeLayout("c", rp, needle.EMPTY_TTL, types.HardDriveType)
		vl.accessLock.RLock()
		defer vl.accessLock.RUnlock()
		return vl.vid2location[needle.VolumeId(1)].AnyReadOnly()
	}

	// the full list, which a server sends first and whenever the digests disagree
	topo.SyncDataNodeRegistration(report(false), dn)
	if flag() {
		t.Fatal("volume registered writable is flagged read-only")
	}
	topo.SyncDataNodeRegistration(report(true), dn)
	if !flag() {
		t.Fatal("full heartbeat turned the volume read-only but the layout flag did not follow")
	}
	topo.SyncDataNodeRegistration(report(false), dn)
	if flag() {
		t.Fatal("full heartbeat turned the volume writable again but the layout flag stayed read-only")
	}

	// the changed-volumes delta, which is what a running server normally sends
	topo.ApplyVolumeChanges(report(true), dn)
	if !flag() {
		t.Fatal("delta heartbeat turned the volume read-only but the layout flag did not follow")
	}
	topo.ApplyVolumeChanges(report(false), dn)
	if flag() {
		t.Fatal("delta heartbeat turned the volume writable again but the layout flag stayed read-only")
	}
}