mirror of
https://github.com/seaweedfs/seaweedfs.git
synced 2026-09-14 02:20:41 +02:00
Chris Lu
af09e1ec7e
When an earlier EC encoding succeeded but the post-encode source-delete left a regular replica behind on one of the servers, the next detection cycle proposes the same volume again. The new encode tries to redistribute shards to targets that already have them mounted, the volume server returns `ec volume %d is mounted; refusing overwrite`, the task fails, and detection re-queues the volume. The cycle repeats forever — issue #9448. The existing `metric.IsECVolume` skip catches the case where the canonical metric is reported on the EC-shard side of the heartbeat, but when the master sees BOTH a regular replica AND its EC shards in the same volume list, the canonical metric we pick is the regular replica and IsECVolume is false. Add a second guard that checks the topology directly via `findExistingECShards` (already present and indexed) and skip the volume when any shards exist, logging a warning that points the admin at the stuck source. This breaks the loop. Auto-cleanup of the orphaned replica is left as follow-up work — deleting a source replica from inside the detector is only safe with a re-verification step right before the delete, plus a config opt-in, and is best done in its own change.
369 lines
11 KiB
Go
369 lines
11 KiB
Go
package erasure_coding
|
||
|
||
import (
|
||
"context"
|
||
"fmt"
|
||
"testing"
|
||
"time"
|
||
|
||
"github.com/seaweedfs/seaweedfs/weed/admin/topology"
|
||
"github.com/seaweedfs/seaweedfs/weed/pb/master_pb"
|
||
"github.com/seaweedfs/seaweedfs/weed/storage/erasure_coding"
|
||
"github.com/seaweedfs/seaweedfs/weed/worker/types"
|
||
"github.com/stretchr/testify/assert"
|
||
"github.com/stretchr/testify/require"
|
||
)
|
||
|
||
func TestECPlacementPlannerApplyReservations(t *testing.T) {
|
||
activeTopology := buildActiveTopology(t, 1, []string{"hdd"}, 10, 0)
|
||
|
||
planner := newECPlacementPlanner(activeTopology, nil)
|
||
require.NotNil(t, planner)
|
||
|
||
key := ecDiskKey("10.0.0.1:8080", 0)
|
||
candidate, ok := planner.candidateByKey[key]
|
||
require.True(t, ok)
|
||
assert.Equal(t, 10, candidate.FreeSlots)
|
||
assert.Equal(t, 0, candidate.ShardCount)
|
||
assert.Equal(t, 0, candidate.LoadCount)
|
||
|
||
shardImpact := topology.CalculateECShardStorageImpact(1, 1)
|
||
destinations := make([]topology.TaskDestinationSpec, 10)
|
||
for i := 0; i < 10; i++ {
|
||
destinations[i] = topology.TaskDestinationSpec{
|
||
ServerID: "10.0.0.1:8080",
|
||
DiskID: 0,
|
||
StorageImpact: &shardImpact,
|
||
}
|
||
}
|
||
|
||
planner.applyTaskReservations(1024, nil, destinations)
|
||
|
||
candidate = planner.candidateByKey[key]
|
||
assert.Equal(t, 9, candidate.FreeSlots, "10 shard slots should reduce available volume slots by 1")
|
||
assert.Equal(t, 10, candidate.ShardCount)
|
||
assert.Equal(t, 1, candidate.LoadCount, "load should only be incremented once per disk")
|
||
}
|
||
|
||
func TestPlanECDestinationsUsesPlanner(t *testing.T) {
|
||
activeTopology := buildActiveTopology(t, 7, []string{"hdd", "ssd"}, 100, 0)
|
||
planner := newECPlacementPlanner(activeTopology, nil)
|
||
require.NotNil(t, planner)
|
||
|
||
metric := &types.VolumeHealthMetrics{
|
||
VolumeID: 1,
|
||
Server: "10.0.0.1:8080",
|
||
Size: 100 * 1024 * 1024,
|
||
Collection: "",
|
||
}
|
||
|
||
plan, err := planECDestinations(planner, metric, NewDefaultConfig(), erasure_coding.DataShardsCount, erasure_coding.ParityShardsCount)
|
||
require.NoError(t, err)
|
||
require.NotNil(t, plan)
|
||
assert.Equal(t, erasure_coding.TotalShardsCount, len(plan.Plans))
|
||
}
|
||
|
||
func TestECPlacementPlannerPrefersTaggedDisks(t *testing.T) {
|
||
activeTopology := buildActiveTopology(t, 3, []string{"hdd"}, 10, 0)
|
||
topo := activeTopology.GetTopologyInfo()
|
||
for _, dc := range topo.DataCenterInfos {
|
||
for _, rack := range dc.RackInfos {
|
||
for k, node := range rack.DataNodeInfos {
|
||
for diskType := range node.DiskInfos {
|
||
if k < 2 {
|
||
node.DiskInfos[diskType].Tags = []string{"fast"}
|
||
} else {
|
||
node.DiskInfos[diskType].Tags = []string{"slow"}
|
||
}
|
||
}
|
||
}
|
||
}
|
||
}
|
||
require.NoError(t, activeTopology.UpdateTopology(topo))
|
||
|
||
planner := newECPlacementPlanner(activeTopology, []string{"fast"})
|
||
require.NotNil(t, planner)
|
||
|
||
selected, err := planner.selectDestinations("", "", "", 2)
|
||
require.NoError(t, err)
|
||
require.Len(t, selected, 2)
|
||
|
||
for _, candidate := range selected {
|
||
key := ecDiskKey(candidate.NodeID, candidate.DiskID)
|
||
assert.True(t, diskHasTag(planner.diskTags[key], "fast"))
|
||
}
|
||
}
|
||
|
||
func TestECPlacementPlannerFallsBackWhenTagsInsufficient(t *testing.T) {
|
||
activeTopology := buildActiveTopology(t, 3, []string{"hdd"}, 10, 0)
|
||
topo := activeTopology.GetTopologyInfo()
|
||
for _, dc := range topo.DataCenterInfos {
|
||
for _, rack := range dc.RackInfos {
|
||
for i, node := range rack.DataNodeInfos {
|
||
for diskType := range node.DiskInfos {
|
||
if i == 0 {
|
||
node.DiskInfos[diskType].Tags = []string{"fast"}
|
||
}
|
||
}
|
||
}
|
||
}
|
||
}
|
||
require.NoError(t, activeTopology.UpdateTopology(topo))
|
||
|
||
planner := newECPlacementPlanner(activeTopology, []string{"fast"})
|
||
require.NotNil(t, planner)
|
||
|
||
selected, err := planner.selectDestinations("", "", "", 3)
|
||
require.NoError(t, err)
|
||
require.Len(t, selected, 3)
|
||
|
||
taggedCount := 0
|
||
for _, candidate := range selected {
|
||
key := ecDiskKey(candidate.NodeID, candidate.DiskID)
|
||
if diskHasTag(planner.diskTags[key], "fast") {
|
||
taggedCount++
|
||
}
|
||
}
|
||
assert.Less(t, taggedCount, len(selected))
|
||
}
|
||
|
||
// TestDetectionSkipsWhenECShardsAlreadyExist guards against issue #9448: a
|
||
// regular replica that survived a previous successful EC encode (source
|
||
// delete didn't clean it up for some reason) gets re-proposed for encoding,
|
||
// the new encode collides with the already-mounted shards on the targets
|
||
// ("ec volume %d is mounted; refusing overwrite"), and detection loops
|
||
// forever on the same volume. Detection must see the existing shards and
|
||
// skip the volume so an admin can clean it up out-of-band.
|
||
func TestDetectionSkipsWhenECShardsAlreadyExist(t *testing.T) {
|
||
const volumeID uint32 = 42
|
||
const collection = ""
|
||
|
||
activeTopology := topology.NewActiveTopology(10)
|
||
nodes := make([]*master_pb.DataNodeInfo, 0, erasure_coding.TotalShardsCount)
|
||
for i := 0; i < erasure_coding.TotalShardsCount; i++ {
|
||
nodeID := fmt.Sprintf("127.0.0.1:%d", 8080+i)
|
||
diskInfo := &master_pb.DiskInfo{
|
||
DiskId: 0,
|
||
VolumeCount: 1,
|
||
MaxVolumeCount: 100,
|
||
EcShardInfos: []*master_pb.VolumeEcShardInformationMessage{{
|
||
Id: volumeID,
|
||
Collection: collection,
|
||
EcIndexBits: uint32(1) << uint(i),
|
||
DiskId: 0,
|
||
}},
|
||
}
|
||
if i == 0 {
|
||
// The orphaned source replica that the previous encode's source
|
||
// delete didn't remove.
|
||
diskInfo.VolumeInfos = []*master_pb.VolumeInformationMessage{{
|
||
Id: volumeID,
|
||
Collection: collection,
|
||
DiskId: 0,
|
||
DiskType: "hdd",
|
||
Size: 200 * 1024 * 1024,
|
||
}}
|
||
}
|
||
nodes = append(nodes, &master_pb.DataNodeInfo{
|
||
Id: nodeID,
|
||
DiskInfos: map[string]*master_pb.DiskInfo{"hdd": diskInfo},
|
||
})
|
||
}
|
||
require.NoError(t, activeTopology.UpdateTopology(&master_pb.TopologyInfo{
|
||
DataCenterInfos: []*master_pb.DataCenterInfo{{
|
||
Id: "dc1",
|
||
RackInfos: []*master_pb.RackInfo{{
|
||
Id: "rack1",
|
||
DataNodeInfos: nodes,
|
||
}},
|
||
}},
|
||
}))
|
||
|
||
clusterInfo := &types.ClusterInfo{ActiveTopology: activeTopology}
|
||
metrics := []*types.VolumeHealthMetrics{{
|
||
VolumeID: volumeID,
|
||
Server: nodes[0].Id,
|
||
Size: 200 * 1024 * 1024,
|
||
Collection: collection,
|
||
FullnessRatio: 0.9,
|
||
LastModified: time.Now().Add(-time.Hour),
|
||
Age: 10 * time.Minute,
|
||
}}
|
||
|
||
results, hasMore, err := Detection(context.Background(), metrics, clusterInfo, NewDefaultConfig(), 0)
|
||
require.NoError(t, err)
|
||
require.False(t, hasMore)
|
||
require.Empty(t, results, "stuck source replica must not produce a new EC encoding proposal")
|
||
}
|
||
|
||
func TestDetectionContextCancellation(t *testing.T) {
|
||
activeTopology := buildActiveTopology(t, 5, []string{"hdd", "ssd"}, 50, 0)
|
||
clusterInfo := &types.ClusterInfo{ActiveTopology: activeTopology}
|
||
metrics := buildVolumeMetricsForIDs(50)
|
||
|
||
ctx, cancel := context.WithCancel(context.Background())
|
||
cancel()
|
||
|
||
_, _, err := Detection(ctx, metrics, clusterInfo, NewDefaultConfig(), 0)
|
||
require.ErrorIs(t, err, context.Canceled)
|
||
}
|
||
|
||
func TestDetectionMaxResultsHonorsLimit(t *testing.T) {
|
||
// One node per shard so each shard gets its own disk (#9369).
|
||
activeTopology := buildActiveTopology(t, erasure_coding.TotalShardsCount, []string{"hdd"}, 20, 0)
|
||
clusterInfo := &types.ClusterInfo{ActiveTopology: activeTopology}
|
||
metrics := buildVolumeMetricsForIDs(3)
|
||
|
||
results, hasMore, err := Detection(context.Background(), metrics, clusterInfo, NewDefaultConfig(), 1)
|
||
require.NoError(t, err)
|
||
assert.Len(t, results, 1)
|
||
assert.True(t, hasMore)
|
||
}
|
||
|
||
// #9369: 7 servers × 2 physical HDDs must yield 14 distinct (server, disk_id)
|
||
// destinations, not 7 destinations doubled up on the same disk.
|
||
func TestPlanECDestinationsSpreadsAcrossPhysicalDisks(t *testing.T) {
|
||
const numServers = 7
|
||
const disksPerServer = 2
|
||
|
||
activeTopology := topology.NewActiveTopology(10)
|
||
nodes := make([]*master_pb.DataNodeInfo, 0, numServers)
|
||
for i := 1; i <= numServers; i++ {
|
||
volumeInfos := make([]*master_pb.VolumeInformationMessage, 0, disksPerServer)
|
||
for d := uint32(0); d < disksPerServer; d++ {
|
||
volumeInfos = append(volumeInfos, &master_pb.VolumeInformationMessage{
|
||
Id: uint32(i*100 + int(d)),
|
||
DiskId: d,
|
||
DiskType: "hdd",
|
||
})
|
||
}
|
||
nodes = append(nodes, &master_pb.DataNodeInfo{
|
||
Id: fmt.Sprintf("127.0.0.1:%d", 8080+i),
|
||
DiskInfos: map[string]*master_pb.DiskInfo{
|
||
"hdd": {
|
||
DiskId: 0,
|
||
VolumeCount: int64(disksPerServer),
|
||
MaxVolumeCount: 200,
|
||
VolumeInfos: volumeInfos,
|
||
},
|
||
},
|
||
})
|
||
}
|
||
require.NoError(t, activeTopology.UpdateTopology(&master_pb.TopologyInfo{
|
||
DataCenterInfos: []*master_pb.DataCenterInfo{{
|
||
Id: "dc1",
|
||
RackInfos: []*master_pb.RackInfo{{
|
||
Id: "rack1",
|
||
DataNodeInfos: nodes,
|
||
}},
|
||
}},
|
||
}))
|
||
|
||
planner := newECPlacementPlanner(activeTopology, nil)
|
||
require.NotNil(t, planner)
|
||
|
||
metric := &types.VolumeHealthMetrics{
|
||
VolumeID: 42,
|
||
Server: "127.0.0.1:8081",
|
||
Size: 100 * 1024 * 1024,
|
||
Collection: "",
|
||
}
|
||
|
||
plan, err := planECDestinations(planner, metric, NewDefaultConfig(), erasure_coding.DataShardsCount, erasure_coding.ParityShardsCount)
|
||
require.NoError(t, err)
|
||
require.NotNil(t, plan)
|
||
require.Equal(t, erasure_coding.TotalShardsCount, len(plan.Plans))
|
||
|
||
seen := make(map[string]bool, len(plan.Plans))
|
||
for _, p := range plan.Plans {
|
||
key := fmt.Sprintf("%s:%d", p.TargetNode, p.TargetDisk)
|
||
assert.False(t, seen[key], "duplicate (server,disk_id) target %s", key)
|
||
seen[key] = true
|
||
}
|
||
}
|
||
|
||
func TestPlanECDestinationsFailsWithInsufficientCapacity(t *testing.T) {
|
||
activeTopology := buildActiveTopology(t, 1, []string{"hdd"}, 1, 1)
|
||
planner := newECPlacementPlanner(activeTopology, nil)
|
||
require.NotNil(t, planner)
|
||
|
||
metric := &types.VolumeHealthMetrics{
|
||
VolumeID: 2,
|
||
Server: "10.0.0.1:8080",
|
||
Size: 10 * 1024 * 1024,
|
||
Collection: "",
|
||
}
|
||
|
||
_, err := planECDestinations(planner, metric, NewDefaultConfig(), erasure_coding.DataShardsCount, erasure_coding.ParityShardsCount)
|
||
require.Error(t, err)
|
||
}
|
||
|
||
func buildVolumeMetricsForIDs(count int) []*types.VolumeHealthMetrics {
|
||
metrics := make([]*types.VolumeHealthMetrics, 0, count)
|
||
now := time.Now()
|
||
for id := 1; id <= count; id++ {
|
||
metrics = append(metrics, &types.VolumeHealthMetrics{
|
||
VolumeID: uint32(id),
|
||
Server: "10.0.0.1:8080",
|
||
Size: 200 * 1024 * 1024,
|
||
Collection: "",
|
||
FullnessRatio: 0.9,
|
||
LastModified: now.Add(-time.Hour),
|
||
Age: 10 * time.Minute,
|
||
})
|
||
}
|
||
return metrics
|
||
}
|
||
|
||
func buildActiveTopology(t *testing.T, nodeCount int, diskTypes []string, maxVolumeCount, usedVolumeCount int64) *topology.ActiveTopology {
|
||
t.Helper()
|
||
activeTopology := topology.NewActiveTopology(10)
|
||
|
||
nodes := make([]*master_pb.DataNodeInfo, 0, nodeCount)
|
||
for i := 1; i <= nodeCount; i++ {
|
||
diskInfos := make(map[string]*master_pb.DiskInfo)
|
||
for diskIndex, diskType := range diskTypes {
|
||
used := usedVolumeCount
|
||
if used > maxVolumeCount {
|
||
used = maxVolumeCount
|
||
}
|
||
volumeInfos := make([]*master_pb.VolumeInformationMessage, 0, 200)
|
||
for vid := 1; vid <= 200; vid++ {
|
||
volumeInfos = append(volumeInfos, &master_pb.VolumeInformationMessage{
|
||
Id: uint32(vid),
|
||
Collection: "",
|
||
DiskId: uint32(diskIndex),
|
||
})
|
||
}
|
||
diskInfos[diskType] = &master_pb.DiskInfo{
|
||
DiskId: uint32(diskIndex),
|
||
VolumeCount: used,
|
||
MaxVolumeCount: maxVolumeCount,
|
||
VolumeInfos: volumeInfos,
|
||
}
|
||
}
|
||
|
||
nodes = append(nodes, &master_pb.DataNodeInfo{
|
||
Id: fmt.Sprintf("10.0.0.%d:8080", i),
|
||
DiskInfos: diskInfos,
|
||
})
|
||
}
|
||
|
||
topologyInfo := &master_pb.TopologyInfo{
|
||
DataCenterInfos: []*master_pb.DataCenterInfo{
|
||
{
|
||
Id: "dc1",
|
||
RackInfos: []*master_pb.RackInfo{
|
||
{
|
||
Id: "rack1",
|
||
DataNodeInfos: nodes,
|
||
},
|
||
},
|
||
},
|
||
},
|
||
}
|
||
|
||
require.NoError(t, activeTopology.UpdateTopology(topologyInfo))
|
||
return activeTopology
|
||
}
|