mirror of
https://github.com/seaweedfs/seaweedfs.git
synced 2026-10-23 22:40:39 +02:00
Chris Lu
4d06742664
* master: detect and quarantine duplicate volume ids across layouts (#11763) * shell: volume.check.duplicate_ids audit command (#11763) * master: persist duplicate-vid quarantine across heartbeats (#11763) * master: quarantine all copies of a duplicated volume id Near-simultaneous registrations let each side quarantine only the other, so arrival order decided which copy stayed writable. A confirmed cross-layout collision means either copy could be the impostor, so flag every colliding copy read-only regardless of registration order. * topology: return a copy from VolumeLayout.Lookup Callers iterating the returned slice raced with registrations and removals replacing the location list in place. * master: release duplicate-vid quarantine when the conflict resolves A cross-node volume.move briefly shows the same id on two nodes under different disk layouts; permanent quarantine would pin a valid target read-only. Keep the first-reported state per quarantined copy and lift the marks once the conflicting copy unregisters, so only a persistent collision stays read-only. * shell: include ec shards in volume.check.duplicate_ids Ec-only volumes share the global id space. Compare shard collections against regular ones while allowing shards and regular copies of the same encoded volume. * master: keep quarantine release current and cover all removal paths Track the latest server-reported read-only flag while quarantined so a release restores the server s true state, and run the release check on collection deletion and data-node removal too, not only explicit volume unregistration. * shell: do not flag ec-only volumes as duplicates An ec-only volume has no regular copy, so its shard collections had nothing to compare against and always read as foreign. * master: quarantine every conflicting layout at the layout level (#11763) - collect all conflicting layouts, not just the first, so a vid spanning three layouts does not leave one writable - mark quarantine on the location list only: folding it into the stored report made the volume digest permanently disagree with the server and forced a full volume list every heartbeat - isAllWritable now honors the location read-only mark so a quarantined copy stays out of writables through resyncs and re-registration - re-check the conflict before keeping marks, and do not clear a node's quarantine while it still holds the vid in another layout - log the full layout keys on both sides of the conflict * master: push duplicate-vid quarantine to the volume server (#11763) A quarantined copy stayed writable on the volume server itself for a client holding an existing file id, since the master-only mark never reached the server's local write path. Quarantine now also calls VolumeMarkReadonly (can_delete so expiry still drains), is re-pushed if the server comes back reporting it writable, and VolumeMarkWritable runs when the conflict resolves and the server had reported writable. * master: retry unenforced quarantine marks and restore server writability on release (#11763) Dial the advertised grpc port, return mark errors, and keep a quarantine entry until the writable push succeeds so a released copy does not stay read-only on the server. * master: treat rp-only layout differences as an in-flight replication change (#11763) volume.configure.replication rolls through nodes one at a time, so a volume legitimately reports the old and new replica placement while it settles. Layouts differing only in rp within the same collection are a supported transition, not a reused id. * shell: group ec shards by collection in the duplicate id audit (#11763) EC shards of one volume legitimately span disk types when DiskTypePrefer spills, so grouping by disk type reports healthy volumes as duplicates.
188 lines
6.4 KiB
Go
188 lines
6.4 KiB
Go
package topology
|
|
|
|
import (
|
|
"testing"
|
|
|
|
"github.com/seaweedfs/seaweedfs/weed/pb/master_pb"
|
|
"github.com/seaweedfs/seaweedfs/weed/storage"
|
|
"github.com/seaweedfs/seaweedfs/weed/storage/needle"
|
|
"github.com/seaweedfs/seaweedfs/weed/storage/super_block"
|
|
"github.com/seaweedfs/seaweedfs/weed/storage/types"
|
|
)
|
|
|
|
func isWritable(t *testing.T, topo *Topology, v storage.VolumeInfo) bool {
|
|
t.Helper()
|
|
vl := topo.FindVolumeLayout(v.Collection, v.ReplicaPlacement, v.Ttl, types.ToDiskType(v.DiskType))
|
|
for _, id := range vl.CloneWritableVolumes() {
|
|
if id == v.Id {
|
|
return true
|
|
}
|
|
}
|
|
return false
|
|
}
|
|
|
|
func TestDuplicateVolumeIdAcrossCollections(t *testing.T) {
|
|
topo := NewTopology("topo", nil, 128, 3, false)
|
|
rp, _ := super_block.NewReplicaPlacementFromString("000")
|
|
|
|
dn1 := NewDataNode("server1")
|
|
dn2 := NewDataNode("server2")
|
|
|
|
v1 := storage.VolumeInfo{Id: 7, Collection: "a", ReplicaPlacement: rp, Ttl: needle.EMPTY_TTL}
|
|
v2 := storage.VolumeInfo{Id: 7, Collection: "b", ReplicaPlacement: rp, Ttl: needle.EMPTY_TTL}
|
|
|
|
dn1.UpdateVolumes([]storage.VolumeInfo{v1})
|
|
dn2.UpdateVolumes([]storage.VolumeInfo{v2})
|
|
topo.RegisterVolumeLayout(v1, dn1)
|
|
topo.RegisterVolumeLayout(v2, dn2)
|
|
topo.flagDuplicateVolumeId(v2, dn2)
|
|
|
|
if isWritable(t, topo, v2) {
|
|
t.Fatal("duplicate newcomer should be quarantined out of the writable list")
|
|
}
|
|
if isWritable(t, topo, v1) {
|
|
t.Fatal("either copy could be the impostor; the earlier copy must be quarantined too")
|
|
}
|
|
}
|
|
|
|
func TestSameLayoutReplicaIsNotDuplicate(t *testing.T) {
|
|
topo := NewTopology("topo", nil, 128, 3, false)
|
|
rp, _ := super_block.NewReplicaPlacementFromString("001")
|
|
|
|
dn1 := NewDataNode("server1")
|
|
dn1.Ip = "10.0.0.1"
|
|
dn1.Port = 8080
|
|
dn2 := NewDataNode("server2")
|
|
dn2.Ip = "10.0.0.2"
|
|
dn2.Port = 8080
|
|
|
|
v := storage.VolumeInfo{Id: 7, Collection: "a", ReplicaPlacement: rp, Ttl: needle.EMPTY_TTL}
|
|
dn1.UpdateVolumes([]storage.VolumeInfo{v})
|
|
dn2.UpdateVolumes([]storage.VolumeInfo{v})
|
|
topo.RegisterVolumeLayout(v, dn1)
|
|
topo.RegisterVolumeLayout(v, dn2)
|
|
topo.flagDuplicateVolumeId(v, dn2)
|
|
|
|
if !isWritable(t, topo, v) {
|
|
t.Fatal("a same-layout replica is a legitimate copy, not a duplicate")
|
|
}
|
|
}
|
|
|
|
func TestSameNodeVolumeMoveIsNotDuplicate(t *testing.T) {
|
|
topo := NewTopology("topo", nil, 128, 3, false)
|
|
rp, _ := super_block.NewReplicaPlacementFromString("000")
|
|
dn := NewDataNode("server1")
|
|
|
|
old := storage.VolumeInfo{Id: 7, Collection: "c", ReplicaPlacement: rp, Ttl: needle.EMPTY_TTL}
|
|
moved := storage.VolumeInfo{Id: 7, Collection: "c", ReplicaPlacement: rp, Ttl: needle.EMPTY_TTL, DiskType: "ssd"}
|
|
dn.UpdateVolumes([]storage.VolumeInfo{old})
|
|
topo.RegisterVolumeLayout(old, dn)
|
|
|
|
// the move registers the new layout before the old entry is removed
|
|
dn.UpdateVolumes([]storage.VolumeInfo{moved})
|
|
topo.RegisterVolumeLayout(moved, dn)
|
|
topo.flagDuplicateVolumeId(moved, dn)
|
|
|
|
if !isWritable(t, topo, moved) {
|
|
t.Fatal("a same-node disk move is not a duplicate")
|
|
}
|
|
}
|
|
|
|
func TestQuarantineReleasedWhenConflictResolves(t *testing.T) {
|
|
topo := NewTopology("topo", nil, 128, 3, false)
|
|
rp, _ := super_block.NewReplicaPlacementFromString("000")
|
|
|
|
dn1 := NewDataNode("server1")
|
|
dn2 := NewDataNode("server2")
|
|
v1 := storage.VolumeInfo{Id: 7, Collection: "a", ReplicaPlacement: rp, Ttl: needle.EMPTY_TTL}
|
|
v2 := storage.VolumeInfo{Id: 7, Collection: "b", ReplicaPlacement: rp, Ttl: needle.EMPTY_TTL}
|
|
dn1.UpdateVolumes([]storage.VolumeInfo{v1})
|
|
dn2.UpdateVolumes([]storage.VolumeInfo{v2})
|
|
topo.RegisterVolumeLayout(v1, dn1)
|
|
topo.RegisterVolumeLayout(v2, dn2)
|
|
topo.flagDuplicateVolumeId(v2, dn2)
|
|
|
|
topo.UnRegisterVolumeLayout(v2, dn2)
|
|
|
|
if !isWritable(t, topo, v1) {
|
|
t.Fatal("the surviving copy must be released once the conflict is gone")
|
|
}
|
|
}
|
|
|
|
func TestQuarantineSurvivesResync(t *testing.T) {
|
|
topo := NewTopology("topo", nil, 128, 3, false)
|
|
rp, _ := super_block.NewReplicaPlacementFromString("000")
|
|
|
|
dn1 := NewDataNode("server1")
|
|
dn2 := NewDataNode("server2")
|
|
v1 := storage.VolumeInfo{Id: 7, Collection: "a", ReplicaPlacement: rp, Ttl: needle.EMPTY_TTL}
|
|
v2 := storage.VolumeInfo{Id: 7, Collection: "b", ReplicaPlacement: rp, Ttl: needle.EMPTY_TTL}
|
|
dn1.UpdateVolumes([]storage.VolumeInfo{v1})
|
|
dn2.UpdateVolumes([]storage.VolumeInfo{v2})
|
|
topo.RegisterVolumeLayout(v1, dn1)
|
|
topo.RegisterVolumeLayout(v2, dn2)
|
|
topo.flagDuplicateVolumeId(v2, dn2)
|
|
|
|
// a resync from the same server re-reports the volume as writable;
|
|
// the quarantine must keep it out of the writable list
|
|
topo.SyncDataNodeRegistration([]*master_pb.VolumeInformationMessage{{
|
|
Id: 7, Collection: "b", ReplicaPlacement: 0, Version: 3,
|
|
}}, dn2)
|
|
|
|
if isWritable(t, topo, v2) {
|
|
t.Fatal("resync must not clear the quarantine")
|
|
}
|
|
}
|
|
|
|
func TestReplicationChangeIsNotDuplicate(t *testing.T) {
|
|
topo := NewTopology("topo", nil, 128, 3, false)
|
|
rpOld, _ := super_block.NewReplicaPlacementFromString("000")
|
|
rpNew, _ := super_block.NewReplicaPlacementFromString("001")
|
|
|
|
dn1 := NewDataNode("server1")
|
|
dn1.Ip = "10.0.0.1"
|
|
dn1.Port = 8080
|
|
dn2 := NewDataNode("server2")
|
|
dn2.Ip = "10.0.0.2"
|
|
dn2.Port = 8080
|
|
|
|
vOld := storage.VolumeInfo{Id: 7, Collection: "a", ReplicaPlacement: rpOld, Ttl: needle.EMPTY_TTL}
|
|
vNew := storage.VolumeInfo{Id: 7, Collection: "a", ReplicaPlacement: rpNew, Ttl: needle.EMPTY_TTL}
|
|
|
|
dn1.UpdateVolumes([]storage.VolumeInfo{vOld})
|
|
dn2.UpdateVolumes([]storage.VolumeInfo{vNew})
|
|
topo.RegisterVolumeLayout(vOld, dn1)
|
|
topo.RegisterVolumeLayout(vNew, dn2)
|
|
topo.flagDuplicateVolumeId(vNew, dn2)
|
|
|
|
if topo.isQuarantined(dn2, vNew.Id) || topo.isQuarantined(dn1, vOld.Id) {
|
|
t.Fatal("an rp-only layout difference is a replication change in flight, not a duplicate")
|
|
}
|
|
}
|
|
|
|
func TestDuplicateVolumeIdAcrossTtls(t *testing.T) {
|
|
topo := NewTopology("topo", nil, 128, 3, false)
|
|
rp, _ := super_block.NewReplicaPlacementFromString("000")
|
|
ttl, _ := needle.ReadTTL("3d")
|
|
|
|
dn1 := NewDataNode("server1")
|
|
dn1.Ip = "10.0.0.1"
|
|
dn1.Port = 8080
|
|
dn2 := NewDataNode("server2")
|
|
dn2.Ip = "10.0.0.2"
|
|
dn2.Port = 8080
|
|
|
|
v1 := storage.VolumeInfo{Id: 7, Collection: "a", ReplicaPlacement: rp, Ttl: needle.EMPTY_TTL}
|
|
v2 := storage.VolumeInfo{Id: 7, Collection: "a", ReplicaPlacement: rp, Ttl: ttl}
|
|
|
|
dn1.UpdateVolumes([]storage.VolumeInfo{v1})
|
|
dn2.UpdateVolumes([]storage.VolumeInfo{v2})
|
|
topo.RegisterVolumeLayout(v1, dn1)
|
|
topo.RegisterVolumeLayout(v2, dn2)
|
|
topo.flagDuplicateVolumeId(v2, dn2)
|
|
|
|
if !topo.isQuarantined(dn2, v2.Id) || !topo.isQuarantined(dn1, v1.Id) {
|
|
t.Fatal("same collection with a different ttl is still a reused volume id")
|
|
}
|
|
}
|