Files
seaweedfs/weed/server/master_grpc_server_heartbeat_auth_test.go
T
Chris Lu f28c7ce6df master: bind heartbeat claims to the connecting peer (#9443)
SendHeartbeat used to accept whatever Ip/Port/Volumes the caller put on
the wire. Three changes tighten that:

- Reject heartbeats whose Ip does not match the gRPC peer's source
  address. Loopback peers are still trusted; operators behind a proxy
  can opt out with -master.allowUntrustedHeartbeat.
- Track which (ip, port) first claimed a volume id or an ec shard slot
  and drop foreign re-claims. Non-EC volume claims are bounded by the
  replica copy count so legitimate replicas still register. EC
  ownership is keyed by (vid, shard_id) so the same vid can legitimately
  be split across many peers as long as their EcIndexBits are disjoint;
  rejected bits are cleared from the bitmap and the parallel ShardSizes
  array is compacted in lock-step.
- Maintain reverse indexes owner -> volumes and owner -> ec shard slots
  so disconnect cleanup is O(M) in what that peer held rather than O(N)
  over the whole map.

Bindings are also released when a heartbeat reports that the peer no
longer holds an id, either via explicit Deleted{Volumes,EcShards}
entries or by omitting it from a full snapshot. Without this, a planned
rebalance that moved a vid or an ec shard from peer A to peer B would
leave B's heartbeats permanently filtered out until A disconnected,
breaking ec encode/decode flows that delete shards on the source as
soon as the move completes.

The (vid -> owners) binding still does not track which replica slot
each peer occupies, so the first N claims under the copy count win;
strict per-slot mapping is a follow-up.
2026-05-12 15:38:52 -07:00

614 lines
22 KiB
Go

package weed_server
import (
"context"
"net"
"testing"
"google.golang.org/grpc/peer"
"github.com/seaweedfs/seaweedfs/weed/pb/master_pb"
"github.com/seaweedfs/seaweedfs/weed/sequence"
"github.com/seaweedfs/seaweedfs/weed/topology"
)
func TestHeartbeatOwnership_FirstClaimWins(t *testing.T) {
o := newHeartbeatOwnership()
if !o.claimVolume(1, "10.0.0.1", 8080, 1) {
t.Fatalf("first claim must succeed")
}
if o.claimVolume(1, "10.6.6.6", 8080, 1) {
t.Fatalf("attacker claim from a different peer must be rejected")
}
if !o.claimVolume(1, "10.0.0.1", 8080, 1) {
t.Fatalf("re-claim from the original peer must succeed")
}
}
func TestHeartbeatOwnership_RespectsReplicaCount(t *testing.T) {
o := newHeartbeatOwnership()
// 3 legitimate replicas
if !o.claimVolume(2, "10.0.0.1", 8080, 3) {
t.Fatal()
}
if !o.claimVolume(2, "10.0.0.2", 8080, 3) {
t.Fatal()
}
if !o.claimVolume(2, "10.0.0.3", 8080, 3) {
t.Fatal()
}
// 4th distinct peer is over count
if o.claimVolume(2, "10.6.6.6", 8080, 3) {
t.Fatalf("over-count claim must be rejected")
}
}
func TestHeartbeatOwnership_Release(t *testing.T) {
o := newHeartbeatOwnership()
if !o.claimVolume(3, "10.0.0.1", 8080, 1) {
t.Fatal()
}
o.release("10.0.0.1", 8080)
if !o.claimVolume(3, "10.0.0.2", 8080, 1) {
t.Fatalf("after release, another peer should be able to take the slot")
}
}
// TestHeartbeatOwnership_EcShardsPerSlot verifies the core EC correctness
// property: each (vid, shard_id) pair has exactly one owner, but the same
// vid may legitimately be split across multiple peers when they claim
// disjoint shard bitmaps. The previous vid-only model rejected those.
func TestHeartbeatOwnership_EcShardsPerSlot(t *testing.T) {
o := newHeartbeatOwnership()
// Peer A claims shards 0,1,2 of vid 7.
if got := o.claimEcShards(7, 0b0111, "10.0.0.1", 8080); got != 0b0111 {
t.Fatalf("peer A should own shards 0-2, got 0x%x", got)
}
// Re-claim from the owning peer is fine.
if got := o.claimEcShards(7, 0b0111, "10.0.0.1", 8080); got != 0b0111 {
t.Fatalf("re-claim from owner should succeed, got 0x%x", got)
}
// Peer B claims shards 3,4 of the same vid: must succeed because the
// slots are disjoint.
if got := o.claimEcShards(7, 0b11000, "10.0.0.2", 8080); got != 0b11000 {
t.Fatalf("peer B should own shards 3-4, got 0x%x", got)
}
// Peer C tries to hijack shard 0 (owned by A) and claim shard 5: only
// the unowned bit comes through.
if got := o.claimEcShards(7, 0b100001, "10.6.6.6", 8080); got != 0b100000 {
t.Fatalf("hijack of shard 0 must be rejected; shard 5 must be granted; got 0x%x", got)
}
}
// TestHeartbeatOwnership_EcShardReleaseIsolatesPeers asserts that
// disconnecting peer A frees only A's bindings, leaving B's intact.
func TestHeartbeatOwnership_EcShardReleaseIsolatesPeers(t *testing.T) {
o := newHeartbeatOwnership()
o.claimEcShards(9, 0b0111, "10.0.0.1", 8080)
o.claimEcShards(9, 0b11000, "10.0.0.2", 8080)
o.release("10.0.0.1", 8080)
// B's shards still locked.
if got := o.claimEcShards(9, 0b11000, "10.6.6.6", 8080); got != 0 {
t.Fatalf("B's bindings must survive A's release, got 0x%x", got)
}
// A's freed shards may be reclaimed by anyone.
if got := o.claimEcShards(9, 0b0111, "10.6.6.6", 8080); got != 0b0111 {
t.Fatalf("A's freed shards should be reclaimable, got 0x%x", got)
}
}
// TestHeartbeatOwnership_ReleaseScopedToOwner checks that volume bindings
// of peer B are not collateral damage when peer A disconnects.
func TestHeartbeatOwnership_ReleaseScopedToOwner(t *testing.T) {
o := newHeartbeatOwnership()
o.claimVolume(10, "10.0.0.1", 8080, 2)
o.claimVolume(10, "10.0.0.2", 8080, 2)
o.release("10.0.0.1", 8080)
// Slot vacated by A is reclaimable by a third peer.
if !o.claimVolume(10, "10.0.0.3", 8080, 2) {
t.Fatalf("A's slot should be reclaimable after release")
}
// B still owns its slot; a fourth peer must not push past the cap.
if o.claimVolume(10, "10.6.6.6", 8080, 2) {
t.Fatalf("over-count claim must be rejected; B's slot was not freed")
}
}
type fakeAddr struct {
network string
s string
}
func (f fakeAddr) Network() string { return f.network }
func (f fakeAddr) String() string { return f.s }
func tcpPeerContext(ip string, port int) context.Context {
return peer.NewContext(context.Background(), &peer.Peer{
Addr: &net.TCPAddr{IP: net.ParseIP(ip), Port: port},
})
}
func TestAuthorizeHeartbeatPeer_AcceptsMatchingIp(t *testing.T) {
ms := &MasterServer{option: &MasterOption{}}
ctx := tcpPeerContext("10.0.0.1", 55555)
if err := ms.authorizeHeartbeatPeer(ctx, &master_pb.Heartbeat{Ip: "10.0.0.1", Port: 8080}); err != nil {
t.Fatalf("unexpected rejection: %v", err)
}
}
func TestAuthorizeHeartbeatPeer_RejectsMismatch(t *testing.T) {
ms := &MasterServer{option: &MasterOption{}}
ctx := tcpPeerContext("10.0.0.1", 55555)
if err := ms.authorizeHeartbeatPeer(ctx, &master_pb.Heartbeat{Ip: "10.6.6.6", Port: 8080}); err == nil {
t.Fatalf("attacker should be rejected when source ip does not match claim")
}
}
func TestAuthorizeHeartbeatPeer_LoopbackAccepted(t *testing.T) {
ms := &MasterServer{option: &MasterOption{}}
ctx := tcpPeerContext("127.0.0.1", 55555)
// A loopback peer is allowed to advertise its external hostname.
if err := ms.authorizeHeartbeatPeer(ctx, &master_pb.Heartbeat{Ip: "host.example", Port: 8080}); err != nil {
t.Fatalf("loopback peer should be trusted: %v", err)
}
}
// TestAuthorizeHeartbeatPeer_InProcessAccepted covers the `weed server` case
// where master + volume run in the same OS process and the volume server
// reaches the master through an in-process / bufconn gRPC connection. Those
// peers surface with a non-TCP net.Addr (e.g. "@") and cannot be spoofed by
// a remote attacker, so they must be trusted regardless of heartbeat.Ip.
func TestAuthorizeHeartbeatPeer_InProcessAccepted(t *testing.T) {
ms := &MasterServer{option: &MasterOption{}}
ctx := peer.NewContext(context.Background(), &peer.Peer{
Addr: fakeAddr{network: "bufconn", s: "@"},
})
if err := ms.authorizeHeartbeatPeer(ctx, &master_pb.Heartbeat{Ip: "host.example", Port: 8080}); err != nil {
t.Fatalf("in-process peer should be trusted: %v", err)
}
}
// TestFilterVolumeOwnership_RejectsSecondPeerOnSameVolumeId verifies the
// end-to-end behaviour for non-EC: with replica count 1 there is exactly one
// legal owner of a vid, and a second peer's claim is stripped before the
// topology sees it.
func TestFilterVolumeOwnership_RejectsSecondPeerOnSameVolumeId(t *testing.T) {
topo := topology.NewTopology("t", sequence.NewMemorySequencer(), 32*1024, 5, false)
dc := topo.GetOrCreateDataCenter("dc")
rack := dc.GetOrCreateRack("rack")
ms := &MasterServer{
option: &MasterOption{},
Topo: topo,
heartbeatOwnership: newHeartbeatOwnership(),
}
dnA := rack.GetOrCreateDataNode("10.0.0.1", 8080, 8081, "10.0.0.1", "A", map[string]uint32{"": 10})
dnB := rack.GetOrCreateDataNode("10.6.6.6", 8080, 8081, "10.6.6.6", "B", map[string]uint32{"": 10})
hbA := &master_pb.Heartbeat{
Ip: "10.0.0.1",
Port: 8080,
Volumes: []*master_pb.VolumeInformationMessage{
{Id: 1, ReplicaPlacement: 0},
},
}
ms.filterVolumeOwnership(dnA, hbA)
if len(hbA.Volumes) != 1 {
t.Fatalf("peer A should retain its claim, got %d volumes", len(hbA.Volumes))
}
hbB := &master_pb.Heartbeat{
Ip: "10.6.6.6",
Port: 8080,
Volumes: []*master_pb.VolumeInformationMessage{
{Id: 1, ReplicaPlacement: 0},
},
}
ms.filterVolumeOwnership(dnB, hbB)
if len(hbB.Volumes) != 0 {
t.Fatalf("peer B's hijack claim should be stripped, got %d volumes", len(hbB.Volumes))
}
}
// TestFilterVolumeOwnership_ReleaseLetsNextPeerTakeOver covers the
// disconnect-and-replace flow: when peer A's stream ends, ownership is freed,
// and a different peer may then claim the same vid (e.g. after planned
// migration).
func TestFilterVolumeOwnership_ReleaseLetsNextPeerTakeOver(t *testing.T) {
ms := &MasterServer{
option: &MasterOption{},
heartbeatOwnership: newHeartbeatOwnership(),
}
topo := topology.NewTopology("t", sequence.NewMemorySequencer(), 32*1024, 5, false)
dc := topo.GetOrCreateDataCenter("dc")
rack := dc.GetOrCreateRack("rack")
dnA := rack.GetOrCreateDataNode("10.0.0.1", 8080, 8081, "10.0.0.1", "A", map[string]uint32{"": 10})
dnB := rack.GetOrCreateDataNode("10.0.0.2", 8080, 8081, "10.0.0.2", "B", map[string]uint32{"": 10})
hb := func(ip string) *master_pb.Heartbeat {
return &master_pb.Heartbeat{Ip: ip, Port: 8080, Volumes: []*master_pb.VolumeInformationMessage{{Id: 5}}}
}
a := hb("10.0.0.1")
ms.filterVolumeOwnership(dnA, a)
if len(a.Volumes) != 1 {
t.Fatalf("peer A should own the volume")
}
b := hb("10.0.0.2")
ms.filterVolumeOwnership(dnB, b)
if len(b.Volumes) != 0 {
t.Fatalf("peer B must not hijack while peer A still owns the id")
}
ms.heartbeatOwnership.release(dnA.Ip, uint32(dnA.Port))
c := hb("10.0.0.2")
ms.filterVolumeOwnership(dnB, c)
if len(c.Volumes) != 1 {
t.Fatalf("after release, peer B should be able to take over")
}
}
// TestFilterVolumeOwnership_EcShardsSplitAcrossPeers exercises the realistic
// EC topology where shards of the same vid live on different volume servers.
// Both peers must keep their disjoint shard bitmaps after filtering.
func TestFilterVolumeOwnership_EcShardsSplitAcrossPeers(t *testing.T) {
ms := &MasterServer{
option: &MasterOption{},
heartbeatOwnership: newHeartbeatOwnership(),
}
topo := topology.NewTopology("t", sequence.NewMemorySequencer(), 32*1024, 5, false)
dc := topo.GetOrCreateDataCenter("dc")
rack := dc.GetOrCreateRack("rack")
dnA := rack.GetOrCreateDataNode("10.0.0.1", 8080, 8081, "10.0.0.1", "A", map[string]uint32{"": 10})
dnB := rack.GetOrCreateDataNode("10.0.0.2", 8080, 8081, "10.0.0.2", "B", map[string]uint32{"": 10})
hbA := &master_pb.Heartbeat{
Ip: "10.0.0.1",
Port: 8080,
EcShards: []*master_pb.VolumeEcShardInformationMessage{
{Id: 100, EcIndexBits: 0x7F, ShardSizes: []int64{1, 2, 3, 4, 5, 6, 7}}, // shards 0..6
},
}
ms.filterVolumeOwnership(dnA, hbA)
if len(hbA.EcShards) != 1 || hbA.EcShards[0].EcIndexBits != 0x7F {
t.Fatalf("peer A should keep shards 0-6, got %+v", hbA.EcShards)
}
if len(hbA.EcShards[0].ShardSizes) != 7 {
t.Fatalf("shard sizes for peer A should be preserved, got %v", hbA.EcShards[0].ShardSizes)
}
hbB := &master_pb.Heartbeat{
Ip: "10.0.0.2",
Port: 8080,
EcShards: []*master_pb.VolumeEcShardInformationMessage{
{Id: 100, EcIndexBits: 0x3F80, ShardSizes: []int64{8, 9, 10, 11, 12, 13, 14}}, // shards 7..13
},
}
ms.filterVolumeOwnership(dnB, hbB)
if len(hbB.EcShards) != 1 || hbB.EcShards[0].EcIndexBits != 0x3F80 {
t.Fatalf("peer B should keep shards 7-13, got %+v", hbB.EcShards)
}
if len(hbB.EcShards[0].ShardSizes) != 7 {
t.Fatalf("shard sizes for peer B should be preserved, got %v", hbB.EcShards[0].ShardSizes)
}
}
// TestFilterVolumeOwnership_EcShardHijackStripped checks the partial-overlap
// case: the granted bits are kept (with their sizes), the rejected bits are
// stripped, and ShardSizes is compacted in lock-step with EcIndexBits.
func TestFilterVolumeOwnership_EcShardHijackStripped(t *testing.T) {
ms := &MasterServer{
option: &MasterOption{},
heartbeatOwnership: newHeartbeatOwnership(),
}
topo := topology.NewTopology("t", sequence.NewMemorySequencer(), 32*1024, 5, false)
dc := topo.GetOrCreateDataCenter("dc")
rack := dc.GetOrCreateRack("rack")
dnA := rack.GetOrCreateDataNode("10.0.0.1", 8080, 8081, "10.0.0.1", "A", map[string]uint32{"": 10})
dnAttacker := rack.GetOrCreateDataNode("10.6.6.6", 8080, 8081, "10.6.6.6", "X", map[string]uint32{"": 10})
hbA := &master_pb.Heartbeat{
Ip: "10.0.0.1",
Port: 8080,
EcShards: []*master_pb.VolumeEcShardInformationMessage{
{Id: 200, EcIndexBits: 0b0111, ShardSizes: []int64{10, 20, 30}}, // shards 0,1,2
},
}
ms.filterVolumeOwnership(dnA, hbA)
hbAttacker := &master_pb.Heartbeat{
Ip: "10.6.6.6",
Port: 8080,
EcShards: []*master_pb.VolumeEcShardInformationMessage{
// claim shards 1 (taken), 3 (free), 5 (free), with sizes in
// shard-id order.
{Id: 200, EcIndexBits: 0b101010, ShardSizes: []int64{77, 88, 99}},
},
}
ms.filterVolumeOwnership(dnAttacker, hbAttacker)
if len(hbAttacker.EcShards) != 1 {
t.Fatalf("attacker should keep the message with the granted bits only, got %d", len(hbAttacker.EcShards))
}
got := hbAttacker.EcShards[0]
if got.EcIndexBits != 0b101000 {
t.Fatalf("expected only shards 3 and 5 to survive, got 0x%x", got.EcIndexBits)
}
if len(got.ShardSizes) != 2 || got.ShardSizes[0] != 88 || got.ShardSizes[1] != 99 {
t.Fatalf("ShardSizes must compact alongside EcIndexBits, got %v", got.ShardSizes)
}
}
// TestHeartbeatOwnership_InvariantsAfterRelease covers the reverse-index
// bookkeeping: after release, the owner-keyed maps are empty for that peer.
func TestHeartbeatOwnership_InvariantsAfterRelease(t *testing.T) {
o := newHeartbeatOwnership()
owner := heartbeatOwner{ip: "10.0.0.1", port: 8080}
o.claimVolume(1, owner.ip, owner.port, 1)
o.claimVolume(2, owner.ip, owner.port, 1)
o.claimEcShards(3, 0b101, owner.ip, owner.port)
o.release(owner.ip, owner.port)
o.mu.Lock()
defer o.mu.Unlock()
if len(o.volumes) != 0 || len(o.ecShards) != 0 {
t.Fatalf("forward maps must be empty after sole owner releases: volumes=%v ecShards=%v", o.volumes, o.ecShards)
}
if _, ok := o.ownerToVolumes[owner]; ok {
t.Fatalf("owner-volume reverse index leaked")
}
if _, ok := o.ownerToEcShards[owner]; ok {
t.Fatalf("owner-ecshard reverse index leaked")
}
}
func TestAuthorizeHeartbeatPeer_OptOut(t *testing.T) {
ms := &MasterServer{option: &MasterOption{AllowUntrustedHeartbeat: true}}
ctx := tcpPeerContext("10.0.0.1", 55555)
if err := ms.authorizeHeartbeatPeer(ctx, &master_pb.Heartbeat{Ip: "10.6.6.6", Port: 8080}); err != nil {
t.Fatalf("opt-out must skip the check: %v", err)
}
}
// TestFilterVolumeOwnership_DeletedEcShardsReleaseBinding covers the
// rebalance flow: peer A heartbeats shard 0 of vid 5, then a follow-up
// heartbeat from A reports the shard as deleted. The binding must release so
// peer B can claim shard 0 on its next heartbeat.
func TestFilterVolumeOwnership_DeletedEcShardsReleaseBinding(t *testing.T) {
ms := &MasterServer{
option: &MasterOption{},
heartbeatOwnership: newHeartbeatOwnership(),
}
topo := topology.NewTopology("t", sequence.NewMemorySequencer(), 32*1024, 5, false)
dc := topo.GetOrCreateDataCenter("dc")
rack := dc.GetOrCreateRack("rack")
dnA := rack.GetOrCreateDataNode("10.0.0.1", 8080, 8081, "10.0.0.1", "A", map[string]uint32{"": 10})
dnB := rack.GetOrCreateDataNode("10.0.0.2", 8080, 8081, "10.0.0.2", "B", map[string]uint32{"": 10})
// Peer A claims shard 0 of vid 5.
hbAClaim := &master_pb.Heartbeat{
Ip: "10.0.0.1",
Port: 8080,
NewEcShards: []*master_pb.VolumeEcShardInformationMessage{
{Id: 5, EcIndexBits: 0b1, ShardSizes: []int64{100}},
},
}
ms.filterVolumeOwnership(dnA, hbAClaim)
if len(hbAClaim.NewEcShards) != 1 || hbAClaim.NewEcShards[0].EcIndexBits != 0b1 {
t.Fatalf("peer A initial claim must succeed, got %+v", hbAClaim.NewEcShards)
}
// Peer B tries to claim shard 0 first; the binding still belongs to A so
// the claim is filtered out.
hbBHijack := &master_pb.Heartbeat{
Ip: "10.0.0.2",
Port: 8080,
NewEcShards: []*master_pb.VolumeEcShardInformationMessage{
{Id: 5, EcIndexBits: 0b1, ShardSizes: []int64{100}},
},
}
ms.filterVolumeOwnership(dnB, hbBHijack)
if len(hbBHijack.NewEcShards) != 0 {
t.Fatalf("peer B must not hijack shard 0 while A owns it, got %+v", hbBHijack.NewEcShards)
}
// Peer A heartbeat reports shard 0 deleted. This must release the
// binding so a different peer can take ownership next.
hbADelete := &master_pb.Heartbeat{
Ip: "10.0.0.1",
Port: 8080,
DeletedEcShards: []*master_pb.VolumeEcShardInformationMessage{
{Id: 5, EcIndexBits: 0b1},
},
}
ms.filterVolumeOwnership(dnA, hbADelete)
// Peer B re-tries the claim; this time the slot is free.
hbBTakeover := &master_pb.Heartbeat{
Ip: "10.0.0.2",
Port: 8080,
NewEcShards: []*master_pb.VolumeEcShardInformationMessage{
{Id: 5, EcIndexBits: 0b1, ShardSizes: []int64{100}},
},
}
ms.filterVolumeOwnership(dnB, hbBTakeover)
if len(hbBTakeover.NewEcShards) != 1 || hbBTakeover.NewEcShards[0].EcIndexBits != 0b1 {
t.Fatalf("peer B should own shard 0 after A's release, got %+v", hbBTakeover.NewEcShards)
}
}
// TestFilterVolumeOwnership_FullSnapshotDropsStaleEcBits covers the case
// where a peer drops a shard between heartbeats without sending an explicit
// DeletedEcShards entry (e.g. after a restart that reloaded fewer shards).
// The full EcShards snapshot must release the shard so another peer can
// claim it.
func TestFilterVolumeOwnership_FullSnapshotDropsStaleEcBits(t *testing.T) {
ms := &MasterServer{
option: &MasterOption{},
heartbeatOwnership: newHeartbeatOwnership(),
}
topo := topology.NewTopology("t", sequence.NewMemorySequencer(), 32*1024, 5, false)
dc := topo.GetOrCreateDataCenter("dc")
rack := dc.GetOrCreateRack("rack")
dnA := rack.GetOrCreateDataNode("10.0.0.1", 8080, 8081, "10.0.0.1", "A", map[string]uint32{"": 10})
dnB := rack.GetOrCreateDataNode("10.0.0.2", 8080, 8081, "10.0.0.2", "B", map[string]uint32{"": 10})
// Peer A snapshots shards {0, 1, 3} = 0b1011 for vid 5.
hbAFull := &master_pb.Heartbeat{
Ip: "10.0.0.1",
Port: 8080,
EcShards: []*master_pb.VolumeEcShardInformationMessage{
{Id: 5, EcIndexBits: 0b1011, ShardSizes: []int64{10, 20, 30}},
},
}
ms.filterVolumeOwnership(dnA, hbAFull)
if hbAFull.EcShards[0].EcIndexBits != 0b1011 {
t.Fatalf("peer A initial snapshot should keep all three bits, got 0x%x", hbAFull.EcShards[0].EcIndexBits)
}
// Peer A re-snapshots with shard 1 dropped: 0b1001.
hbAShrink := &master_pb.Heartbeat{
Ip: "10.0.0.1",
Port: 8080,
EcShards: []*master_pb.VolumeEcShardInformationMessage{
{Id: 5, EcIndexBits: 0b1001, ShardSizes: []int64{10, 30}},
},
}
ms.filterVolumeOwnership(dnA, hbAShrink)
if hbAShrink.EcShards[0].EcIndexBits != 0b1001 {
t.Fatalf("peer A shrink snapshot should keep the two surviving bits, got 0x%x", hbAShrink.EcShards[0].EcIndexBits)
}
// Peer B can now claim shard 1 because the diff released that bit.
hbBTakeover := &master_pb.Heartbeat{
Ip: "10.0.0.2",
Port: 8080,
NewEcShards: []*master_pb.VolumeEcShardInformationMessage{
{Id: 5, EcIndexBits: 0b0010, ShardSizes: []int64{20}},
},
}
ms.filterVolumeOwnership(dnB, hbBTakeover)
if len(hbBTakeover.NewEcShards) != 1 || hbBTakeover.NewEcShards[0].EcIndexBits != 0b0010 {
t.Fatalf("peer B should pick up shard 1 after A's snapshot diff, got %+v", hbBTakeover.NewEcShards)
}
// Bits 0 and 3 must still be locked to A; peer B cannot hijack them.
hbBHijack := &master_pb.Heartbeat{
Ip: "10.0.0.2",
Port: 8080,
NewEcShards: []*master_pb.VolumeEcShardInformationMessage{
{Id: 5, EcIndexBits: 0b1001, ShardSizes: []int64{10, 30}},
},
}
ms.filterVolumeOwnership(dnB, hbBHijack)
if len(hbBHijack.NewEcShards) != 0 {
t.Fatalf("peer B must not steal bits A still owns, got %+v", hbBHijack.NewEcShards)
}
}
// TestFilterVolumeOwnership_FullSnapshotDropsStaleVolumes covers the same
// pattern for non-EC volumes: when peer A no longer lists a vid in a full
// snapshot, the binding must release so another peer can take over.
func TestFilterVolumeOwnership_FullSnapshotDropsStaleVolumes(t *testing.T) {
ms := &MasterServer{
option: &MasterOption{},
heartbeatOwnership: newHeartbeatOwnership(),
}
topo := topology.NewTopology("t", sequence.NewMemorySequencer(), 32*1024, 5, false)
dc := topo.GetOrCreateDataCenter("dc")
rack := dc.GetOrCreateRack("rack")
dnA := rack.GetOrCreateDataNode("10.0.0.1", 8080, 8081, "10.0.0.1", "A", map[string]uint32{"": 10})
dnB := rack.GetOrCreateDataNode("10.0.0.2", 8080, 8081, "10.0.0.2", "B", map[string]uint32{"": 10})
hbAFull := &master_pb.Heartbeat{
Ip: "10.0.0.1",
Port: 8080,
Volumes: []*master_pb.VolumeInformationMessage{
{Id: 7, ReplicaPlacement: 0},
{Id: 8, ReplicaPlacement: 0},
},
}
ms.filterVolumeOwnership(dnA, hbAFull)
// Snapshot drops vid 7.
hbAShrink := &master_pb.Heartbeat{
Ip: "10.0.0.1",
Port: 8080,
Volumes: []*master_pb.VolumeInformationMessage{
{Id: 8, ReplicaPlacement: 0},
},
}
ms.filterVolumeOwnership(dnA, hbAShrink)
// Peer B claims vid 7; the binding has been released by the diff.
hbB := &master_pb.Heartbeat{
Ip: "10.0.0.2",
Port: 8080,
Volumes: []*master_pb.VolumeInformationMessage{
{Id: 7, ReplicaPlacement: 0},
},
}
ms.filterVolumeOwnership(dnB, hbB)
if len(hbB.Volumes) != 1 {
t.Fatalf("peer B should pick up vid 7 after A's snapshot diff, got %+v", hbB.Volumes)
}
// Peer B cannot hijack vid 8 — A still owns it.
hbBHijack := &master_pb.Heartbeat{
Ip: "10.0.0.2",
Port: 8080,
Volumes: []*master_pb.VolumeInformationMessage{
{Id: 8, ReplicaPlacement: 0},
},
}
ms.filterVolumeOwnership(dnB, hbBHijack)
if len(hbBHijack.Volumes) != 0 {
t.Fatalf("peer B must not hijack vid 8 while A owns it, got %+v", hbBHijack.Volumes)
}
}
// TestHeartbeatOwnership_ReleaseEcShardsIgnoresForeign confirms that a
// malicious peer cannot use a forged DeletedEcShards entry to evict the
// legitimate owner of a shard slot.
func TestHeartbeatOwnership_ReleaseEcShardsIgnoresForeign(t *testing.T) {
o := newHeartbeatOwnership()
if got := o.claimEcShards(11, 0b1, "10.0.0.1", 8080); got != 0b1 {
t.Fatalf("peer A must own shard 0")
}
// Attacker tries to release A's binding.
o.releaseEcShards(11, 0b1, "10.6.6.6", 8080)
// Peer C still cannot claim it.
if got := o.claimEcShards(11, 0b1, "10.0.0.3", 8080); got != 0 {
t.Fatalf("foreign release must be a no-op, got 0x%x", got)
}
// Owner can still release its own binding.
o.releaseEcShards(11, 0b1, "10.0.0.1", 8080)
if got := o.claimEcShards(11, 0b1, "10.0.0.3", 8080); got != 0b1 {
t.Fatalf("after owner release, slot must be reclaimable, got 0x%x", got)
}
}