mirror of
https://github.com/seaweedfs/seaweedfs.git
synced 2026-10-26 06:50:45 +01:00
Chris Lu
53834117a6
* mini: lift the post-mint allocation hold on first inventory report DeferAllocation armed a fixed warmup window whenever the leader minted a fresh topology id, so the first write on a brand-new weed mini blocked for the whole window even though no prior cluster state could exist. The window still has to cover a rebuilt master's veteran servers, which report on their first post-reconnect heartbeat. Mini has exactly one possible reporter — the embedded volume server, whose first heartbeat carries the full inventory — so hold allocation only until that report is applied. The bound raise and topology-id freeze checks run while the heartbeat is processed, so a wiped master state still raises the bound or trips the freeze before the hold lifts. Standalone weed master and weed server keep the fixed window: volume servers outside the process can still be on their way. * master: apply queued topology-id reports before opening the allocation gate SetTopologyInitPending(false) cleared the pending flag before replaying topology ids stashed while the local id was still being minted. A NextVolumeId landing in the drain window saw no gate and no freeze yet. The post-mint allocation hold used to cover the gap by delaying the winner past the freeze; once the hold can release on first inventory, the gate must stay closed until the queued checks have run. * mini: gate the first-report release on single-master mode releaseAllocOnFirstReport assumes the embedded volume server is the only possible reporter of prior cluster state. A mini started with -master.peers joins other masters, whose own volume servers can still be on their way when the first report lands, so keep the regular quiet-pulse hold there. * master: check reported topology id on empty full reports A HasNoVolumes heartbeat carries the same LastSeenTopologyId a populated list does, but the mismatch freeze only ran for non-empty reports. An EC-only veteran could therefore release the post-mint allocation hold through NoteInventoryApplied while its stale topology id never reached CheckReportedTopologyId. * master: drain the topology-id stash inside the init-gate lock CheckReportedTopologyId only held reportedTopologyIdsLock while stashing, so a report in flight during SetTopologyInitPending(false) could land its freeze after topologyInitPending cleared, letting a NextVolumeId slip through the gap. The whole check now runs under the lock, and the gate is released inside the same critical section as the drain. * mini, master: pin and enforce the single volume-server reporter isSingleMasterMode only checks master peers; SendHeartbeat still accepted any volume server, so a late foreign report could carry prior cluster state after the first-report release dropped the hold. Mini now pins the embedded volume server's address and the master refuses registration from any other reporter while ReleaseAllocOnFirstReport is set. * master: admit the volume-server reporter before applying heartbeat state * test: cover SendHeartbeat admission ordering A refused reporter must not freeze growth or raise the id bound, and the pinned reporter's stale topology id still must. * mini: re-derive the single-reporter release on every run The package-level master options survive in-process reruns, so a multi-master mini following a single-master one must not inherit the first-report release or a stale reporter pin. * master: admit addressless first heartbeats under the reporter pin An unregistered heartbeat with no Ip skipped the single-reporter admission check but still reached the seen-bound raise and the topology-id freeze before the delta-heartbeat skip. Admission now runs for every unregistered heartbeat; an empty address can never equal the pin.
111 lines
3.5 KiB
Go
111 lines
3.5 KiB
Go
package weed_server
|
|
|
|
import (
|
|
"io"
|
|
"testing"
|
|
|
|
"github.com/seaweedfs/raft"
|
|
"github.com/seaweedfs/seaweedfs/weed/pb/master_pb"
|
|
"github.com/stretchr/testify/assert"
|
|
"github.com/stretchr/testify/require"
|
|
"google.golang.org/grpc"
|
|
)
|
|
|
|
// admitTestRaftServer answers as leader and swallows Do so a growth freeze
|
|
// lands its local flag without needing a real raft log.
|
|
type admitTestRaftServer struct {
|
|
raft.Server
|
|
}
|
|
|
|
func (admitTestRaftServer) State() string { return raft.Leader }
|
|
func (admitTestRaftServer) Do(raft.Command) (interface{}, error) { return nil, nil }
|
|
|
|
type fakeHeartbeatStream struct {
|
|
grpc.ServerStream
|
|
heartbeats []*master_pb.Heartbeat
|
|
sent []*master_pb.HeartbeatResponse
|
|
}
|
|
|
|
func (s *fakeHeartbeatStream) Recv() (*master_pb.Heartbeat, error) {
|
|
if len(s.heartbeats) == 0 {
|
|
return nil, io.EOF
|
|
}
|
|
hb := s.heartbeats[0]
|
|
s.heartbeats = s.heartbeats[1:]
|
|
return hb, nil
|
|
}
|
|
|
|
func (s *fakeHeartbeatStream) Send(resp *master_pb.HeartbeatResponse) error {
|
|
s.sent = append(s.sent, resp)
|
|
return nil
|
|
}
|
|
|
|
// A server the single-reporter admission refuses must not leave any applied
|
|
// state behind: its stale topology id cannot freeze growth and its seen bound
|
|
// cannot raise the master's.
|
|
func TestSendHeartbeatRejectsForeignReporterBeforeApplyingState(t *testing.T) {
|
|
ms := newLeaderMaster()
|
|
ms.Topo.RaftServer = admitTestRaftServer{}
|
|
ms.Topo.SetGrowthFreezeConfig(true, 0)
|
|
ms.Topo.SetTopologyId("cluster-b")
|
|
ms.Topo.ReleaseAllocationOnFirstReport("10.0.0.1:8080")
|
|
|
|
stream := &fakeHeartbeatStream{heartbeats: []*master_pb.Heartbeat{{
|
|
Ip: "10.0.0.2",
|
|
Port: 8080,
|
|
MaxVolumeIdSeen: 7,
|
|
LastSeenTopologyId: "cluster-a",
|
|
Volumes: []*master_pb.VolumeInformationMessage{{Id: 3}},
|
|
}}}
|
|
|
|
err := ms.SendHeartbeat(stream)
|
|
require.ErrorContains(t, err, "rejected")
|
|
assert.False(t, ms.Topo.IsGrowthFrozen())
|
|
assert.Zero(t, uint32(ms.Topo.GetMaxVolumeId()))
|
|
assert.Empty(t, stream.sent)
|
|
}
|
|
|
|
// The pinned reporter is still subject to the mismatch freeze: admission gates
|
|
// who may report, not what a legitimate report can do.
|
|
func TestSendHeartbeatAdmittedReporterStillFreezesOnMismatch(t *testing.T) {
|
|
ms := newLeaderMaster()
|
|
ms.Topo.RaftServer = admitTestRaftServer{}
|
|
ms.Topo.SetGrowthFreezeConfig(true, 0)
|
|
ms.Topo.SetTopologyId("cluster-b")
|
|
ms.Topo.ReleaseAllocationOnFirstReport("10.0.0.1:8080")
|
|
|
|
stream := &fakeHeartbeatStream{heartbeats: []*master_pb.Heartbeat{{
|
|
Ip: "10.0.0.1",
|
|
Port: 8080,
|
|
MaxVolumeIdSeen: 7,
|
|
LastSeenTopologyId: "cluster-a",
|
|
Volumes: []*master_pb.VolumeInformationMessage{{Id: 3}},
|
|
}}}
|
|
|
|
_ = ms.SendHeartbeat(stream)
|
|
assert.True(t, ms.Topo.IsGrowthFrozen())
|
|
}
|
|
|
|
// An unregistered heartbeat with no address can never be the pinned reporter:
|
|
// its stale topology id and seen bound must not reach the topology on the way
|
|
// to the delta-heartbeat skip.
|
|
func TestSendHeartbeatRejectsAddresslessFirstHeartbeat(t *testing.T) {
|
|
ms := newLeaderMaster()
|
|
ms.Topo.RaftServer = admitTestRaftServer{}
|
|
ms.Topo.SetGrowthFreezeConfig(true, 0)
|
|
ms.Topo.SetTopologyId("cluster-b")
|
|
ms.Topo.ReleaseAllocationOnFirstReport("10.0.0.1:8080")
|
|
|
|
stream := &fakeHeartbeatStream{heartbeats: []*master_pb.Heartbeat{{
|
|
MaxVolumeIdSeen: 7,
|
|
LastSeenTopologyId: "cluster-a",
|
|
Volumes: []*master_pb.VolumeInformationMessage{{Id: 3}},
|
|
}}}
|
|
|
|
err := ms.SendHeartbeat(stream)
|
|
require.ErrorContains(t, err, "rejected")
|
|
assert.False(t, ms.Topo.IsGrowthFrozen())
|
|
assert.Zero(t, uint32(ms.Topo.GetMaxVolumeId()))
|
|
assert.Empty(t, stream.sent)
|
|
}
|