mirror of
https://github.com/seaweedfs/seaweedfs.git
synced 2026-11-21 03:43:41 +01:00
Compare commits
10
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
65685ee915 | ||
|
|
60c23ae5e0 | ||
|
|
acc1075fd5 | ||
|
|
cbbc9e6efe | ||
|
|
70bbd27638 | ||
|
|
630dc8aee1 | ||
|
|
ca4e1c76d8 | ||
|
|
d9236a9c7f | ||
|
|
9cc1f0dc36 | ||
|
|
9ded580993 |
No files matched your search
+16
-9
@@ -81,6 +81,12 @@ type MasterOptions struct {
|
||||
// is cancelled. Used by integration tests and by weed mini; nil for
|
||||
// standalone weed master.
|
||||
shutdownCtx context.Context
|
||||
// releaseAllocOnFirstReport lifts the post-mint allocation hold as soon as
|
||||
// one full volume-server inventory has been applied. weed mini sets it:
|
||||
// the embedded volume server is the only possible reporter, so the warmup
|
||||
// window after a fresh topology id is dead time. Standalone weed master /
|
||||
// weed server can face late-arriving veterans and must keep the window.
|
||||
releaseAllocOnFirstReport bool
|
||||
}
|
||||
|
||||
func init() {
|
||||
@@ -480,14 +486,15 @@ func (m *MasterOptions) toMasterOption(whiteList []string) *weed_server.MasterOp
|
||||
VacuumIntervalSeconds: *m.vacuumIntervalSeconds,
|
||||
VacuumDeleteEmptyAfterSeconds: *m.vacuumDeleteEmptyAfterSeconds,
|
||||
// PulseSeconds: *m.pulseSeconds,
|
||||
DefaultReplicaPlacement: *m.defaultReplication,
|
||||
GarbageThreshold: *m.garbageThreshold,
|
||||
WhiteList: whiteList,
|
||||
DisableHttp: *m.disableHttp,
|
||||
MetricsAddress: *m.metricsAddress,
|
||||
MetricsIntervalSec: *m.metricsIntervalSec,
|
||||
TelemetryUrl: *m.telemetryUrl,
|
||||
TelemetryEnabled: *m.telemetryEnabled,
|
||||
SingleMaster: isSingleMasterMode(*m.peers),
|
||||
DefaultReplicaPlacement: *m.defaultReplication,
|
||||
GarbageThreshold: *m.garbageThreshold,
|
||||
WhiteList: whiteList,
|
||||
DisableHttp: *m.disableHttp,
|
||||
MetricsAddress: *m.metricsAddress,
|
||||
MetricsIntervalSec: *m.metricsIntervalSec,
|
||||
TelemetryUrl: *m.telemetryUrl,
|
||||
TelemetryEnabled: *m.telemetryEnabled,
|
||||
SingleMaster: isSingleMasterMode(*m.peers),
|
||||
ReleaseAllocOnFirstReport: m.releaseAllocOnFirstReport,
|
||||
}
|
||||
}
|
||||
@@ -1521,6 +1521,11 @@ func runMini(cmd *Command, args []string) bool {
|
||||
// tear those services down too. On Ctrl+C they rely on their own
|
||||
// OnInterrupt hooks (see grace's LIFO ordering).
|
||||
miniMasterOptions.shutdownCtx = MiniClusterCtx
|
||||
// Mini's embedded volume server is the only possible reporter of prior
|
||||
// cluster state; its first heartbeat already carries the full inventory, so
|
||||
// the post-mint allocation hold ends there instead of running the whole
|
||||
// warmup window on every fresh start.
|
||||
miniMasterOptions.releaseAllocOnFirstReport = true
|
||||
miniOptions.v.shutdownCtx = MiniClusterCtx
|
||||
miniFilerOptions.shutdownCtx = MiniClusterCtx
|
||||
// Mini is a small/dev setup with short-lived RPCs; cap the filer's
|
||||
|
||||
@@ -63,6 +63,11 @@ copy_other = 1 # create n x 1 = n actual volumes
|
||||
threshold = 0.9 # create threshold
|
||||
reservation_timeout = "5m"# capacity reservation timeout before unreleased reservations expire
|
||||
disable = false # disables volume growth if true
|
||||
# allocation_deferral = "15s" # hold the first volume id allocation after a fresh
|
||||
# topology id is minted, giving a rebuilt master's
|
||||
# veteran volume servers time to report. The hold
|
||||
# lifts early once heartbeats stay clean for two
|
||||
# pulses. "0s" disables it (e.g. test setups).
|
||||
|
||||
# configuration flags for replication
|
||||
[master.replication]
|
||||
|
||||
@@ -103,6 +103,55 @@ func announceVolume(message *master_pb.VolumeLocation, vid uint32, isRemote, isR
|
||||
}
|
||||
}
|
||||
|
||||
// reportsPriorClusterState tells whether a heartbeat could come from a server
|
||||
// still carrying cluster state from before this leader's term: it holds a
|
||||
// volume or shard id above the bound this master has issued, has seen a
|
||||
// higher volume id bound, or last synced with a different topology id.
|
||||
// Echoes of this term's own allocations stay under the bound and do not count.
|
||||
func reportsPriorClusterState(heartbeat *master_pb.Heartbeat, topologyId string, bound needle.VolumeId) bool {
|
||||
if needle.VolumeId(heartbeat.MaxVolumeIdSeen) > bound ||
|
||||
(heartbeat.LastSeenTopologyId != "" && heartbeat.LastSeenTopologyId != topologyId) {
|
||||
return true
|
||||
}
|
||||
unissued := func(id uint32) bool { return needle.VolumeId(id) > bound }
|
||||
for _, v := range heartbeat.Volumes {
|
||||
if unissued(v.Id) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
for _, v := range heartbeat.NewVolumes {
|
||||
if unissued(v.Id) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
for _, v := range heartbeat.ChangedVolumes {
|
||||
if unissued(v.Id) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
for _, v := range heartbeat.DeletedVolumes {
|
||||
if unissued(v.Id) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
for _, s := range heartbeat.EcShards {
|
||||
if unissued(s.Id) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
for _, s := range heartbeat.NewEcShards {
|
||||
if unissued(s.Id) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
for _, s := range heartbeat.DeletedEcShards {
|
||||
if unissued(s.Id) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func (ms *MasterServer) SendHeartbeat(stream master_pb.Seaweed_SendHeartbeatServer) error {
|
||||
var dn *topology.DataNode
|
||||
|
||||
@@ -182,6 +231,9 @@ func (ms *MasterServer) SendHeartbeat(stream master_pb.Seaweed_SendHeartbeatServ
|
||||
continue
|
||||
}
|
||||
|
||||
if (dn == nil && heartbeat.Ip != "") || reportsPriorClusterState(heartbeat, ms.Topo.GetTopologyId(), ms.Topo.GetMaxVolumeId()) {
|
||||
ms.Topo.NoteAllocationEvidence()
|
||||
}
|
||||
ms.Topo.Sequence.SetMax(heartbeat.MaxFileKey)
|
||||
ms.Topo.UpAdjustMaxVolumeId(needle.VolumeId(heartbeat.MaxVolumeIdSeen))
|
||||
if len(heartbeat.Volumes) > 0 {
|
||||
@@ -304,6 +356,13 @@ func (ms *MasterServer) SendHeartbeat(stream master_pb.Seaweed_SendHeartbeatServ
|
||||
}
|
||||
}
|
||||
|
||||
// A full volume list is the heartbeat section that settles what a
|
||||
// reporter holds; once applied, a post-mint allocation hold kept for
|
||||
// prior cluster state has nothing left to wait for.
|
||||
if len(heartbeat.Volumes) > 0 || heartbeat.HasNoVolumes {
|
||||
ms.Topo.NoteInventoryApplied()
|
||||
}
|
||||
|
||||
if len(heartbeat.NewEcShards) > 0 || len(heartbeat.DeletedEcShards) > 0 {
|
||||
stats.MasterReceivedHeartbeatCounter.WithLabelValues("newEcShards").Inc()
|
||||
// update master internal volume layouts
|
||||
|
||||
@@ -199,27 +199,15 @@ func (ms *MasterServer) Assign(ctx context.Context, req *master_pb.AssignRequest
|
||||
if dn == nil {
|
||||
continue
|
||||
}
|
||||
var replicas []*master_pb.Location
|
||||
for _, r := range dnList.Rest() {
|
||||
replicas = append(replicas, &master_pb.Location{
|
||||
Url: r.Url(),
|
||||
PublicUrl: r.PublicUrl,
|
||||
GrpcPort: uint32(r.GrpcPort),
|
||||
DataCenter: r.GetDataCenterId(),
|
||||
})
|
||||
return assignResponse(fid, count, dnList, ms.guard), nil
|
||||
}
|
||||
// The last in-loop pick can predate the growth it triggered concluding:
|
||||
// growth registers its volumes before clearing the pending flag, so pick
|
||||
// once more before shedding.
|
||||
if fid, count, dnList, _, err := ms.Topo.PickForWrite(req.Count, option, vl, req.ExpectedDataSize); err == nil {
|
||||
if dnList.Head() != nil {
|
||||
return assignResponse(fid, count, dnList, ms.guard), nil
|
||||
}
|
||||
return &master_pb.AssignResponse{
|
||||
Fid: fid,
|
||||
Location: &master_pb.Location{
|
||||
Url: dn.Url(),
|
||||
PublicUrl: dn.PublicUrl,
|
||||
GrpcPort: uint32(dn.GrpcPort),
|
||||
DataCenter: dn.GetDataCenterId(),
|
||||
},
|
||||
Count: count,
|
||||
Auth: string(security.GenJwtForVolumeServer(ms.guard.SigningKey(), ms.guard.ExpiresAfterSec(), fid)),
|
||||
Replicas: replicas,
|
||||
}, nil
|
||||
}
|
||||
// The initiator timed out with its growth still pending: shed retryably
|
||||
// rather than failing the write that growth is about to satisfy.
|
||||
@@ -235,6 +223,31 @@ func (ms *MasterServer) Assign(ctx context.Context, req *master_pb.AssignRequest
|
||||
return nil, lastErr
|
||||
}
|
||||
|
||||
func assignResponse(fid string, count uint64, dnList *topology.VolumeLocationList, guard *security.Guard) *master_pb.AssignResponse {
|
||||
var replicas []*master_pb.Location
|
||||
for _, r := range dnList.Rest() {
|
||||
replicas = append(replicas, &master_pb.Location{
|
||||
Url: r.Url(),
|
||||
PublicUrl: r.PublicUrl,
|
||||
GrpcPort: uint32(r.GrpcPort),
|
||||
DataCenter: r.GetDataCenterId(),
|
||||
})
|
||||
}
|
||||
dn := dnList.Head()
|
||||
return &master_pb.AssignResponse{
|
||||
Fid: fid,
|
||||
Location: &master_pb.Location{
|
||||
Url: dn.Url(),
|
||||
PublicUrl: dn.PublicUrl,
|
||||
GrpcPort: uint32(dn.GrpcPort),
|
||||
DataCenter: dn.GetDataCenterId(),
|
||||
},
|
||||
Count: count,
|
||||
Auth: string(security.GenJwtForVolumeServer(guard.SigningKey(), guard.ExpiresAfterSec(), fid)),
|
||||
Replicas: replicas,
|
||||
}
|
||||
}
|
||||
|
||||
// describeDiskLayout names the disk layout an assign targets. The empty disk
|
||||
// type is the legacy unlabeled layout on servers that were never started with
|
||||
// -disk; naming it "hdd" sends operators looking for servers that were never
|
||||
|
||||
@@ -0,0 +1,53 @@
|
||||
package weed_server
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb/master_pb"
|
||||
"github.com/stretchr/testify/assert"
|
||||
)
|
||||
|
||||
func TestReportsPriorClusterState(t *testing.T) {
|
||||
topoId := "topo-1"
|
||||
|
||||
// a quiet heartbeat from a synced server is not evidence
|
||||
assert.False(t, reportsPriorClusterState(&master_pb.Heartbeat{
|
||||
LastSeenTopologyId: topoId,
|
||||
}, topoId, 5))
|
||||
|
||||
// echoes of volume ids this master issued are not evidence
|
||||
assert.False(t, reportsPriorClusterState(&master_pb.Heartbeat{
|
||||
MaxVolumeIdSeen: 5,
|
||||
NewVolumes: []*master_pb.VolumeShortInformationMessage{{Id: 5}},
|
||||
ChangedVolumes: []*master_pb.VolumeInformationMessage{{Id: 3}},
|
||||
LastSeenTopologyId: topoId,
|
||||
}, topoId, 5))
|
||||
|
||||
// ids above the issued bound come from before this term
|
||||
assert.True(t, reportsPriorClusterState(&master_pb.Heartbeat{
|
||||
MaxVolumeIdSeen: 6,
|
||||
}, topoId, 5))
|
||||
assert.True(t, reportsPriorClusterState(&master_pb.Heartbeat{
|
||||
Volumes: []*master_pb.VolumeInformationMessage{{Id: 6}},
|
||||
}, topoId, 5))
|
||||
assert.True(t, reportsPriorClusterState(&master_pb.Heartbeat{
|
||||
NewVolumes: []*master_pb.VolumeShortInformationMessage{{Id: 6}},
|
||||
}, topoId, 5))
|
||||
assert.True(t, reportsPriorClusterState(&master_pb.Heartbeat{
|
||||
DeletedVolumes: []*master_pb.VolumeShortInformationMessage{{Id: 6}},
|
||||
}, topoId, 5))
|
||||
assert.True(t, reportsPriorClusterState(&master_pb.Heartbeat{
|
||||
EcShards: []*master_pb.VolumeEcShardInformationMessage{{Id: 42}},
|
||||
}, topoId, 5))
|
||||
assert.True(t, reportsPriorClusterState(&master_pb.Heartbeat{
|
||||
NewEcShards: []*master_pb.VolumeEcShardInformationMessage{{Id: 42}},
|
||||
}, topoId, 5))
|
||||
assert.True(t, reportsPriorClusterState(&master_pb.Heartbeat{
|
||||
DeletedEcShards: []*master_pb.VolumeEcShardInformationMessage{{Id: 42}},
|
||||
}, topoId, 5))
|
||||
|
||||
// a heartbeat last synced under another topology id is prior state
|
||||
assert.True(t, reportsPriorClusterState(&master_pb.Heartbeat{
|
||||
LastSeenTopologyId: "topo-0",
|
||||
}, topoId, 5))
|
||||
}
|
||||
@@ -70,6 +70,10 @@ type MasterOption struct {
|
||||
TelemetryUrl string
|
||||
TelemetryEnabled bool
|
||||
VolumeGrowthDisabled bool
|
||||
// ReleaseAllocOnFirstReport lifts the post-mint allocation hold as soon as
|
||||
// one full volume-server inventory has been applied, for setups whose
|
||||
// embedded volume server is the only possible reporter (weed mini).
|
||||
ReleaseAllocOnFirstReport bool
|
||||
}
|
||||
|
||||
type MasterServer struct {
|
||||
@@ -175,6 +179,16 @@ func NewMasterServer(r *mux.Router, option *MasterOption, peers map[string]pb.Se
|
||||
ms.Topo.SetGrowthFreezeConfig(
|
||||
v.GetBool("master.volume_growth.freeze_on_topology_mismatch"),
|
||||
v.GetDuration("master.volume_growth.state_loss_recovery_wait"))
|
||||
if str := strings.TrimSpace(v.GetString("master.volume_growth.allocation_deferral")); str != "" {
|
||||
if d, err := time.ParseDuration(str); err != nil {
|
||||
glog.Warningf("invalid master.volume_growth.allocation_deferral %q: %v", str, err)
|
||||
} else {
|
||||
ms.Topo.SetAllocationDeferral(d)
|
||||
}
|
||||
}
|
||||
if option.ReleaseAllocOnFirstReport {
|
||||
ms.Topo.ReleaseAllocationOnFirstReport()
|
||||
}
|
||||
ms.Topo.QuarantineVolumeFn = func(dn *topology.DataNode, vid needle.VolumeId, readOnly bool) error {
|
||||
err := operation.WithVolumeServerClient(false, dn.ServerAddress(), ms.grpcDialOption, func(client volume_server_pb.VolumeServerClient) error {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
|
||||
|
||||
@@ -261,6 +261,16 @@ func (ms *MasterServer) dirAssignHandler(w http.ResponseWriter, r *http.Request)
|
||||
}
|
||||
}
|
||||
|
||||
// The last in-loop pick can predate the growth it triggered concluding:
|
||||
// growth registers its volumes before clearing the pending flag, so pick
|
||||
// once more before shedding.
|
||||
if fid, count, dnList, _, err := ms.Topo.PickForWrite(requestedCount, option, vl, expectedDataSize); err == nil {
|
||||
if dn := dnList.Head(); dn != nil {
|
||||
ms.maybeAddJwtAuthorization(w, fid, true)
|
||||
writeJsonQuiet(w, r, http.StatusOK, operation.AssignResult{Fid: fid, Url: dn.Url(), PublicUrl: dn.PublicUrl, Count: count})
|
||||
return
|
||||
}
|
||||
}
|
||||
// See Assign: initiator that timed out with growth still pending stays retryable.
|
||||
if initiatedGrow && vl.HasGrowRequest() && ms.Topo.AvailableSpaceFor(option) > 0 {
|
||||
w.Header().Set("Retry-After", "1")
|
||||
|
||||
@@ -65,7 +65,7 @@ func EnsureTopologyId(topo *topology.Topology, checkLeaderFn func() bool, persis
|
||||
} else {
|
||||
glog.V(0).Infof("TopologyId generated: %s", topologyId)
|
||||
}
|
||||
topo.DeferAllocation(topo.WarmupDuration())
|
||||
topo.DeferAllocation(topo.AllocationDeferralDuration())
|
||||
break
|
||||
}
|
||||
}
|
||||
@@ -144,3 +144,76 @@ func TestAllocationDelay(t *testing.T) {
|
||||
t.Fatalf("expired deferral should not delay: %v", wait)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAllocationDelayEarlyRelease(t *testing.T) {
|
||||
topo := NewTopology("topo", nil, 128, 1, false)
|
||||
|
||||
topo.DeferAllocation(time.Minute)
|
||||
wait := topo.allocationDelay()
|
||||
if wait <= 0 || wait > 2*time.Second {
|
||||
t.Fatalf("quiet window should bound the deferral near two pulses, got %v", wait)
|
||||
}
|
||||
|
||||
time.Sleep(2100 * time.Millisecond)
|
||||
if wait := topo.allocationDelay(); wait > 0 {
|
||||
t.Fatalf("two quiet pulses should release the deferral early, got %v", wait)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAllocationDelayExtendedByEvidence(t *testing.T) {
|
||||
topo := NewTopology("topo", nil, 128, 1, false)
|
||||
|
||||
topo.DeferAllocation(time.Minute)
|
||||
time.Sleep(1100 * time.Millisecond)
|
||||
topo.NoteAllocationEvidence()
|
||||
time.Sleep(1200 * time.Millisecond)
|
||||
if wait := topo.allocationDelay(); wait <= 0 {
|
||||
t.Fatal("evidence should extend the deferral past the original quiet deadline")
|
||||
}
|
||||
}
|
||||
|
||||
func TestAllocationDeferralDuration(t *testing.T) {
|
||||
topo := NewTopology("topo", nil, 128, 3, false)
|
||||
|
||||
if d := topo.AllocationDeferralDuration(); d != topo.WarmupDuration() {
|
||||
t.Fatalf("unset deferral should default to the warmup window, got %v", d)
|
||||
}
|
||||
topo.SetAllocationDeferral(0)
|
||||
if d := topo.AllocationDeferralDuration(); d != 0 {
|
||||
t.Fatalf("zero deferral should disable the hold, got %v", d)
|
||||
}
|
||||
topo.SetAllocationDeferral(2 * time.Second)
|
||||
if d := topo.AllocationDeferralDuration(); d != 2*time.Second {
|
||||
t.Fatalf("configured deferral should win, got %v", d)
|
||||
}
|
||||
topo.SetAllocationDeferral(-1)
|
||||
if d := topo.AllocationDeferralDuration(); d != topo.WarmupDuration() {
|
||||
t.Fatalf("negative deferral should restore the default, got %v", d)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAllocationDeferralReleasedByInventory(t *testing.T) {
|
||||
topo := NewTopology("topo", nil, 128, 3, false)
|
||||
topo.ReleaseAllocationOnFirstReport()
|
||||
|
||||
topo.DeferAllocation(time.Minute)
|
||||
topo.NoteInventoryApplied()
|
||||
if wait := topo.allocationDelay(); wait > 0 {
|
||||
t.Fatalf("applied inventory should lift the deferral, still waiting %v", wait)
|
||||
}
|
||||
|
||||
fresh := NewTopology("topo", nil, 128, 3, false)
|
||||
fresh.ReleaseAllocationOnFirstReport()
|
||||
fresh.NoteInventoryApplied()
|
||||
fresh.DeferAllocation(time.Minute)
|
||||
if wait := fresh.allocationDelay(); wait > 0 {
|
||||
t.Fatalf("a report applied before the mint should keep the hold unarmed, got %v", wait)
|
||||
}
|
||||
|
||||
walled := NewTopology("topo", nil, 128, 3, false)
|
||||
walled.DeferAllocation(time.Minute)
|
||||
walled.NoteInventoryApplied()
|
||||
if wait := walled.allocationDelay(); wait <= 0 {
|
||||
t.Fatal("without opt-in an applied inventory must not lift the deferral")
|
||||
}
|
||||
}
|
||||
@@ -92,12 +92,22 @@ type Topology struct {
|
||||
lastPersistedTopologyId atomic.Value
|
||||
lastPersistedGrowthFrozen atomic.Bool
|
||||
|
||||
growthFrozen atomic.Bool
|
||||
growthFrozenAt atomic.Int64
|
||||
freezeOnMismatch atomic.Bool
|
||||
stateLossRecoveryWait atomic.Int64
|
||||
allocationNotBefore atomic.Int64
|
||||
topologyInitPending atomic.Bool
|
||||
growthFrozen atomic.Bool
|
||||
growthFrozenAt atomic.Int64
|
||||
freezeOnMismatch atomic.Bool
|
||||
stateLossRecoveryWait atomic.Int64
|
||||
allocationNotBefore atomic.Int64
|
||||
allocationEvidenceSince atomic.Int64
|
||||
allocationDeferral atomic.Int64
|
||||
topologyInitPending atomic.Bool
|
||||
|
||||
// releaseAllocOnReport lifts a post-mint allocation hold as soon as a full
|
||||
// volume-server inventory has been applied, for single-reporter setups
|
||||
// (weed mini) whose embedded volume server is the only possible source of
|
||||
// prior cluster state. inventoryApplied records that at least one full
|
||||
// volume list has been synced into this topology.
|
||||
releaseAllocOnReport atomic.Bool
|
||||
inventoryApplied atomic.Bool
|
||||
|
||||
reportedTopologyIdsLock sync.Mutex
|
||||
pendingReportedTopologyIds map[string]struct{}
|
||||
@@ -324,6 +334,8 @@ func NewTopology(id string, seq sequence.Sequencer, volumeSizeLimit uint64, puls
|
||||
t.chanFullVolumes = make(chan storage.VolumeInfo)
|
||||
t.chanCrowdedVolumes = make(chan storage.VolumeInfo)
|
||||
|
||||
t.allocationDeferral.Store(-1)
|
||||
|
||||
t.Configuration = &Configuration{}
|
||||
t.dataNodeIndex = make(map[string]*DataNode)
|
||||
t.quarantinePushLocks = make(map[quarantinedVolume]*quarantinePushLock)
|
||||
@@ -621,6 +633,11 @@ func (t *Topology) NextVolumeId() (needle.VolumeId, error) {
|
||||
}
|
||||
if wait := t.allocationDelay(); wait > 0 {
|
||||
glog.V(0).Infof("deferring volume id allocation for %v", wait)
|
||||
}
|
||||
for wait := t.allocationDelay(); wait > 0; wait = t.allocationDelay() {
|
||||
if wait > 500*time.Millisecond {
|
||||
wait = 500 * time.Millisecond
|
||||
}
|
||||
time.Sleep(wait)
|
||||
if t.IsGrowthFrozen() {
|
||||
return 0, fmt.Errorf("volume growth is frozen after master state loss; verify the cluster inventory, then run: volume.growth.resume")
|
||||
@@ -1573,15 +1590,77 @@ func (t *Topology) SetTopologyInitPending(pending bool) {
|
||||
}
|
||||
}
|
||||
|
||||
// ReleaseAllocationOnFirstReport opts the topology into lifting the post-mint
|
||||
// allocation hold once one full volume-server inventory has been applied. Only
|
||||
// valid where a single volume server can carry prior cluster state (weed
|
||||
// mini's embedded server): its first heartbeat already raises the id bound or
|
||||
// trips the growth freeze, so the quiet-pulse tail is dead time.
|
||||
func (t *Topology) ReleaseAllocationOnFirstReport() {
|
||||
t.releaseAllocOnReport.Store(true)
|
||||
}
|
||||
|
||||
// NoteInventoryApplied marks a heartbeat's full volume list as applied to this
|
||||
// topology. Under ReleaseAllocationOnFirstReport it also lifts the post-mint
|
||||
// allocation hold: the bound raise and topology-id freeze checks run while the
|
||||
// heartbeat is processed, so nothing protected remains pending afterwards.
|
||||
func (t *Topology) NoteInventoryApplied() {
|
||||
t.inventoryApplied.Store(true)
|
||||
if t.releaseAllocOnReport.Load() {
|
||||
t.allocationNotBefore.Store(0)
|
||||
}
|
||||
}
|
||||
|
||||
// DeferAllocation holds off new id allocation for d. Called when the leader
|
||||
// mints a fresh topology id: on a rebuilt master that window gives veteran
|
||||
// servers a pulse to raise the bound or trip the freeze.
|
||||
func (t *Topology) DeferAllocation(d time.Duration) {
|
||||
t.allocationNotBefore.Store(time.Now().Add(d).UnixNano())
|
||||
// Under release-on-report a hold is pointless once inventory has landed:
|
||||
// the veteran heartbeat that mattered has already been applied.
|
||||
if t.releaseAllocOnReport.Load() && t.inventoryApplied.Load() {
|
||||
return
|
||||
}
|
||||
now := time.Now()
|
||||
t.allocationNotBefore.Store(now.Add(d).UnixNano())
|
||||
t.allocationEvidenceSince.Store(now.UnixNano())
|
||||
// Close the race with a report applied between the check above and the
|
||||
// store, which would otherwise leave the full window armed.
|
||||
if t.releaseAllocOnReport.Load() && t.inventoryApplied.Load() {
|
||||
t.allocationNotBefore.Store(0)
|
||||
}
|
||||
}
|
||||
|
||||
// NoteAllocationEvidence records a heartbeat signal that could still carry
|
||||
// prior cluster state — a server joining or reporting held volumes, a raised
|
||||
// id bound, or a foreign topology id. The deferral lifts two pulses after the
|
||||
// last such signal, so a cluster with nothing left to report stops waiting.
|
||||
func (t *Topology) NoteAllocationEvidence() {
|
||||
t.allocationEvidenceSince.Store(time.Now().UnixNano())
|
||||
}
|
||||
|
||||
// SetAllocationDeferral overrides the post-mint allocation hold: zero disables
|
||||
// it, a positive duration sets the cap, negative restores the warmup window.
|
||||
func (t *Topology) SetAllocationDeferral(d time.Duration) {
|
||||
t.allocationDeferral.Store(int64(d))
|
||||
}
|
||||
|
||||
// AllocationDeferralDuration returns the configured post-mint hold, or one
|
||||
// warmup window when unset.
|
||||
func (t *Topology) AllocationDeferralDuration() time.Duration {
|
||||
if d := t.allocationDeferral.Load(); d >= 0 {
|
||||
return time.Duration(d)
|
||||
}
|
||||
return t.WarmupDuration()
|
||||
}
|
||||
|
||||
func (t *Topology) allocationDelay() time.Duration {
|
||||
return time.Until(time.Unix(0, t.allocationNotBefore.Load()))
|
||||
deadline := t.allocationNotBefore.Load()
|
||||
if deadline == 0 {
|
||||
return 0
|
||||
}
|
||||
if quietUntil := t.allocationEvidenceSince.Load() + 2*int64(t.pulse)*int64(time.Second); quietUntil < deadline {
|
||||
deadline = quietUntil
|
||||
}
|
||||
return time.Until(time.Unix(0, deadline))
|
||||
}
|
||||
|
||||
// SetMaxVolumeIdFile mirrors the allocation bound to a file outside raft, so
|
||||
|
||||
Reference in new issue
Block a user