From d3b8030a69de22ac06cd8e90dad0eeddb194fc45 Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Sun, 30 Aug 2026 11:08:53 -0700 Subject: [PATCH] master: shed assigns retryably until volume servers register capacity (#11032) An assign arriving before any volume server has heartbeated saw zero available space and failed outright with a plain error no client retries, so the first write to a fresh bucket answered 500 while the cluster was still starting. Distinguish a topology with no registered capacity from a genuinely full one: fail fast only when registered capacity is exhausted, and shed ResourceExhausted otherwise so the client's retry budget rides out the startup window. Claude-Session: https://claude.ai/code/session_018G9kWFgy8BaBAEkYV3YL9n --- weed/server/master_grpc_server_assign.go | 11 ++++- weed/server/master_grpc_server_assign_test.go | 41 ++++++++++++++++++- weed/topology/node.go | 7 ++++ 3 files changed, 55 insertions(+), 4 deletions(-) diff --git a/weed/server/master_grpc_server_assign.go b/weed/server/master_grpc_server_assign.go index 0f037dcd1..02f45269a 100644 --- a/weed/server/master_grpc_server_assign.go +++ b/weed/server/master_grpc_server_assign.go @@ -117,7 +117,7 @@ func (ms *MasterServer) Assign(ctx context.Context, req *master_pb.AssignRequest fid, count, dnList, shouldGrow, err := ms.Topo.PickForWrite(req.Count, option, vl, req.ExpectedDataSize) if shouldGrow && !initiatedGrow && !ms.option.VolumeGrowthDisabled && vl.AddGrowRequestIfAbsent() { initiatedGrow = true - if err != nil && ms.Topo.AvailableSpaceFor(option) <= 0 { + if err != nil && ms.Topo.AvailableSpaceFor(option) <= 0 && ms.Topo.CapacityFor(option) > 0 { err = fmt.Errorf("%s and no free volumes left for %s", err.Error(), option.String()) } ms.volumeGrowthRequestChan <- &topology.VolumeGrowRequest{ @@ -136,7 +136,14 @@ func (ms *MasterServer) Assign(ctx context.Context, req *master_pb.AssignRequest } if shouldGrow { if ms.Topo.AvailableSpaceFor(option) <= 0 { - break // out of space: surface the real error, not a retryable shed + if ms.Topo.CapacityFor(option) > 0 { + break // out of space: surface the real error, not a retryable shed + } + // No capacity registered for this disk type yet, typically a + // just-started cluster whose volume servers have not + // heartbeated. Shed retryably so the first write rides out + // the startup window instead of failing outright. + return nil, status.Errorf(codes.ResourceExhausted, "no volume server capacity registered yet for %s", option.String()) } // Only the initiator waits, and only while the growth it triggered // is still pending: followers shed fast so a herd doesn't pin a diff --git a/weed/server/master_grpc_server_assign_test.go b/weed/server/master_grpc_server_assign_test.go index 7b65002f8..acd8ea60a 100644 --- a/weed/server/master_grpc_server_assign_test.go +++ b/weed/server/master_grpc_server_assign_test.go @@ -175,9 +175,24 @@ func TestAssignAbortsOnCancel(t *testing.T) { // Out of space, Assign fails fast with the real error rather than masking it as // a retryable "growth in progress". func TestAssignFailsFastWhenOutOfSpace(t *testing.T) { - ms := newLeaderMaster() // no data nodes -> no free space + ms := newLeaderMaster() + // One slot, already taken by another collection's volume: capacity is + // registered but genuinely exhausted. + dn := ms.Topo.GetOrCreateDataCenter("dc1").GetOrCreateRack("rack1"). + GetOrCreateDataNode("127.0.0.1", 8080, 18080, "127.0.0.1", "dn1", map[string]uint32{"": 1}) + rp, err := super_block.NewReplicaPlacementFromString("000") + require.NoError(t, err) + v := storage.VolumeInfo{ + Id: needle.VolumeId(1), + Collection: "other", + Version: needle.GetCurrentVersion(), + ReplicaPlacement: rp, + Ttl: needle.EMPTY_TTL, + } + dn.UpdateVolumes([]storage.VolumeInfo{v}) + ms.Topo.RegisterVolumeLayout(v, dn) - req := &master_pb.AssignRequest{Count: 1, Replication: "000"} + req := &master_pb.AssignRequest{Count: 1, Replication: "000", Collection: "fresh"} start := time.Now() resp, err := ms.Assign(context.Background(), req) @@ -187,7 +202,29 @@ func TestAssignFailsFastWhenOutOfSpace(t *testing.T) { require.Nil(t, resp) if st, ok := status.FromError(err); ok { assert.NotEqual(t, codes.Unavailable, st.Code()) + assert.NotEqual(t, codes.ResourceExhausted, st.Code()) } assert.Contains(t, err.Error(), "no free volumes left") assert.Less(t, elapsed, 2*time.Second) } + +// A topology with no registered capacity is a cluster whose volume servers have +// not heartbeated yet, not one that is full: the first write to a fresh bucket +// races the volume server registration at startup, so Assign must shed with a +// retryable code instead of failing the write outright. +func TestAssignShedsRetryablyBeforeCapacityRegisters(t *testing.T) { + ms := newLeaderMaster() // no data nodes: nothing has heartbeated yet + + req := &master_pb.AssignRequest{Count: 1, Replication: "000"} + + start := time.Now() + resp, err := ms.Assign(context.Background(), req) + elapsed := time.Since(start) + + require.Error(t, err) + require.Nil(t, resp) + st, ok := status.FromError(err) + require.True(t, ok) + assert.Equal(t, codes.ResourceExhausted, st.Code()) + assert.Less(t, elapsed, 2*time.Second) +} diff --git a/weed/topology/node.go b/weed/topology/node.go index dc5300789..8d82fdfe2 100644 --- a/weed/topology/node.go +++ b/weed/topology/node.go @@ -313,6 +313,13 @@ func (n *NodeImpl) AvailableSpaceFor(option *VolumeGrowOption) int64 { return freeVolumeSlotCount } +// CapacityFor is the total registered volume slots for the option's disk type; +// zero means no volume server has reported capacity for it yet. +func (n *NodeImpl) CapacityFor(option *VolumeGrowOption) int64 { + t := n.getOrCreateDisk(option.DiskType) + return atomic.LoadInt64(&t.maxVolumeCount) + atomic.LoadInt64(&t.remoteVolumeCount) +} + // AvailableSpaceForReservation returns available space considering existing reservations func (n *NodeImpl) AvailableSpaceForReservation(option *VolumeGrowOption) int64 { baseAvailable := n.AvailableSpaceFor(option)