master: shed assigns retryably until volume servers register capacity (#11032)

An assign arriving before any volume server has heartbeated saw zero
available space and failed outright with a plain error no client retries,
so the first write to a fresh bucket answered 500 while the cluster was
still starting. Distinguish a topology with no registered capacity from a
genuinely full one: fail fast only when registered capacity is exhausted,
and shed ResourceExhausted otherwise so the client's retry budget rides
out the startup window.

Claude-Session: https://claude.ai/code/session_018G9kWFgy8BaBAEkYV3YL9n

weed/server/master_grpc_server_assign.go

@@ -117,7 +117,7 @@ func (ms *MasterServer) Assign(ctx context.Context, req *master_pb.AssignRequest
 		fid, count, dnList, shouldGrow, err := ms.Topo.PickForWrite(req.Count, option, vl, req.ExpectedDataSize)
 		if shouldGrow && !initiatedGrow && !ms.option.VolumeGrowthDisabled && vl.AddGrowRequestIfAbsent() {
 			initiatedGrow = true
-			if err != nil && ms.Topo.AvailableSpaceFor(option) <= 0 {
+			if err != nil && ms.Topo.AvailableSpaceFor(option) <= 0 && ms.Topo.CapacityFor(option) > 0 {
 				err = fmt.Errorf("%s and no free volumes left for %s", err.Error(), option.String())
 			}
 			ms.volumeGrowthRequestChan <- &topology.VolumeGrowRequest{
@@ -136,7 +136,14 @@ func (ms *MasterServer) Assign(ctx context.Context, req *master_pb.AssignRequest
 			}
 			if shouldGrow {
 				if ms.Topo.AvailableSpaceFor(option) <= 0 {
-					break // out of space: surface the real error, not a retryable shed
+					if ms.Topo.CapacityFor(option) > 0 {
+						break // out of space: surface the real error, not a retryable shed
+					}
+					// No capacity registered for this disk type yet, typically a
+					// just-started cluster whose volume servers have not
+					// heartbeated. Shed retryably so the first write rides out
+					// the startup window instead of failing outright.
+					return nil, status.Errorf(codes.ResourceExhausted, "no volume server capacity registered yet for %s", option.String())
 				}
 				// Only the initiator waits, and only while the growth it triggered
 				// is still pending: followers shed fast so a herd doesn't pin a

weed/server/master_grpc_server_assign_test.go

@@ -175,9 +175,24 @@ func TestAssignAbortsOnCancel(t *testing.T) {
 // Out of space, Assign fails fast with the real error rather than masking it as
 // a retryable "growth in progress".
 func TestAssignFailsFastWhenOutOfSpace(t *testing.T) {
-	ms := newLeaderMaster() // no data nodes -> no free space
+	ms := newLeaderMaster()
+	// One slot, already taken by another collection's volume: capacity is
+	// registered but genuinely exhausted.
+	dn := ms.Topo.GetOrCreateDataCenter("dc1").GetOrCreateRack("rack1").
+		GetOrCreateDataNode("127.0.0.1", 8080, 18080, "127.0.0.1", "dn1", map[string]uint32{"": 1})
+	rp, err := super_block.NewReplicaPlacementFromString("000")
+	require.NoError(t, err)
+	v := storage.VolumeInfo{
+		Id:               needle.VolumeId(1),
+		Collection:       "other",
+		Version:          needle.GetCurrentVersion(),
+		ReplicaPlacement: rp,
+		Ttl:              needle.EMPTY_TTL,
+	}
+	dn.UpdateVolumes([]storage.VolumeInfo{v})
+	ms.Topo.RegisterVolumeLayout(v, dn)
 
-	req := &master_pb.AssignRequest{Count: 1, Replication: "000"}
+	req := &master_pb.AssignRequest{Count: 1, Replication: "000", Collection: "fresh"}
 
 	start := time.Now()
 	resp, err := ms.Assign(context.Background(), req)
@@ -187,7 +202,29 @@ func TestAssignFailsFastWhenOutOfSpace(t *testing.T) {
 	require.Nil(t, resp)
 	if st, ok := status.FromError(err); ok {
 		assert.NotEqual(t, codes.Unavailable, st.Code())
+		assert.NotEqual(t, codes.ResourceExhausted, st.Code())
 	}
 	assert.Contains(t, err.Error(), "no free volumes left")
 	assert.Less(t, elapsed, 2*time.Second)
 }
+
+// A topology with no registered capacity is a cluster whose volume servers have
+// not heartbeated yet, not one that is full: the first write to a fresh bucket
+// races the volume server registration at startup, so Assign must shed with a
+// retryable code instead of failing the write outright.
+func TestAssignShedsRetryablyBeforeCapacityRegisters(t *testing.T) {
+	ms := newLeaderMaster() // no data nodes: nothing has heartbeated yet
+
+	req := &master_pb.AssignRequest{Count: 1, Replication: "000"}
+
+	start := time.Now()
+	resp, err := ms.Assign(context.Background(), req)
+	elapsed := time.Since(start)
+
+	require.Error(t, err)
+	require.Nil(t, resp)
+	st, ok := status.FromError(err)
+	require.True(t, ok)
+	assert.Equal(t, codes.ResourceExhausted, st.Code())
+	assert.Less(t, elapsed, 2*time.Second)
+}

weed/topology/node.go

@@ -313,6 +313,13 @@ func (n *NodeImpl) AvailableSpaceFor(option *VolumeGrowOption) int64 {
 	return freeVolumeSlotCount
 }
 
+// CapacityFor is the total registered volume slots for the option's disk type;
+// zero means no volume server has reported capacity for it yet.
+func (n *NodeImpl) CapacityFor(option *VolumeGrowOption) int64 {
+	t := n.getOrCreateDisk(option.DiskType)
+	return atomic.LoadInt64(&t.maxVolumeCount) + atomic.LoadInt64(&t.remoteVolumeCount)
+}
+
 // AvailableSpaceForReservation returns available space considering existing reservations
 func (n *NodeImpl) AvailableSpaceForReservation(option *VolumeGrowOption) int64 {
 	baseAvailable := n.AvailableSpaceFor(option)