diff --git a/weed/server/master_grpc_server_assign.go b/weed/server/master_grpc_server_assign.go index 02f45269a..cbd4bc0f0 100644 --- a/weed/server/master_grpc_server_assign.go +++ b/weed/server/master_grpc_server_assign.go @@ -136,13 +136,24 @@ func (ms *MasterServer) Assign(ctx context.Context, req *master_pb.AssignRequest } if shouldGrow { if ms.Topo.AvailableSpaceFor(option) <= 0 { - if ms.Topo.CapacityFor(option) > 0 { - break // out of space: surface the real error, not a retryable shed + // Fail fast whenever any capacity is registered: full for + // this medium, or a medium no volume server serves — a + // state a heartbeat won't change, so a retryable shed + // would loop until the client's deadline. Shed retryably + // only while nothing at all has registered, a just-started + // cluster whose volume servers have not heartbeated, so + // the first write rides out the startup window instead of + // failing outright. + if ms.Topo.CapacityForAnyDisk() > 0 { + if ms.Topo.CapacityFor(option) <= 0 { + // Wrapped here, not beside the "no free volumes left" + // wrap above, so followers and growth-disabled + // masters name the unserved medium too — the + // initiator block is skipped for both. + lastErr = fmt.Errorf("%s and no volume server carries disk type %q for %s", err.Error(), option.DiskType.ReadableString(), option.String()) + } + break // surface the real error, not a retryable shed } - // No capacity registered for this disk type yet, typically a - // just-started cluster whose volume servers have not - // heartbeated. Shed retryably so the first write rides out - // the startup window instead of failing outright. return nil, status.Errorf(codes.ResourceExhausted, "no volume server capacity registered yet for %s", option.String()) } // Only the initiator waits, and only while the growth it triggered diff --git a/weed/server/master_grpc_server_assign_test.go b/weed/server/master_grpc_server_assign_test.go index acd8ea60a..8446b18e6 100644 --- a/weed/server/master_grpc_server_assign_test.go +++ b/weed/server/master_grpc_server_assign_test.go @@ -228,3 +228,48 @@ func TestAssignShedsRetryablyBeforeCapacityRegisters(t *testing.T) { assert.Equal(t, codes.ResourceExhausted, st.Code()) assert.Less(t, elapsed, 2*time.Second) } + +// A cluster serving only other media is not one still starting up: capacity for +// the requested disk type will never register, so the startup shed would loop +// until the client's deadline. Assign must fail fast with the real error and +// name the unserved medium — for the growth initiator, for a follower whose +// growth is already in flight, and when growth is disabled outright. +func TestAssignFailsFastWhenDiskTypeUnserved(t *testing.T) { + tests := []struct { + name string + setup func(t *testing.T, ms *MasterServer, req *master_pb.AssignRequest) + }{ + {"initiator", func(t *testing.T, ms *MasterServer, req *master_pb.AssignRequest) {}}, + {"follower joins growth in flight", func(t *testing.T, ms *MasterServer, req *master_pb.AssignRequest) { + markGrowthInFlight(t, ms.Topo, req) + }}, + {"growth disabled", func(t *testing.T, ms *MasterServer, req *master_pb.AssignRequest) { + ms.option.VolumeGrowthDisabled = true + }}, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + ms := newLeaderMaster() + // ssd capacity registered, but the request asks for the default (hdd). + ms.Topo.GetOrCreateDataCenter("dc1").GetOrCreateRack("rack1"). + GetOrCreateDataNode("127.0.0.1", 8080, 18080, "127.0.0.1", "dn1", map[string]uint32{"ssd": 1}) + + req := &master_pb.AssignRequest{Count: 1, Replication: "000", Collection: "fresh"} + tc.setup(t, ms, req) + + start := time.Now() + resp, err := ms.Assign(context.Background(), req) + elapsed := time.Since(start) + + require.Error(t, err) + require.Nil(t, resp) + if st, ok := status.FromError(err); ok { + assert.NotEqual(t, codes.Unavailable, st.Code()) + assert.NotEqual(t, codes.ResourceExhausted, st.Code()) + } + assert.Contains(t, err.Error(), topology.NoWritableVolumes) + assert.Contains(t, err.Error(), `no volume server carries disk type "hdd"`) + assert.Less(t, elapsed, 2*time.Second) + }) + } +} diff --git a/weed/topology/node.go b/weed/topology/node.go index 8d82fdfe2..f0bb84297 100644 --- a/weed/topology/node.go +++ b/weed/topology/node.go @@ -320,6 +320,18 @@ func (n *NodeImpl) CapacityFor(option *VolumeGrowOption) int64 { return atomic.LoadInt64(&t.maxVolumeCount) + atomic.LoadInt64(&t.remoteVolumeCount) } +// CapacityForAnyDisk is the total registered volume slots across every disk +// type. CapacityFor answers zero both while a cluster is still starting and +// when it never serves the option's medium; this tells the two apart. +func (n *NodeImpl) CapacityForAnyDisk() (total int64) { + n.diskUsages.RLock() + defer n.diskUsages.RUnlock() + for _, t := range n.diskUsages.usages { + total += atomic.LoadInt64(&t.maxVolumeCount) + atomic.LoadInt64(&t.remoteVolumeCount) + } + return +} + // AvailableSpaceForReservation returns available space considering existing reservations func (n *NodeImpl) AvailableSpaceForReservation(option *VolumeGrowOption) int64 { baseAvailable := n.AvailableSpaceFor(option)