diff --git a/test/testutil/ports.go b/test/testutil/ports.go index 867f20761..103f5a440 100644 --- a/test/testutil/ports.go +++ b/test/testutil/ports.go @@ -11,6 +11,23 @@ import ( // GrpcPortOffset is the offset weed mini uses to derive gRPC ports from HTTP ports. const GrpcPortOffset = 10000 +// linuxEphemeralPortFloor is the Linux default ip_local_port_range lower bound. +// Ports at or above it can be claimed as the source port of a transient +// outbound connection during mini's own startup (e.g. volume dialing master +// gRPC) in the window between allocation and the listener bind, surfacing as +// "bind: address already in use". +const linuxEphemeralPortFloor = 32768 + +// miniPortMin/miniPortMax bound the random HTTP port range for weed mini. +// The cap keeps both the HTTP port and its +GrpcPortOffset gRPC counterpart +// below linuxEphemeralPortFloor, out of the range the kernel draws ephemeral +// source ports from, so the steal above cannot happen. Deriving the cap from +// GrpcPortOffset keeps the two in lockstep if the offset ever changes. +const ( + miniPortMin = 10000 + miniPortMax = linuxEphemeralPortFloor - GrpcPortOffset +) + // miniDefaultPorts are the weed mini flag defaults (see weed/command/mini.go). // A test only overrides services it uses; unspecified services still bind // these defaults, so allocation must avoid handing them out (or any value @@ -83,8 +100,8 @@ func MustAllocatePorts(t *testing.T, count int) []int { // wrong port and hang. func AllocateMiniPorts(count int) ([]int, error) { const ( - minPort = 10000 - maxPort = 55000 + minPort = miniPortMin + maxPort = miniPortMax ) reserved := reservedMiniPorts() ports := make([]int, 0, count) @@ -155,8 +172,8 @@ func MustFreeMiniPort(t *testing.T, name string) int { // allocated, so a mini gRPC port cannot be recycled as a regular port mid-batch. func AllocatePortSet(miniCount, regularCount int) (mini []int, regular []int, err error) { const ( - minPort = 10000 - maxPort = 55000 + minPort = miniPortMin + maxPort = miniPortMax ) reserved := reservedMiniPorts() mini = make([]int, 0, miniCount) diff --git a/test/testutil/ports_test.go b/test/testutil/ports_test.go index 5c02e20a3..8d22c51e7 100644 --- a/test/testutil/ports_test.go +++ b/test/testutil/ports_test.go @@ -25,6 +25,27 @@ func TestAllocateMiniPortsAvoidsMiniDefaults(t *testing.T) { } } +// AllocateMiniPorts must keep every port it hands out, and that port's gRPC +// counterpart, below 32768 - the Linux default ip_local_port_range floor. +// A real failure: filer was allocated 44204, then lost the bind to a transient +// outbound connection that had grabbed 44204 as an ephemeral source port +// during mini startup ("bind: address already in use"). +func TestAllocateMiniPortsBelowEphemeralFloor(t *testing.T) { + const ephemeralFloor = linuxEphemeralPortFloor + for iter := 0; iter < 200; iter++ { + ports, err := AllocateMiniPorts(4) + if err != nil { + t.Fatalf("iter %d: AllocateMiniPorts: %v", iter, err) + } + for _, p := range ports { + if p+GrpcPortOffset >= ephemeralFloor { + t.Fatalf("iter %d: port %d (gRPC %d) reaches the ephemeral range floor %d", + iter, p, p+GrpcPortOffset, ephemeralFloor) + } + } + } +} + func TestAllocatePortSetNoGrpcCollision(t *testing.T) { // Run a few iterations to catch the OS-recycles-just-closed-port race // that previously hit regular ports when the mini gRPC offset was freed