test: consolidate port allocation into shared test/testutil package (#8982)

* test: consolidate port allocation into shared test/testutil package

Move duplicated port allocation logic from 15+ test files into a single
shared package at test/testutil/. This fixes a port collision bug where
independently allocated ports could overlap via the gRPC offset
(port+10000), causing weed mini to reject the configuration.

The shared package provides:
- AllocatePorts: atomic allocation of N unique ports
- AllocateMiniPorts/MustFreeMiniPorts: gRPC-offset-aware allocation
  that prevents port A+10000 == port B collisions
- WaitForPort, WaitForService, FindBindIP, WriteIAMConfig, HasDocker

* test: address review feedback and fix FUSE build

- Revert fuse_integration change: it has its own go.mod and cannot
  import the shared testutil package
- AllocateMiniPorts: hold all listeners open until the entire batch is
  allocated, preventing race conditions where other processes steal ports
- HasDocker: add 5s context timeout to avoid hanging on stalled Docker
- WaitForService: only treat 2xx HTTP status codes as ready

* test: use global rand in AllocateMiniPorts for better seeding

Go 1.20+ auto-seeds the global rand generator. Using it avoids
identical sequences when multiple tests call at the same nanosecond.

* test: revert WaitForService status code check

S3 endpoints return non-2xx (e.g. 403) on bare GET requests, so
requiring 2xx caused the S3 integration test to time out. Any HTTP
response is sufficient proof that the service is running.

* test: fix gofmt formatting in s3tables test files
This commit is contained in:
Chris Lu authored and GitHub committed 2026-04-08 11:30:02 -07:00
1 parent ac12a735c7
commit fbe758efa8
26 files changed
+308 -716

No files matched your search

+16 -100
View File
@@ -9,7 +9,6 @@ import (
"flag"
"fmt"
"io"
"net"
"net/http"
"os"
"os/exec"
@@ -17,6 +16,8 @@ import (
"strings"
"testing"
"time"
"github.com/seaweedfs/seaweedfs/test/testutil"
)
// sharedEnv is the single TestEnvironment shared across all tests in this package.
@@ -69,23 +70,6 @@ type TestEnvironment struct {
dockerAvailable bool
}
// hasDocker checks if Docker is available
func hasDocker() bool {
cmd := exec.Command("docker", "version")
return cmd.Run() == nil
}
// getFreePort returns an available ephemeral port and its listener
func getFreePort() (int, net.Listener, error) {
listener, err := net.Listen("tcp", "127.0.0.1:0")
if err != nil {
return 0, nil, err
}
addr := listener.Addr().(*net.TCPAddr)
return addr.Port, listener, nil
}
// newTestEnvironmentForMain creates a TestEnvironment without calling t.Fatalf so it
// can be used from TestMain (which has no *testing.T).
func newTestEnvironmentForMain() (*TestEnvironment, error) {
@@ -118,95 +102,27 @@ func newTestEnvironmentForMain() (*TestEnvironment, error) {
return nil, fmt.Errorf("create temp dir: %w", err)
}
// Allocate free ephemeral ports for each service
var listeners []net.Listener
closeListeners := func() {
for _, l := range listeners {
l.Close()
}
}
var l net.Listener
s3Port, l, err := getFreePort()
// Allocate 9 unique ports atomically: s3, iceberg, s3Grpc, master, masterGrpc,
// filer, filerGrpc, volume, volumeGrpc
ports, err := testutil.AllocatePorts(9)
if err != nil {
closeListeners()
return nil, fmt.Errorf("get free port for S3: %w", err)
return nil, fmt.Errorf("allocate ports: %w", err)
}
listeners = append(listeners, l)
icebergPort, l, err := getFreePort()
if err != nil {
closeListeners()
return nil, fmt.Errorf("get free port for Iceberg: %w", err)
}
listeners = append(listeners, l)
s3GrpcPort, l, err := getFreePort()
if err != nil {
closeListeners()
return nil, fmt.Errorf("get free port for S3 gRPC: %w", err)
}
listeners = append(listeners, l)
masterPort, l, err := getFreePort()
if err != nil {
closeListeners()
return nil, fmt.Errorf("get free port for Master: %w", err)
}
listeners = append(listeners, l)
masterGrpcPort, l, err := getFreePort()
if err != nil {
closeListeners()
return nil, fmt.Errorf("get free port for Master gRPC: %w", err)
}
listeners = append(listeners, l)
filerPort, l, err := getFreePort()
if err != nil {
closeListeners()
return nil, fmt.Errorf("get free port for Filer: %w", err)
}
listeners = append(listeners, l)
filerGrpcPort, l, err := getFreePort()
if err != nil {
closeListeners()
return nil, fmt.Errorf("get free port for Filer gRPC: %w", err)
}
listeners = append(listeners, l)
volumePort, l, err := getFreePort()
if err != nil {
closeListeners()
return nil, fmt.Errorf("get free port for Volume: %w", err)
}
listeners = append(listeners, l)
volumeGrpcPort, l, err := getFreePort()
if err != nil {
closeListeners()
return nil, fmt.Errorf("get free port for Volume gRPC: %w", err)
}
listeners = append(listeners, l)
// Release the port reservations so weed mini can bind to them
closeListeners()
return &TestEnvironment{
seaweedDir: seaweedDir,
weedBinary: weedBinary,
dataDir: dataDir,
s3Port: s3Port,
s3GrpcPort: s3GrpcPort,
icebergPort: icebergPort,
masterPort: masterPort,
masterGrpcPort: masterGrpcPort,
filerPort: filerPort,
filerGrpcPort: filerGrpcPort,
volumePort: volumePort,
volumeGrpcPort: volumeGrpcPort,
dockerAvailable: hasDocker(),
s3Port: ports[0],
s3GrpcPort: ports[1],
icebergPort: ports[2],
masterPort: ports[3],
masterGrpcPort: ports[4],
filerPort: ports[5],
filerGrpcPort: ports[6],
volumePort: ports[7],
volumeGrpcPort: ports[8],
dockerAvailable: testutil.HasDocker(),
}, nil
}
@@ -18,7 +18,7 @@ import (
v4 "github.com/aws/aws-sdk-go-v2/aws/signer/v4"
"github.com/aws/aws-sdk-go-v2/credentials"
"github.com/seaweedfs/seaweedfs/test/s3tables/testutil"
"github.com/seaweedfs/seaweedfs/test/testutil"
)
var (
@@ -114,11 +114,12 @@ func (env *TestEnvironment) StartSeaweedFS(t *testing.T) {
t.Fatalf("failed to create temp directory: %v", err)
}
env.masterPort = testutil.MustFreeMiniPort(t, "Master")
env.filerPort = testutil.MustFreeMiniPort(t, "Filer")
env.s3Port = testutil.MustFreeMiniPort(t, "S3")
env.icebergRestPort = testutil.MustFreeMiniPort(t, "Iceberg")
env.risingwavePort = testutil.MustFreeMiniPort(t, "RisingWave")
ports := testutil.MustFreeMiniPorts(t, []string{"Master", "Filer", "S3", "Iceberg", "RisingWave"})
env.masterPort = ports[0]
env.filerPort = ports[1]
env.s3Port = ports[2]
env.icebergRestPort = ports[3]
env.risingwavePort = ports[4]
env.bindIP = testutil.FindBindIP()
+6 -5
View File
@@ -16,7 +16,7 @@ import (
"github.com/aws/aws-sdk-go-v2/aws"
"github.com/aws/aws-sdk-go-v2/credentials"
"github.com/aws/aws-sdk-go-v2/service/s3"
"github.com/seaweedfs/seaweedfs/test/s3tables/testutil"
"github.com/seaweedfs/seaweedfs/test/testutil"
"github.com/testcontainers/testcontainers-go"
)
@@ -90,10 +90,11 @@ func (env *TestEnvironment) StartSeaweedFS(t *testing.T) {
t.Fatalf("failed to create temp directory: %v", err)
}
env.masterPort = testutil.MustFreeMiniPort(t, "Master")
env.filerPort = testutil.MustFreeMiniPort(t, "Filer")
env.s3Port = testutil.MustFreeMiniPort(t, "S3")
env.icebergRestPort = testutil.MustFreeMiniPort(t, "Iceberg")
ports := testutil.MustFreeMiniPorts(t, []string{"Master", "Filer", "S3", "Iceberg"})
env.masterPort = ports[0]
env.filerPort = ports[1]
env.s3Port = ports[2]
env.icebergRestPort = ports[3]
bindIP := testutil.FindBindIP()
@@ -17,7 +17,7 @@ import (
"github.com/aws/aws-sdk-go-v2/aws"
"github.com/aws/aws-sdk-go-v2/credentials"
"github.com/aws/aws-sdk-go-v2/service/s3"
"github.com/seaweedfs/seaweedfs/test/s3tables/testutil"
"github.com/seaweedfs/seaweedfs/test/testutil"
)
type TestEnvironment struct {
@@ -40,7 +40,6 @@ type TestEnvironment struct {
dockerAvailable bool
accessKey string
secretKey string
closers []io.Closer
}
func TestTrinoIcebergCatalog(t *testing.T) {
@@ -214,19 +213,24 @@ func NewTestEnvironment(t *testing.T) *TestEnvironment {
bindIP := testutil.FindBindIP()
env := &TestEnvironment{
seaweedDir: seaweedDir,
weedBinary: weedBinary,
dataDir: dataDir,
bindIP: bindIP,
closers: []io.Closer{},
}
// 9 ports: master(2), volume(2), filer(2), s3(2), iceberg(1)
ports := testutil.MustAllocatePorts(t, 9)
env.masterPort, env.masterGrpcPort = env.mustFreePortPair("Master")
env.volumePort, env.volumeGrpcPort = env.mustFreePortPair("Volume")
env.filerPort, env.filerGrpcPort = env.mustFreePortPair("Filer")
env.s3Port, env.s3GrpcPort = env.mustFreePortPair("S3")
env.icebergPort = env.mustFreePort("Iceberg")
env := &TestEnvironment{
seaweedDir: seaweedDir,
weedBinary: weedBinary,
dataDir: dataDir,
bindIP: bindIP,
masterPort: ports[0],
masterGrpcPort: ports[1],
volumePort: ports[2],
volumeGrpcPort: ports[3],
filerPort: ports[4],
filerGrpcPort: ports[5],
s3Port: ports[6],
s3GrpcPort: ports[7],
icebergPort: ports[8],
}
env.dockerAvailable = hasDocker()
env.accessKey = "AKIAIOSFODNN7EXAMPLE"
@@ -235,47 +239,6 @@ func NewTestEnvironment(t *testing.T) *TestEnvironment {
return env
}
func (env *TestEnvironment) mustFreePort(name string) int {
port, closer, err := getFreePort()
if err != nil {
panic(fmt.Sprintf("Failed to get free port for %s: %v", name, err))
}
env.closers = append(env.closers, closer)
return port
}
func (env *TestEnvironment) mustFreePortPair(name string) (int, int) {
httpPort, httpCloser, grpcPort, grpcCloser, err := findAvailablePortPair()
if err != nil {
panic(fmt.Sprintf("Failed to get free port pair for %s: %v", name, err))
}
env.closers = append(env.closers, httpCloser, grpcCloser)
return httpPort, grpcPort
}
func mustFreePort(t *testing.T, name string) int {
t.Helper()
port, closer, err := getFreePort()
if err != nil {
t.Fatalf("Failed to get free port for %s: %v", name, err)
}
closer.Close()
return port
}
func mustFreePortPair(t *testing.T, name string) (int, int) {
t.Helper()
httpPort, httpCloser, grpcPort, grpcCloser, err := findAvailablePortPair()
if err != nil {
t.Fatalf("Failed to get free port pair for %s: %v", name, err)
}
httpCloser.Close()
grpcCloser.Close()
return httpPort, grpcPort
}
func (env *TestEnvironment) StartSeaweedFS(t *testing.T) {
t.Helper()
@@ -293,12 +256,6 @@ func (env *TestEnvironment) StartSeaweedFS(t *testing.T) {
ctx, cancel := context.WithCancel(context.Background())
env.weedCancel = cancel
// Close all port listeners right before starting the weed process
for _, closer := range env.closers {
closer.Close()
}
env.closers = nil
cmd := exec.CommandContext(ctx, env.weedBinary, "mini",
"-master.port", fmt.Sprintf("%d", env.masterPort),
"-master.port.grpc", fmt.Sprintf("%d", env.masterGrpcPort),
@@ -631,29 +588,6 @@ func hasDocker() bool {
return cmd.Run() == nil
}
func findAvailablePortPair() (int, io.Closer, int, io.Closer, error) {
httpPort, httpCloser, err := getFreePort()
if err != nil {
return 0, nil, 0, nil, err
}
grpcPort, grpcCloser, err := getFreePort()
if err != nil {
httpCloser.Close()
return 0, nil, 0, nil, err
}
return httpPort, httpCloser, grpcPort, grpcCloser, nil
}
func getFreePort() (int, io.Closer, error) {
listener, err := net.Listen("tcp", "0.0.0.0:0")
if err != nil {
return 0, nil, err
}
addr := listener.Addr().(*net.TCPAddr)
return addr.Port, listener, nil
}
func randomString(length int) string {
const charset = "abcdefghijklmnopqrstuvwxyz0123456789"
b := make([]byte, length)
+1 -1
View File
@@ -25,7 +25,7 @@ import (
s3types "github.com/aws/aws-sdk-go-v2/service/s3/types"
"github.com/aws/aws-sdk-go-v2/service/sts"
"github.com/seaweedfs/seaweedfs/test/s3tables/testutil"
"github.com/seaweedfs/seaweedfs/test/testutil"
"github.com/seaweedfs/seaweedfs/weed/s3api/s3tables"
)
@@ -16,7 +16,6 @@ import (
"flag"
"fmt"
"io"
"net"
"net/http"
"os"
"path"
@@ -40,6 +39,7 @@ import (
"google.golang.org/grpc"
"google.golang.org/grpc/credentials/insecure"
"github.com/seaweedfs/seaweedfs/test/testutil"
"github.com/seaweedfs/seaweedfs/weed/command"
"github.com/seaweedfs/seaweedfs/weed/filer"
"github.com/seaweedfs/seaweedfs/weed/glog"
@@ -92,7 +92,7 @@ func TestMain(m *testing.M) {
}
func startCluster(testDir string, extraArgs []string) (*testCluster, error) {
ports, err := findPorts(10)
ports, err := testutil.AllocatePorts(10)
if err != nil {
return nil, err
}
@@ -201,26 +201,6 @@ func (c *testCluster) filerConn(t *testing.T) (*grpc.ClientConn, filer_pb.Seawee
return conn, filer_pb.NewSeaweedFilerClient(conn)
}
func findPorts(n int) ([]int, error) {
ls := make([]*net.TCPListener, n)
ps := make([]int, n)
for i := 0; i < n; i++ {
l, err := net.Listen("tcp", "127.0.0.1:0")
if err != nil {
for j := 0; j < i; j++ {
ls[j].Close()
}
return nil, err
}
ls[i] = l.(*net.TCPListener)
ps[i] = ls[i].Addr().(*net.TCPAddr).Port
}
for _, l := range ls {
l.Close()
}
return ps, nil
}
func waitReady(endpoint string, timeout time.Duration) error {
client := &http.Client{Timeout: 1 * time.Second}
deadline := time.Now().Add(timeout)
+1 -1
View File
@@ -19,7 +19,7 @@ import (
"github.com/aws/aws-sdk-go-v2/credentials"
"github.com/aws/aws-sdk-go-v2/service/s3"
"github.com/seaweedfs/seaweedfs/test/s3tables/testutil"
"github.com/seaweedfs/seaweedfs/test/testutil"
)
const (
@@ -9,7 +9,7 @@ import (
"testing"
"time"
"github.com/seaweedfs/seaweedfs/test/s3tables/testutil"
"github.com/seaweedfs/seaweedfs/test/testutil"
)
// TestEnvironment mirrors the one in trino_catalog_test.go but simplified
@@ -3,7 +3,6 @@ package s3tables
import (
"context"
"fmt"
"net"
"net/http"
"os"
"path/filepath"
@@ -18,6 +17,7 @@ import (
"flag"
"github.com/seaweedfs/seaweedfs/test/testutil"
"github.com/seaweedfs/seaweedfs/weed/command"
"github.com/seaweedfs/seaweedfs/weed/glog"
"github.com/seaweedfs/seaweedfs/weed/s3api/s3tables"
@@ -590,40 +590,12 @@ func testTargetOperations(t *testing.T, client *S3TablesClient) {
// Helper functions
// findAvailablePorts finds n available ports by binding to port 0 multiple times
// It keeps the listeners open until all ports are found to ensure uniqueness
func findAvailablePorts(n int) ([]int, error) {
listeners := make([]*net.TCPListener, n)
ports := make([]int, n)
// Open all listeners to ensure we get unique ports
for i := 0; i < n; i++ {
listener, err := net.Listen("tcp", "127.0.0.1:0")
if err != nil {
// Close valid listeners before returning error
for j := 0; j < i; j++ {
listeners[j].Close()
}
return nil, err
}
listeners[i] = listener.(*net.TCPListener)
ports[i] = listeners[i].Addr().(*net.TCPAddr).Port
}
// Close all listeners
for _, l := range listeners {
l.Close()
}
return ports, nil
}
// startMiniClusterInDir starts a weed mini instance using testDir as the data
// directory. It does not require a *testing.T so it can be called from TestMain.
// extraArgs are appended to the default mini command flags.
func startMiniClusterInDir(testDir string, extraArgs []string) (*TestCluster, error) {
// We need 10 unique ports: Master(2), Volume(2), Filer(2), S3(2), Admin(2)
ports, err := findAvailablePorts(10)
ports, err := testutil.AllocatePorts(10)
if err != nil {
return nil, fmt.Errorf("failed to find available ports: %v", err)
}
-92
View File
@@ -1,92 +0,0 @@
package testutil
import (
"context"
"fmt"
"net"
"net/http"
"os/exec"
"testing"
"time"
)
func HasDocker() bool {
cmd := exec.Command("docker", "version")
return cmd.Run() == nil
}
// MustFreePortPair is a convenience wrapper for tests that only need a single pair.
// Prefer MustAllocatePorts when allocating multiple pairs to guarantee uniqueness.
func MustFreePortPair(t *testing.T, name string) (int, int) {
ports := MustAllocatePorts(t, 2)
return ports[0], ports[1]
}
// MustAllocatePorts allocates count unique free ports atomically.
// All listeners are held open until every port is obtained, preventing
// the OS from recycling a port between successive allocations.
func MustAllocatePorts(t *testing.T, count int) []int {
t.Helper()
ports, err := AllocatePorts(count)
if err != nil {
t.Fatalf("Failed to allocate %d free ports: %v", count, err)
}
return ports
}
// AllocatePorts allocates count unique free ports atomically.
func AllocatePorts(count int) ([]int, error) {
listeners := make([]net.Listener, 0, count)
ports := make([]int, 0, count)
for i := 0; i < count; i++ {
l, err := net.Listen("tcp", "0.0.0.0:0")
if err != nil {
for _, ll := range listeners {
_ = ll.Close()
}
return nil, err
}
listeners = append(listeners, l)
ports = append(ports, l.Addr().(*net.TCPAddr).Port)
}
for _, l := range listeners {
_ = l.Close()
}
return ports, nil
}
func WaitForService(url string, timeout time.Duration) bool {
client := &http.Client{Timeout: 2 * time.Second}
ctx, cancel := context.WithTimeout(context.Background(), timeout)
defer cancel()
ticker := time.NewTicker(500 * time.Millisecond)
defer ticker.Stop()
for {
select {
case <-ctx.Done():
return false
case <-ticker.C:
resp, err := client.Get(url)
if err == nil {
resp.Body.Close()
return true
}
}
}
}
func WaitForPort(port int, timeout time.Duration) bool {
deadline := time.Now().Add(timeout)
address := fmt.Sprintf("127.0.0.1:%d", port)
for time.Now().Before(deadline) {
conn, err := net.DialTimeout("tcp", address, 500*time.Millisecond)
if err == nil {
_ = conn.Close()
return true
}
time.Sleep(100 * time.Millisecond)
}
return false
}
-93
View File
@@ -1,93 +0,0 @@
package testutil
import (
"fmt"
"math/rand"
"net"
"os"
"path/filepath"
"testing"
"time"
)
const SeaweedMiniStartupTimeout = 45 * time.Second
func FindBindIP() string {
addrs, err := net.InterfaceAddrs()
if err != nil {
return "127.0.0.1"
}
for _, addr := range addrs {
ipNet, ok := addr.(*net.IPNet)
if !ok || ipNet.IP == nil {
continue
}
ip := ipNet.IP.To4()
if ip == nil || ip.IsLoopback() || ip.IsLinkLocalUnicast() {
continue
}
return ip.String()
}
return "127.0.0.1"
}
func WriteIAMConfig(dir, accessKey, secretKey string) (string, error) {
iamConfigPath := filepath.Join(dir, "iam_config.json")
iamConfig := fmt.Sprintf(`{
"identities": [
{
"name": "admin",
"credentials": [
{
"accessKey": "%s",
"secretKey": "%s"
}
],
"actions": [
"Admin",
"Read",
"List",
"Tagging",
"Write"
]
}
]
}`, accessKey, secretKey)
if err := os.WriteFile(iamConfigPath, []byte(iamConfig), 0644); err != nil {
return "", err
}
return iamConfigPath, nil
}
func MustFreeMiniPort(t *testing.T, name string) int {
t.Helper()
const (
minPort = 10000
maxPort = 55000
)
r := rand.New(rand.NewSource(time.Now().UnixNano()))
for i := 0; i < 1000; i++ {
port := minPort + r.Intn(maxPort-minPort)
listener, err := net.Listen("tcp", fmt.Sprintf("127.0.0.1:%d", port))
if err != nil {
continue
}
_ = listener.Close()
grpcPort := port + 10000
grpcListener, err := net.Listen("tcp", fmt.Sprintf("127.0.0.1:%d", grpcPort))
if err != nil {
continue
}
_ = grpcListener.Close()
return port
}
t.Fatalf("failed to get free weed mini port for %s", name)
return 0
}