seaweedfs/weed/pb/grpc_client_server_test.go

package pb

import (
	"context"
	"fmt"
	"runtime"
	"testing"

	"google.golang.org/grpc"
	"google.golang.org/grpc/codes"
	"google.golang.org/grpc/status"
)

// TestShouldInvalidateConnection_MarshalErrorIsPerRequest ensures that a
// client-side proto marshal failure does NOT cause the shared cached
// ClientConn to be torn down. Tearing it down would cancel every other
// in-flight RPC (seaweedfs#9139: one file with invalid-UTF-8 bytes triggered
// an avalanche of "connection is closing" errors on unrelated operations).
func TestShouldInvalidateConnection_MarshalErrorIsPerRequest(t *testing.T) {
	// Reproduces the exact error gRPC returns when a string field in the
	// outgoing request contains invalid UTF-8 bytes.
	marshalErr := status.Error(codes.Internal,
		"grpc: error while marshaling: string field contains invalid UTF-8")
	if shouldInvalidateConnection(context.Background(), marshalErr) {
		t.Fatalf("client-side marshal error must not invalidate the shared connection")
	}

	// Same error wrapped with fmt.Errorf (common when callers add context).
	wrapped := fmt.Errorf("upload data: %w", marshalErr)
	if shouldInvalidateConnection(context.Background(), wrapped) {
		t.Fatalf("wrapped marshal error must not invalidate the shared connection")
	}
}

// TestShouldInvalidateConnection_CallerContextExpiryIsPerRequest ensures that a
// Canceled/DeadlineExceeded caused by the caller's own context expiring does NOT
// tear down the shared cached ClientConn. Doing so would cancel every other
// in-flight RPC on it with "the client connection is closing" — the cascade
// that turned one slow chunk assign into a flood of failures during a
// high-concurrency upload (seaweedfs#9765).
func TestShouldInvalidateConnection_CallerContextExpiryIsPerRequest(t *testing.T) {
	expired, cancel := context.WithCancel(context.Background())
	cancel()
	for _, code := range []codes.Code{codes.Canceled, codes.DeadlineExceeded} {
		err := status.Error(code, "context expired")
		if shouldInvalidateConnection(expired, err) {
			t.Fatalf("%v with an expired caller context must not invalidate the shared connection", code)
		}
	}
}

// TestShouldInvalidateConnection_StaleChannelStillInvalidates ensures the
// carve-out above is gated on the RPC's context: a Canceled/DeadlineExceeded
// while that context is still live is the genuine stale-channel signal (e.g. a
// peer restart behind a k8s Service VIP) and must still invalidate so the next
// attempt dials fresh.
func TestShouldInvalidateConnection_StaleChannelStillInvalidates(t *testing.T) {
	live, cancel := context.WithCancel(context.Background())
	defer cancel()
	for _, code := range []codes.Code{codes.Canceled, codes.DeadlineExceeded} {
		err := status.Error(code, "stale channel")
		if !shouldInvalidateConnection(live, err) {
			t.Fatalf("%v with a live attempt context must still invalidate the connection", code)
		}
	}
}

// TestShouldInvalidateConnection_NonCancellableContextIsNoEvidence covers
// seaweedfs#10947. Background/TODO never expires, so Err() stays nil forever and
// the guard above answered "stale channel" for every caller that has no deadline
// to honor — which is almost all of them, the S3 gateway included. One client
// abandoning a request then closed the shared filer channel and every concurrent
// multipart part died with "the client connection is closing".
func TestShouldInvalidateConnection_NonCancellableContextIsNoEvidence(t *testing.T) {
	for _, ctx := range []context.Context{context.Background(), context.TODO(), nil} {
		for _, code := range []codes.Code{codes.Canceled, codes.DeadlineExceeded} {
			err := status.Error(code, "context canceled")
			if shouldInvalidateConnection(ctx, err) {
				t.Fatalf("%v must not invalidate the shared connection on a non-cancellable context", code)
			}
		}
	}
}

// TestShouldInvalidateConnection_ClientConnClosingIsNotStale ensures a caller
// whose RPC was rejected because this process is already closing the shared
// ClientConn does not close it again. gRPC raises ErrClientConnClosing locally,
// so it is only ever the footprint of another goroutine's teardown - reading it
// as a stale-channel signal lets one teardown re-arm itself across the herd of
// callers it just cancelled.
func TestShouldInvalidateConnection_ClientConnClosingIsNotStale(t *testing.T) {
	live, cancel := context.WithCancel(context.Background())
	defer cancel()
	for _, err := range []error{grpc.ErrClientConnClosing, fmt.Errorf("assign volume: %w", grpc.ErrClientConnClosing)} {
		if shouldInvalidateConnection(live, err) {
			t.Fatalf("%v must not invalidate the shared connection", err)
		}
	}
}

// TestShouldInvalidateConnection_ResourceExhaustedIsPerRequest ensures the
// master's growth-in-progress assign shed (codes.ResourceExhausted) does NOT
// tear down the shared cached ClientConn. The shed fires per-request across a
// herd of concurrent assigns; invalidating on it would cancel every other
// in-flight assign with "the client connection is closing" — the cascade in
// seaweedfs#10118. It is retried by the caller without touching the channel.
func TestShouldInvalidateConnection_ResourceExhaustedIsPerRequest(t *testing.T) {
	shed := status.Error(codes.ResourceExhausted, "no writable volumes for x, volume growth in progress")
	if shouldInvalidateConnection(context.Background(), shed) {
		t.Fatalf("ResourceExhausted shed must not invalidate the shared connection")
	}
	if shouldInvalidateConnection(nil, shed) {
		t.Fatalf("ResourceExhausted shed must not invalidate the shared connection (nil ctx)")
	}
}

// TestShouldInvalidateConnection_GenuineInternalStillInvalidates ensures the
// marshal-error carve-out does not swallow real server-side Internal errors,
// which previously caused — and should continue to cause — connection
// invalidation.
func TestShouldInvalidateConnection_GenuineInternalStillInvalidates(t *testing.T) {
	serverInternal := status.Error(codes.Internal, "stream terminated by RST_STREAM with code 2")
	if !shouldInvalidateConnection(context.Background(), serverInternal) {
		t.Fatalf("genuine server-side Internal must still invalidate the connection")
	}
}

// TestShouldInvalidateConnection_TransportErrorsStillInvalidate is a
// regression guard for the string-matching fallback path (e.g. a raw
// "connection refused" from net.Dial that never acquired a gRPC status).
func TestShouldInvalidateConnection_TransportErrorsStillInvalidate(t *testing.T) {
	for _, msg := range []string{
		"rpc error: code = Unavailable desc = transport is closing",
		"dial tcp: connection refused",
		"read: connection reset by peer",
	} {
		if !shouldInvalidateConnection(context.Background(), fmt.Errorf("%s", msg)) {
			t.Fatalf("transport error %q must still invalidate", msg)
		}
	}
}

// TestIsClientSideMarshalError_RequiresGrpcStatus ensures the carve-out is
// type-based (via errors.As on the grpc status interface), not a naive
// string match against arbitrary errors that happen to mention marshaling.
// A plain errors.New(...) with the same prefix must NOT be treated as a
// per-request marshal error — we have no evidence the connection is healthy.
func TestIsClientSideMarshalError_RequiresGrpcStatus(t *testing.T) {
	impostor := fmt.Errorf("grpc: error while marshaling: synthetic non-status error")
	if isClientSideMarshalError(impostor) {
		t.Fatalf("plain error must not match the marshal-error carve-out")
	}
}

// TestResolveLocalGrpcSocket_RemotePortCollision is a regression test for
// issue #9254. A `weed server` process registers a Unix socket for its
// in-process volume server on host A. A standalone `weed volume` on host B
// happens to use the same gRPC port. Dials from the master to host B must
// continue out over TCP — they must NOT be hijacked into host A's local
// socket on the basis of port match alone.
func TestResolveLocalGrpcSocket_RemotePortCollision(t *testing.T) {
	if runtime.GOOS == "windows" {
		t.Skip("Unix-socket routing is disabled on Windows (#9430)")
	}
	// Snapshot and restore global state so the test does not leak into others.
	localGrpcSocketsLock.Lock()
	prevSockets := localGrpcSockets
	prevHosts := localGrpcHosts
	localGrpcSockets = make(map[int]string)
	localGrpcHosts = make(map[int]map[string]struct{})
	localGrpcSocketsLock.Unlock()
	t.Cleanup(func() {
		localGrpcSocketsLock.Lock()
		localGrpcSockets = prevSockets
		localGrpcHosts = prevHosts
		localGrpcSocketsLock.Unlock()
	})

	const localHost = "10.0.0.2"
	const remoteHost = "10.0.0.3"
	const collidingPort = 17334
	const socketPath = "/tmp/seaweedfs-volume-grpc-17334.sock"

	RegisterLocalGrpcSocket(localHost, collidingPort, socketPath)

	cases := []struct {
		name    string
		address string
		want    string
	}{
		{"local advertised host routes to socket", localHost + ":17334", socketPath},
		{"loopback v4 routes to socket", "127.0.0.1:17334", socketPath},
		{"localhost routes to socket", "localhost:17334", socketPath},
		{"loopback v6 routes to socket", "[::1]:17334", socketPath},
		{"empty host (bare port) routes to socket", ":17334", socketPath},
		{"remote host with same port stays on TCP", remoteHost + ":17334", ""},
		{"unrelated host with same port stays on TCP", "192.168.1.5:17334", ""},
		{"unregistered port stays on TCP", localHost + ":17335", ""},
		{"malformed address stays on TCP", "not-a-host-port", ""},
	}
	for _, tc := range cases {
		t.Run(tc.name, func(t *testing.T) {
			if got := resolveLocalGrpcSocket(tc.address); got != tc.want {
				t.Fatalf("resolveLocalGrpcSocket(%q) = %q, want %q", tc.address, got, tc.want)
			}
		})
	}
}