diff --git a/weed/pb/grpc_client_server.go b/weed/pb/grpc_client_server.go index 2a119c075e..6fb1c79b54 100644 --- a/weed/pb/grpc_client_server.go +++ b/weed/pb/grpc_client_server.go @@ -167,14 +167,58 @@ func ServeGrpcOnLocalSocket(grpcServer *grpc.Server, grpcPort int) { if listener == nil { return } - listener = &localSocketListener{Listener: listener} glog.V(0).Infof("gRPC also listening on Unix socket %s", socketPath) - go func() { - if err := grpcServer.Serve(listener); err != nil && err != grpc.ErrServerStopped { - glog.Errorf("gRPC Unix socket server error on %s: %v", socketPath, err) + go serveGrpcOnLocalListener(grpcServer, socketPath, listener) +} + +// localSocketCheckInterval is how often a served socket path is checked for +// removal by something else, such as a temp cleaner aging /tmp. +var localSocketCheckInterval = time.Minute + +// serveGrpcOnLocalListener serves the socket until the gRPC server stops, +// rebinding it when the socket file is removed underneath the listener. +func serveGrpcOnLocalListener(grpcServer *grpc.Server, socketPath string, listener net.Listener) { + served := make(chan error, 1) + serve := func(l net.Listener) { + // A replaced listener must not unlink the path its successor binds; + // every exit below removes the socket file itself. + if unixListener, ok := l.(*net.UnixListener); ok { + unixListener.SetUnlinkOnClose(false) } - os.Remove(socketPath) - }() + go func() { served <- grpcServer.Serve(&localSocketListener{Listener: l}) }() + } + serve(listener) + ticker := time.NewTicker(localSocketCheckInterval) + defer ticker.Stop() + for { + select { + case err := <-served: + if err != nil && err != grpc.ErrServerStopped { + glog.Errorf("gRPC Unix socket server error on %s: %v", socketPath, err) + } + os.Remove(socketPath) + return + case <-ticker.C: + if _, err := os.Lstat(socketPath); !os.IsNotExist(err) { + continue + } + // Bind before closing the old listener, so a failed bind is + // retried on the next check and shutdown still ends this loop. + replacement := ListenOnLocalSocket(socketPath) + if replacement == nil { + continue + } + glog.Warningf("gRPC Unix socket %s was removed and has been recreated", socketPath) + listener.Close() + if err := <-served; err == nil || err == grpc.ErrServerStopped { + replacement.Close() + os.Remove(socketPath) + return + } + listener = replacement + serve(listener) + } + } } // localSocketListener re-applies the buffer sizes to every accepted connection. @@ -233,22 +277,21 @@ func NewGrpcServer(opts ...grpc.ServerOption) *grpc.Server { func GrpcDial(ctx context.Context, address string, waitForReady bool, opts ...grpc.DialOption) (*grpc.ClientConn, error) { var options []grpc.DialOption - // Route through Unix socket if one is registered for this address's port + // Route through Unix socket if one is registered for this address's port. + // The socket file can vanish under a running server (temp cleaners age + // /tmp), so a failed socket dial falls back to TCP. if socketPath := resolveLocalGrpcSocket(address); socketPath != "" { - options = append(options, grpc.WithContextDialer(func(ctx context.Context, _ string) (net.Conn, error) { + options = append(options, grpc.WithContextDialer(func(ctx context.Context, addr string) (net.Conn, error) { d := net.Dialer{Control: setLocalSocketBuffers} - return d.DialContext(ctx, "unix", socketPath) + conn, err := d.DialContext(ctx, "unix", socketPath) + if err != nil && ctx.Err() == nil { + glog.V(1).Infof("dial unix %s: %v, dialing %s over TCP", socketPath, err, addr) + return dialGrpcTcp(ctx, addr) + } + return conn, err })) } else { - // Always install so a conn cached before SetOutboundLocalIP binds once set; - // the stock net.Dialer until then preserves gRPC's default dial behavior. - options = append(options, grpc.WithContextDialer(func(ctx context.Context, addr string) (net.Conn, error) { - if util.OutboundLocalAddr() == nil { - var d net.Dialer - return d.DialContext(ctx, "tcp", addr) - } - return util.OutboundDialContext(ctx, "tcp", addr) - })) + options = append(options, grpc.WithContextDialer(dialGrpcTcp)) } options = append(options, @@ -272,6 +315,17 @@ func GrpcDial(ctx context.Context, address string, waitForReady bool, opts ...gr return grpc.DialContext(ctx, address, options...) } +// dialGrpcTcp is installed even before SetOutboundLocalIP so a cached conn +// binds once it is set; the stock net.Dialer until then preserves gRPC's +// default dial behavior. +func dialGrpcTcp(ctx context.Context, addr string) (net.Conn, error) { + if util.OutboundLocalAddr() == nil { + var d net.Dialer + return d.DialContext(ctx, "tcp", addr) + } + return util.OutboundDialContext(ctx, "tcp", addr) +} + func getOrCreateConnection(address string, waitForReady bool, opts ...grpc.DialOption) (*versionedGrpcClient, error) { grpcClientsLock.Lock() diff --git a/weed/pb/grpc_local_socket_test.go b/weed/pb/grpc_local_socket_test.go new file mode 100644 index 0000000000..e354e169dd --- /dev/null +++ b/weed/pb/grpc_local_socket_test.go @@ -0,0 +1,142 @@ +package pb + +import ( + "context" + "net" + "os" + "path/filepath" + "runtime" + "strconv" + "testing" + "time" + + "google.golang.org/grpc" + "google.golang.org/grpc/credentials/insecure" + "google.golang.org/grpc/health" + healthpb "google.golang.org/grpc/health/grpc_health_v1" +) + +func isolateLocalGrpcSockets(t *testing.T) { + t.Helper() + if runtime.GOOS == "windows" { + t.Skip("Unix-socket routing is disabled on Windows") + } + localGrpcSocketsLock.Lock() + prevSockets, prevHosts := localGrpcSockets, localGrpcHosts + localGrpcSockets = make(map[int]string) + localGrpcHosts = make(map[int]map[string]struct{}) + localGrpcSocketsLock.Unlock() + t.Cleanup(func() { + localGrpcSocketsLock.Lock() + localGrpcSockets, localGrpcHosts = prevSockets, prevHosts + localGrpcSocketsLock.Unlock() + }) +} + +// localSocketDir is short because TMPDIR can push socket paths past the +// sockaddr_un limit. +func localSocketDir(t *testing.T) string { + t.Helper() + dir, err := os.MkdirTemp("/tmp", "swsock") + if err != nil { + t.Fatalf("create socket dir: %v", err) + } + t.Cleanup(func() { os.RemoveAll(dir) }) + return dir +} + +func startHealthGrpcServer(t *testing.T) (*grpc.Server, int) { + t.Helper() + listener, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatalf("listen: %v", err) + } + grpcServer := grpc.NewServer() + healthpb.RegisterHealthServer(grpcServer, health.NewServer()) + go grpcServer.Serve(listener) + t.Cleanup(grpcServer.Stop) + return grpcServer, listener.Addr().(*net.TCPAddr).Port +} + +func checkGrpcHealth(t *testing.T, address string) { + t.Helper() + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + conn, err := GrpcDial(ctx, address, false, grpc.WithTransportCredentials(insecure.NewCredentials())) + if err != nil { + t.Fatalf("dial %s: %v", address, err) + } + defer conn.Close() + if _, err := healthpb.NewHealthClient(conn).Check(ctx, &healthpb.HealthCheckRequest{}); err != nil { + t.Fatalf("health check over %s: %v", address, err) + } +} + +func TestGrpcDialFallsBackToTcpWhenLocalSocketIsMissing(t *testing.T) { + isolateLocalGrpcSockets(t) + _, port := startHealthGrpcServer(t) + RegisterLocalGrpcSocket("127.0.0.1", port, filepath.Join(localSocketDir(t), "removed.sock")) + + checkGrpcHealth(t, net.JoinHostPort("127.0.0.1", strconv.Itoa(port))) +} + +func TestServeGrpcOnLocalSocketRecreatesRemovedSocket(t *testing.T) { + isolateLocalGrpcSockets(t) + prevInterval := localSocketCheckInterval + localSocketCheckInterval = 10 * time.Millisecond + t.Cleanup(func() { localSocketCheckInterval = prevInterval }) + + grpcServer, port := startHealthGrpcServer(t) + socketDir := localSocketDir(t) + socketPath := filepath.Join(socketDir, "grpc.sock") + RegisterLocalGrpcSocket("127.0.0.1", port, socketPath) + ServeGrpcOnLocalSocket(grpcServer, port) + waitForSocketFile(t, socketPath) + + // Rebinding fails while the folder is gone and must be retried later. + if err := os.RemoveAll(socketDir); err != nil { + t.Fatalf("remove socket dir: %v", err) + } + time.Sleep(10 * localSocketCheckInterval) + if err := os.Mkdir(socketDir, 0o700); err != nil { + t.Fatalf("recreate socket dir: %v", err) + } + waitForSocketFile(t, socketPath) + + conn, err := grpc.NewClient("unix:"+socketPath, grpc.WithTransportCredentials(insecure.NewCredentials())) + if err != nil { + t.Fatalf("dial recreated socket: %v", err) + } + defer conn.Close() + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + if _, err := healthpb.NewHealthClient(conn).Check(ctx, &healthpb.HealthCheckRequest{}); err != nil { + t.Fatalf("health check over the recreated socket: %v", err) + } + + grpcServer.Stop() + deadline := time.Now().Add(5 * time.Second) + for { + if _, err := os.Lstat(socketPath); os.IsNotExist(err) { + break + } + if time.Now().After(deadline) { + t.Fatalf("socket %s left behind after the server stopped", socketPath) + } + time.Sleep(10 * time.Millisecond) + } +} + +func waitForSocketFile(t *testing.T, socketPath string) { + t.Helper() + deadline := time.Now().Add(5 * time.Second) + for { + if info, err := os.Lstat(socketPath); err == nil && info.Mode()&os.ModeSocket != 0 { + return + } + if time.Now().After(deadline) { + t.Fatalf("socket %s was not created", socketPath) + } + time.Sleep(10 * time.Millisecond) + } +}