From cd97c93deb7a5ad2df8261a69e9219505cdf21e3 Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Fri, 23 Oct 2026 16:07:32 -0700 Subject: [PATCH] command: keep filer, s3, and sftp up when the local unix socket cannot bind (#11999) * pb: share the stale-socket tolerant unix listener ServeGrpcOnLocalSocket already treats the local socket as optional; pull the remove-and-listen block out so the filer, s3, and sftp http sockets can use it too. * command: keep filer, s3, and sftp up when the local socket cannot bind A stale socket file owned by another uid (e.g. left in /tmp by a containerized job) made os.Remove fail with EPERM and the Fatalf took the whole server down. The gRPC local socket already treats the unix listener as optional; do the same for the http/sftp sockets. * filer: clear shared localSocket flag when the unix socket cannot bind s3.localFilerSocket and (in weed server) the sftp local socket share the filer's -localSocket flag pointer. If the filer cannot bind the socket it now keeps running without it, so consumers still holding the configured path would dial a dead or foreign-owned socket. * command: keep sftp on its own localSocket flag under weed server sftpOptions.localSocket was overwritten with the filer's -localSocket flag, so -sftp.localSocket was ignored and an explicit filer socket path made the sftp server try to bind the same socket the filer already held. * s3: fall back to TCP when the filer local socket cannot be dialed The embedded S3 server copies the filer's -localSocket flag before the filer tries to bind it, so a socket that never comes up left every filer HTTP request dialing a dead path. Try the unix socket first and fall back to the request's TCP address on dial failure. * filer: keep the shared localSocket flag read-only after startup Clearing *fo.localSocket raced with the embedded s3 goroutine reading the same flag pointer and could not help a reader that had already copied it. The s3 filer client's TCP fallback covers the unbound-socket case without mutating shared flag state. --- weed/command/filer.go | 18 ++---------------- weed/command/s3.go | 17 ++--------------- weed/command/server.go | 1 - weed/command/sftp.go | 28 +++++++--------------------- weed/pb/grpc_client_server.go | 25 ++++++++++++++++++------- weed/s3api/s3api_server.go | 8 ++++++-- 6 files changed, 35 insertions(+), 62 deletions(-) diff --git a/weed/command/filer.go b/weed/command/filer.go index 7a9c1300f2..5be17a26af 100644 --- a/weed/command/filer.go +++ b/weed/command/filer.go @@ -5,7 +5,6 @@ import ( "crypto/tls" "crypto/x509" "fmt" - "net" "net/http" "os" "runtime" @@ -522,27 +521,14 @@ func (fo *FilerOptions) startFiler() { } stopGrpcServer := func() { gracefulStopGrpc(grpcS, gracefulTimeout) } - var socketServer *http.Server if runtime.GOOS != "windows" { localSocket := *fo.localSocket explicit := localSocket != "" if !explicit { localSocket = fmt.Sprintf("/tmp/seaweedfs-filer-%d.sock", *fo.port) } - if err := os.Remove(localSocket); err != nil && !os.IsNotExist(err) { - if explicit { - glog.Fatalf("Failed to remove %s, error: %s", localSocket, err.Error()) - } - glog.Warningf("Failed to remove %s, error: %s", localSocket, err.Error()) - } - filerSocketListener, err := net.Listen("unix", localSocket) - if err != nil { - if explicit { - glog.Fatalf("Failed to listen on %s: %v", localSocket, err) - } - glog.Errorf("Failed to listen on %s, skipping local socket server: %v", localSocket, err) - } else { - socketServer = newHttpServer(defaultHandler, nil) + if filerSocketListener := pb.ListenOnLocalSocket(localSocket); filerSocketListener != nil { + socketServer := newHttpServer(defaultHandler, nil) httpServers = append(httpServers, socketServer) go socketServer.Serve(filerSocketListener) } diff --git a/weed/command/s3.go b/weed/command/s3.go index a37d7c8179..e2d377c64e 100644 --- a/weed/command/s3.go +++ b/weed/command/s3.go @@ -6,7 +6,6 @@ import ( "crypto/x509" "fmt" "io/ioutil" - "net" "net/http" "os" "runtime" @@ -433,24 +432,12 @@ func (s3opt *S3Options) startS3Server() bool { if !explicit { localSocket = fmt.Sprintf("/tmp/seaweedfs-s3-%d.sock", *s3opt.port) } - if err := os.Remove(localSocket); err != nil && !os.IsNotExist(err) { - if explicit { - glog.Fatalf("Failed to remove %s, error: %s", localSocket, err.Error()) - } - glog.Warningf("Failed to remove %s, error: %s", localSocket, err.Error()) - } - s3SocketListener, err := net.Listen("unix", localSocket) - if err != nil { - if explicit { - glog.Fatalf("Failed to listen on %s: %v", localSocket, err) - } - glog.Errorf("Failed to listen on %s, skipping local socket server: %v", localSocket, err) - } else { + if s3SocketListener := pb.ListenOnLocalSocket(localSocket); s3SocketListener != nil { socketServer := newHttpServer(router, nil) httpServers = append(httpServers, socketServer) go func() { if err := socketServer.Serve(s3SocketListener); err != nil && err != http.ErrServerClosed { - glog.Errorf("Failed to start S3 http server on %s: %v", localSocket, err) + glog.Fatalf("Failed to start S3 http server: %v", err) } }() } diff --git a/weed/command/server.go b/weed/command/server.go index a9b27bc064..5c800b3004 100644 --- a/weed/command/server.go +++ b/weed/command/server.go @@ -422,7 +422,6 @@ func runServer(cmd *Command, args []string) bool { if *isStartingSftp { go func() { time.Sleep(2 * time.Second) - sftpOptions.localSocket = filerOptions.localSocket sftpOptions.startSftpServer() }() } diff --git a/weed/command/sftp.go b/weed/command/sftp.go index 054096392d..eb482d018b 100644 --- a/weed/command/sftp.go +++ b/weed/command/sftp.go @@ -3,8 +3,6 @@ package command import ( "context" "fmt" - "net" - "os" "runtime" "time" @@ -184,26 +182,14 @@ func (sftpOpt *SftpOptions) startSftpServer() bool { if !explicit { localSocket = fmt.Sprintf("/tmp/seaweedfs-sftp-%d.sock", *sftpOpt.port) } - if err := os.Remove(localSocket); err != nil && !os.IsNotExist(err) { - if explicit { - glog.Fatalf("Failed to remove %s, error: %s", localSocket, err.Error()) - } - glog.Warningf("Failed to remove %s, error: %s", localSocket, err.Error()) - } - go func() { - // start on local unix socket - sftpSocketListener, err := net.Listen("unix", localSocket) - if err != nil { - if explicit { - glog.Fatalf("Failed to listen on %s: %v", localSocket, err) + if sftpSocketListener := pb.ListenOnLocalSocket(localSocket); sftpSocketListener != nil { + go func() { + // start on local unix socket + if err := service.Serve(sftpSocketListener); err != nil { + glog.Fatalf("Failed to serve SFTP on socket %s: %v", localSocket, err) } - glog.Errorf("Failed to listen on %s, skipping local socket server: %v", localSocket, err) - return - } - if err := service.Serve(sftpSocketListener); err != nil { - glog.Errorf("Failed to serve SFTP on socket %s: %v", localSocket, err) - } - }() + }() + } } // Start the SFTP service on TCP diff --git a/weed/pb/grpc_client_server.go b/weed/pb/grpc_client_server.go index 8ba3a16f33..2a119c075e 100644 --- a/weed/pb/grpc_client_server.go +++ b/weed/pb/grpc_client_server.go @@ -140,6 +140,22 @@ func resolveLocalGrpcSocket(address string) string { return localGrpcSockets[port] } +// ListenOnLocalSocket binds a unix socket listener for a local fast path +// next to the TCP listener. A stale socket file is removed first; a path +// that cannot be cleared or bound is skipped, since the socket is optional. +func ListenOnLocalSocket(socketPath string) net.Listener { + if err := os.Remove(socketPath); err != nil && !os.IsNotExist(err) { + glog.Warningf("Failed to remove old socket %s: %v", socketPath, err) + } + lc := net.ListenConfig{Control: setLocalSocketBuffers} + listener, err := lc.Listen(context.Background(), "unix", socketPath) + if err != nil { + glog.Errorf("Failed to listen on Unix socket %s: %v", socketPath, err) + return nil + } + return listener +} + // ServeGrpcOnLocalSocket starts serving a gRPC server on a Unix socket // if one is registered for the given port. func ServeGrpcOnLocalSocket(grpcServer *grpc.Server, grpcPort int) { @@ -147,13 +163,8 @@ func ServeGrpcOnLocalSocket(grpcServer *grpc.Server, grpcPort int) { if socketPath == "" { return } - if err := os.Remove(socketPath); err != nil && !os.IsNotExist(err) { - glog.Warningf("Failed to remove old gRPC socket %s: %v", socketPath, err) - } - lc := net.ListenConfig{Control: setLocalSocketBuffers} - listener, err := lc.Listen(context.Background(), "unix", socketPath) - if err != nil { - glog.Errorf("Failed to listen on gRPC Unix socket %s: %v", socketPath, err) + listener := ListenOnLocalSocket(socketPath) + if listener == nil { return } listener = &localSocketListener{Listener: listener} diff --git a/weed/s3api/s3api_server.go b/weed/s3api/s3api_server.go index 86b1025081..c10e311e3d 100644 --- a/weed/s3api/s3api_server.go +++ b/weed/s3api/s3api_server.go @@ -507,8 +507,12 @@ func NewS3ApiServerWithStore(router *mux.Router, option *S3ApiServerOption, expl } else { s3ApiServer.client = &http.Client{ Transport: &http.Transport{ - DialContext: func(_ context.Context, _, _ string) (net.Conn, error) { - return net.Dial("unix", option.LocalFilerSocket) + DialContext: func(ctx context.Context, _, addr string) (net.Conn, error) { + conn, err := (&net.Dialer{}).DialContext(ctx, "unix", option.LocalFilerSocket) + if err != nil { + return util.OutboundDialContext(ctx, "tcp", addr) + } + return conn, nil }, }, }