command: keep filer, s3, and sftp up when the local unix socket cannot bind (#11999)

* pb: share the stale-socket tolerant unix listener

ServeGrpcOnLocalSocket already treats the local socket as optional; pull the remove-and-listen block out so the filer, s3, and sftp http sockets can use it too.

* command: keep filer, s3, and sftp up when the local socket cannot bind

A stale socket file owned by another uid (e.g. left in /tmp by a containerized job) made os.Remove fail with EPERM and the Fatalf took the whole server down. The gRPC local socket already treats the unix listener as optional; do the same for the http/sftp sockets.

* filer: clear shared localSocket flag when the unix socket cannot bind

s3.localFilerSocket and (in weed server) the sftp local socket share the
filer's -localSocket flag pointer. If the filer cannot bind the socket it
now keeps running without it, so consumers still holding the configured
path would dial a dead or foreign-owned socket.

* command: keep sftp on its own localSocket flag under weed server

sftpOptions.localSocket was overwritten with the filer's -localSocket
flag, so -sftp.localSocket was ignored and an explicit filer socket path
made the sftp server try to bind the same socket the filer already held.

* s3: fall back to TCP when the filer local socket cannot be dialed

The embedded S3 server copies the filer's -localSocket flag before the filer tries to bind it, so a socket that never comes up left every filer HTTP request dialing a dead path. Try the unix socket first and fall back to the request's TCP address on dial failure.

* filer: keep the shared localSocket flag read-only after startup

Clearing *fo.localSocket raced with the embedded s3 goroutine reading the same flag pointer and could not help a reader that had already copied it. The s3 filer client's TCP fallback covers the unbound-socket case without mutating shared flag state.

weed/command/filer.go

@@ -5,7 +5,6 @@ import (
 	"crypto/tls"
 	"crypto/x509"
 	"fmt"
-	"net"
 	"net/http"
 	"os"
 	"runtime"
@@ -522,27 +521,14 @@ func (fo *FilerOptions) startFiler() {
 	}
 	stopGrpcServer := func() { gracefulStopGrpc(grpcS, gracefulTimeout) }
 
-	var socketServer *http.Server
 	if runtime.GOOS != "windows" {
 		localSocket := *fo.localSocket
 		explicit := localSocket != ""
 		if !explicit {
 			localSocket = fmt.Sprintf("/tmp/seaweedfs-filer-%d.sock", *fo.port)
 		}
-		if err := os.Remove(localSocket); err != nil && !os.IsNotExist(err) {
-			if explicit {
-				glog.Fatalf("Failed to remove %s, error: %s", localSocket, err.Error())
-			}
-			glog.Warningf("Failed to remove %s, error: %s", localSocket, err.Error())
-		}
-		filerSocketListener, err := net.Listen("unix", localSocket)
-		if err != nil {
-			if explicit {
-				glog.Fatalf("Failed to listen on %s: %v", localSocket, err)
-			}
-			glog.Errorf("Failed to listen on %s, skipping local socket server: %v", localSocket, err)
-		} else {
-			socketServer = newHttpServer(defaultHandler, nil)
+		if filerSocketListener := pb.ListenOnLocalSocket(localSocket); filerSocketListener != nil {
+			socketServer := newHttpServer(defaultHandler, nil)
 			httpServers = append(httpServers, socketServer)
 			go socketServer.Serve(filerSocketListener)
 		}

weed/command/s3.go

@@ -6,7 +6,6 @@ import (
 	"crypto/x509"
 	"fmt"
 	"io/ioutil"
-	"net"
 	"net/http"
 	"os"
 	"runtime"
@@ -433,24 +432,12 @@ func (s3opt *S3Options) startS3Server() bool {
 		if !explicit {
 			localSocket = fmt.Sprintf("/tmp/seaweedfs-s3-%d.sock", *s3opt.port)
 		}
-		if err := os.Remove(localSocket); err != nil && !os.IsNotExist(err) {
-			if explicit {
-				glog.Fatalf("Failed to remove %s, error: %s", localSocket, err.Error())
-			}
-			glog.Warningf("Failed to remove %s, error: %s", localSocket, err.Error())
-		}
-		s3SocketListener, err := net.Listen("unix", localSocket)
-		if err != nil {
-			if explicit {
-				glog.Fatalf("Failed to listen on %s: %v", localSocket, err)
-			}
-			glog.Errorf("Failed to listen on %s, skipping local socket server: %v", localSocket, err)
-		} else {
+		if s3SocketListener := pb.ListenOnLocalSocket(localSocket); s3SocketListener != nil {
 			socketServer := newHttpServer(router, nil)
 			httpServers = append(httpServers, socketServer)
 			go func() {
 				if err := socketServer.Serve(s3SocketListener); err != nil && err != http.ErrServerClosed {
-					glog.Errorf("Failed to start S3 http server on %s: %v", localSocket, err)
+					glog.Fatalf("Failed to start S3 http server: %v", err)
 				}
 			}()
 		}

weed/command/server.go

@@ -422,7 +422,6 @@ func runServer(cmd *Command, args []string) bool {
 	if *isStartingSftp {
 		go func() {
 			time.Sleep(2 * time.Second)
-			sftpOptions.localSocket = filerOptions.localSocket
 			sftpOptions.startSftpServer()
 		}()
 	}

weed/command/sftp.go

@@ -3,8 +3,6 @@ package command
 import (
 	"context"
 	"fmt"
-	"net"
-	"os"
 	"runtime"
 	"time"
 
@@ -184,26 +182,14 @@ func (sftpOpt *SftpOptions) startSftpServer() bool {
 		if !explicit {
 			localSocket = fmt.Sprintf("/tmp/seaweedfs-sftp-%d.sock", *sftpOpt.port)
 		}
-		if err := os.Remove(localSocket); err != nil && !os.IsNotExist(err) {
-			if explicit {
-				glog.Fatalf("Failed to remove %s, error: %s", localSocket, err.Error())
-			}
-			glog.Warningf("Failed to remove %s, error: %s", localSocket, err.Error())
-		}
-		go func() {
-			// start on local unix socket
-			sftpSocketListener, err := net.Listen("unix", localSocket)
-			if err != nil {
-				if explicit {
-					glog.Fatalf("Failed to listen on %s: %v", localSocket, err)
+		if sftpSocketListener := pb.ListenOnLocalSocket(localSocket); sftpSocketListener != nil {
+			go func() {
+				// start on local unix socket
+				if err := service.Serve(sftpSocketListener); err != nil {
+					glog.Fatalf("Failed to serve SFTP on socket %s: %v", localSocket, err)
 				}
-				glog.Errorf("Failed to listen on %s, skipping local socket server: %v", localSocket, err)
-				return
-			}
-			if err := service.Serve(sftpSocketListener); err != nil {
-				glog.Errorf("Failed to serve SFTP on socket %s: %v", localSocket, err)
-			}
-		}()
+			}()
+		}
 	}
 
 	// Start the SFTP service on TCP

weed/pb/grpc_client_server.go

@@ -140,6 +140,22 @@ func resolveLocalGrpcSocket(address string) string {
 	return localGrpcSockets[port]
 }
 
+// ListenOnLocalSocket binds a unix socket listener for a local fast path
+// next to the TCP listener. A stale socket file is removed first; a path
+// that cannot be cleared or bound is skipped, since the socket is optional.
+func ListenOnLocalSocket(socketPath string) net.Listener {
+	if err := os.Remove(socketPath); err != nil && !os.IsNotExist(err) {
+		glog.Warningf("Failed to remove old socket %s: %v", socketPath, err)
+	}
+	lc := net.ListenConfig{Control: setLocalSocketBuffers}
+	listener, err := lc.Listen(context.Background(), "unix", socketPath)
+	if err != nil {
+		glog.Errorf("Failed to listen on Unix socket %s: %v", socketPath, err)
+		return nil
+	}
+	return listener
+}
+
 // ServeGrpcOnLocalSocket starts serving a gRPC server on a Unix socket
 // if one is registered for the given port.
 func ServeGrpcOnLocalSocket(grpcServer *grpc.Server, grpcPort int) {
@@ -147,13 +163,8 @@ func ServeGrpcOnLocalSocket(grpcServer *grpc.Server, grpcPort int) {
 	if socketPath == "" {
 		return
 	}
-	if err := os.Remove(socketPath); err != nil && !os.IsNotExist(err) {
-		glog.Warningf("Failed to remove old gRPC socket %s: %v", socketPath, err)
-	}
-	lc := net.ListenConfig{Control: setLocalSocketBuffers}
-	listener, err := lc.Listen(context.Background(), "unix", socketPath)
-	if err != nil {
-		glog.Errorf("Failed to listen on gRPC Unix socket %s: %v", socketPath, err)
+	listener := ListenOnLocalSocket(socketPath)
+	if listener == nil {
 		return
 	}
 	listener = &localSocketListener{Listener: listener}

weed/s3api/s3api_server.go

@@ -507,8 +507,12 @@ func NewS3ApiServerWithStore(router *mux.Router, option *S3ApiServerOption, expl
 	} else {
 		s3ApiServer.client = &http.Client{
 			Transport: &http.Transport{
-				DialContext: func(_ context.Context, _, _ string) (net.Conn, error) {
-					return net.Dial("unix", option.LocalFilerSocket)
+				DialContext: func(ctx context.Context, _, addr string) (net.Conn, error) {
+					conn, err := (&net.Dialer{}).DialContext(ctx, "unix", option.LocalFilerSocket)
+					if err != nil {
+						return util.OutboundDialContext(ctx, "tcp", addr)
+					}
+					return conn, nil
 				},
 			},
 		}