mirror of
https://github.com/seaweedfs/seaweedfs.git
synced 2026-11-18 09:46:24 +01:00
pb: recover when a local gRPC socket file is deleted (#12694)
* pb: fall back to TCP when the local gRPC socket cannot be dialed A temp cleaner can delete a local gRPC socket file while its server keeps running. Every later dial to that server then failed with "no such file or directory", so the master could no longer reach its own volume server and volume growth stopped. Dial the server's TCP address when the socket dial fails, as the S3 gateway already does for the filer's HTTP socket. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * pb: recreate a local gRPC socket whose file was removed Once its socket file was deleted, a server kept accepting on an unreachable listener until restart, so same-host RPCs stayed on the TCP fallback. Check the path every minute and, when the file is gone, close the old listener and bind the socket again. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * pb: retry a failed local socket rebind on the next check A failed rebind ended the watcher, so the socket stayed missing after the cause cleared. Bind the replacement before closing the old listener: a failed bind keeps the old listener serving and is retried on the next check, and the old Serve still reports server shutdown. The old listener no longer unlinks the path on close, since the replacement now owns it. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * pb: set the local socket unlink policy before serving the listener Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> --------- Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
b74971b6d4
commit
dbb2fdb93b
2 files changed
+214
-18
No files matched your search
@@ -167,14 +167,58 @@ func ServeGrpcOnLocalSocket(grpcServer *grpc.Server, grpcPort int) {
|
||||
if listener == nil {
|
||||
return
|
||||
}
|
||||
listener = &localSocketListener{Listener: listener}
|
||||
glog.V(0).Infof("gRPC also listening on Unix socket %s", socketPath)
|
||||
go func() {
|
||||
if err := grpcServer.Serve(listener); err != nil && err != grpc.ErrServerStopped {
|
||||
glog.Errorf("gRPC Unix socket server error on %s: %v", socketPath, err)
|
||||
go serveGrpcOnLocalListener(grpcServer, socketPath, listener)
|
||||
}
|
||||
|
||||
// localSocketCheckInterval is how often a served socket path is checked for
|
||||
// removal by something else, such as a temp cleaner aging /tmp.
|
||||
var localSocketCheckInterval = time.Minute
|
||||
|
||||
// serveGrpcOnLocalListener serves the socket until the gRPC server stops,
|
||||
// rebinding it when the socket file is removed underneath the listener.
|
||||
func serveGrpcOnLocalListener(grpcServer *grpc.Server, socketPath string, listener net.Listener) {
|
||||
served := make(chan error, 1)
|
||||
serve := func(l net.Listener) {
|
||||
// A replaced listener must not unlink the path its successor binds;
|
||||
// every exit below removes the socket file itself.
|
||||
if unixListener, ok := l.(*net.UnixListener); ok {
|
||||
unixListener.SetUnlinkOnClose(false)
|
||||
}
|
||||
os.Remove(socketPath)
|
||||
}()
|
||||
go func() { served <- grpcServer.Serve(&localSocketListener{Listener: l}) }()
|
||||
}
|
||||
serve(listener)
|
||||
ticker := time.NewTicker(localSocketCheckInterval)
|
||||
defer ticker.Stop()
|
||||
for {
|
||||
select {
|
||||
case err := <-served:
|
||||
if err != nil && err != grpc.ErrServerStopped {
|
||||
glog.Errorf("gRPC Unix socket server error on %s: %v", socketPath, err)
|
||||
}
|
||||
os.Remove(socketPath)
|
||||
return
|
||||
case <-ticker.C:
|
||||
if _, err := os.Lstat(socketPath); !os.IsNotExist(err) {
|
||||
continue
|
||||
}
|
||||
// Bind before closing the old listener, so a failed bind is
|
||||
// retried on the next check and shutdown still ends this loop.
|
||||
replacement := ListenOnLocalSocket(socketPath)
|
||||
if replacement == nil {
|
||||
continue
|
||||
}
|
||||
glog.Warningf("gRPC Unix socket %s was removed and has been recreated", socketPath)
|
||||
listener.Close()
|
||||
if err := <-served; err == nil || err == grpc.ErrServerStopped {
|
||||
replacement.Close()
|
||||
os.Remove(socketPath)
|
||||
return
|
||||
}
|
||||
listener = replacement
|
||||
serve(listener)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// localSocketListener re-applies the buffer sizes to every accepted connection.
|
||||
@@ -233,22 +277,21 @@ func NewGrpcServer(opts ...grpc.ServerOption) *grpc.Server {
|
||||
func GrpcDial(ctx context.Context, address string, waitForReady bool, opts ...grpc.DialOption) (*grpc.ClientConn, error) {
|
||||
var options []grpc.DialOption
|
||||
|
||||
// Route through Unix socket if one is registered for this address's port
|
||||
// Route through Unix socket if one is registered for this address's port.
|
||||
// The socket file can vanish under a running server (temp cleaners age
|
||||
// /tmp), so a failed socket dial falls back to TCP.
|
||||
if socketPath := resolveLocalGrpcSocket(address); socketPath != "" {
|
||||
options = append(options, grpc.WithContextDialer(func(ctx context.Context, _ string) (net.Conn, error) {
|
||||
options = append(options, grpc.WithContextDialer(func(ctx context.Context, addr string) (net.Conn, error) {
|
||||
d := net.Dialer{Control: setLocalSocketBuffers}
|
||||
return d.DialContext(ctx, "unix", socketPath)
|
||||
conn, err := d.DialContext(ctx, "unix", socketPath)
|
||||
if err != nil && ctx.Err() == nil {
|
||||
glog.V(1).Infof("dial unix %s: %v, dialing %s over TCP", socketPath, err, addr)
|
||||
return dialGrpcTcp(ctx, addr)
|
||||
}
|
||||
return conn, err
|
||||
}))
|
||||
} else {
|
||||
// Always install so a conn cached before SetOutboundLocalIP binds once set;
|
||||
// the stock net.Dialer until then preserves gRPC's default dial behavior.
|
||||
options = append(options, grpc.WithContextDialer(func(ctx context.Context, addr string) (net.Conn, error) {
|
||||
if util.OutboundLocalAddr() == nil {
|
||||
var d net.Dialer
|
||||
return d.DialContext(ctx, "tcp", addr)
|
||||
}
|
||||
return util.OutboundDialContext(ctx, "tcp", addr)
|
||||
}))
|
||||
options = append(options, grpc.WithContextDialer(dialGrpcTcp))
|
||||
}
|
||||
|
||||
options = append(options,
|
||||
@@ -272,6 +315,17 @@ func GrpcDial(ctx context.Context, address string, waitForReady bool, opts ...gr
|
||||
return grpc.DialContext(ctx, address, options...)
|
||||
}
|
||||
|
||||
// dialGrpcTcp is installed even before SetOutboundLocalIP so a cached conn
|
||||
// binds once it is set; the stock net.Dialer until then preserves gRPC's
|
||||
// default dial behavior.
|
||||
func dialGrpcTcp(ctx context.Context, addr string) (net.Conn, error) {
|
||||
if util.OutboundLocalAddr() == nil {
|
||||
var d net.Dialer
|
||||
return d.DialContext(ctx, "tcp", addr)
|
||||
}
|
||||
return util.OutboundDialContext(ctx, "tcp", addr)
|
||||
}
|
||||
|
||||
func getOrCreateConnection(address string, waitForReady bool, opts ...grpc.DialOption) (*versionedGrpcClient, error) {
|
||||
|
||||
grpcClientsLock.Lock()
|
||||
|
||||
@@ -0,0 +1,142 @@
|
||||
package pb
|
||||
|
||||
import (
|
||||
"context"
|
||||
"net"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"runtime"
|
||||
"strconv"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"google.golang.org/grpc"
|
||||
"google.golang.org/grpc/credentials/insecure"
|
||||
"google.golang.org/grpc/health"
|
||||
healthpb "google.golang.org/grpc/health/grpc_health_v1"
|
||||
)
|
||||
|
||||
func isolateLocalGrpcSockets(t *testing.T) {
|
||||
t.Helper()
|
||||
if runtime.GOOS == "windows" {
|
||||
t.Skip("Unix-socket routing is disabled on Windows")
|
||||
}
|
||||
localGrpcSocketsLock.Lock()
|
||||
prevSockets, prevHosts := localGrpcSockets, localGrpcHosts
|
||||
localGrpcSockets = make(map[int]string)
|
||||
localGrpcHosts = make(map[int]map[string]struct{})
|
||||
localGrpcSocketsLock.Unlock()
|
||||
t.Cleanup(func() {
|
||||
localGrpcSocketsLock.Lock()
|
||||
localGrpcSockets, localGrpcHosts = prevSockets, prevHosts
|
||||
localGrpcSocketsLock.Unlock()
|
||||
})
|
||||
}
|
||||
|
||||
// localSocketDir is short because TMPDIR can push socket paths past the
|
||||
// sockaddr_un limit.
|
||||
func localSocketDir(t *testing.T) string {
|
||||
t.Helper()
|
||||
dir, err := os.MkdirTemp("/tmp", "swsock")
|
||||
if err != nil {
|
||||
t.Fatalf("create socket dir: %v", err)
|
||||
}
|
||||
t.Cleanup(func() { os.RemoveAll(dir) })
|
||||
return dir
|
||||
}
|
||||
|
||||
func startHealthGrpcServer(t *testing.T) (*grpc.Server, int) {
|
||||
t.Helper()
|
||||
listener, err := net.Listen("tcp", "127.0.0.1:0")
|
||||
if err != nil {
|
||||
t.Fatalf("listen: %v", err)
|
||||
}
|
||||
grpcServer := grpc.NewServer()
|
||||
healthpb.RegisterHealthServer(grpcServer, health.NewServer())
|
||||
go grpcServer.Serve(listener)
|
||||
t.Cleanup(grpcServer.Stop)
|
||||
return grpcServer, listener.Addr().(*net.TCPAddr).Port
|
||||
}
|
||||
|
||||
func checkGrpcHealth(t *testing.T, address string) {
|
||||
t.Helper()
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
|
||||
defer cancel()
|
||||
conn, err := GrpcDial(ctx, address, false, grpc.WithTransportCredentials(insecure.NewCredentials()))
|
||||
if err != nil {
|
||||
t.Fatalf("dial %s: %v", address, err)
|
||||
}
|
||||
defer conn.Close()
|
||||
if _, err := healthpb.NewHealthClient(conn).Check(ctx, &healthpb.HealthCheckRequest{}); err != nil {
|
||||
t.Fatalf("health check over %s: %v", address, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestGrpcDialFallsBackToTcpWhenLocalSocketIsMissing(t *testing.T) {
|
||||
isolateLocalGrpcSockets(t)
|
||||
_, port := startHealthGrpcServer(t)
|
||||
RegisterLocalGrpcSocket("127.0.0.1", port, filepath.Join(localSocketDir(t), "removed.sock"))
|
||||
|
||||
checkGrpcHealth(t, net.JoinHostPort("127.0.0.1", strconv.Itoa(port)))
|
||||
}
|
||||
|
||||
func TestServeGrpcOnLocalSocketRecreatesRemovedSocket(t *testing.T) {
|
||||
isolateLocalGrpcSockets(t)
|
||||
prevInterval := localSocketCheckInterval
|
||||
localSocketCheckInterval = 10 * time.Millisecond
|
||||
t.Cleanup(func() { localSocketCheckInterval = prevInterval })
|
||||
|
||||
grpcServer, port := startHealthGrpcServer(t)
|
||||
socketDir := localSocketDir(t)
|
||||
socketPath := filepath.Join(socketDir, "grpc.sock")
|
||||
RegisterLocalGrpcSocket("127.0.0.1", port, socketPath)
|
||||
ServeGrpcOnLocalSocket(grpcServer, port)
|
||||
waitForSocketFile(t, socketPath)
|
||||
|
||||
// Rebinding fails while the folder is gone and must be retried later.
|
||||
if err := os.RemoveAll(socketDir); err != nil {
|
||||
t.Fatalf("remove socket dir: %v", err)
|
||||
}
|
||||
time.Sleep(10 * localSocketCheckInterval)
|
||||
if err := os.Mkdir(socketDir, 0o700); err != nil {
|
||||
t.Fatalf("recreate socket dir: %v", err)
|
||||
}
|
||||
waitForSocketFile(t, socketPath)
|
||||
|
||||
conn, err := grpc.NewClient("unix:"+socketPath, grpc.WithTransportCredentials(insecure.NewCredentials()))
|
||||
if err != nil {
|
||||
t.Fatalf("dial recreated socket: %v", err)
|
||||
}
|
||||
defer conn.Close()
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
|
||||
defer cancel()
|
||||
if _, err := healthpb.NewHealthClient(conn).Check(ctx, &healthpb.HealthCheckRequest{}); err != nil {
|
||||
t.Fatalf("health check over the recreated socket: %v", err)
|
||||
}
|
||||
|
||||
grpcServer.Stop()
|
||||
deadline := time.Now().Add(5 * time.Second)
|
||||
for {
|
||||
if _, err := os.Lstat(socketPath); os.IsNotExist(err) {
|
||||
break
|
||||
}
|
||||
if time.Now().After(deadline) {
|
||||
t.Fatalf("socket %s left behind after the server stopped", socketPath)
|
||||
}
|
||||
time.Sleep(10 * time.Millisecond)
|
||||
}
|
||||
}
|
||||
|
||||
func waitForSocketFile(t *testing.T, socketPath string) {
|
||||
t.Helper()
|
||||
deadline := time.Now().Add(5 * time.Second)
|
||||
for {
|
||||
if info, err := os.Lstat(socketPath); err == nil && info.Mode()&os.ModeSocket != 0 {
|
||||
return
|
||||
}
|
||||
if time.Now().After(deadline) {
|
||||
t.Fatalf("socket %s was not created", socketPath)
|
||||
}
|
||||
time.Sleep(10 * time.Millisecond)
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user