mirror of
https://github.com/seaweedfs/seaweedfs.git
synced 2026-11-18 09:46:24 +01:00
server: add -grpc.localSocketDir for the local gRPC sockets (#12693)
* server: add -grpc.localSocketDir for the local gRPC sockets `weed server` and `weed mini` hard-coded each local gRPC Unix socket under /tmp. systemd-tmpfiles ages /tmp after 10 days and macOS after 3, so a long-running server loses its socket, and RegisterLocalGrpcSocket keeps routing local dials to the missing path: the master can no longer reach its volume server. Add -grpc.localSocketDir to both commands, defaulting to /tmp so an existing setup is unchanged, and build every socket path from it with filepath.Join. A folder that does not exist or is not a directory is refused at startup rather than at the first local RPC. A folder whose socket paths exceed the platform's sockaddr_un limit is refused too, because every bind then fails with EINVAL while the server keeps running and each local RPC dials a socket that does not exist. Refs #12692 * server: refuse a -grpc.localSocketDir the server cannot bind in An existing folder the server's user cannot write to passed the startup check, and then every socket bind failed: ListenOnLocalSocket only logs the error while GrpcDial keeps routing local dials to a socket that was never created. After the path-length check, create and remove a probe Unix socket in the folder, and refuse startup with the bind error if that fails. * server: give the -grpc.localSocketDir probe a random name A probe named only by the process id could be precreated by another user in a shared /tmp, or left by a crashed process whose pid was reused, and the server cannot remove such a file, so startup failed although the real socket paths were free. Name the probe with 8 random bytes, retry a name that is taken, and rely on the listener's close to unlink it. * server: keep the -grpc.localSocketDir probe name within the service socket lengths The probe was 5 bytes longer than the master and volume socket names, so a folder whose service sockets fit the sockaddr_un limit could still be refused. A 23-byte probe name is shorter than any service socket name, and a process with no local sockets skips the check. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * server: trim the -grpc.localSocketDir comments and drop issue references Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> --------- Co-authored-by: Brent Pease <179639673+brentDSL@users.noreply.github.com> Co-authored-by: Chris Lu <chrislusf@users.noreply.github.com> Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
4 files changed
+308
-6
No files matched your search
@@ -0,0 +1,85 @@
|
||||
package command
|
||||
|
||||
import (
|
||||
"crypto/rand"
|
||||
"errors"
|
||||
"fmt"
|
||||
"net"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"runtime"
|
||||
"syscall"
|
||||
)
|
||||
|
||||
// grpcLocalSocket is a service name and the resolved gRPC port it listens on.
|
||||
type grpcLocalSocket struct {
|
||||
name string
|
||||
port int
|
||||
}
|
||||
|
||||
// grpcLocalSocketPath returns the Unix socket path for local RPCs to a gRPC
|
||||
// service, inside dir.
|
||||
func grpcLocalSocketPath(dir, name string, grpcPort int) string {
|
||||
return filepath.Join(dir, fmt.Sprintf("seaweedfs-%s-grpc-%d.sock", name, grpcPort))
|
||||
}
|
||||
|
||||
// grpcLocalSocketPathLimit returns the longest Unix socket path the platform
|
||||
// accepts: sockaddr_un's sun_path less the terminating NUL.
|
||||
func grpcLocalSocketPathLimit() int {
|
||||
return len(syscall.RawSockaddrUnix{}.Path) - 1
|
||||
}
|
||||
|
||||
// checkGrpcLocalSocketDir rejects a socket folder that does not exist, is not
|
||||
// a directory, is too long for one of its socket paths, or refuses a socket
|
||||
// bind, since local RPCs keep dialing a registered socket that was never bound.
|
||||
//
|
||||
// The folder is unused, and not checked, on Windows (see
|
||||
// pb.RegisterLocalGrpcSocket) or when no service registers a socket.
|
||||
func checkGrpcLocalSocketDir(dir string, sockets []grpcLocalSocket) error {
|
||||
if runtime.GOOS == "windows" || len(sockets) == 0 {
|
||||
return nil
|
||||
}
|
||||
info, err := os.Stat(dir)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if !info.IsDir() {
|
||||
return errors.New("not a directory")
|
||||
}
|
||||
limit := grpcLocalSocketPathLimit()
|
||||
for _, socket := range sockets {
|
||||
path := grpcLocalSocketPath(dir, socket.name, socket.port)
|
||||
if len(path) > limit {
|
||||
return fmt.Errorf("socket path %q is %d bytes and does not fit the platform's %d-byte Unix socket path limit; use a shorter -grpc.localSocketDir", path, len(path), limit)
|
||||
}
|
||||
}
|
||||
return probeGrpcLocalSocketDir(dir)
|
||||
}
|
||||
|
||||
// probeGrpcLocalSocketDir binds and removes a randomly named Unix socket in
|
||||
// dir, so a file another user left in a shared folder cannot block startup.
|
||||
// The 23-byte name is shorter than any service socket name.
|
||||
func probeGrpcLocalSocketDir(dir string) error {
|
||||
const attempts = 5
|
||||
var lastErr error
|
||||
for i := 0; i < attempts; i++ {
|
||||
suffix := make([]byte, 4)
|
||||
if _, err := rand.Read(suffix); err != nil {
|
||||
return fmt.Errorf("cannot pick a Unix socket probe name: %v", err)
|
||||
}
|
||||
probePath := filepath.Join(dir, fmt.Sprintf("seaweedfs-%x.sock", suffix))
|
||||
listener, err := net.Listen("unix", probePath)
|
||||
if errors.Is(err, syscall.EADDRINUSE) {
|
||||
lastErr = err
|
||||
continue
|
||||
}
|
||||
if err != nil {
|
||||
return fmt.Errorf("cannot create a Unix socket in it: %v; give the server's user write access or choose another -grpc.localSocketDir", err)
|
||||
}
|
||||
if err := listener.Close(); err != nil {
|
||||
return fmt.Errorf("cannot close the Unix socket probe %q: %v", probePath, err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
return fmt.Errorf("cannot create a Unix socket in it: %d random probe names were all taken, last: %v", attempts, lastErr)
|
||||
}
|
||||
@@ -0,0 +1,200 @@
|
||||
package command
|
||||
|
||||
import (
|
||||
"net"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"runtime"
|
||||
"strconv"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// TestGrpcLocalSocketPath checks that -grpc.localSocketDir defaults to /tmp
|
||||
// for `weed server` and `weed mini` and that every role's socket lands in it.
|
||||
func TestGrpcLocalSocketPath(t *testing.T) {
|
||||
if runtime.GOOS == "windows" {
|
||||
t.Skip("Unix-socket routing is disabled on Windows")
|
||||
}
|
||||
|
||||
for _, tc := range []struct {
|
||||
command string
|
||||
dir *string
|
||||
}{
|
||||
{"weed server", serverGrpcLocalSocketDir},
|
||||
{"weed mini", miniGrpcLocalSocketDir},
|
||||
} {
|
||||
t.Run(tc.command, func(t *testing.T) {
|
||||
if *tc.dir != "/tmp" {
|
||||
t.Fatalf("-grpc.localSocketDir default = %q, want /tmp", *tc.dir)
|
||||
}
|
||||
if got, want := grpcLocalSocketPath(*tc.dir, "volume", 17334), "/tmp/seaweedfs-volume-grpc-17334.sock"; got != want {
|
||||
t.Fatalf("default socket path = %q, want %q", got, want)
|
||||
}
|
||||
|
||||
dir := t.TempDir()
|
||||
prev := *tc.dir
|
||||
*tc.dir = dir
|
||||
t.Cleanup(func() { *tc.dir = prev })
|
||||
|
||||
for _, role := range []string{"master", "volume", "filer", "s3", "admin"} {
|
||||
path := grpcLocalSocketPath(*tc.dir, role, 17334)
|
||||
if got := filepath.Dir(path); got != dir {
|
||||
t.Fatalf("%s socket path %q is not in %q", role, path, dir)
|
||||
}
|
||||
if got, want := filepath.Base(path), "seaweedfs-"+role+"-grpc-17334.sock"; got != want {
|
||||
t.Fatalf("%s socket file name = %q, want %q", role, got, want)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// TestGrpcLocalSocketDir checks that -grpc.localSocketDir is refused when one
|
||||
// of its socket paths cannot fit in the platform's sockaddr_un.
|
||||
func TestGrpcLocalSocketDir(t *testing.T) {
|
||||
if runtime.GOOS == "windows" {
|
||||
t.Skip("Unix-socket routing is disabled on Windows")
|
||||
}
|
||||
|
||||
sockets := []grpcLocalSocket{{"volume", 17334}}
|
||||
limit := grpcLocalSocketPathLimit()
|
||||
|
||||
// /tmp is the historical default and short enough on every Unix for any
|
||||
// socket name; t.TempDir() is not, since macOS roots it under a long
|
||||
// /var/folders path.
|
||||
if err := checkGrpcLocalSocketDir("/tmp", sockets); err != nil {
|
||||
t.Fatalf("checkGrpcLocalSocketDir(%q): %v", "/tmp", err)
|
||||
}
|
||||
|
||||
dir := t.TempDir()
|
||||
for len(grpcLocalSocketPath(dir, "volume", 17334)) <= limit {
|
||||
dir = filepath.Join(dir, "abcdefghijklmnop")
|
||||
if err := os.Mkdir(dir, 0o755); err != nil {
|
||||
t.Fatalf("create %q: %v", dir, err)
|
||||
}
|
||||
}
|
||||
err := checkGrpcLocalSocketDir(dir, sockets)
|
||||
if err == nil {
|
||||
t.Fatalf("checkGrpcLocalSocketDir accepted %q, whose socket path is longer than the %d-byte limit", dir, limit)
|
||||
}
|
||||
if path := grpcLocalSocketPath(dir, "volume", 17334); !strings.Contains(err.Error(), path) {
|
||||
t.Fatalf("error %q does not name the socket path %q", err, path)
|
||||
}
|
||||
if !strings.Contains(err.Error(), strconv.Itoa(limit)) {
|
||||
t.Fatalf("error %q does not name the %d-byte limit", err, limit)
|
||||
}
|
||||
}
|
||||
|
||||
// TestGrpcLocalSocketDirAtLimit checks that a folder whose longest socket
|
||||
// path is exactly the platform limit is accepted and that the socket binds.
|
||||
func TestGrpcLocalSocketDirAtLimit(t *testing.T) {
|
||||
if runtime.GOOS == "windows" {
|
||||
t.Skip("Unix-socket routing is disabled on Windows")
|
||||
}
|
||||
|
||||
base, err := os.MkdirTemp("/tmp", "seaweedfs-grpc-limit-")
|
||||
if err != nil {
|
||||
t.Fatalf("create folder: %v", err)
|
||||
}
|
||||
t.Cleanup(func() {
|
||||
if err := os.RemoveAll(base); err != nil {
|
||||
t.Errorf("remove %q: %v", base, err)
|
||||
}
|
||||
})
|
||||
|
||||
sockets := []grpcLocalSocket{{"s3", 18333}, {"volume", 18080}}
|
||||
limit := grpcLocalSocketPathLimit()
|
||||
name := filepath.Base(grpcLocalSocketPath(base, "volume", 18080))
|
||||
dir := filepath.Join(base, strings.Repeat("d", limit-len(base)-len(name)-2))
|
||||
if err := os.Mkdir(dir, 0o755); err != nil {
|
||||
t.Fatalf("create %q: %v", dir, err)
|
||||
}
|
||||
path := grpcLocalSocketPath(dir, "volume", 18080)
|
||||
if len(path) != limit {
|
||||
t.Fatalf("socket path %q is %d bytes, want %d", path, len(path), limit)
|
||||
}
|
||||
|
||||
if err := checkGrpcLocalSocketDir(dir, sockets); err != nil {
|
||||
t.Fatalf("checkGrpcLocalSocketDir(%q): %v", dir, err)
|
||||
}
|
||||
listener, err := net.Listen("unix", path)
|
||||
if err != nil {
|
||||
t.Fatalf("bind %q: %v", path, err)
|
||||
}
|
||||
if err := listener.Close(); err != nil {
|
||||
t.Fatalf("close %q: %v", path, err)
|
||||
}
|
||||
}
|
||||
|
||||
// TestGrpcLocalSocketDirUnwritable checks that -grpc.localSocketDir is
|
||||
// refused when the server's user cannot create a socket in the folder.
|
||||
func TestGrpcLocalSocketDirUnwritable(t *testing.T) {
|
||||
if runtime.GOOS == "windows" {
|
||||
t.Skip("Unix-socket routing is disabled on Windows")
|
||||
}
|
||||
if os.Geteuid() == 0 {
|
||||
t.Skip("root bypasses folder mode bits")
|
||||
}
|
||||
|
||||
// A folder directly under /tmp keeps the probe socket path well inside the
|
||||
// sockaddr_un limit, which t.TempDir() does not on macOS.
|
||||
dir, err := os.MkdirTemp("/tmp", "seaweedfs-grpc-unwritable-")
|
||||
if err != nil {
|
||||
t.Fatalf("create folder: %v", err)
|
||||
}
|
||||
t.Cleanup(func() {
|
||||
if err := os.Chmod(dir, 0o755); err != nil {
|
||||
t.Errorf("restore mode of %q: %v", dir, err)
|
||||
}
|
||||
if err := os.RemoveAll(dir); err != nil {
|
||||
t.Errorf("remove %q: %v", dir, err)
|
||||
}
|
||||
})
|
||||
if err := os.Chmod(dir, 0o555); err != nil {
|
||||
t.Fatalf("chmod %q: %v", dir, err)
|
||||
}
|
||||
|
||||
err = checkGrpcLocalSocketDir(dir, []grpcLocalSocket{{"volume", 17334}})
|
||||
if err == nil {
|
||||
t.Fatalf("checkGrpcLocalSocketDir accepted %q, where the server cannot create a socket", dir)
|
||||
}
|
||||
if !strings.Contains(err.Error(), dir) {
|
||||
t.Fatalf("error %q does not name the folder %q", err, dir)
|
||||
}
|
||||
if !strings.Contains(err.Error(), "cannot create a Unix socket") {
|
||||
t.Fatalf("error %q does not report the failed socket bind", err)
|
||||
}
|
||||
}
|
||||
|
||||
// TestGrpcLocalSocketDirProbeLeavesNothing checks that the startup probe
|
||||
// removes its socket, so repeated starts never fill a shared folder.
|
||||
func TestGrpcLocalSocketDirProbeLeavesNothing(t *testing.T) {
|
||||
if runtime.GOOS == "windows" {
|
||||
t.Skip("Unix-socket routing is disabled on Windows")
|
||||
}
|
||||
|
||||
// See TestGrpcLocalSocketDirUnwritable for why the folder is under /tmp.
|
||||
dir, err := os.MkdirTemp("/tmp", "seaweedfs-grpc-probe-")
|
||||
if err != nil {
|
||||
t.Fatalf("create folder: %v", err)
|
||||
}
|
||||
t.Cleanup(func() {
|
||||
if err := os.RemoveAll(dir); err != nil {
|
||||
t.Errorf("remove %q: %v", dir, err)
|
||||
}
|
||||
})
|
||||
|
||||
for i := 0; i < 3; i++ {
|
||||
if err := checkGrpcLocalSocketDir(dir, []grpcLocalSocket{{"volume", 17334}}); err != nil {
|
||||
t.Fatalf("checkGrpcLocalSocketDir(%q): %v", dir, err)
|
||||
}
|
||||
}
|
||||
entries, err := os.ReadDir(dir)
|
||||
if err != nil {
|
||||
t.Fatalf("read %q: %v", dir, err)
|
||||
}
|
||||
if len(entries) != 0 {
|
||||
t.Fatalf("probe left %d entries in %q, first %q", len(entries), dir, entries[0].Name())
|
||||
}
|
||||
}
|
||||
+14
-5
@@ -410,6 +410,7 @@ var (
|
||||
miniDataFolders = cmdMini.Flag.String("dir", ".", "directory to store data files")
|
||||
miniMetricsHttpPort = cmdMini.Flag.Int("metricsPort", 0, "Prometheus metrics listen port")
|
||||
miniMetricsHttpIp = cmdMini.Flag.String("metricsIp", "", "metrics listen ip. If empty, default to same as -ip.bind option.")
|
||||
miniGrpcLocalSocketDir = cmdMini.Flag.String("grpc.localSocketDir", "/tmp", "directory for the local gRPC Unix sockets; default to /tmp. Point it at a directory a temp cleaner does not sweep, such as /run/seaweedfs")
|
||||
miniS3Config = cmdMini.Flag.String("s3.config", "", "path to the S3 config file")
|
||||
miniIamConfig = cmdMini.Flag.String("s3.iam.config", "", "path to the advanced IAM config file for S3")
|
||||
miniS3AllowDeleteBucketNotEmpty = cmdMini.Flag.Bool("s3.allowDeleteBucketNotEmpty", false, "allow recursive deleting all entries along with bucket")
|
||||
@@ -1568,13 +1569,21 @@ func runMini(cmd *Command, args []string) bool {
|
||||
|
||||
// Register Unix socket paths for gRPC services so local inter-service
|
||||
// communication goes through Unix sockets instead of TCP.
|
||||
pb.RegisterLocalGrpcSocket(*miniIp, *miniMasterOptions.portGrpc, fmt.Sprintf("/tmp/seaweedfs-master-grpc-%d.sock", *miniMasterOptions.portGrpc))
|
||||
pb.RegisterLocalGrpcSocket(*miniIp, *miniOptions.v.portGrpc, fmt.Sprintf("/tmp/seaweedfs-volume-grpc-%d.sock", *miniOptions.v.portGrpc))
|
||||
pb.RegisterLocalGrpcSocket(*miniIp, *miniFilerOptions.portGrpc, fmt.Sprintf("/tmp/seaweedfs-filer-grpc-%d.sock", *miniFilerOptions.portGrpc))
|
||||
localSockets := []grpcLocalSocket{
|
||||
{"master", *miniMasterOptions.portGrpc},
|
||||
{"volume", *miniOptions.v.portGrpc},
|
||||
{"filer", *miniFilerOptions.portGrpc},
|
||||
{"admin", *miniAdminOptions.grpcPort},
|
||||
}
|
||||
if *miniS3Options.portGrpc > 0 {
|
||||
pb.RegisterLocalGrpcSocket(*miniIp, *miniS3Options.portGrpc, fmt.Sprintf("/tmp/seaweedfs-s3-grpc-%d.sock", *miniS3Options.portGrpc))
|
||||
localSockets = append(localSockets, grpcLocalSocket{"s3", *miniS3Options.portGrpc})
|
||||
}
|
||||
if err := checkGrpcLocalSocketDir(*miniGrpcLocalSocketDir, localSockets); err != nil {
|
||||
glog.Fatalf("Check gRPC local socket folder (-grpc.localSocketDir=%q): %v", *miniGrpcLocalSocketDir, err)
|
||||
}
|
||||
for _, socket := range localSockets {
|
||||
pb.RegisterLocalGrpcSocket(*miniIp, socket.port, grpcLocalSocketPath(*miniGrpcLocalSocketDir, socket.name, socket.port))
|
||||
}
|
||||
pb.RegisterLocalGrpcSocket(*miniIp, *miniAdminOptions.grpcPort, fmt.Sprintf("/tmp/seaweedfs-admin-grpc-%d.sock", *miniAdminOptions.grpcPort))
|
||||
|
||||
go stats_collect.StartMetricsServer(*miniMetricsHttpIp, *miniMetricsHttpPort)
|
||||
|
||||
|
||||
@@ -74,6 +74,7 @@ var (
|
||||
volumeMinFreeSpace = cmdServer.Flag.String("volume.minFreeSpace", "", "min free disk space (value<=100 as percentage like 1, other as human readable bytes, like 10GiB). Low disk space will mark all volumes as ReadOnly.")
|
||||
serverMetricsHttpPort = cmdServer.Flag.Int("metricsPort", 0, "Prometheus metrics listen port")
|
||||
serverMetricsHttpIp = cmdServer.Flag.String("metricsIp", "", "metrics listen ip. If empty, default to same as -ip.bind option.")
|
||||
serverGrpcLocalSocketDir = cmdServer.Flag.String("grpc.localSocketDir", "/tmp", "directory for the local gRPC Unix sockets; default to /tmp. Point it at a directory a temp cleaner does not sweep, such as /run/seaweedfs")
|
||||
|
||||
// pulseSeconds = cmdServer.Flag.Int("pulseSeconds", 5, "number of seconds between heartbeats")
|
||||
isStartingMasterServer = cmdServer.Flag.Bool("master", true, "whether to start master server")
|
||||
@@ -433,6 +434,7 @@ func runServer(cmd *Command, args []string) bool {
|
||||
// Register Unix socket paths for gRPC services running in this process
|
||||
// so local inter-service communication uses Unix sockets instead of TCP.
|
||||
// Resolve gRPC ports early (same calculation each service does internally).
|
||||
var localSockets []grpcLocalSocket
|
||||
for _, svc := range []struct {
|
||||
starting *bool
|
||||
portGrpc *int
|
||||
@@ -448,9 +450,15 @@ func runServer(cmd *Command, args []string) bool {
|
||||
if *svc.portGrpc == 0 {
|
||||
*svc.portGrpc = 10000 + *svc.port
|
||||
}
|
||||
pb.RegisterLocalGrpcSocket(*serverIp, *svc.portGrpc, fmt.Sprintf("/tmp/seaweedfs-%s-grpc-%d.sock", svc.name, *svc.portGrpc))
|
||||
localSockets = append(localSockets, grpcLocalSocket{svc.name, *svc.portGrpc})
|
||||
}
|
||||
}
|
||||
if err := checkGrpcLocalSocketDir(*serverGrpcLocalSocketDir, localSockets); err != nil {
|
||||
glog.Fatalf("Check gRPC local socket folder (-grpc.localSocketDir=%q): %v", *serverGrpcLocalSocketDir, err)
|
||||
}
|
||||
for _, socket := range localSockets {
|
||||
pb.RegisterLocalGrpcSocket(*serverIp, socket.port, grpcLocalSocketPath(*serverGrpcLocalSocketDir, socket.name, socket.port))
|
||||
}
|
||||
|
||||
serverWhiteList := util.StringSplit(*serverWhiteListOption, ",")
|
||||
|
||||
|
||||
Reference in new issue
Block a user