diff --git a/weed/command/grpc_local_socket.go b/weed/command/grpc_local_socket.go new file mode 100644 index 0000000000..c668326bc2 --- /dev/null +++ b/weed/command/grpc_local_socket.go @@ -0,0 +1,85 @@ +package command + +import ( + "crypto/rand" + "errors" + "fmt" + "net" + "os" + "path/filepath" + "runtime" + "syscall" +) + +// grpcLocalSocket is a service name and the resolved gRPC port it listens on. +type grpcLocalSocket struct { + name string + port int +} + +// grpcLocalSocketPath returns the Unix socket path for local RPCs to a gRPC +// service, inside dir. +func grpcLocalSocketPath(dir, name string, grpcPort int) string { + return filepath.Join(dir, fmt.Sprintf("seaweedfs-%s-grpc-%d.sock", name, grpcPort)) +} + +// grpcLocalSocketPathLimit returns the longest Unix socket path the platform +// accepts: sockaddr_un's sun_path less the terminating NUL. +func grpcLocalSocketPathLimit() int { + return len(syscall.RawSockaddrUnix{}.Path) - 1 +} + +// checkGrpcLocalSocketDir rejects a socket folder that does not exist, is not +// a directory, is too long for one of its socket paths, or refuses a socket +// bind, since local RPCs keep dialing a registered socket that was never bound. +// +// The folder is unused, and not checked, on Windows (see +// pb.RegisterLocalGrpcSocket) or when no service registers a socket. +func checkGrpcLocalSocketDir(dir string, sockets []grpcLocalSocket) error { + if runtime.GOOS == "windows" || len(sockets) == 0 { + return nil + } + info, err := os.Stat(dir) + if err != nil { + return err + } + if !info.IsDir() { + return errors.New("not a directory") + } + limit := grpcLocalSocketPathLimit() + for _, socket := range sockets { + path := grpcLocalSocketPath(dir, socket.name, socket.port) + if len(path) > limit { + return fmt.Errorf("socket path %q is %d bytes and does not fit the platform's %d-byte Unix socket path limit; use a shorter -grpc.localSocketDir", path, len(path), limit) + } + } + return probeGrpcLocalSocketDir(dir) +} + +// probeGrpcLocalSocketDir binds and removes a randomly named Unix socket in +// dir, so a file another user left in a shared folder cannot block startup. +// The 23-byte name is shorter than any service socket name. +func probeGrpcLocalSocketDir(dir string) error { + const attempts = 5 + var lastErr error + for i := 0; i < attempts; i++ { + suffix := make([]byte, 4) + if _, err := rand.Read(suffix); err != nil { + return fmt.Errorf("cannot pick a Unix socket probe name: %v", err) + } + probePath := filepath.Join(dir, fmt.Sprintf("seaweedfs-%x.sock", suffix)) + listener, err := net.Listen("unix", probePath) + if errors.Is(err, syscall.EADDRINUSE) { + lastErr = err + continue + } + if err != nil { + return fmt.Errorf("cannot create a Unix socket in it: %v; give the server's user write access or choose another -grpc.localSocketDir", err) + } + if err := listener.Close(); err != nil { + return fmt.Errorf("cannot close the Unix socket probe %q: %v", probePath, err) + } + return nil + } + return fmt.Errorf("cannot create a Unix socket in it: %d random probe names were all taken, last: %v", attempts, lastErr) +} diff --git a/weed/command/grpc_local_socket_test.go b/weed/command/grpc_local_socket_test.go new file mode 100644 index 0000000000..cd7c8596b1 --- /dev/null +++ b/weed/command/grpc_local_socket_test.go @@ -0,0 +1,200 @@ +package command + +import ( + "net" + "os" + "path/filepath" + "runtime" + "strconv" + "strings" + "testing" +) + +// TestGrpcLocalSocketPath checks that -grpc.localSocketDir defaults to /tmp +// for `weed server` and `weed mini` and that every role's socket lands in it. +func TestGrpcLocalSocketPath(t *testing.T) { + if runtime.GOOS == "windows" { + t.Skip("Unix-socket routing is disabled on Windows") + } + + for _, tc := range []struct { + command string + dir *string + }{ + {"weed server", serverGrpcLocalSocketDir}, + {"weed mini", miniGrpcLocalSocketDir}, + } { + t.Run(tc.command, func(t *testing.T) { + if *tc.dir != "/tmp" { + t.Fatalf("-grpc.localSocketDir default = %q, want /tmp", *tc.dir) + } + if got, want := grpcLocalSocketPath(*tc.dir, "volume", 17334), "/tmp/seaweedfs-volume-grpc-17334.sock"; got != want { + t.Fatalf("default socket path = %q, want %q", got, want) + } + + dir := t.TempDir() + prev := *tc.dir + *tc.dir = dir + t.Cleanup(func() { *tc.dir = prev }) + + for _, role := range []string{"master", "volume", "filer", "s3", "admin"} { + path := grpcLocalSocketPath(*tc.dir, role, 17334) + if got := filepath.Dir(path); got != dir { + t.Fatalf("%s socket path %q is not in %q", role, path, dir) + } + if got, want := filepath.Base(path), "seaweedfs-"+role+"-grpc-17334.sock"; got != want { + t.Fatalf("%s socket file name = %q, want %q", role, got, want) + } + } + }) + } +} + +// TestGrpcLocalSocketDir checks that -grpc.localSocketDir is refused when one +// of its socket paths cannot fit in the platform's sockaddr_un. +func TestGrpcLocalSocketDir(t *testing.T) { + if runtime.GOOS == "windows" { + t.Skip("Unix-socket routing is disabled on Windows") + } + + sockets := []grpcLocalSocket{{"volume", 17334}} + limit := grpcLocalSocketPathLimit() + + // /tmp is the historical default and short enough on every Unix for any + // socket name; t.TempDir() is not, since macOS roots it under a long + // /var/folders path. + if err := checkGrpcLocalSocketDir("/tmp", sockets); err != nil { + t.Fatalf("checkGrpcLocalSocketDir(%q): %v", "/tmp", err) + } + + dir := t.TempDir() + for len(grpcLocalSocketPath(dir, "volume", 17334)) <= limit { + dir = filepath.Join(dir, "abcdefghijklmnop") + if err := os.Mkdir(dir, 0o755); err != nil { + t.Fatalf("create %q: %v", dir, err) + } + } + err := checkGrpcLocalSocketDir(dir, sockets) + if err == nil { + t.Fatalf("checkGrpcLocalSocketDir accepted %q, whose socket path is longer than the %d-byte limit", dir, limit) + } + if path := grpcLocalSocketPath(dir, "volume", 17334); !strings.Contains(err.Error(), path) { + t.Fatalf("error %q does not name the socket path %q", err, path) + } + if !strings.Contains(err.Error(), strconv.Itoa(limit)) { + t.Fatalf("error %q does not name the %d-byte limit", err, limit) + } +} + +// TestGrpcLocalSocketDirAtLimit checks that a folder whose longest socket +// path is exactly the platform limit is accepted and that the socket binds. +func TestGrpcLocalSocketDirAtLimit(t *testing.T) { + if runtime.GOOS == "windows" { + t.Skip("Unix-socket routing is disabled on Windows") + } + + base, err := os.MkdirTemp("/tmp", "seaweedfs-grpc-limit-") + if err != nil { + t.Fatalf("create folder: %v", err) + } + t.Cleanup(func() { + if err := os.RemoveAll(base); err != nil { + t.Errorf("remove %q: %v", base, err) + } + }) + + sockets := []grpcLocalSocket{{"s3", 18333}, {"volume", 18080}} + limit := grpcLocalSocketPathLimit() + name := filepath.Base(grpcLocalSocketPath(base, "volume", 18080)) + dir := filepath.Join(base, strings.Repeat("d", limit-len(base)-len(name)-2)) + if err := os.Mkdir(dir, 0o755); err != nil { + t.Fatalf("create %q: %v", dir, err) + } + path := grpcLocalSocketPath(dir, "volume", 18080) + if len(path) != limit { + t.Fatalf("socket path %q is %d bytes, want %d", path, len(path), limit) + } + + if err := checkGrpcLocalSocketDir(dir, sockets); err != nil { + t.Fatalf("checkGrpcLocalSocketDir(%q): %v", dir, err) + } + listener, err := net.Listen("unix", path) + if err != nil { + t.Fatalf("bind %q: %v", path, err) + } + if err := listener.Close(); err != nil { + t.Fatalf("close %q: %v", path, err) + } +} + +// TestGrpcLocalSocketDirUnwritable checks that -grpc.localSocketDir is +// refused when the server's user cannot create a socket in the folder. +func TestGrpcLocalSocketDirUnwritable(t *testing.T) { + if runtime.GOOS == "windows" { + t.Skip("Unix-socket routing is disabled on Windows") + } + if os.Geteuid() == 0 { + t.Skip("root bypasses folder mode bits") + } + + // A folder directly under /tmp keeps the probe socket path well inside the + // sockaddr_un limit, which t.TempDir() does not on macOS. + dir, err := os.MkdirTemp("/tmp", "seaweedfs-grpc-unwritable-") + if err != nil { + t.Fatalf("create folder: %v", err) + } + t.Cleanup(func() { + if err := os.Chmod(dir, 0o755); err != nil { + t.Errorf("restore mode of %q: %v", dir, err) + } + if err := os.RemoveAll(dir); err != nil { + t.Errorf("remove %q: %v", dir, err) + } + }) + if err := os.Chmod(dir, 0o555); err != nil { + t.Fatalf("chmod %q: %v", dir, err) + } + + err = checkGrpcLocalSocketDir(dir, []grpcLocalSocket{{"volume", 17334}}) + if err == nil { + t.Fatalf("checkGrpcLocalSocketDir accepted %q, where the server cannot create a socket", dir) + } + if !strings.Contains(err.Error(), dir) { + t.Fatalf("error %q does not name the folder %q", err, dir) + } + if !strings.Contains(err.Error(), "cannot create a Unix socket") { + t.Fatalf("error %q does not report the failed socket bind", err) + } +} + +// TestGrpcLocalSocketDirProbeLeavesNothing checks that the startup probe +// removes its socket, so repeated starts never fill a shared folder. +func TestGrpcLocalSocketDirProbeLeavesNothing(t *testing.T) { + if runtime.GOOS == "windows" { + t.Skip("Unix-socket routing is disabled on Windows") + } + + // See TestGrpcLocalSocketDirUnwritable for why the folder is under /tmp. + dir, err := os.MkdirTemp("/tmp", "seaweedfs-grpc-probe-") + if err != nil { + t.Fatalf("create folder: %v", err) + } + t.Cleanup(func() { + if err := os.RemoveAll(dir); err != nil { + t.Errorf("remove %q: %v", dir, err) + } + }) + + for i := 0; i < 3; i++ { + if err := checkGrpcLocalSocketDir(dir, []grpcLocalSocket{{"volume", 17334}}); err != nil { + t.Fatalf("checkGrpcLocalSocketDir(%q): %v", dir, err) + } + } + entries, err := os.ReadDir(dir) + if err != nil { + t.Fatalf("read %q: %v", dir, err) + } + if len(entries) != 0 { + t.Fatalf("probe left %d entries in %q, first %q", len(entries), dir, entries[0].Name()) + } +} diff --git a/weed/command/mini.go b/weed/command/mini.go index 4edc19673e..f42db8827d 100644 --- a/weed/command/mini.go +++ b/weed/command/mini.go @@ -410,6 +410,7 @@ var ( miniDataFolders = cmdMini.Flag.String("dir", ".", "directory to store data files") miniMetricsHttpPort = cmdMini.Flag.Int("metricsPort", 0, "Prometheus metrics listen port") miniMetricsHttpIp = cmdMini.Flag.String("metricsIp", "", "metrics listen ip. If empty, default to same as -ip.bind option.") + miniGrpcLocalSocketDir = cmdMini.Flag.String("grpc.localSocketDir", "/tmp", "directory for the local gRPC Unix sockets; default to /tmp. Point it at a directory a temp cleaner does not sweep, such as /run/seaweedfs") miniS3Config = cmdMini.Flag.String("s3.config", "", "path to the S3 config file") miniIamConfig = cmdMini.Flag.String("s3.iam.config", "", "path to the advanced IAM config file for S3") miniS3AllowDeleteBucketNotEmpty = cmdMini.Flag.Bool("s3.allowDeleteBucketNotEmpty", false, "allow recursive deleting all entries along with bucket") @@ -1568,13 +1569,21 @@ func runMini(cmd *Command, args []string) bool { // Register Unix socket paths for gRPC services so local inter-service // communication goes through Unix sockets instead of TCP. - pb.RegisterLocalGrpcSocket(*miniIp, *miniMasterOptions.portGrpc, fmt.Sprintf("/tmp/seaweedfs-master-grpc-%d.sock", *miniMasterOptions.portGrpc)) - pb.RegisterLocalGrpcSocket(*miniIp, *miniOptions.v.portGrpc, fmt.Sprintf("/tmp/seaweedfs-volume-grpc-%d.sock", *miniOptions.v.portGrpc)) - pb.RegisterLocalGrpcSocket(*miniIp, *miniFilerOptions.portGrpc, fmt.Sprintf("/tmp/seaweedfs-filer-grpc-%d.sock", *miniFilerOptions.portGrpc)) + localSockets := []grpcLocalSocket{ + {"master", *miniMasterOptions.portGrpc}, + {"volume", *miniOptions.v.portGrpc}, + {"filer", *miniFilerOptions.portGrpc}, + {"admin", *miniAdminOptions.grpcPort}, + } if *miniS3Options.portGrpc > 0 { - pb.RegisterLocalGrpcSocket(*miniIp, *miniS3Options.portGrpc, fmt.Sprintf("/tmp/seaweedfs-s3-grpc-%d.sock", *miniS3Options.portGrpc)) + localSockets = append(localSockets, grpcLocalSocket{"s3", *miniS3Options.portGrpc}) + } + if err := checkGrpcLocalSocketDir(*miniGrpcLocalSocketDir, localSockets); err != nil { + glog.Fatalf("Check gRPC local socket folder (-grpc.localSocketDir=%q): %v", *miniGrpcLocalSocketDir, err) + } + for _, socket := range localSockets { + pb.RegisterLocalGrpcSocket(*miniIp, socket.port, grpcLocalSocketPath(*miniGrpcLocalSocketDir, socket.name, socket.port)) } - pb.RegisterLocalGrpcSocket(*miniIp, *miniAdminOptions.grpcPort, fmt.Sprintf("/tmp/seaweedfs-admin-grpc-%d.sock", *miniAdminOptions.grpcPort)) go stats_collect.StartMetricsServer(*miniMetricsHttpIp, *miniMetricsHttpPort) diff --git a/weed/command/server.go b/weed/command/server.go index 04aac2f76f..e62378219d 100644 --- a/weed/command/server.go +++ b/weed/command/server.go @@ -74,6 +74,7 @@ var ( volumeMinFreeSpace = cmdServer.Flag.String("volume.minFreeSpace", "", "min free disk space (value<=100 as percentage like 1, other as human readable bytes, like 10GiB). Low disk space will mark all volumes as ReadOnly.") serverMetricsHttpPort = cmdServer.Flag.Int("metricsPort", 0, "Prometheus metrics listen port") serverMetricsHttpIp = cmdServer.Flag.String("metricsIp", "", "metrics listen ip. If empty, default to same as -ip.bind option.") + serverGrpcLocalSocketDir = cmdServer.Flag.String("grpc.localSocketDir", "/tmp", "directory for the local gRPC Unix sockets; default to /tmp. Point it at a directory a temp cleaner does not sweep, such as /run/seaweedfs") // pulseSeconds = cmdServer.Flag.Int("pulseSeconds", 5, "number of seconds between heartbeats") isStartingMasterServer = cmdServer.Flag.Bool("master", true, "whether to start master server") @@ -433,6 +434,7 @@ func runServer(cmd *Command, args []string) bool { // Register Unix socket paths for gRPC services running in this process // so local inter-service communication uses Unix sockets instead of TCP. // Resolve gRPC ports early (same calculation each service does internally). + var localSockets []grpcLocalSocket for _, svc := range []struct { starting *bool portGrpc *int @@ -448,9 +450,15 @@ func runServer(cmd *Command, args []string) bool { if *svc.portGrpc == 0 { *svc.portGrpc = 10000 + *svc.port } - pb.RegisterLocalGrpcSocket(*serverIp, *svc.portGrpc, fmt.Sprintf("/tmp/seaweedfs-%s-grpc-%d.sock", svc.name, *svc.portGrpc)) + localSockets = append(localSockets, grpcLocalSocket{svc.name, *svc.portGrpc}) } } + if err := checkGrpcLocalSocketDir(*serverGrpcLocalSocketDir, localSockets); err != nil { + glog.Fatalf("Check gRPC local socket folder (-grpc.localSocketDir=%q): %v", *serverGrpcLocalSocketDir, err) + } + for _, socket := range localSockets { + pb.RegisterLocalGrpcSocket(*serverIp, socket.port, grpcLocalSocketPath(*serverGrpcLocalSocketDir, socket.name, socket.port)) + } serverWhiteList := util.StringSplit(*serverWhiteListOption, ",")