fuse integration: run posix suites on weed mini and allow allow_other mounts (#11984)

The separate master/volume/filer processes in framework.go bound fixed
ports, discarded child output, and never noticed an early process exit,
so a slow or dead volume server burned the whole 30s timeout with no
logs. Switch to the same weed-mini approach the fuse_test package
framework already uses: dynamic filer ports, child output captured to
log files, and readiness waits that bail out the moment the process
dies. The external-mount path (TEST_MOUNT_POINT) is unchanged.

The posix-compliance workflow mounts with -allowOthers, but fusermount
refuses allow_other unless user_allow_other is set in /etc/fuse.conf,
mirroring the fuse-integration workflow FUSE setup.

.github/workflows/posix-compliance.yml

@@ -98,7 +98,11 @@ jobs:
         
         # Set permissions on FUSE device
         sudo chmod 666 /dev/fuse
-        
+
+        # Allow non-root mounts to pass -o allow_other (weed -allowOthers)
+        echo 'user_allow_other' | sudo tee -a /etc/fuse.conf
+        sudo chmod 644 /etc/fuse.conf
+
         # Ensure fuse module is loaded
         sudo modprobe fuse || true
         
@@ -680,6 +684,8 @@ jobs:
         sudo apt-get install -y fuse libfuse-dev fio
         sudo usermod -a -G fuse $USER
         sudo chmod 666 /dev/fuse
+        # The posix suites mount with -allowOthers; allow_other needs this
+        echo 'user_allow_other' | sudo tee -a /etc/fuse.conf
     
     - name: Build SeaweedFS
       run: make
@@ -729,6 +735,8 @@ jobs:
         sudo apt-get update
         sudo apt-get install -y fuse libfuse-dev
         pip3 install --user nfstest
+        # The posix suites mount with -allowOthers; allow_other needs this
+        echo 'user_allow_other' | sudo tee -a /etc/fuse.conf
     
     - name: Build SeaweedFS
       run: make

test/fuse_integration/framework.go

@@ -7,6 +7,7 @@ import (
 	"os"
 	"os/exec"
 	"path/filepath"
+	"strconv"
 	"syscall"
 	"testing"
 	"time"
@@ -14,19 +15,19 @@ import (
 	"github.com/stretchr/testify/require"
 )
 
-// FuseTestFramework provides utilities for FUSE integration testing
+// FuseTestFramework provides utilities for FUSE integration testing.
+// It starts a single "weed mini" process (master+volume+filer in one)
+// and a separate "weed mount" process for the FUSE filesystem.
 type FuseTestFramework struct {
 	t             *testing.T
 	tempDir       string
 	mountPoint    string
 	dataDir       string
-	masterProcess *os.Process
-	volumeProcess *os.Process
-	filerProcess  *os.Process
-	mountProcess  *os.Process
-	masterAddr    string
-	volumeAddr    string
+	logDir        string
+	miniProcess   *managedProcess
+	mountProcess  *managedProcess
 	filerAddr     string
+	filerPort     int
 	weedBinary    string
 	isSetup       bool
 	config        *TestConfig
@@ -35,14 +36,18 @@ type FuseTestFramework struct {
 
 // TestConfig holds configuration for FUSE tests
 type TestConfig struct {
-	Collection   string
-	Replication  string
-	ChunkSizeMB  int
-	CacheSizeMB  int
-	NumVolumes   int
-	EnableDebug  bool
-	MountOptions []string
-	SkipCleanup  bool // for debugging failed tests
+	Collection  string
+	Replication string
+	ChunkSizeMB int
+	CacheSizeMB int
+	NumVolumes  int
+	EnableDebug bool
+	// MountGlobalOptions are glog flags (-v, -logtostderr, ...) registered on
+	// weed's global flagset; they must precede the subcommand name or the
+	// mount process dies at flag parsing.
+	MountGlobalOptions []string
+	MountOptions       []string
+	SkipCleanup        bool // for debugging failed tests
 }
 
 // DefaultTestConfig returns a default configuration for FUSE tests
@@ -59,7 +64,7 @@ func DefaultTestConfig() *TestConfig {
 	}
 }
 
-// NewFuseTestFramework creates a new FUSE testing framework
+// NewFuseTestFramework creates a new FUSE testing framework.
 func NewFuseTestFramework(t *testing.T, config *TestConfig) *FuseTestFramework {
 	if config == nil {
 		config = DefaultTestConfig()
@@ -68,20 +73,63 @@ func NewFuseTestFramework(t *testing.T, config *TestConfig) *FuseTestFramework {
 	tempDir, err := os.MkdirTemp("", "seaweedfs_fuse_test_")
 	require.NoError(t, err)
 
+	filerPort := freePort(t)
+
 	return &FuseTestFramework{
 		t:          t,
 		tempDir:    tempDir,
 		mountPoint: filepath.Join(tempDir, "mount"),
 		dataDir:    filepath.Join(tempDir, "data"),
-		masterAddr: "127.0.0.1:19333",
-		volumeAddr: "127.0.0.1:18080",
-		filerAddr:  "127.0.0.1:18888",
+		logDir:     filepath.Join(tempDir, "logs"),
+		filerPort:  filerPort,
+		filerAddr:  fmt.Sprintf("127.0.0.1:%d", filerPort),
 		weedBinary: findWeedBinary(),
 		isSetup:    false,
 		config:     config,
 	}
 }
 
+// grpcPortOffset mirrors weed's HTTP->gRPC port convention (gRPC = HTTP + offset);
+// "weed mount" derives the filer gRPC port from the filer HTTP address the same way.
+const grpcPortOffset = 10000
+
+// freePort returns a free filer HTTP port whose gRPC sibling (port+grpcPortOffset)
+// is also free. Both must stay below the Linux ephemeral floor (32768): a gRPC port
+// above it can be transiently grabbed by an outbound connection, forcing mini to
+// relocate its filer gRPC port while "weed mount" keeps dialing HTTP+10000 — the
+// mount then never connects and the test times out. Capping HTTP at 22000 keeps the
+// gRPC port at or below 32000.
+func freePort(t *testing.T) int {
+	t.Helper()
+	const (
+		minServicePort = 20000
+		maxServicePort = 22000
+	)
+
+	portCount := maxServicePort - minServicePort + 1
+	start := minServicePort + int(time.Now().UnixNano()%int64(portCount))
+
+	for attempt := 0; attempt < portCount; attempt++ {
+		port := minServicePort + (start-minServicePort+attempt)%portCount
+		if portIsFree(port) && portIsFree(port+grpcPortOffset) {
+			return port
+		}
+	}
+
+	t.Fatalf("failed to allocate a free HTTP/gRPC port pair in [%d,%d]", minServicePort, maxServicePort)
+	return 0
+}
+
+// portIsFree reports whether a TCP port can currently be bound on 127.0.0.1.
+func portIsFree(port int) bool {
+	l, err := net.Listen("tcp", net.JoinHostPort("127.0.0.1", strconv.Itoa(port)))
+	if err != nil {
+		return false
+	}
+	l.Close()
+	return true
+}
+
 // Setup starts SeaweedFS cluster and mounts FUSE filesystem
 func (f *FuseTestFramework) Setup(config *TestConfig) error {
 	if f.isSetup {
@@ -159,41 +207,22 @@ func (f *FuseTestFramework) Setup(config *TestConfig) error {
 	}
 
 	// Create directories for full cluster setup
-	dirs := []string{f.mountPoint, f.dataDir}
+	dirs := []string{f.mountPoint, f.logDir, f.dataDir}
 	for _, dir := range dirs {
 		if err := os.MkdirAll(dir, 0755); err != nil {
 			return fmt.Errorf("failed to create directory %s: %v", dir, err)
 		}
 	}
 
-	// Start master
-	if err := f.startMaster(config); err != nil {
-		return fmt.Errorf("failed to start master: %v", err)
+	// Start weed mini (master + volume + filer in one process)
+	if err := f.startMini(config); err != nil {
+		return fmt.Errorf("failed to start weed mini: %v", err)
 	}
 
-	// Wait for master to be ready
-	if err := f.waitForService(f.masterAddr, 30*time.Second); err != nil {
-		return fmt.Errorf("master not ready: %v", err)
-	}
-
-	// Start volume servers
-	if err := f.startVolumeServers(config); err != nil {
-		return fmt.Errorf("failed to start volume servers: %v", err)
-	}
-
-	// Wait for volume server to be ready
-	if err := f.waitForService(f.volumeAddr, 30*time.Second); err != nil {
-		return fmt.Errorf("volume server not ready: %v", err)
-	}
-
-	// Start filer
-	if err := f.startFiler(config); err != nil {
-		return fmt.Errorf("failed to start filer: %v", err)
-	}
-
-	// Wait for filer to be ready
-	if err := f.waitForService(f.filerAddr, 30*time.Second); err != nil {
-		return fmt.Errorf("filer not ready: %v", err)
+	// Wait for filer to be ready (mini starts all services on filerPort)
+	if err := f.waitForService(f.miniProcess, f.filerAddr, 30*time.Second); err != nil {
+		f.dumpLog("mini")
+		return fmt.Errorf("weed mini not ready: %v", err)
 	}
 
 	// Skip rather than fail on a host with no FUSE device to mount through
@@ -208,6 +237,7 @@ func (f *FuseTestFramework) Setup(config *TestConfig) error {
 
 	// Wait for mount to be ready
 	if err := f.waitForMount(30 * time.Second); err != nil {
+		f.dumpLog("mount")
 		return fmt.Errorf("FUSE mount not ready: %v", err)
 	}
 
@@ -215,7 +245,8 @@ func (f *FuseTestFramework) Setup(config *TestConfig) error {
 	return nil
 }
 
-// Cleanup stops all processes and removes temporary files
+// Cleanup stops all processes and removes temporary files.
+// If the test failed, it dumps logs automatically.
 func (f *FuseTestFramework) Cleanup() {
 	// An external mount needs no teardown; only this run's work dir is ours.
 	// A TEST_SKIP_CLUSTER_SETUP that fell through to a self-started cluster
@@ -228,25 +259,32 @@ func (f *FuseTestFramework) Cleanup() {
 		return
 	}
 
-	if f.mountProcess != nil {
-		f.unmountFuse()
+	if f.t.Failed() {
+		f.DumpLogs()
 	}
 
 	// Stop processes in reverse order
-	processes := []*os.Process{f.mountProcess, f.filerProcess, f.volumeProcess, f.masterProcess}
-	for _, proc := range processes {
-		if proc != nil {
-			proc.Signal(syscall.SIGTERM)
-			proc.Wait()
-		}
+	f.unmountFuse()
+	if f.miniProcess != nil {
+		f.miniProcess.stop()
+		f.miniProcess = nil
 	}
 
+	f.copyLogsForCI()
+
 	// Remove temp directory
 	if !f.config.SkipCleanup {
 		os.RemoveAll(f.tempDir)
 	}
 }
 
+// DumpLogs prints the tail of all SeaweedFS process logs to test output.
+func (f *FuseTestFramework) DumpLogs() {
+	for _, name := range []string{"mini", "mount"} {
+		f.dumpLog(name)
+	}
+}
+
 // GetMountPoint returns the FUSE mount point path
 func (f *FuseTestFramework) GetMountPoint() string {
 	return f.mountPoint
@@ -257,88 +295,136 @@ func (f *FuseTestFramework) GetFilerAddr() string {
 	return f.filerAddr
 }
 
-// startMaster starts the SeaweedFS master server
-func (f *FuseTestFramework) startMaster(config *TestConfig) error {
+// managedProcess is a started weed sub-command whose exit is watched, so a wait
+// for it to come up ends the moment it dies instead of burning its full timeout.
+type managedProcess struct {
+	cmd  *exec.Cmd
+	done chan struct{}
+	err  error // exit error, read only after done is closed
+}
+
+// exited returns the exit error once the process is gone, nil while it runs.
+func (p *managedProcess) exited() error {
+	select {
+	case <-p.done:
+		if p.err != nil {
+			return p.err
+		}
+		return fmt.Errorf("exit status 0")
+	default:
+		return nil
+	}
+}
+
+// stop asks the process to terminate and waits for it to go away.
+func (p *managedProcess) stop() {
+	// Signal fails with os.ErrProcessDone when the child is already gone, which
+	// is exactly the case the select below handles.
+	_ = p.cmd.Process.Signal(syscall.SIGTERM)
+	select {
+	case <-p.done:
+	case <-time.After(10 * time.Second):
+		p.cmd.Process.Kill()
+		<-p.done
+	}
+}
+
+// startProcess is a helper that starts a weed sub-command with output captured
+// to a log file in f.logDir.
+func (f *FuseTestFramework) startProcess(name string, args []string) (*managedProcess, error) {
+	logFile, err := os.Create(filepath.Join(f.logDir, name+".log"))
+	if err != nil {
+		return nil, fmt.Errorf("create log file: %v", err)
+	}
+	cmd := exec.Command(f.weedBinary, args...)
+	cmd.Dir = f.tempDir
+	cmd.Stdout = logFile
+	cmd.Stderr = logFile
+	if err := cmd.Start(); err != nil {
+		logFile.Close()
+		return nil, err
+	}
+	// Close the file handle — the child process inherited it.
+	logFile.Close()
+
+	p := &managedProcess{cmd: cmd, done: make(chan struct{})}
+	go func() {
+		p.err = cmd.Wait()
+		close(p.done)
+	}()
+	return p, nil
+}
+
+// dumpLog prints the last lines of a process log file to the test output
+// for debugging when a service fails to start or a test fails.
+func (f *FuseTestFramework) dumpLog(name string) {
+	data, err := os.ReadFile(filepath.Join(f.logDir, name+".log"))
+	if err != nil {
+		f.t.Logf("[%s log] (not available: %v)", name, err)
+		return
+	}
+	// Show last 16KB on failure for meaningful context.
+	const maxTail = 16 * 1024
+	if len(data) > maxTail {
+		data = data[len(data)-maxTail:]
+	}
+	f.t.Logf("[%s log tail (%d bytes)]\n%s", name, len(data), string(data))
+}
+
+// copyLogsForCI copies SeaweedFS process logs to /tmp/seaweedfs-fuse-logs/
+// so the CI workflow can upload them as artifacts.
+func (f *FuseTestFramework) copyLogsForCI() {
+	ciLogDir := "/tmp/seaweedfs-fuse-logs"
+	os.MkdirAll(ciLogDir, 0755)
+	for _, name := range []string{"mini", "mount"} {
+		src := filepath.Join(f.logDir, name+".log")
+		data, err := os.ReadFile(src)
+		if err != nil {
+			continue
+		}
+		os.WriteFile(filepath.Join(ciLogDir, name+".log"), data, 0644)
+	}
+}
+
+// startMini starts "weed mini" which runs master+volume+filer in one process.
+func (f *FuseTestFramework) startMini(config *TestConfig) error {
 	var args []string
 	if config.EnableDebug {
 		// -v is a global flag; before the subcommand or flag parsing dies
 		args = append(args, "-v=4")
 	}
 	args = append(args,
-		"master",
+		"mini",
+		"-dir="+f.dataDir,
 		"-ip=127.0.0.1",
-		"-port=19333",
-		"-mdir=" + filepath.Join(f.dataDir, "master"),
-		"-raftBootstrap",
+		"-ip.bind=127.0.0.1",
+		"-filer.port="+strconv.Itoa(f.filerPort),
+		"-s3=false",
+		"-webdav=false",
+		"-admin.ui=false",
 	)
 
-	cmd := exec.Command(f.weedBinary, args...)
-	cmd.Dir = f.tempDir
-	if err := cmd.Start(); err != nil {
+	proc, err := f.startProcess("mini", args)
+	if err != nil {
 		return err
 	}
-	f.masterProcess = cmd.Process
-	return nil
-}
-
-// startVolumeServers starts SeaweedFS volume servers
-func (f *FuseTestFramework) startVolumeServers(config *TestConfig) error {
-	var args []string
-	if config.EnableDebug {
-		args = append(args, "-v=4")
-	}
-	args = append(args,
-		"volume",
-		"-mserver=" + f.masterAddr,
-		"-ip=127.0.0.1",
-		"-port=18080",
-		"-dir=" + filepath.Join(f.dataDir, "volume"),
-		fmt.Sprintf("-max=%d", config.NumVolumes),
-	)
-
-	cmd := exec.Command(f.weedBinary, args...)
-	cmd.Dir = f.tempDir
-	if err := cmd.Start(); err != nil {
-		return err
-	}
-	f.volumeProcess = cmd.Process
-	return nil
-}
-
-// startFiler starts the SeaweedFS filer server
-func (f *FuseTestFramework) startFiler(config *TestConfig) error {
-	var args []string
-	if config.EnableDebug {
-		args = append(args, "-v=4")
-	}
-	args = append(args,
-		"filer",
-		"-master=" + f.masterAddr,
-		"-ip=127.0.0.1",
-		"-port=18888",
-	)
-
-	cmd := exec.Command(f.weedBinary, args...)
-	cmd.Dir = f.tempDir
-	if err := cmd.Start(); err != nil {
-		return err
-	}
-	f.filerProcess = cmd.Process
+	f.miniProcess = proc
 	return nil
 }
 
 // mountFuse mounts the SeaweedFS FUSE filesystem
 func (f *FuseTestFramework) mountFuse(config *TestConfig) error {
-	var args []string
+	args := append([]string{}, config.MountGlobalOptions...)
 	if config.EnableDebug {
 		args = append(args, "-v=4")
 	}
 	args = append(args,
 		"mount",
-		"-filer=" + f.filerAddr,
-		"-dir=" + f.mountPoint,
+		"-filer="+f.filerAddr,
+		"-dir="+f.mountPoint,
 		"-filer.path=/",
 		"-dirAutoCreate",
+		"-allowOthers=false",
 	)
 
 	if config.Collection != "" {
@@ -351,35 +437,35 @@ func (f *FuseTestFramework) mountFuse(config *TestConfig) error {
 		args = append(args, fmt.Sprintf("-chunkSizeLimitMB=%d", config.ChunkSizeMB))
 	}
 	if config.CacheSizeMB > 0 {
-		args = append(args, fmt.Sprintf("-cacheSizeMB=%d", config.CacheSizeMB))
+		args = append(args, fmt.Sprintf("-cacheCapacityMB=%d", config.CacheSizeMB))
 	}
 
 	args = append(args, config.MountOptions...)
 
-	cmd := exec.Command(f.weedBinary, args...)
-	cmd.Dir = f.tempDir
-	if err := cmd.Start(); err != nil {
+	proc, err := f.startProcess("mount", args)
+	if err != nil {
 		return err
 	}
-	f.mountProcess = cmd.Process
+	f.mountProcess = proc
 	return nil
 }
 
 // unmountFuse unmounts the FUSE filesystem
 func (f *FuseTestFramework) unmountFuse() error {
 	if f.mountProcess != nil {
-		f.mountProcess.Signal(syscall.SIGTERM)
-		f.mountProcess.Wait()
+		f.mountProcess.stop()
 		f.mountProcess = nil
 	}
 
 	// Also try system unmount as backup
+	exec.Command("fusermount3", "-u", f.mountPoint).Run()
+	exec.Command("fusermount", "-u", f.mountPoint).Run()
 	exec.Command("umount", f.mountPoint).Run()
 	return nil
 }
 
 // waitForService waits for a service to be available
-func (f *FuseTestFramework) waitForService(addr string, timeout time.Duration) error {
+func (f *FuseTestFramework) waitForService(proc *managedProcess, addr string, timeout time.Duration) error {
 	deadline := time.Now().Add(timeout)
 	for time.Now().Before(deadline) {
 		conn, err := net.DialTimeout("tcp", addr, 1*time.Second)
@@ -387,19 +473,23 @@ func (f *FuseTestFramework) waitForService(addr string, timeout time.Duration) e
 			conn.Close()
 			return nil
 		}
+		if exitErr := proc.exited(); exitErr != nil {
+			return fmt.Errorf("process exited (%v) before %s accepted connections", exitErr, addr)
+		}
 		time.Sleep(100 * time.Millisecond)
 	}
 	return fmt.Errorf("service at %s not ready within timeout", addr)
 }
 
-// waitForMount waits for the FUSE mount to be ready
+// waitForMount waits for the FUSE mount to be ready. A stat/ReadDir probe
+// alone is not enough: the bare mount point directory passes both before the
+// mount process finishes starting, letting tests race ahead and write to the
+// local disk underneath the mount. The mount point's device ID differing from
+// its parent's confirms a filesystem is actually mounted there.
 func (f *FuseTestFramework) waitForMount(timeout time.Duration) error {
-	// The mount point is created before weed mount starts, so Stat/ReadDir
-	// alone succeed on the plain directory. The device id only differs from
-	// its parent's once a filesystem is actually mounted on it.
 	parentDev, err := deviceID(filepath.Dir(f.mountPoint))
 	if err != nil {
-		return fmt.Errorf("stat mount parent: %w", err)
+		return fmt.Errorf("stat mount point parent: %w", err)
 	}
 	deadline := time.Now().Add(timeout)
 	for time.Now().Before(deadline) {
@@ -408,6 +498,11 @@ func (f *FuseTestFramework) waitForMount(timeout time.Duration) error {
 				return nil
 			}
 		}
+		// A mount that cannot mount at all (no /dev/fuse, fusermount not setuid)
+		// dies within a second; reporting that beats waiting out the timeout.
+		if exitErr := f.mountProcess.exited(); exitErr != nil {
+			return fmt.Errorf("mount process exited (%v)", exitErr)
+		}
 		time.Sleep(100 * time.Millisecond)
 	}
 	return fmt.Errorf("mount point not ready within timeout")