diff --git a/.github/workflows/nfs-tests.yml b/.github/workflows/nfs-tests.yml index 1167d3311..92fd0a7ea 100644 --- a/.github/workflows/nfs-tests.yml +++ b/.github/workflows/nfs-tests.yml @@ -66,12 +66,44 @@ jobs: # Install test dependencies go mod download - # Run all NFS tests - go test -v -timeout=${{ env.TEST_TIMEOUT }} ./... + # Run the protocol-layer tests. The kernel-mount tests require root + # for mount(2) and are exercised in their own privileged step below; + # skip them here so a "skipped because not root" line doesn't show + # up as noise on every CI run. + go test -v -timeout=${{ env.TEST_TIMEOUT }} -skip '^TestKernelMount' ./... echo "============================================" echo "NFS integration tests completed" + - name: Install kernel NFS client + run: | + # nfs-common provides mount.nfs; netbase provides /etc/protocols + # which mount.nfs's protocol-name lookups (`tcp`, `udp`) need. + sudo apt-get update + sudo apt-get install -y nfs-common netbase + + - name: Run kernel-mount E2E tests + run: | + cd test/nfs + + echo "Running kernel-mount end-to-end tests..." + echo "These mount the running 'weed nfs' subprocess via the actual" + echo "Linux NFS client to catch protocol regressions invisible to" + echo "the go-nfs-client-based tests above." + echo "============================================" + + # mount(2) is privileged. Preserve PATH so 'go' (and the weed + # binary that test/nfs/framework.go locates via $PATH) resolve + # correctly under sudo, and pass through the Go module/cache dirs + # so we don't redownload modules under root. + sudo env "PATH=$PATH" \ + GOMODCACHE="$(go env GOMODCACHE)" \ + GOCACHE="$(go env GOCACHE)" \ + go test -v -timeout=${{ env.TEST_TIMEOUT }} -run '^TestKernelMount' ./... + + echo "============================================" + echo "Kernel-mount E2E tests completed" + - name: Test Summary if: always() run: | @@ -91,7 +123,15 @@ jobs: echo "- **Sequential Append**: Two-part concatenation" >> $GITHUB_STEP_SUMMARY echo "- **ReadDir After Remove**: Meta cache does not serve stale entries" >> $GITHUB_STEP_SUMMARY echo "" >> $GITHUB_STEP_SUMMARY + echo "### Kernel-Mount E2E Coverage" >> $GITHUB_STEP_SUMMARY + echo "- **V3 over TCP**: baseline NFSv3 mount + readdir" >> $GITHUB_STEP_SUMMARY + echo "- **V3 with mountproto=udp**: regression test for UDP MOUNT v3 responder" >> $GITHUB_STEP_SUMMARY + echo "- **V4 rejects cleanly**: regression test for the v4 PROG_MISMATCH path (#9262)" >> $GITHUB_STEP_SUMMARY + echo "" >> $GITHUB_STEP_SUMMARY echo "### Harness" >> $GITHUB_STEP_SUMMARY - echo "Each test boots its own master + volume + filer + nfs subprocess" >> $GITHUB_STEP_SUMMARY - echo "stack on loopback and drives it via the NFSv3 RPC protocol using" >> $GITHUB_STEP_SUMMARY - echo "go-nfs-client. No kernel NFS mount or privileged port is required." >> $GITHUB_STEP_SUMMARY + echo "Most tests boot their own master + volume + filer + nfs subprocess" >> $GITHUB_STEP_SUMMARY + echo "stack on loopback and drive it via the NFSv3 RPC protocol using" >> $GITHUB_STEP_SUMMARY + echo "go-nfs-client. The kernel-mount E2E tests reuse the same harness" >> $GITHUB_STEP_SUMMARY + echo "but mount the export through the in-tree Linux NFS client to" >> $GITHUB_STEP_SUMMARY + echo "catch protocol regressions a Go-only client can't see; they run" >> $GITHUB_STEP_SUMMARY + echo "in a separate privileged step (mount(2) requires root)." >> $GITHUB_STEP_SUMMARY diff --git a/test/nfs/kernel_mount_test.go b/test/nfs/kernel_mount_test.go new file mode 100644 index 000000000..d78059191 --- /dev/null +++ b/test/nfs/kernel_mount_test.go @@ -0,0 +1,193 @@ +//go:build linux + +package nfs + +// End-to-end mount tests that drive the real Linux NFS client (mount.nfs + +// in-tree kernel) against a running `weed nfs` subprocess. These exist to +// catch regressions that the existing framework can't see, because the +// framework drives the server with willscott/go-nfs-client — the same RPC +// library the server uses internally — so any bug shared between the two +// (XDR layout, version dispatch, RPC framing) round-trips invisibly. +// +// Two real bugs hit recently were exactly that shape: +// 1. NFSv4 mis-routed to the v3 SETATTR handler (#9262). The client +// library never sends NFSv4, so the test suite never noticed; the +// Linux kernel mount path did notice, with EIO. +// 2. UDP MOUNT v3 missing. Only TCP MOUNT was advertised; the kernel +// defaults mountproto=udp in many setups, so the in-tree client +// surfaced EPROTONOSUPPORT during MOUNT setup. +// +// These tests mount over the actual loopback interface using mount.nfs and +// shell out to /bin/mount and /bin/umount. They require root (mount(2) is +// privileged) and Linux (the in-tree NFS client is what's being exercised); +// they t.Skip cleanly when either prerequisite is missing. +// +// Run locally with: +// +// cd test/nfs +// sudo go test -v -run TestKernelMount ./... +// +// CI runs them via .github/workflows/nfs-tests.yml after installing +// nfs-common (mount.nfs + helpers). + +import ( + "errors" + "fmt" + "net" + "os" + "os/exec" + "strings" + "testing" +) + +// kernelMountSkipIfUnsupported skips the test when the host can't run a +// real NFS mount. The combined check belongs in one place so the three +// kernel-mount tests stay focused on what they're actually verifying. +func kernelMountSkipIfUnsupported(t *testing.T) { + t.Helper() + if os.Geteuid() != 0 { + t.Skip("kernel mount test requires root; mount(2) is privileged") + } + if _, err := exec.LookPath("mount.nfs"); err != nil { + t.Skipf("mount.nfs not installed: %v (CI installs the nfs-common package)", err) + } +} + +// kernelMount runs /bin/mount with the given options against the framework's +// running NFS server, returns the mountpoint and an unmount closure. We pass +// explicit port=/mountport= options so the kernel never queries portmap. +// That keeps the harness honest about what it's testing — the NFS / MOUNT +// wire protocol — and avoids colliding with a system rpcbind on shared CI +// runners (port 111 is privileged and frequently in use already). +func kernelMount(t *testing.T, fw *NfsTestFramework, optsTemplate string) (string, func()) { + t.Helper() + host, portStr, err := net.SplitHostPort(fw.NfsAddr()) + if err != nil { + t.Fatalf("split nfs addr %q: %v", fw.NfsAddr(), err) + } + mountpoint, err := os.MkdirTemp("", "weed-nfs-kmount-") + if err != nil { + t.Fatalf("mkdtemp: %v", err) + } + opts := strings.ReplaceAll(optsTemplate, "{port}", portStr) + target := fmt.Sprintf("%s:%s", host, fw.ExportRoot()) + cmd := exec.Command("mount", "-t", "nfs", "-o", opts, target, mountpoint) + if out, err := cmd.CombinedOutput(); err != nil { + _ = os.RemoveAll(mountpoint) + t.Fatalf("mount %s -o %s failed: %v\nmount output:\n%s", target, opts, err, out) + } + teardown := func() { + // -f to bail out faster if the server's already gone. + _ = exec.Command("umount", "-f", mountpoint).Run() + _ = os.RemoveAll(mountpoint) + } + return mountpoint, teardown +} + +func newKernelMountFramework(t *testing.T) *NfsTestFramework { + t.Helper() + cfg := DefaultTestConfig() + fw := NewNfsTestFramework(t, cfg) + if err := fw.Setup(cfg); err != nil { + fw.Cleanup() + t.Fatalf("framework setup: %v", err) + } + t.Cleanup(fw.Cleanup) + return fw +} + +// TestKernelMountV3TCP exercises the most common mount form: NFSv3 + MOUNT +// v3, both over TCP. This is what the existing go-nfs-client tests cover at +// the protocol layer, but running it through mount.nfs and the kernel +// confirms that the wire format we emit decodes cleanly under a different +// XDR/RPC parser. +func TestKernelMountV3TCP(t *testing.T) { + kernelMountSkipIfUnsupported(t) + fw := newKernelMountFramework(t) + + mountpoint, undo := kernelMount(t, fw, + "nfsvers=3,nolock,port={port},mountport={port},proto=tcp,mountproto=tcp") + defer undo() + + if _, err := os.Stat(mountpoint); err != nil { + t.Errorf("stat mountpoint: %v", err) + } + if _, err := os.ReadDir(mountpoint); err != nil { + t.Errorf("readdir mountpoint: %v", err) + } +} + +// TestKernelMountV3MountProtoUDP is the regression test for the UDP MOUNT +// v3 responder. mountproto=udp forces the kernel to call MOUNT over UDP +// only; before the responder existed the kernel hit nothing (MOUNT was +// advertised TCP-only) and surfaced EPROTONOSUPPORT during mount setup. +func TestKernelMountV3MountProtoUDP(t *testing.T) { + kernelMountSkipIfUnsupported(t) + fw := newKernelMountFramework(t) + + mountpoint, undo := kernelMount(t, fw, + "nfsvers=3,nolock,port={port},mountport={port},proto=tcp,mountproto=udp") + defer undo() + + if _, err := os.Stat(mountpoint); err != nil { + t.Errorf("stat mountpoint: %v", err) + } +} + +// TestKernelMountV4RejectsCleanly is the regression test for the NFSv4 +// PROG_MISMATCH path (#9262). The server only speaks NFSv3, but the +// previous behaviour was to mis-route v4 COMPOUND to the v3 SETATTR +// handler and write garbage; the kernel surfaced EIO instead of a +// version-mismatch error and (depending on distro) didn't fall back to +// v3. The version filter now answers PROG_MISMATCH so the kernel sees +// "v4 not supported" cleanly. +// +// The test asserts: +// 1. mount.nfs exits non-zero (no silent success against a v3 server); +// 2. the failure message mentions protocol/version/io, which is what the +// kernel surfaces when it gets PROG_MISMATCH instead of garbage. A +// pre-fix server returns "mount system call failed" with no further +// context, so a regression collapses the assertion onto that branch. +func TestKernelMountV4RejectsCleanly(t *testing.T) { + kernelMountSkipIfUnsupported(t) + fw := newKernelMountFramework(t) + + host, portStr, err := net.SplitHostPort(fw.NfsAddr()) + if err != nil { + t.Fatalf("split nfs addr: %v", err) + } + mountpoint, err := os.MkdirTemp("", "weed-nfs-kmount-v4-") + if err != nil { + t.Fatalf("mkdtemp: %v", err) + } + defer os.RemoveAll(mountpoint) + + target := fmt.Sprintf("%s:%s", host, fw.ExportRoot()) + cmd := exec.Command("mount", "-t", "nfs", "-o", + fmt.Sprintf("vers=4,port=%s", portStr), + target, mountpoint) + out, err := cmd.CombinedOutput() + defer exec.Command("umount", "-f", mountpoint).Run() + + if err == nil { + t.Fatalf("v4 mount unexpectedly succeeded against v3-only server\nmount output:\n%s", out) + } + // Don't pin the exact error string — different distros print slightly + // different things — but require some hint that the kernel saw a + // protocol-level failure rather than a generic "mount system call + // failed". Without the version filter, mount.nfs prints the latter + // alone; with it, the former. + lower := strings.ToLower(string(out)) + if !strings.Contains(lower, "protocol") && + !strings.Contains(lower, "version") && + !strings.Contains(lower, "i/o") { + t.Errorf("v4 mount failure didn't mention protocol/version/io; output:\n%s", out) + } + // Also require a non-zero exit so a future change that makes mount(2) + // silently succeed (e.g. by relaxing the version filter) shows up + // here even if the message phrasing changes. + var ee *exec.ExitError + if !errors.As(err, &ee) { + t.Errorf("expected mount to exit non-zero with ExitError, got %v", err) + } +} diff --git a/weed/command/nfs.go b/weed/command/nfs.go index b064a0fe8..a2af58b2d 100644 --- a/weed/command/nfs.go +++ b/weed/command/nfs.go @@ -73,6 +73,10 @@ or enable the built-in portmap responder on the server: weed nfs ... -portmap.bind=0.0.0.0 +With the responder enabled MOUNT v3 is answered over both TCP and UDP, +so the plain mount form above just works — no mountproto override is +required even on clients whose default mountproto is UDP. + Binding port 111 requires root or CAP_NET_BIND_SERVICE and must not collide with a system rpcbind. `, diff --git a/weed/server/nfs/mount_udp.go b/weed/server/nfs/mount_udp.go new file mode 100644 index 000000000..649f15354 --- /dev/null +++ b/weed/server/nfs/mount_udp.go @@ -0,0 +1,265 @@ +package nfs + +import ( + "encoding/binary" + "fmt" + "net" + "sync" + "time" + + "github.com/seaweedfs/seaweedfs/weed/filer" + "github.com/seaweedfs/seaweedfs/weed/glog" + "github.com/seaweedfs/seaweedfs/weed/util" +) + +// The upstream willscott/go-nfs library only serves the MOUNT protocol over +// TCP. Linux's mount.nfs and the in-kernel NFS client default `mountproto` to +// UDP in many configurations, so against a stock `weed nfs` deployment the +// kernel queries portmap for "MOUNT v3 UDP", gets port=0 ("not registered"), +// and either falls back inconsistently or surfaces EPROTONOSUPPORT +// ("requested NFS version or transport protocol is not supported"). The user +// either has to add `mountproto=tcp` / `mountport=2049` to their mount +// options or guess that their distro happens to fall back to TCP on its own. +// +// This responder closes that gap. It speaks just enough of MOUNT v3 to handle +// MOUNT_NULL / MOUNT_MNT / MOUNT_UMNT over UDP — the only procedures the +// kernel actually invokes during mount setup and teardown — so plain +// `mount -t nfs : /mnt` works without any client-side protocol +// hints. The protocol layout is intentionally identical to the TCP MOUNT +// handler in handler.go's Mount() so the two paths return the same +// filehandle and the same set of auth flavors for the same export. +// +// References: RFC 1813 §5 (NFSv3/MOUNTv3), RFC 5531 (RPC). + +const ( + mountUDPMaxRecord = 32 * 1024 + + // mountUDPRetryBackoff mirrors portmapRetryBackoff so the two + // listening goroutines back off identically under host pressure. + mountUDPRetryBackoff = 50 * time.Millisecond + + mountVersion = 3 + + mountProcNull = 0 + mountProcMnt = 1 + mountProcUmnt = 3 + + // MOUNT v3 status codes (mountstat3 in RFC 1813 §5.1.1). + mnt3StatOK uint32 = 0 + mnt3ErrAcces uint32 = 13 + mnt3ErrNoEnt uint32 = 2 + mnt3ErrNotDir uint32 = 20 + + // XDR opaque length cap for dirpath. RFC 1813 §5.1 limits MNTPATHLEN + // to 1024; cap a bit higher for headroom and reject anything beyond. + mountUDPMaxPathLen = 4096 + + // AuthFlavor numeric IDs (matches go-nfs and RFC 5531 §8). + authFlavorNull = 0 + authFlavorUnix = 1 +) + +// mountUDPServer answers MOUNT v3 RPCs over UDP. It listens on the same port +// the NFS TCP server uses (2049 by default), since that's what we advertise +// via portmap, and shares the parent Server's exportRoot, exportID, and +// client allowlist so the UDP MOUNT path applies the same access policy as +// the TCP path. +type mountUDPServer struct { + bindIP string + port int + server *Server + + udpConn *net.UDPConn + + mu sync.Mutex + closed bool + done chan struct{} + wg sync.WaitGroup +} + +func newMountUDPServer(bindIP string, port int, server *Server) *mountUDPServer { + return &mountUDPServer{ + bindIP: bindIP, + port: port, + server: server, + done: make(chan struct{}), + } +} + +func (m *mountUDPServer) Start() error { + addr := net.JoinHostPort(m.bindIP, fmt.Sprintf("%d", m.port)) + udpAddr, err := net.ResolveUDPAddr("udp", addr) + if err != nil { + return fmt.Errorf("mount udp resolve %s: %w", addr, err) + } + udpConn, err := net.ListenUDP("udp", udpAddr) + if err != nil { + return fmt.Errorf("mount udp listen %s: %w", addr, err) + } + m.udpConn = udpConn + m.wg.Add(1) + go func() { + defer m.wg.Done() + m.serve() + }() + return nil +} + +func (m *mountUDPServer) Close() error { + m.mu.Lock() + if m.closed { + m.mu.Unlock() + return nil + } + m.closed = true + close(m.done) + m.mu.Unlock() + if m.udpConn != nil { + _ = m.udpConn.Close() + } + m.wg.Wait() + return nil +} + +func (m *mountUDPServer) isClosed() bool { + m.mu.Lock() + defer m.mu.Unlock() + return m.closed +} + +func (m *mountUDPServer) serve() { + buf := make([]byte, mountUDPMaxRecord) + for { + n, addr, err := m.udpConn.ReadFromUDP(buf) + if err != nil { + if m.isClosed() { + return + } + // Transient read failure: log, back off, keep the + // responder alive — same pattern as portmap UDP. + glog.V(1).Infof("mount udp read: %v", err) + select { + case <-m.done: + return + case <-time.After(mountUDPRetryBackoff): + continue + } + } + // Apply the parent server's client allowlist before we even + // look at the RPC bytes, mirroring the TCP path's + // allowlistListener wrapping. + if m.server != nil && m.server.clientAuthorizer != nil && !m.server.clientAuthorizer.isAllowedAddr(addr) { + glog.V(1).Infof("mount udp: rejecting unauthorized client %s", addr) + continue + } + reply := m.handleCall(buf[:n], addr) + if reply == nil { + continue + } + if _, err := m.udpConn.WriteToUDP(reply, addr); err != nil { + glog.V(1).Infof("mount udp write to %s: %v", addr, err) + } + } +} + +// handleCall classifies one RPC CALL message and returns the encoded reply, +// or nil if the call is malformed enough to drop silently. +func (m *mountUDPServer) handleCall(callBuf []byte, addr *net.UDPAddr) []byte { + xid, prog, vers, proc, args, err := parseRPCCall(callBuf) + if err != nil { + return nil + } + if prog != mountProgram { + return encodeAcceptedReply(xid, rpcAcceptProgUnavail, nil) + } + if vers != mountVersion { + // Mismatch — advertise the v3..v3 we actually support. + body := make([]byte, 8) + binary.BigEndian.PutUint32(body[0:4], mountVersion) + binary.BigEndian.PutUint32(body[4:8], mountVersion) + return encodeAcceptedReply(xid, rpcAcceptProgMismatch, body) + } + + switch proc { + case mountProcNull: + return encodeAcceptedReply(xid, rpcAcceptSuccess, nil) + case mountProcMnt: + return m.handleMount(xid, args, addr) + case mountProcUmnt: + // Stateless server: there's nothing to forget, just acknowledge. + // The client sends back the dirpath in args; we don't need to + // validate it here because UMNT has no return data. + return encodeAcceptedReply(xid, rpcAcceptSuccess, nil) + default: + // MOUNT v3 also defines DUMP / EXPORT / UMNTALL but the kernel + // mount path doesn't invoke them. Returning PROC_UNAVAIL is + // the protocol-correct response. + return encodeAcceptedReply(xid, rpcAcceptProcUnavail, nil) + } +} + +// handleMount implements MOUNT v3 MNT. The wire format is RFC 1813 §5.1.4: +// +// MOUNT3args { dirpath3 dirpath; } // XDR opaque +// MOUNT3res { mountstat3 status; if OK { handle, auth_flavors[] } } +// +// We mirror handler.go's Mount(): export-path mismatch returns NoEnt; root +// inode is encoded as a synthetic directory filehandle so it round-trips with +// the TCP MOUNT path without an extra filer round-trip per UDP MOUNT call. +func (m *mountUDPServer) handleMount(xid uint32, args []byte, addr *net.UDPAddr) []byte { + if len(args) < 4 { + return encodeAcceptedReply(xid, rpcAcceptGarbageArgs, nil) + } + pathLen := binary.BigEndian.Uint32(args[0:4]) + if pathLen > mountUDPMaxPathLen { + return encodeAcceptedReply(xid, rpcAcceptGarbageArgs, nil) + } + padded := (pathLen + 3) &^ 3 + if uint32(len(args)) < 4+padded { + return encodeAcceptedReply(xid, rpcAcceptGarbageArgs, nil) + } + dirpath := string(args[4 : 4+pathLen]) + + requestedPath := normalizeExportRoot(util.FullPath(dirpath)) + if requestedPath != m.server.exportRoot { + glog.V(1).Infof("mount udp: client %s requested %q but export is %q", addr, dirpath, m.server.exportRoot) + return encodeMountStatus(xid, mnt3ErrNoEnt) + } + + rootHandle := NewFileHandle(m.server.exportID, FileHandleKindDirectory, 0, filer.InodeIndexInitialGeneration).Encode() + flavors := []uint32{authFlavorNull, authFlavorUnix} + return encodeMountSuccess(xid, rootHandle, flavors) +} + +// encodeMountStatus returns a MOUNT MNT reply carrying just an error status. +// Per RFC 1813 §5.1.4 a non-OK status terminates the response — no handle or +// flavors follow. +func encodeMountStatus(xid, status uint32) []byte { + body := make([]byte, 4) + binary.BigEndian.PutUint32(body, status) + return encodeAcceptedReply(xid, rpcAcceptSuccess, body) +} + +// encodeMountSuccess builds the OK MOUNT MNT reply: status=OK, file handle +// (XDR opaque), and the supported auth_flavors list. +func encodeMountSuccess(xid uint32, handle []byte, flavors []uint32) []byte { + handleLen := uint32(len(handle)) + handlePadded := (handleLen + 3) &^ 3 + bodyLen := 4 + 4 + handlePadded + 4 + 4*uint32(len(flavors)) + + body := make([]byte, bodyLen) + binary.BigEndian.PutUint32(body[0:4], mnt3StatOK) + binary.BigEndian.PutUint32(body[4:8], handleLen) + copy(body[8:8+handleLen], handle) + // Trailing pad bytes are already zero from make(). + + pos := 8 + handlePadded + binary.BigEndian.PutUint32(body[pos:pos+4], uint32(len(flavors))) + pos += 4 + for _, fl := range flavors { + binary.BigEndian.PutUint32(body[pos:pos+4], fl) + pos += 4 + } + + return encodeAcceptedReply(xid, rpcAcceptSuccess, body) +} diff --git a/weed/server/nfs/mount_udp_test.go b/weed/server/nfs/mount_udp_test.go new file mode 100644 index 000000000..802c37b70 --- /dev/null +++ b/weed/server/nfs/mount_udp_test.go @@ -0,0 +1,306 @@ +package nfs + +import ( + "encoding/binary" + "net" + "testing" + "time" + + "github.com/seaweedfs/seaweedfs/weed/util" +) + +// buildMountCallFrame constructs a MOUNT v3 RPC CALL with an opaque dirpath +// argument. The shape matches RFC 5531 §9: xid + msg_type=CALL + rpcvers=2 + +// prog + vers + proc + cred(AUTH_NONE) + verf(AUTH_NONE) + arg. +func buildMountCallFrame(xid, prog, vers, proc uint32, dirpath string) []byte { + // RPC CALL header (24 bytes) + 2 × AUTH_NONE opaque_auth (16 bytes) + + // dirpath as XDR opaque (4-byte length + padded body). + dpLen := uint32(len(dirpath)) + dpPadded := (dpLen + 3) &^ 3 + out := make([]byte, 24+16+4+dpPadded) + binary.BigEndian.PutUint32(out[0:4], xid) + binary.BigEndian.PutUint32(out[4:8], rpcMsgCall) + binary.BigEndian.PutUint32(out[8:12], 2) // rpcvers + binary.BigEndian.PutUint32(out[12:16], prog) + binary.BigEndian.PutUint32(out[16:20], vers) + binary.BigEndian.PutUint32(out[20:24], proc) + // cred + verf both AUTH_NONE / length 0 (already zero-filled). + binary.BigEndian.PutUint32(out[40:44], dpLen) + copy(out[44:44+dpLen], dirpath) + return out +} + +func newMountUDPTestServer(t *testing.T, exportPath string) (*mountUDPServer, *net.UDPConn) { + t.Helper() + + // Build a minimal Server with just the fields the UDP MOUNT path + // uses: exportRoot, exportID, and a permissive clientAuthorizer. + exportRoot := normalizeExportRoot(util.FullPath(exportPath)) + authz, err := newClientAuthorizer(nil) + if err != nil { + t.Fatal(err) + } + srv := &Server{ + option: &Option{}, + exportRoot: exportRoot, + exportID: exportIDForRoot(exportRoot), + clientAuthorizer: authz, + } + + udpAddr, err := net.ResolveUDPAddr("udp", "127.0.0.1:0") + if err != nil { + t.Fatal(err) + } + conn, err := net.ListenUDP("udp", udpAddr) + if err != nil { + t.Fatal(err) + } + + m := &mountUDPServer{ + bindIP: "127.0.0.1", + port: conn.LocalAddr().(*net.UDPAddr).Port, + server: srv, + udpConn: conn, + done: make(chan struct{}), + } + m.wg.Add(1) + go func() { + defer m.wg.Done() + m.serve() + }() + t.Cleanup(func() { + _ = m.Close() + }) + return m, conn +} + +func sendMountUDP(t *testing.T, target *net.UDPAddr, payload []byte) []byte { + t.Helper() + c, err := net.DialUDP("udp", nil, target) + if err != nil { + t.Fatal(err) + } + defer c.Close() + if _, err := c.Write(payload); err != nil { + t.Fatal(err) + } + _ = c.SetReadDeadline(time.Now().Add(2 * time.Second)) + buf := make([]byte, 4096) + n, err := c.Read(buf) + if err != nil { + t.Fatalf("read reply: %v", err) + } + return buf[:n] +} + +// parseRPCReply pulls xid, accept_stat, and the body that follows accept_stat +// out of a MSG_ACCEPTED reply. Unlike the TCP path there is no fragment +// marker — the entire UDP datagram is the reply. +func parseRPCReply(t *testing.T, reply []byte) (xid, acceptStat uint32, body []byte) { + t.Helper() + if len(reply) < 24 { + t.Fatalf("reply too short: %d bytes", len(reply)) + } + xid = binary.BigEndian.Uint32(reply[0:4]) + if mt := binary.BigEndian.Uint32(reply[4:8]); mt != rpcMsgReply { + t.Fatalf("msg_type=%d want REPLY(1)", mt) + } + if rs := binary.BigEndian.Uint32(reply[8:12]); rs != rpcMsgAccepted { + t.Fatalf("reply_stat=%d want MSG_ACCEPTED(0)", rs) + } + acceptStat = binary.BigEndian.Uint32(reply[20:24]) + body = reply[24:] + return +} + +func TestMountUDPNullReturnsSuccess(t *testing.T) { + m, conn := newMountUDPTestServer(t, "/exports") + target := conn.LocalAddr().(*net.UDPAddr) + + reply := sendMountUDP(t, target, buildMountCallFrame(7, mountProgram, 3, mountProcNull, "")) + xid, astat, body := parseRPCReply(t, reply) + + if xid != 7 { + t.Errorf("xid=%d want 7", xid) + } + if astat != rpcAcceptSuccess { + t.Errorf("accept_stat=%d want SUCCESS(0)", astat) + } + if len(body) != 0 { + t.Errorf("NULL reply body should be empty, got %d bytes", len(body)) + } + _ = m +} + +func TestMountUDPMntReturnsHandleAndFlavors(t *testing.T) { + m, conn := newMountUDPTestServer(t, "/exports") + target := conn.LocalAddr().(*net.UDPAddr) + + reply := sendMountUDP(t, target, buildMountCallFrame(42, mountProgram, 3, mountProcMnt, "/exports")) + xid, astat, body := parseRPCReply(t, reply) + + if xid != 42 { + t.Errorf("xid=%d want 42", xid) + } + if astat != rpcAcceptSuccess { + t.Fatalf("accept_stat=%d want SUCCESS(0)", astat) + } + if len(body) < 4 { + t.Fatalf("body too short: %d bytes", len(body)) + } + status := binary.BigEndian.Uint32(body[0:4]) + if status != mnt3StatOK { + t.Fatalf("mountstat3=%d want OK(0)", status) + } + + // fhandle3: uint32 length + padded opaque bytes. + if len(body) < 8 { + t.Fatalf("body missing handle length: %d bytes", len(body)) + } + handleLen := binary.BigEndian.Uint32(body[4:8]) + handlePadded := (handleLen + 3) &^ 3 + if uint32(len(body)) < 8+handlePadded+4 { + t.Fatalf("body truncated: have %d, need at least %d", len(body), 8+handlePadded+4) + } + handle := body[8 : 8+handleLen] + if _, err := DecodeFileHandle(handle); err != nil { + t.Fatalf("returned handle does not decode: %v", err) + } + + flavorOff := 8 + handlePadded + count := binary.BigEndian.Uint32(body[flavorOff : flavorOff+4]) + if count != 2 { + t.Errorf("flavor count=%d want 2 (NULL + UNIX)", count) + } + got := []uint32{ + binary.BigEndian.Uint32(body[flavorOff+4 : flavorOff+8]), + binary.BigEndian.Uint32(body[flavorOff+8 : flavorOff+12]), + } + if got[0] != authFlavorNull || got[1] != authFlavorUnix { + t.Errorf("flavors=%v want [%d %d]", got, authFlavorNull, authFlavorUnix) + } + _ = m +} + +func TestMountUDPMntRejectsWrongPath(t *testing.T) { + _, conn := newMountUDPTestServer(t, "/exports") + target := conn.LocalAddr().(*net.UDPAddr) + + reply := sendMountUDP(t, target, buildMountCallFrame(99, mountProgram, 3, mountProcMnt, "/somewhere/else")) + _, astat, body := parseRPCReply(t, reply) + + if astat != rpcAcceptSuccess { + t.Fatalf("accept_stat=%d want SUCCESS(0); MNT3ERR is in the body, not at the RPC layer", astat) + } + if len(body) < 4 { + t.Fatalf("body too short: %d bytes", len(body)) + } + if status := binary.BigEndian.Uint32(body[0:4]); status != mnt3ErrNoEnt { + t.Errorf("mountstat3=%d want NoEnt(2)", status) + } + if len(body) != 4 { + t.Errorf("error reply should carry only the status; got %d trailing bytes", len(body)-4) + } +} + +func TestMountUDPRejectsWrongVersion(t *testing.T) { + // Same defence-in-depth as the TCP version filter: don't speak v1/v4 + // MOUNT — return PROG_MISMATCH advertising 3..3 so the client knows + // to retry with v3. + _, conn := newMountUDPTestServer(t, "/exports") + target := conn.LocalAddr().(*net.UDPAddr) + + reply := sendMountUDP(t, target, buildMountCallFrame(1, mountProgram, 4, mountProcNull, "")) + _, astat, body := parseRPCReply(t, reply) + + if astat != rpcAcceptProgMismatch { + t.Fatalf("accept_stat=%d want PROG_MISMATCH(2)", astat) + } + if len(body) != 8 { + t.Fatalf("PROG_MISMATCH body=%d bytes want 8", len(body)) + } + low := binary.BigEndian.Uint32(body[0:4]) + high := binary.BigEndian.Uint32(body[4:8]) + if low != 3 || high != 3 { + t.Errorf("supported range=(%d,%d) want (3,3)", low, high) + } +} + +func TestMountUDPRejectsWrongProgram(t *testing.T) { + _, conn := newMountUDPTestServer(t, "/exports") + target := conn.LocalAddr().(*net.UDPAddr) + + // 100021 is NLM, which we don't run here. + reply := sendMountUDP(t, target, buildMountCallFrame(1, 100021, 4, mountProcNull, "")) + _, astat, _ := parseRPCReply(t, reply) + if astat != rpcAcceptProgUnavail { + t.Errorf("accept_stat=%d want PROG_UNAVAIL(1)", astat) + } +} + +func TestMountUDPUmntAcknowledges(t *testing.T) { + _, conn := newMountUDPTestServer(t, "/exports") + target := conn.LocalAddr().(*net.UDPAddr) + + // UMNT carries a dirpath but the server is stateless and ignores it. + reply := sendMountUDP(t, target, buildMountCallFrame(8, mountProgram, 3, mountProcUmnt, "/exports")) + _, astat, body := parseRPCReply(t, reply) + if astat != rpcAcceptSuccess { + t.Errorf("accept_stat=%d want SUCCESS(0)", astat) + } + if len(body) != 0 { + t.Errorf("UMNT reply body should be empty, got %d bytes", len(body)) + } +} + +func TestMountUDPRejectsTruncatedMntArgs(t *testing.T) { + _, conn := newMountUDPTestServer(t, "/exports") + target := conn.LocalAddr().(*net.UDPAddr) + + // Hand-craft an MNT call whose dirpath length field claims 16 bytes + // but no body follows. Using buildMountCallFrame would also emit a + // trailing length=0 from the empty-string default; we need exactly + // "length, no body" so the GARBAGE_ARGS path actually fires. + frame := make([]byte, 24+16+4) // header + auth + 4-byte length only + binary.BigEndian.PutUint32(frame[0:4], 1) // xid + binary.BigEndian.PutUint32(frame[4:8], rpcMsgCall) // msg_type + binary.BigEndian.PutUint32(frame[8:12], 2) // rpcvers + binary.BigEndian.PutUint32(frame[12:16], mountProgram) + binary.BigEndian.PutUint32(frame[16:20], 3) // mount vers + binary.BigEndian.PutUint32(frame[20:24], mountProcMnt) + // auth = two AUTH_NONE / length-0 stanzas (already zero from make). + binary.BigEndian.PutUint32(frame[40:44], 16) // dirpath length=16, no bytes follow + reply := sendMountUDP(t, target, frame) + _, astat, _ := parseRPCReply(t, reply) + if astat != rpcAcceptGarbageArgs { + t.Errorf("accept_stat=%d want GARBAGE_ARGS(4)", astat) + } +} + +func TestMountUDPCloseStopsServing(t *testing.T) { + m, conn := newMountUDPTestServer(t, "/exports") + target := conn.LocalAddr().(*net.UDPAddr) + + // Sanity: NULL works before close. + _ = sendMountUDP(t, target, buildMountCallFrame(1, mountProgram, 3, mountProcNull, "")) + + if err := m.Close(); err != nil { + t.Fatalf("Close: %v", err) + } + // After Close the socket is shut, so a fresh send should fail to + // read a reply within the deadline rather than producing a + // well-formed response. + c, err := net.DialUDP("udp", nil, target) + if err != nil { + // Some platforms refuse the dial outright after Close — that's + // also acceptable: the server is gone either way. + return + } + defer c.Close() + _, _ = c.Write(buildMountCallFrame(2, mountProgram, 3, mountProcNull, "")) + _ = c.SetReadDeadline(time.Now().Add(200 * time.Millisecond)) + buf := make([]byte, 1024) + if _, err := c.Read(buf); err == nil { + t.Error("Close should have stopped the responder, but a reply still arrived") + } +} diff --git a/weed/server/nfs/portmap.go b/weed/server/nfs/portmap.go index 899100dff..45c715fef 100644 --- a/weed/server/nfs/portmap.go +++ b/weed/server/nfs/portmap.go @@ -103,10 +103,13 @@ type portmapServer struct { } // newPortmapServer builds a responder advertising the NFS services the caller -// runs on nfsTCPPort. We expose NFS v3 TCP and MOUNT v3 TCP only: the -// underlying library does not handle UDP or older MOUNT versions, so it would -// be misleading to advertise them. -func newPortmapServer(bindIP string, port int, nfsTCPPort uint32) *portmapServer { +// runs on nfsPort. NFS itself is TCP-only here (the upstream go-nfs library +// doesn't speak NFS UDP). MOUNT, however, is served over both TCP (via +// go-nfs) and UDP (via mountUDPServer in mount_udp.go), so we advertise +// both — that's what makes plain `mount -t nfs : /mnt` work +// against Linux clients whose default mountproto is UDP without needing +// mountproto=tcp / mountport=2049 mount options. +func newPortmapServer(bindIP string, port int, nfsPort uint32) *portmapServer { if port <= 0 { port = portmapPort } @@ -115,8 +118,9 @@ func newPortmapServer(bindIP string, port int, nfsTCPPort uint32) *portmapServer port: port, done: make(chan struct{}), entries: []portmapEntry{ - {Program: nfsProgram, Version: 3, Protocol: ipProtoTCP, Port: nfsTCPPort}, - {Program: mountProgram, Version: 3, Protocol: ipProtoTCP, Port: nfsTCPPort}, + {Program: nfsProgram, Version: 3, Protocol: ipProtoTCP, Port: nfsPort}, + {Program: mountProgram, Version: 3, Protocol: ipProtoTCP, Port: nfsPort}, + {Program: mountProgram, Version: 3, Protocol: ipProtoUDP, Port: nfsPort}, }, } } diff --git a/weed/server/nfs/portmap_test.go b/weed/server/nfs/portmap_test.go index ed470959b..cc3ec8df5 100644 --- a/weed/server/nfs/portmap_test.go +++ b/weed/server/nfs/portmap_test.go @@ -137,6 +137,7 @@ func TestHandleCall_GetPort_HitAndMiss(t *testing.T) { }{ {"nfs-v3-tcp-hit", nfsProgram, 3, ipProtoTCP, 2049}, {"mount-v3-tcp-hit", mountProgram, 3, ipProtoTCP, 2049}, + {"mount-v3-udp-hit", mountProgram, 3, ipProtoUDP, 2049}, {"mount-v1-tcp-miss", mountProgram, 1, ipProtoTCP, 0}, {"nfs-v3-udp-miss", nfsProgram, 3, ipProtoUDP, 0}, {"nlm-miss", 100021, 4, ipProtoTCP, 0}, @@ -190,12 +191,13 @@ func TestHandleCall_Dump(t *testing.T) { }) p += 16 } - if len(entries) != 2 { - t.Fatalf("got %d dump entries, want 2: %+v", len(entries), entries) + if len(entries) != 3 { + t.Fatalf("got %d dump entries, want 3: %+v", len(entries), entries) } wantSet := map[portmapEntry]bool{ {Program: nfsProgram, Version: 3, Protocol: ipProtoTCP, Port: 2049}: false, {Program: mountProgram, Version: 3, Protocol: ipProtoTCP, Port: 2049}: false, + {Program: mountProgram, Version: 3, Protocol: ipProtoUDP, Port: 2049}: false, } for _, e := range entries { if _, ok := wantSet[e]; !ok { diff --git a/weed/server/nfs/server.go b/weed/server/nfs/server.go index 7b99d815c..6538882fc 100644 --- a/weed/server/nfs/server.go +++ b/weed/server/nfs/server.go @@ -98,6 +98,21 @@ func (s *Server) Start() error { return fmt.Errorf("listen nfs on %s:%d: %w", s.option.BindIp, s.option.Port, err) } + // MOUNT v3 over UDP runs alongside the TCP NFS listener on the same + // port. The kernel default for mountproto is UDP in many setups, so + // without this responder a plain `mount -t nfs : /mnt` + // gets EPROTONOSUPPORT during the MOUNT phase even though the TCP + // NFS path is fine. + mountUDP := newMountUDPServer(s.option.BindIp, s.option.Port, s) + if err := mountUDP.Start(); err != nil { + _ = listener.Close() + return fmt.Errorf("start mount udp: %w", err) + } + defer func() { + _ = mountUDP.Close() + }() + glog.V(0).Infof("MOUNT v3 UDP responder listening on %s:%d", s.option.BindIp, s.option.Port) + var portmap *portmapServer if s.option.PortmapBind != "" { portmap = newPortmapServer(s.option.PortmapBind, portmapPort, uint32(s.option.Port)) @@ -105,8 +120,8 @@ func (s *Server) Start() error { _ = listener.Close() return fmt.Errorf("start portmap: %w", pmErr) } - glog.V(0).Infof("NFS portmap responder listening on %s:%d (NFS v3 tcp=%d, MOUNT v3 tcp=%d)", - s.option.PortmapBind, portmapPort, s.option.Port, s.option.Port) + glog.V(0).Infof("NFS portmap responder listening on %s:%d (NFS v3 tcp=%d, MOUNT v3 tcp=%d, MOUNT v3 udp=%d)", + s.option.PortmapBind, portmapPort, s.option.Port, s.option.Port, s.option.Port) defer func() { if portmap != nil { _ = portmap.Close() @@ -119,13 +134,18 @@ func (s *Server) Start() error { } // logMountHint prints a copy-pasteable Linux mount command so operators can -// see at startup how to mount the export from a client. The go-nfs library -// does not run portmap, so without -portmap.bind the client must bypass -// portmap via -o port=,mountport=,proto=tcp,mountproto=tcp. +// see at startup how to mount the export from a client. +// +// With -portmap.bind set, MOUNT is now answered over both TCP and UDP, so a +// plain `mount -t nfs host:/export /mnt` works — there is no longer any +// kernel-default mountproto path that fails. Without -portmap.bind the +// client still has to bypass portmap entirely via the explicit +// port=/mountport=/proto=/mountproto= options. func (s *Server) logMountHint() { exportPath := string(s.exportRoot) if s.option.PortmapBind != "" { glog.V(0).Infof("mount example: mount -t nfs -o nfsvers=3,nolock :%s ", exportPath) + glog.V(0).Infof("(MOUNT v3 is served over both TCP and UDP, so no mountproto override is needed.)") return } glog.V(0).Infof("mount example (bypasses portmap): mount -t nfs -o nfsvers=3,nolock,noacl,port=%d,mountport=%d,proto=tcp,mountproto=tcp :%s ",