Merge remote-tracking branch 'origin/master' into codex-rust-volume-server-bootstrap

# Conflicts:
#	.github/workflows/volume-server-integration-tests.yml
#	test/volume_server/framework/cluster.go
#	test/volume_server/framework/cluster_dual.go
#	test/volume_server/framework/volume_fixture.go
#	test/volume_server/grpc/data_stream_success_test.go
#	test/volume_server/http/public_cors_methods_test.go
#	test/volume_server/matrix/config_profiles.go

test/benchmark/fuse_db/README.md

@@ -0,0 +1,99 @@
+# FUSE database load / durability / performance benchmark
+
+Runs **MySQL (InnoDB)** and **SQLite** with their data files on a SeaweedFS **FUSE mount**,
+with ~1 GB datasets, and answers two questions:
+
+1. **Durability** — does any committed data get lost across normal shutdown, an unexpected
+   `kill -9`, or a crash during active writes?
+2. **Performance** — how much slower is the FUSE mount than the local disk for the same
+   workload (bulk load, OLTP commits, scans, raw fsync)?
+
+Each row carries a CRC32 of an incompressible (so ~1 GB actually hits the volumes)
+deterministic payload; verification checks `integrity_check`/`CHECK TABLE`, exact row count,
+contiguous-id prefix, and a per-row CRC recompute.
+
+## Results (single node, macOS arm64 + macFUSE, weed 4.34, local NVMe, 2026-06-15)
+
+### Durability — 6/6 PASS, no committed data lost, no corruption
+| scenario | what | SQLite | MySQL |
+|---|---|---|---|
+| A normal shutdown | graceful DB stop, unmount, cluster stop, restart, verify | PASS | PASS |
+| B unexpected (`kill -9`) | kill -9 db+mount+cluster, restart, recover, verify | PASS | PASS |
+| C crash during writes | kill -9 mid-load, restart, recover, verify committed prefix | PASS | PASS |
+
+Scenario C confirmed correct recovery: the in-flight transaction rolled back, every
+fsync-committed row survived as a contiguous prefix (InnoDB redo recovery ~9 s).
+
+### Performance — FUSE vs host (same disk, same durability settings)
+Raw filesystem:
+| metric | host | FUSE | FUSE slower |
+|---|---|---|---|
+| seq write +fsync | 2336 MB/s | 369 MB/s | 6.3x |
+| seq read (warm) | 3435 MB/s | 1422 MB/s | 2.4x |
+| fsync latency | 0.13 ms | 1.18 ms | 9x |
+
+SQLite (1 GB, journal=DELETE, synchronous=FULL):
+| metric | host | FUSE | FUSE slower |
+|---|---|---|---|
+| bulk load | 5.8 s (177 MB/s) | 11.6 s (88 MB/s) | 2.0x |
+| OLTP 1-row txns | 1987 tx/s | 171 tx/s | 11.6x |
+| full scan (warm) | 316 MB/s | 427 MB/s | ~equal (python-bound) |
+
+MySQL/InnoDB (1 GB, trx_commit=1, flush_method=fsync):
+| metric | host | FUSE | FUSE slower |
+|---|---|---|---|
+| bulk load | 24.4 s (42 MB/s) | 32.9 s (31 MB/s) | 1.35x |
+| OLTP 1-row commits | 10542 tx/s | 1144 tx/s | 9.2x |
+| full scan | 2419 MB/s | 430 MB/s | 5.6x |
+
+**Takeaway:** the cost is dominated by fsync/commit latency (0.13 → 1.18 ms, ~9x) because
+every durable commit uploads the chunk to the volume + persists the filer chunk-manifest over
+loopback. Small fsync'd transactions are ~9-12x slower; bulk loads (commits amortized) are
+1.3-2x slower; warm/sequential reads are close to host. These are single-node loopback
+numbers — a real cluster (remote volumes, replication) adds network RTT + replica fsync per
+commit.
+
+## Durability caveat (test scope)
+`kill -9` terminates the **processes** while the OS keeps running, so everything that reached
+the OS page cache (all fsync'd data) survives — this faithfully tests process/daemon crashes
+and is what passed 6/6. It does **not** simulate true **power loss** (page cache lost). The
+FUSE mount uploads chunks **without** `fsync=true` (`weed/mount/weedfs_write.go` builds
+`UploadOption` with no `Fsync`), so the volume does not fsync `.dat` per upload and the filer
+leveldb likely does not sync per write; under a real power cut, recently-"committed" data that
+only reached the page cache could be lost. A VM hard-reset / power-loss test on real hardware
+is the follow-up to close that gap.
+
+## Requirements
+- `weed` on `$PATH` (or set `WEED=/path/to/weed`)
+- macFUSE (macOS) or libfuse (Linux)
+- `python3`, `sqlite3`
+- MySQL/MariaDB install; set `MYSQL_BASE` to its prefix (default macOS Homebrew
+  `/opt/homebrew/opt/mysql`; must contain `bin/mysqld`, `bin/mysql`, `bin/mysqladmin`).
+  For `mysql_bench.py` on a non-default install, also set `MYSQL_BIN=/path/to/mysql`.
+
+## Run
+Runtime artifacts (cluster, mount, logs) go to `$SEAWEED_BENCH_WORK`
+(default `/tmp/seaweedfs_fuse_db_bench`), kept out of the repo.
+
+    cd test/benchmark/fuse_db
+    bash bin/run_sqlite.sh   > results/sqlite.log   2>&1   # SQLite durability suite
+    bash bin/run_mysql.sh    > results/mysql.log    2>&1   # MySQL durability suite
+    bash bin/compare.sh      > results/compare.log  2>&1   # FUSE vs host performance
+
+The harness manages only its own processes (via pidfiles) on non-default ports
+(9555/9560/9565, mysqld 3308); it never runs `pkill weed`, so other SeaweedFS instances on
+the box are untouched.
+
+## Files
+    bin/lib.sh           cluster/mount/mysqld lifecycle helpers (start/stop/kill, clean state)
+    bin/run_sqlite.sh    SQLite 1GB load + A/B/C crash scenarios
+    bin/run_mysql.sh     MySQL 1GB load + A/B/C crash scenarios
+    bin/compare.sh       FUSE-vs-host driver
+    bin/sqlite_gen.py    deterministic incompressible SQLite loader (progress/ref files)
+    bin/sqlite_verify.py SQLite integrity + count + prefix + per-row CRC verifier
+    bin/sqlite_bench.py  SQLite load/OLTP/scan timing probe
+    bin/mysql_bench.py   MySQL load/OLTP/scan timing probe (via mysql CLI)
+    bin/fsbench.py       raw seq-write / fsync-latency / seq-read microbenchmark
+
+The numbers above are the reference baseline; rerun the scripts to reproduce. Run output is
+written to results/*.log, which is gitignored (repo-wide *.log rule) -- not checked in.

test/benchmark/fuse_db/bin/compare.sh

@@ -0,0 +1,44 @@
+#!/bin/bash
+# Identical SQLite + MySQL + raw-fs workloads on HOST (local APFS) vs the SeaweedFS
+# FUSE mount. Same disk underneath, same DB durability settings -> the delta is the
+# SeaweedFS+FUSE stack.
+BIN="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+source "$BIN/lib.sh"
+HOSTDIR=$WORK/hostdir          # local disk, NOT under the FUSE mount
+ROWS=262144; SBATCH=4096; MBATCH=8192; OLTP=2000
+
+mysqld_init_at() { rm -rf "$1"; mkdir -p "$1"; "$MYSQLD" --initialize-insecure --datadir="$1" \
+    --basedir="$MYSQL_BASE" --log-error="$LOGS/mysql_init_cmp.log"; }
+mysqld_run() {
+  "$MYSQLD" --datadir="$1" --basedir="$MYSQL_BASE" --socket="$MYSQL_SOCK" --port=$MYSQL_PORT \
+    --bind-address=127.0.0.1 --innodb_flush_log_at_trx_commit=1 --innodb_flush_method=fsync \
+    --innodb_doublewrite=ON --innodb_buffer_pool_size=256M --secure-file-priv="" \
+    --log-error="$LOGS/mysqld_cmp.log" >> "$LOGS/mysqld_cmp.out" 2>&1 &
+  echo $! > "$PIDS/mysql.pid"; mysql_wait; }
+
+echo "###### FUSE vs HOST   $(date) ######"
+echo "ROWS=$ROWS  payload=4096B incompressible  SQLite(journal=DELETE,sync=FULL)  InnoDB(trx_commit=1,flush=fsync)"
+
+# ===================== HOST (local APFS, no SeaweedFS) =====================
+clean_state >/dev/null 2>&1
+rm -rf "$HOSTDIR"; mkdir -p "$HOSTDIR/sqlite" "$HOSTDIR/fsb"
+echo; echo "===== HOST: raw fs ====="; python3 "$BIN/fsbench.py" "$HOSTDIR/fsb"
+echo; echo "===== HOST: SQLite ====="; python3 "$BIN/sqlite_bench.py" "$HOSTDIR/sqlite/b.db" $ROWS $SBATCH $OLTP
+echo; echo "===== HOST: MySQL ====="
+mysqld_init_at "$HOSTDIR/mysql/data"; mysqld_run "$HOSTDIR/mysql/data"
+python3 "$BIN/mysql_bench.py" "$MYSQL_SOCK" cmp $ROWS $MBATCH $OLTP
+mysql_stop_graceful
+
+# ===================== FUSE (SeaweedFS mount) =====================
+cluster_start || exit 1; mount_start || exit 1
+mkdir -p "$MNT/sqlite" "$MNT/fsb"
+echo; echo "===== FUSE: raw fs ====="; python3 "$BIN/fsbench.py" "$MNT/fsb"
+echo; echo "===== FUSE: SQLite ====="; python3 "$BIN/sqlite_bench.py" "$MNT/sqlite/b.db" $ROWS $SBATCH $OLTP
+echo; echo "===== FUSE: MySQL ====="
+mysqld_init_at "$MNT/mysql/data"; mysqld_run "$MNT/mysql/data"
+python3 "$BIN/mysql_bench.py" "$MYSQL_SOCK" cmp $ROWS $MBATCH $OLTP
+mysql_stop_graceful
+
+stop_all
+rm -rf "$HOSTDIR"
+echo; echo "###### compare done ######"

test/benchmark/fuse_db/bin/fsbench.py

@@ -0,0 +1,52 @@
+#!/usr/bin/env python3
+# Raw filesystem microbenchmark in one directory. Prints TSV: METRIC<TAB>value<TAB>unit
+#   SEQWRITE  MB/s  (512MB stream + final fsync)
+#   FSYNC     ops/s (500 x: pwrite 4KB at random offset in a 64MB file, then fsync)
+#   SEQREAD   MB/s  (read the 512MB file back; warm cache)
+import sys, os, time, random
+
+d = sys.argv[1]
+os.makedirs(d, exist_ok=True)
+big = os.path.join(d, "fsb_big.bin")
+sm  = os.path.join(d, "fsb_sync.bin")
+
+# SEQWRITE 512MB + fsync
+n = 512 * 1024 * 1024
+buf = os.urandom(8 * 1024 * 1024)
+t = time.time(); f = open(big, "wb"); w = 0
+while w < n:
+    f.write(buf); w += len(buf)
+f.flush(); os.fsync(f.fileno()); f.close()
+dt = time.time() - t
+print(f"SEQWRITE\t{w/1048576/dt:.0f}\tMB/s")
+
+# FSYNC latency: 500 x (pwrite 4KB random offset + fsync) on a 64MB file
+fd = os.open(sm, os.O_RDWR | os.O_CREAT, 0o644)
+os.ftruncate(fd, 64 * 1024 * 1024)
+os.fsync(fd)
+page = os.urandom(4096)
+N = 500
+rnd = random.Random(12345)
+t = time.time()
+for _ in range(N):
+    off = rnd.randrange(0, 64 * 1024 * 1024 - 4096) & ~0xfff
+    os.pwrite(fd, page, off)
+    os.fsync(fd)
+dt = time.time() - t
+os.close(fd)
+print(f"FSYNC\t{N/dt:.0f}\tops/s")
+print(f"FSYNCLAT\t{dt/N*1000:.2f}\tms")
+
+# SEQREAD (warm)
+t = time.time(); tot = 0
+with open(big, "rb") as f:
+    while True:
+        b = f.read(8 * 1024 * 1024)
+        if not b: break
+        tot += len(b)
+dt = time.time() - t
+print(f"SEQREAD\t{tot/1048576/dt:.0f}\tMB/s")
+
+for p in (big, sm):
+    try: os.remove(p)
+    except OSError: pass

test/benchmark/fuse_db/bin/lib.sh

@@ -0,0 +1,214 @@
+# Shared config + helpers for the SeaweedFS FUSE load/durability test.
+# IMPORTANT: this harness manages ONLY its own processes via pidfiles and its own
+# (non-default) ports. It never runs `pkill weed`, so any other SeaweedFS instances
+# on the box (e.g. default ports 9333/8888) are safe.
+#
+# Config via env:
+#   SEAWEED_BENCH_WORK  runtime dir for cluster/mount/logs (default /tmp/...; kept out of the repo)
+#   WEED                path to the weed binary (default: from $PATH)
+#   MYSQL_BASE          MySQL install prefix (default macOS Homebrew; on Linux e.g. /usr)
+
+set -u
+
+WORK="${SEAWEED_BENCH_WORK:-/tmp/seaweedfs_fuse_db_bench}"
+WEED="${WEED:-$(command -v weed || echo weed)}"
+RUN=$WORK
+CLUSTER=$RUN/cluster        # weed server -dir (master+volume+filer state) -- local disk
+MNT=$RUN/mnt                # FUSE mount point (the SeaweedFS filesystem under test)
+LOGS=$RUN/logs
+LOCAL=$RUN/local            # non-FUSE scratch: pidfiles, mysql socket, references
+PIDS=$LOCAL/pids
+RESULTS=$RUN/results
+
+# Isolated, non-default ports (avoid colliding with any running cluster)
+MASTER_PORT=9555
+VOLUME_PORT=9560
+FILER_PORT=9565
+FILER_GRPC=19565
+
+# MySQL (macOS Homebrew default; override MYSQL_BASE on Linux, e.g. /usr)
+MYSQL_BASE="${MYSQL_BASE:-/opt/homebrew/opt/mysql}"
+MYSQLD=$MYSQL_BASE/bin/mysqld
+MYSQL=$MYSQL_BASE/bin/mysql
+export MYSQL_BIN="${MYSQL_BIN:-$MYSQL}"   # propagated to mysql_bench.py
+MYSQLADMIN=$MYSQL_BASE/bin/mysqladmin
+MYSQL_PORT=3308
+MYSQL_SOCK=$LOCAL/mysql.sock      # socket on LOCAL disk, datadir on FUSE
+MYSQL_DATADIR=$MNT/mysql/data     # on the FUSE mount -- the thing under test
+
+ts() { date +%H:%M:%S; }
+say() { echo "[$(ts)] $*"; }
+
+ensure_dirs() { mkdir -p "$CLUSTER" "$MNT" "$LOGS" "$LOCAL" "$PIDS" "$RESULTS" "$LOCAL/mountcache"; }
+
+# Portable unmount: fusermount (Linux non-root) -> umount -> diskutil (macOS).
+fuse_umount()       { fusermount -u  "$1" 2>/dev/null || fusermount3 -u  "$1" 2>/dev/null || umount    "$1" 2>/dev/null || diskutil unmount       "$1" 2>/dev/null; }
+fuse_umount_force() { fusermount -uz "$1" 2>/dev/null || fusermount3 -uz "$1" 2>/dev/null || umount -f "$1" 2>/dev/null || diskutil unmount force "$1" 2>/dev/null; }
+
+# My (and only my) ports. Used for the kill-by-port safety net in clean_state.
+MY_PORTS="$MASTER_PORT $VOLUME_PORT $FILER_PORT 19555 19560 $FILER_GRPC $MYSQL_PORT"
+port_busy() { lsof -ti TCP:"$1" >/dev/null 2>&1; }
+wait_port_free() { # $1=port $2=max-halfsec
+  local n=${2:-40}; for i in $(seq 1 "$n"); do port_busy "$1" || return 0; sleep 0.5; done; return 1; }
+
+# ---- cluster (master+volume+filer in one process) ----
+cluster_start() {
+  ensure_dirs
+  if ! wait_port_free "$MASTER_PORT" 30; then
+    say "ERROR master port $MASTER_PORT still busy (pid $(lsof -ti TCP:$MASTER_PORT|tr '\n' ' ')); not starting"; return 1
+  fi
+  say "starting weed server (master:$MASTER_PORT volume:$VOLUME_PORT filer:$FILER_PORT)"
+  "$WEED" server -ip=127.0.0.1 -dir="$CLUSTER" \
+    -master.port=$MASTER_PORT -volume.port=$VOLUME_PORT \
+    -filer -filer.port=$FILER_PORT \
+    -volume.max=50 -master.volumeSizeLimitMB=1024 \
+    >> "$LOGS/cluster.log" 2>&1 &
+  echo $! > "$PIDS/cluster.pid"
+  cluster_wait
+}
+cluster_wait() {
+  local pid; pid=$(cat "$PIDS/cluster.pid" 2>/dev/null)
+  for i in $(seq 1 120); do
+    # the process we launched must still be alive -- guards against a foreign
+    # process answering on the port after ours fataled (e.g. bind conflict)
+    if ! kill -0 "$pid" 2>/dev/null; then
+      say "ERROR cluster process $pid died during startup"; tail -15 "$LOGS/cluster.log"; return 1
+    fi
+    if curl -s "http://127.0.0.1:$MASTER_PORT/cluster/status" >/dev/null 2>&1 \
+       && curl -s "http://127.0.0.1:$FILER_PORT/" >/dev/null 2>&1; then
+      say "cluster ready (pid $pid)"; return 0
+    fi
+    sleep 0.5
+  done
+  say "ERROR cluster not ready"; tail -20 "$LOGS/cluster.log"; return 1
+}
+cluster_stop_graceful() {
+  local pid; pid=$(cat "$PIDS/cluster.pid" 2>/dev/null) || return 0
+  [ -n "${pid:-}" ] || return 0
+  say "graceful stop cluster pid $pid (SIGTERM)"
+  kill -TERM "$pid" 2>/dev/null
+  for i in $(seq 1 60); do kill -0 "$pid" 2>/dev/null || { rm -f "$PIDS/cluster.pid"; return 0; }; sleep 0.5; done
+  say "cluster still alive after 30s; SIGKILL"; kill -9 "$pid" 2>/dev/null; rm -f "$PIDS/cluster.pid"
+}
+cluster_kill_hard() {
+  local pid; pid=$(cat "$PIDS/cluster.pid" 2>/dev/null) || return 0
+  [ -n "${pid:-}" ] && { say "HARD kill -9 cluster pid $pid"; kill -9 "$pid" 2>/dev/null; }
+  rm -f "$PIDS/cluster.pid"
+}
+
+# ---- FUSE mount ----
+mount_start() {
+  ensure_dirs
+  say "mounting FUSE at $MNT (filer 127.0.0.1:$FILER_PORT)"
+  "$WEED" mount -filer=127.0.0.1:$FILER_PORT -dir="$MNT" \
+    -dirAutoCreate -cacheDir="$LOCAL/mountcache" \
+    >> "$LOGS/mount.log" 2>&1 &
+  echo $! > "$PIDS/mount.pid"
+  mount_wait
+}
+mount_wait() {
+  local pid; pid=$(cat "$PIDS/mount.pid" 2>/dev/null)
+  for i in $(seq 1 120); do
+    if ! kill -0 "$pid" 2>/dev/null; then
+      say "ERROR mount process $pid died during startup"; tail -15 "$LOGS/mount.log"; return 1
+    fi
+    if mount | grep -q "$MNT" && ls "$MNT" >/dev/null 2>&1; then say "mount ready (pid $pid)"; return 0; fi
+    sleep 0.5
+  done
+  say "ERROR mount not ready"; tail -20 "$LOGS/mount.log"; return 1
+}
+mount_stop_graceful() {
+  say "graceful unmount $MNT"
+  fuse_umount "$MNT"
+  local pid; pid=$(cat "$PIDS/mount.pid" 2>/dev/null)
+  if [ -n "${pid:-}" ]; then
+    for i in $(seq 1 40); do kill -0 "$pid" 2>/dev/null || break; sleep 0.5; done
+    kill -0 "$pid" 2>/dev/null && { kill -TERM "$pid" 2>/dev/null; sleep 1; }
+  fi
+  rm -f "$PIDS/mount.pid"
+}
+mount_kill_hard() {
+  local pid; pid=$(cat "$PIDS/mount.pid" 2>/dev/null)
+  [ -n "${pid:-}" ] && { say "HARD kill -9 mount pid $pid"; kill -9 "$pid" 2>/dev/null; }
+  rm -f "$PIDS/mount.pid"
+  # clean up the stale mountpoint
+  fuse_umount_force "$MNT"
+}
+
+# ---- MySQL ----
+mysql_init() {
+  ensure_dirs
+  rm -rf "$MNT/mysql"; mkdir -p "$MNT/mysql"
+  say "initializing MySQL datadir on FUSE: $MYSQL_DATADIR"
+  "$MYSQLD" --initialize-insecure --datadir="$MYSQL_DATADIR" --basedir="$MYSQL_BASE" \
+    --log-error="$LOGS/mysql-init.log"
+}
+mysql_start() {
+  say "starting mysqld (port $MYSQL_PORT, datadir on FUSE, flush_log_at_trx_commit=1, flush_method=fsync)"
+  "$MYSQLD" --datadir="$MYSQL_DATADIR" --basedir="$MYSQL_BASE" \
+    --socket="$MYSQL_SOCK" --port=$MYSQL_PORT --bind-address=127.0.0.1 \
+    --innodb_flush_log_at_trx_commit=1 --innodb_flush_method=fsync \
+    --innodb_doublewrite=ON --innodb_buffer_pool_size=256M \
+    --secure-file-priv="" --log-error="$LOGS/mysqld.log" \
+    >> "$LOGS/mysqld.out" 2>&1 &
+  echo $! > "$PIDS/mysql.pid"
+  mysql_wait
+}
+mysql_wait() {
+  for i in $(seq 1 120); do
+    "$MYSQLADMIN" --socket="$MYSQL_SOCK" -uroot ping >/dev/null 2>&1 && { say "mysqld ready"; return 0; }
+    sleep 0.5
+  done
+  say "ERROR mysqld not ready"; tail -30 "$LOGS/mysqld.log"; return 1
+}
+mysql_stop_graceful() {
+  say "graceful mysqld shutdown"
+  "$MYSQLADMIN" --socket="$MYSQL_SOCK" -uroot shutdown 2>/dev/null
+  local pid; pid=$(cat "$PIDS/mysql.pid" 2>/dev/null)
+  if [ -n "${pid:-}" ]; then for i in $(seq 1 60); do kill -0 "$pid" 2>/dev/null || break; sleep 0.5; done; fi
+  rm -f "$PIDS/mysql.pid"
+}
+mysql_kill() {
+  local pid; pid=$(cat "$PIDS/mysql.pid" 2>/dev/null)
+  [ -n "${pid:-}" ] && { say "HARD kill -9 mysqld pid $pid"; kill -9 "$pid" 2>/dev/null; }
+  rm -f "$PIDS/mysql.pid"
+}
+mq() { "$MYSQL" --socket="$MYSQL_SOCK" -uroot "$@"; }
+
+# ---- whole-node unexpected shutdown (simultaneous -9) ----
+hard_kill_all() {
+  say "=== UNEXPECTED SHUTDOWN: kill -9 db + mount + cluster ==="
+  mysql_kill
+  mount_kill_hard
+  cluster_kill_hard
+  kill_my_ports   # safety net in case any pidfile was stale
+}
+
+# stop everything I started (graceful, pidfile-based)
+stop_all() { mysql_stop_graceful; mount_stop_graceful; cluster_stop_graceful; }
+
+# safety net: kill anything still bound to MY (non-default) ports
+kill_my_ports() {
+  for p in $MY_PORTS; do
+    local pids; pids=$(lsof -ti TCP:"$p" 2>/dev/null)
+    [ -n "$pids" ] && { say "killing leftover pid(s) $pids on my port $p"; kill -9 $pids 2>/dev/null; }
+  done
+}
+
+# ---- clean slate (wipe all test state; never touches user data) ----
+clean_state() {
+  say "cleaning test state"
+  stop_all
+  fuse_umount_force "$MNT"
+  kill_my_ports
+  for p in $MASTER_PORT $VOLUME_PORT $FILER_PORT; do wait_port_free "$p" 40 || say "WARN port $p still busy"; done
+  rm -rf "$CLUSTER" "$MNT" "$LOCAL/mountcache" "$PIDS" 2>/dev/null
+  ensure_dirs
+}
+
+status_report() {
+  echo "---- status ----"
+  echo "df $MNT:"; df -h "$MNT" 2>/dev/null | tail -1
+  echo "cluster dir size:"; du -sh "$CLUSTER" 2>/dev/null
+  echo "volumes:"; curl -s "http://127.0.0.1:$MASTER_PORT/dir/status" 2>/dev/null | python3 -c 'import sys,json;d=json.load(sys.stdin);print("  topology free/max:",d.get("Topology",{}).get("Free"),d.get("Topology",{}).get("Max"))' 2>/dev/null
+}

test/benchmark/fuse_db/bin/mysql_bench.py

@@ -0,0 +1,48 @@
+#!/usr/bin/env python3
+# MySQL performance probe via the mysql CLI (no driver needed). Times each phase
+# in Python. Prints TSV: METRIC<TAB>secs<TAB>rate
+#   LOAD : bulk-load <rows> rows (4096B incompressible AES payload), <batch>-row txns
+#   OLTP : <oltp> single-row autocommit INSERTs (each fsyncs redo, trx_commit=1) -> tx/s
+#   SCAN : SUM(CRC32(payload)) over the table (reads every payload) -> MB/s
+import os, re, sys, subprocess, time
+
+sock  = sys.argv[1]
+db    = sys.argv[2]
+rows  = int(sys.argv[3])
+batch = int(sys.argv[4])
+oltp  = int(sys.argv[5])
+MYSQL = os.environ.get("MYSQL_BIN", "mysql")   # lib.sh exports MYSQL_BIN; else use PATH
+
+if not re.fullmatch(r"[A-Za-z0-9_]+", db):     # db is interpolated into SQL
+    raise SystemExit(f"invalid database name: {db!r}")
+
+def run(sql):
+    r = subprocess.run([MYSQL, "--socket", sock, "-uroot"], input=sql.encode(),
+                       stdout=subprocess.DEVNULL, stderr=subprocess.PIPE, timeout=1800)
+    if r.returncode != 0:
+        sys.stderr.write(r.stderr.decode()[:500]); raise SystemExit("mysql failed")
+
+run(f"DROP DATABASE IF EXISTS {db}; CREATE DATABASE {db};"
+    f"CREATE TABLE {db}.t(id INT PRIMARY KEY,payload BLOB,chk BIGINT) ENGINE=InnoDB;"
+    f"CREATE TABLE {db}.o(id INT AUTO_INCREMENT PRIMARY KEY,v INT) ENGINE=InnoDB;")
+
+mb = rows * 4096 / 1048576
+stmts = ["SET SESSION cte_max_recursion_depth=1000000;"]
+o = 0
+while o < rows:
+    b = min(batch, rows - o)
+    stmts.append(
+        f"INSERT INTO {db}.t(id,payload,chk) "
+        f"WITH RECURSIVE seq(n) AS (SELECT 0 UNION ALL SELECT n+1 FROM seq WHERE n<{b-1}) "
+        f"SELECT n+{o},AES_ENCRYPT(REPEAT('a',4080),SHA2(n+{o},256)),"
+        f"CRC32(AES_ENCRYPT(REPEAT('a',4080),SHA2(n+{o},256))) FROM seq;")
+    o += b
+t = time.time(); run("\n".join(stmts)); dt = time.time() - t
+print(f"LOAD\t{dt:.1f}\t{mb/dt:.0f} MB/s")
+
+sql = "\n".join(f"INSERT INTO {db}.o(v) VALUES({i});" for i in range(oltp))
+t = time.time(); run(sql); dt = time.time() - t
+print(f"OLTP\t{dt:.2f}\t{oltp/dt:.0f} tx/s")
+
+t = time.time(); run(f"SELECT SUM(CRC32(payload)) FROM {db}.t;"); dt = time.time() - t
+print(f"SCAN\t{dt:.1f}\t{mb/dt:.0f} MB/s")

test/benchmark/fuse_db/bin/run_mysql.sh

@@ -0,0 +1,120 @@
+#!/bin/bash
+# MySQL (InnoDB) ~1GB load test on the SeaweedFS FUSE mount + crash suite.
+# Datadir lives on the FUSE mount; innodb_flush_log_at_trx_commit=1 +
+# innodb_flush_method=fsync => every commit fsyncs the redo log through FUSE.
+BIN="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+source "$BIN/lib.sh"
+
+TOTAL=262144          # 262144 * 4096B payload = ~1GiB row data (InnoDB on-disk larger)
+BATCH=8192            # 32 committed transactions
+PROG=$LOCAL/mysql_progress.txt
+PASS=0; FAIL=0
+chk() { if [ "$1" -eq 0 ]; then echo ">>> $2: PASS"; PASS=$((PASS+1)); else echo ">>> $2: FAIL"; FAIL=$((FAIL+1)); fi; }
+
+mysql_schema() {
+  mq -e "CREATE DATABASE IF NOT EXISTS loadtest;
+         DROP TABLE IF EXISTS loadtest.t;
+         CREATE TABLE loadtest.t(id INT PRIMARY KEY, payload BLOB, chk BIGINT) ENGINE=InnoDB;"
+}
+mysql_load() {  # loads TOTAL rows in BATCH-sized committed inserts; progress -> $PROG
+  local O=0 B rem
+  : > "$PROG"
+  while [ $O -lt $TOTAL ]; do
+    B=$BATCH; rem=$((TOTAL-O)); [ $rem -lt $B ] && B=$rem
+    # incompressible + deterministic payload: AES-128-ECB (no IV -> deterministic)
+    # of a fixed 4080B plaintext, keyed per-id -> 4096B ciphertext that defeats
+    # gzip on the volumes. Identical expr for payload and chk, so CRC32(payload)
+    # recomputed at verify must equal stored chk.
+    mq -e "SET SESSION cte_max_recursion_depth=1000000;
+           INSERT INTO loadtest.t(id,payload,chk)
+           WITH RECURSIVE seq(n) AS (SELECT 0 UNION ALL SELECT n+1 FROM seq WHERE n < $((B-1)))
+           SELECT n+$O,
+                  AES_ENCRYPT(REPEAT('a',4080), SHA2(n+$O,256)),
+                  CRC32(AES_ENCRYPT(REPEAT('a',4080), SHA2(n+$O,256)))
+           FROM seq;" \
+      || { echo "INSERT failed at offset $O"; return 1; }
+    # durable + atomic progress (matches sqlite_gen.py) so the crash-test lower
+    # bound is never stale or torn: write tmp, fsync, rename.
+    O=$((O+B)); printf '%s' "$O" > "$PROG.tmp"
+    python3 -c "import os,sys; fd=os.open(sys.argv[1],os.O_RDONLY); os.fsync(fd); os.close(fd)" "$PROG.tmp"
+    mv -f "$PROG.tmp" "$PROG"
+  done
+}
+mysql_verify() {  # $1=expected $2=mode(exact|atleast) ; returns 0 on PASS
+  local exp=$1 mode=$2 cnt sumchk corrupt minid maxid checktbl ok=0
+  read -r cnt sumchk corrupt minid maxid <<EOF
+$(mq -N -e "SELECT COUNT(*),COALESCE(SUM(chk),0),COALESCE(SUM(chk<>CRC32(payload)),0),COALESCE(MIN(id),-1),COALESCE(MAX(id),-1) FROM loadtest.t;")
+EOF
+  checktbl=$(mq -N -e "CHECK TABLE loadtest.t;" | awk '{print $NF}')
+  echo "  rows=$cnt corrupt=$corrupt minid=$minid maxid=$maxid CHECK=$checktbl (want $mode $exp)"
+  [ "$checktbl" = "OK" ] || { echo "  -> CHECK TABLE failed"; ok=1; }
+  [ "$corrupt" = "0" ]   || { echo "  -> payload corruption in $corrupt rows"; ok=1; }
+  if [ "$mode" = exact ]; then
+    [ "$cnt" = "$exp" ] || { echo "  -> row count mismatch"; ok=1; }
+    { [ "$minid" = 0 ] && [ "$maxid" = "$((exp-1))" ]; } || { echo "  -> not contiguous 0..$((exp-1))"; ok=1; }
+  else
+    [ "$cnt" -ge "$exp" ] 2>/dev/null || { echo "  -> LOST COMMITTED DATA (cnt<$exp)"; ok=1; }
+    { [ "$minid" = 0 ] && [ "$maxid" = "$((cnt-1))" ]; } || { echo "  -> not contiguous prefix"; ok=1; }
+  fi
+  return $ok
+}
+
+echo "######## MySQL FUSE load test  $(date) ########"
+
+# ---------- fresh cluster + mysql + 1GB load ----------
+clean_state; cluster_start || exit 1; mount_start || exit 1
+mysql_init || { echo "mysql_init failed"; tail -20 "$LOGS/mysql-init.log"; exit 1; }
+mysql_start || exit 1
+mysql_schema
+say "loading ~1GB into InnoDB"
+t0=$SECONDS
+mysql_load || echo "load reported an error"
+say "load done in $((SECONDS-t0))s"
+mq -N -e "SELECT CONCAT('innodb rows=',COUNT(*),'  data_len=',ROUND(SUM(LENGTH(payload))/1048576),'MB') FROM loadtest.t;"
+du -sh "$MYSQL_DATADIR"; status_report
+
+# ---------- Scenario A: normal (graceful) shutdown + restart ----------
+echo; echo "==================== SCENARIO A: NORMAL SHUTDOWN ===================="
+mysql_stop_graceful
+mount_stop_graceful
+cluster_stop_graceful
+sleep 1
+cluster_start || exit 1; mount_start || exit 1; mysql_start || exit 1
+mysql_verify $TOTAL exact; chk $? "A/normal-shutdown"
+
+# ---------- Scenario B: unexpected shutdown (kill -9 everything) ----------
+echo; echo "============== SCENARIO B: UNEXPECTED SHUTDOWN (kill -9) =============="
+hard_kill_all
+sleep 1
+cluster_start || exit 1; mount_start || exit 1
+say "restarting mysqld -> InnoDB crash recovery runs"
+mysql_start || { echo "mysqld failed to recover"; tail -30 "$LOGS/mysqld.log"; chk 1 "B/unexpected-shutdown"; }
+if mq -e "SELECT 1" >/dev/null 2>&1; then mysql_verify $TOTAL exact; chk $? "B/unexpected-shutdown"; fi
+
+# ---------- Scenario C: crash DURING active writes ----------
+echo; echo "=========== SCENARIO C: CRASH DURING ACTIVE WRITES (kill -9) ==========="
+mysql_stop_graceful
+clean_state; cluster_start || exit 1; mount_start || exit 1
+mysql_init || exit 1; mysql_start || exit 1; mysql_schema
+say "loading in background; will crash storage + mysqld mid-write"
+mysql_load > "$LOGS/mysql_load_c.log" 2>&1 &
+LPID=$!
+while :; do
+  p=$(cat "$PROG" 2>/dev/null || echo 0); p=${p:-0}
+  [ "$p" -ge $((TOTAL/2)) ] && break
+  kill -0 $LPID 2>/dev/null || { say "loader exited early"; break; }
+  sleep 0.2
+done
+LB=$(cat "$PROG" 2>/dev/null); LB=${LB:-0}   # empty file (killed mid-write) -> 0
+say "durable committed lower-bound at crash = $LB rows; crashing now"
+hard_kill_all
+kill -9 $LPID 2>/dev/null; wait $LPID 2>/dev/null
+sleep 1
+cluster_start || exit 1; mount_start || exit 1
+say "restarting mysqld -> InnoDB recovery; in-flight txn must roll back, committed must survive"
+mysql_start || { echo "recovery failed"; tail -30 "$LOGS/mysqld.log"; chk 1 "C/crash-during-write"; }
+if mq -e "SELECT 1" >/dev/null 2>&1; then mysql_verify "$LB" atleast; chk $? "C/crash-during-write"; fi
+
+echo; echo "######## MySQL RESULT: PASS=$PASS FAIL=$FAIL ########"
+mysql_stop_graceful; mount_stop_graceful; cluster_stop_graceful
+exit $FAIL

test/benchmark/fuse_db/bin/run_sqlite.sh

@@ -0,0 +1,67 @@
+#!/bin/bash
+# SQLite ~1GB load test on the SeaweedFS FUSE mount + crash suite.
+BIN="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+source "$BIN/lib.sh"
+
+TOTAL=262144          # 262144 * 4096B payload = ~1GiB of row data
+BATCH=4096            # 64 committed transactions (each fsync'd, synchronous=FULL)
+DB=$MNT/sqlite/load.db
+PROG=$LOCAL/sqlite_progress.txt
+REF=$LOCAL/sqlite_ref.txt
+PASS=0; FAIL=0
+chk() { if [ "$1" -eq 0 ]; then echo ">>> $2: PASS"; PASS=$((PASS+1)); else echo ">>> $2: FAIL"; FAIL=$((FAIL+1)); fi; }
+
+echo "######## SQLite FUSE load test  $(date) ########"
+
+# ---------- fresh cluster + 1GB load ----------
+clean_state; cluster_start || exit 1; mount_start || exit 1
+mkdir -p "$MNT/sqlite"
+say "loading ~1GB into SQLite (journal=DELETE synchronous=FULL)"
+t0=$SECONDS
+python3 "$BIN/sqlite_gen.py" "$DB" $TOTAL $BATCH "$PROG" "$REF" FULL DELETE | tail -3
+say "load done in $((SECONDS-t0))s; file size:"; ls -lh "$DB"; du -sh "$MNT/sqlite"
+status_report
+
+# ---------- Scenario A: normal (graceful) shutdown + restart ----------
+echo; echo "==================== SCENARIO A: NORMAL SHUTDOWN ===================="
+mount_stop_graceful
+cluster_stop_graceful
+sleep 1
+cluster_start || exit 1; mount_start || exit 1
+say "verifying after graceful restart"
+python3 "$BIN/sqlite_verify.py" "$DB" $TOTAL exact; chk $? "A/normal-shutdown"
+
+# ---------- Scenario B: unexpected shutdown (kill -9) + restart ----------
+echo; echo "============== SCENARIO B: UNEXPECTED SHUTDOWN (kill -9) =============="
+hard_kill_all
+sleep 1
+cluster_start || exit 1; mount_start || exit 1
+say "verifying after kill -9 + restart"
+python3 "$BIN/sqlite_verify.py" "$DB" $TOTAL exact; chk $? "B/unexpected-shutdown"
+
+# ---------- Scenario C: crash DURING active writes ----------
+echo; echo "=========== SCENARIO C: CRASH DURING ACTIVE WRITES (kill -9) ==========="
+clean_state; cluster_start || exit 1; mount_start || exit 1
+mkdir -p "$MNT/sqlite"; : > "$PROG"
+say "starting loader in background; will crash storage mid-write"
+python3 "$BIN/sqlite_gen.py" "$DB" $TOTAL $BATCH "$PROG" "" FULL DELETE > "$LOGS/sqlite_gen_c.log" 2>&1 &
+GENPID=$!
+# wait until at least half is durably committed
+while :; do
+  p=$(cat "$PROG" 2>/dev/null || echo 0); p=${p:-0}
+  [ "$p" -ge $((TOTAL/2)) ] && break
+  kill -0 $GENPID 2>/dev/null || { say "loader exited early"; break; }
+  sleep 0.2
+done
+LB=$(cat "$PROG" 2>/dev/null); LB=${LB:-0}   # empty file (killed mid-write) -> 0
+say "durable committed lower-bound at crash = $LB rows; crashing storage now"
+hard_kill_all                      # storage dies under the still-writing loader
+kill -9 $GENPID 2>/dev/null; wait $GENPID 2>/dev/null
+sleep 1
+cluster_start || exit 1; mount_start || exit 1
+say "verifying recovery (committed prefix must survive, integrity must hold)"
+python3 "$BIN/sqlite_verify.py" "$DB" "$LB" atleast; chk $? "C/crash-during-write"
+
+echo; echo "######## SQLite RESULT: PASS=$PASS FAIL=$FAIL ########"
+mount_stop_graceful; cluster_stop_graceful
+exit $FAIL

test/benchmark/fuse_db/bin/sqlite_bench.py

@@ -0,0 +1,51 @@
+#!/usr/bin/env python3
+# SQLite performance probe at one db path. Prints TSV: METRIC<TAB>secs<TAB>rate
+#   LOAD  : bulk-load <rows> rows (4096B incompressible payload) in <batch>-row txns
+#   OLTP  : <oltp> single-row INSERTs, each its own fsync'd autocommit txn -> tx/s
+#   SCAN  : full table scan reading every payload (warm) -> MB/s
+# journal=DELETE, synchronous=FULL (durable: fsync per commit).
+import sys, os, sqlite3, time, random
+
+db    = sys.argv[1]
+rows  = int(sys.argv[2])
+batch = int(sys.argv[3])
+oltp  = int(sys.argv[4])
+
+for ext in ("", "-journal", "-wal", "-shm"):
+    try: os.remove(db + ext)
+    except OSError: pass
+
+con = sqlite3.connect(db, isolation_level=None)
+con.execute("PRAGMA journal_mode=DELETE;")
+con.execute("PRAGMA synchronous=FULL;")
+con.execute("CREATE TABLE t(id INTEGER PRIMARY KEY, payload BLOB, chk INTEGER);")
+
+# LOAD
+mb = rows * 4096 / 1048576
+t = time.time(); done = 0
+rnd = random.Random(12345)   # reuse one PRNG; this probe doesn't verify payloads
+while done < rows:
+    end = min(done + batch, rows)
+    data = [(i, rnd.randbytes(4096), 0) for i in range(done, end)]
+    con.execute("BEGIN")
+    con.executemany("INSERT INTO t VALUES(?,?,?)", data)
+    con.execute("COMMIT")
+    done = end
+dt = time.time() - t
+print(f"LOAD\t{dt:.1f}\t{mb/dt:.0f} MB/s")
+
+# OLTP: each INSERT autocommits (isolation_level=None) -> 1 fsync'd txn each
+con.execute("CREATE TABLE o(id INTEGER PRIMARY KEY, v INTEGER);")
+t = time.time()
+for i in range(oltp):
+    con.execute("INSERT INTO o(v) VALUES(?)", (i,))
+dt = time.time() - t
+print(f"OLTP\t{dt:.2f}\t{oltp/dt:.0f} tx/s")
+
+# SCAN (warm): read every payload
+t = time.time(); nbytes = 0
+for rid, p, c in con.execute("SELECT id,payload,chk FROM t"):
+    nbytes += len(p)
+dt = time.time() - t
+print(f"SCAN\t{dt:.1f}\t{nbytes/1048576/dt:.0f} MB/s")
+con.close()

test/benchmark/fuse_db/bin/sqlite_gen.py

@@ -0,0 +1,62 @@
+#!/usr/bin/env python3
+# Deterministic ~1GB SQLite load on the FUSE mount.
+# Each row: id, payload (4096 deterministic bytes derived from id), chk=crc32(payload).
+# Commits in batches; after each commit, writes the committed row count to a
+# progress file on LOCAL disk and fsyncs it -> a durable record of what was
+# committed, used to verify "no committed data lost" after a crash.
+import sys, os, sqlite3, zlib, random
+
+db_path      = sys.argv[1]
+total_rows   = int(sys.argv[2])
+batch        = int(sys.argv[3])
+progress_path= sys.argv[4]            # LOCAL disk
+ref_path     = sys.argv[5] if len(sys.argv) > 5 else None
+sync_mode    = sys.argv[6] if len(sys.argv) > 6 else "FULL"
+journal_mode = sys.argv[7] if len(sys.argv) > 7 else "DELETE"
+
+# Validate before interpolating into PRAGMAs: a typo must not silently weaken
+# durability and invalidate the comparison / crash-survival claims.
+if sync_mode.upper() not in {"OFF", "NORMAL", "FULL", "EXTRA"}:
+    raise SystemExit(f"invalid synchronous mode: {sync_mode!r}")
+if journal_mode.upper() not in {"DELETE", "TRUNCATE", "PERSIST", "MEMORY", "WAL", "OFF"}:
+    raise SystemExit(f"invalid journal mode: {journal_mode!r}")
+
+def payload(i: int) -> bytes:
+    # deterministic but incompressible (PRNG seeded by id) -> 4096 bytes that
+    # actually consume ~1GB on the volume servers (defeats gzip storage).
+    return random.Random(i).randbytes(4096)
+
+con = sqlite3.connect(db_path, isolation_level=None)
+con.execute(f"PRAGMA journal_mode={journal_mode};")
+con.execute(f"PRAGMA synchronous={sync_mode};")
+con.execute("CREATE TABLE IF NOT EXISTS t(id INTEGER PRIMARY KEY, payload BLOB, chk INTEGER);")
+
+def write_progress(n):
+    # atomic: write tmp + fsync + rename, so a kill -9 mid-write can't leave an
+    # empty/torn progress file (the crash-test lower bound must stay trustworthy).
+    tmp = progress_path + ".tmp"
+    with open(tmp, "w") as f:
+        f.write(str(n)); f.flush(); os.fsync(f.fileno())
+    os.replace(tmp, progress_path)
+
+done = 0
+sum_chk = 0
+sum_id = 0
+while done < total_rows:
+    end = min(done + batch, total_rows)
+    rows = []
+    for i in range(done, end):
+        p = payload(i); c = zlib.crc32(p)
+        rows.append((i, p, c)); sum_chk += c; sum_id += i
+    con.execute("BEGIN;")
+    con.executemany("INSERT INTO t(id,payload,chk) VALUES(?,?,?);", rows)
+    con.execute("COMMIT;")
+    done = end
+    write_progress(done)
+    print(f"committed {done}/{total_rows}", flush=True)
+
+if ref_path:
+    with open(ref_path, "w") as f:
+        f.write(f"{total_rows} {sum_chk} {sum_id}\n"); f.flush(); os.fsync(f.fileno())
+con.close()
+print("DONE", flush=True)

test/benchmark/fuse_db/bin/sqlite_verify.py

@@ -0,0 +1,70 @@
+#!/usr/bin/env python3
+# Verify a SQLite DB after a crash/restart.
+#  - PRAGMA integrity_check  (structural consistency of the b-tree / pages)
+#  - row count / sum(chk) / sum(id) vs expected
+#  - recompute crc32(payload) for every row vs stored chk (byte corruption)
+#
+# usage: sqlite_verify.py <db> <expected_rows> [mode]
+#   mode=exact  (default): count must == expected_rows (normal/unexpected shutdown
+#               of a fully-loaded DB -- nothing may be lost or extra).
+#   mode=atleast: expected_rows is a LOWER BOUND (durable committed progress at
+#               crash time). count must be >= expected_rows and form a contiguous
+#               prefix 0..count-1 (the in-flight txn may or may not have committed,
+#               but no committed row may be lost and recovery must be consistent).
+import sys, sqlite3, zlib, random
+
+db_path = sys.argv[1]
+expected_rows = int(sys.argv[2])
+mode = sys.argv[3] if len(sys.argv) > 3 else "exact"
+if mode not in ("exact", "atleast"):   # a typo must not silently relax checks
+    raise SystemExit(f"invalid mode: {mode!r} (want exact|atleast)")
+
+def payload(i: int) -> bytes:
+    return random.Random(i).randbytes(4096)
+
+con = sqlite3.connect(db_path)
+ok = True
+
+ic = con.execute("PRAGMA integrity_check;").fetchone()[0]
+print(f"integrity_check: {ic}")
+if ic != "ok": ok = False
+
+cnt    = con.execute("SELECT COUNT(*) FROM t;").fetchone()[0]
+minid  = con.execute("SELECT MIN(id) FROM t;").fetchone()[0]
+maxid  = con.execute("SELECT MAX(id) FROM t;").fetchone()[0]
+sum_chk= con.execute("SELECT COALESCE(SUM(chk),0) FROM t;").fetchone()[0]
+sum_id = con.execute("SELECT COALESCE(SUM(id),0)  FROM t;").fetchone()[0]
+
+if mode == "exact":
+    target = expected_rows
+    if cnt != expected_rows:
+        print(f"rows: got={cnt} expected={expected_rows}  MISMATCH"); ok = False
+    else:
+        print(f"rows: got={cnt} expected={expected_rows}  OK")
+else:  # atleast
+    print(f"rows: got={cnt} committed-lower-bound={expected_rows}  "
+          f"{'OK' if cnt >= expected_rows else 'LOST COMMITTED DATA'}")
+    if cnt < expected_rows: ok = False
+    target = cnt
+    # contiguous prefix 0..cnt-1 ?
+    if cnt > 0 and (minid != 0 or maxid != cnt - 1):
+        print(f"prefix: MIN={minid} MAX={maxid} expected 0..{cnt-1}  NON-CONTIGUOUS"); ok = False
+    else:
+        print(f"prefix: 0..{maxid} contiguous  OK")
+
+exp_sum_chk = sum(zlib.crc32(payload(i)) for i in range(target))
+exp_sum_id  = target * (target - 1) // 2
+print(f"sum(chk): got={sum_chk} expected={exp_sum_chk}  {'OK' if sum_chk==exp_sum_chk else 'MISMATCH'}")
+print(f"sum(id):  got={sum_id} expected={exp_sum_id}  {'OK' if sum_id==exp_sum_id else 'MISMATCH'}")
+if sum_chk != exp_sum_chk or sum_id != exp_sum_id: ok = False
+
+corrupt = 0; scanned = 0
+for rid, p, c in con.execute("SELECT id,payload,chk FROM t;"):
+    scanned += 1
+    if zlib.crc32(p) != c: corrupt += 1
+print(f"payload byte-check: scanned={scanned} corrupt={corrupt}  {'OK' if corrupt==0 else 'CORRUPT'}")
+if corrupt != 0: ok = False
+
+con.close()
+print("VERIFY: " + ("PASS" if ok else "FAIL"))
+sys.exit(0 if ok else 1)

test/erasure_coding/admin_dockertest/ec_integration_test.go

@@ -2,32 +2,37 @@ package admin_dockertest
 
 import (
 	"bytes"
+	crand "crypto/rand"
 	"encoding/json"
 	"fmt"
 	"io"
-	"io/ioutil"
-	"math/rand"
 	"net/http"
-	"net/url"
 	"os"
 	"os/exec"
 	"path/filepath"
 	"regexp"
 	"strings"
+	"sync"
 	"testing"
 	"time"
 )
 
 const (
-	AdminUrl  = "http://localhost:23646"
+	AdminUrl  = "http://localhost:12535"
 	MasterUrl = "http://localhost:9333"
 	FilerUrl  = "http://localhost:8888"
 )
 
-// Helper to run commands in background and track PIDs for cleanup
-var runningCmds []*exec.Cmd
+// Helper to run commands in background and track PIDs for cleanup. Guarded
+// by runningCmdsLock so parallel subprocess startup can append safely.
+var (
+	runningCmds     []*exec.Cmd
+	runningCmdsLock sync.Mutex
+)
 
 func cleanup() {
+	runningCmdsLock.Lock()
+	defer runningCmdsLock.Unlock()
 	for _, cmd := range runningCmds {
 		if cmd.Process != nil {
 			cmd.Process.Kill()
@@ -61,7 +66,9 @@ func startWeed(t *testing.T, name string, args ...string) *exec.Cmd {
 	if err != nil {
 		t.Fatalf("Failed to start weed %v: %v", args, err)
 	}
+	runningCmdsLock.Lock()
 	runningCmds = append(runningCmds, cmd)
+	runningCmdsLock.Unlock()
 	return cmd
 }
 
@@ -109,14 +116,23 @@ func ensureEnvironment(t *testing.T) {
 	waitForUrl(t, MasterUrl+"/cluster/status", 10)
 
 	// 3. Start Volume Server (Worker)
-	// Start 14 volume servers to verify RS(10,4) default EC
+	// Start 14 volume servers to verify RS(10,4) default EC. Fork/exec in
+	// parallel because startWeed is non-blocking and the per-process fork +
+	// mkdir + log-file-open overhead stacks up sequentially on cold CI
+	// disks, eating most of the admin /health wait budget further down.
+	var volWg sync.WaitGroup
 	for i := 1; i <= 14; i++ {
-		volName := fmt.Sprintf("volume%d", i)
-		port := 8080 + i - 1
-		dir := filepath.Join("tmp", volName)
-		os.MkdirAll(dir, 0755)
-		startWeed(t, volName, "volume", "-dir="+dir, "-mserver=localhost:9333", fmt.Sprintf("-port=%d", port), "-ip=localhost")
+		volWg.Add(1)
+		go func(i int) {
+			defer volWg.Done()
+			volName := fmt.Sprintf("volume%d", i)
+			port := 8080 + i - 1
+			dir := filepath.Join("tmp", volName)
+			os.MkdirAll(dir, 0755)
+			startWeed(t, volName, "volume", "-dir="+dir, "-mserver=localhost:9333", fmt.Sprintf("-port=%d", port), "-ip=localhost", "-volume.allowUntrustedRemoteEndpoints")
+		}(i)
 	}
+	volWg.Wait()
 
 	// 4. Start Filer
 	os.MkdirAll(filepath.Join("tmp", "filer"), 0755)
@@ -131,14 +147,19 @@ func ensureEnvironment(t *testing.T) {
 		debugPort := 6060 + i
 		dir, _ := filepath.Abs(filepath.Join("tmp", workerName))
 		os.MkdirAll(dir, 0755)
-		startWeed(t, workerName, "worker", "-admin=localhost:23646", "-workingDir="+dir, fmt.Sprintf("-metricsPort=%d", metricsPort), fmt.Sprintf("-debug.port=%d", debugPort))
+		startWeed(t, workerName, "worker", "-admin=localhost:12535", "-workingDir="+dir, fmt.Sprintf("-metricsPort=%d", metricsPort), fmt.Sprintf("-debug.port=%d", debugPort))
 	}
 
 	// 6. Start Admin
 	os.RemoveAll(filepath.Join("tmp", "admin"))
 	os.MkdirAll(filepath.Join("tmp", "admin"), 0755)
-	startWeed(t, "admin", "admin", "-master=localhost:9333", "-port=23646", "-dataDir=./tmp/admin")
-	waitForUrl(t, AdminUrl+"/health", 60)
+	startWeed(t, "admin", "admin", "-master=localhost:9333", "-port=12535", "-dataDir=./tmp/admin")
+	// Admin is started after master, 14 volume servers, filer and 2 workers,
+	// so under cold CI conditions the wait here has to absorb the tail of
+	// every earlier subprocess coming up. 60s is too tight and has flaked;
+	// 180s gives comfortable headroom without meaningfully extending the
+	// fast path (the first successful /health usually hits well under 30s).
+	waitForUrl(t, AdminUrl+"/health", 180)
 
 	t.Log("Environment started successfully")
 }
@@ -155,135 +176,133 @@ func waitForUrl(t *testing.T, url string, retries int) {
 	t.Fatalf("Timeout waiting for %s", url)
 }
 
+func fetchJSON(url string, out interface{}) error {
+	resp, err := http.Get(url)
+	if err != nil {
+		return err
+	}
+	defer resp.Body.Close()
+	if resp.StatusCode != http.StatusOK {
+		body, _ := io.ReadAll(resp.Body)
+		return fmt.Errorf("status %d: %s", resp.StatusCode, string(body))
+	}
+	return json.NewDecoder(resp.Body).Decode(out)
+}
+
+func mapField(obj map[string]interface{}, key string) (interface{}, bool) {
+	if obj == nil {
+		return nil, false
+	}
+	if value, ok := obj[key]; ok {
+		return value, true
+	}
+	return nil, false
+}
+
+func mapFieldAny(obj map[string]interface{}, keys ...string) (interface{}, bool) {
+	for _, key := range keys {
+		if value, ok := mapField(obj, key); ok {
+			return value, true
+		}
+	}
+	return nil, false
+}
+
 func TestEcEndToEnd(t *testing.T) {
 	defer cleanup()
 	ensureEnvironment(t)
 
 	client := &http.Client{}
 
-	// 1. Configure Global Maintenance (Scan Interval = 1s) via API
-	t.Log("Configuring Global Maintenance via API...")
+	// 1. Configure plugin job types for fast EC detection/execution.
+	t.Log("Configuring plugin job types via API...")
 
-	// 1.1 Fetch current config
-	req, _ := http.NewRequest("GET", AdminUrl+"/api/maintenance/config", nil)
+	// Disable volume balance to reduce interference for this EC-focused test.
+	balanceConfig := map[string]interface{}{
+		"job_type": "volume_balance",
+		"admin_runtime": map[string]interface{}{
+			"enabled": false,
+		},
+	}
+	jsonBody, err := json.Marshal(balanceConfig)
+	if err != nil {
+		t.Fatalf("Failed to marshal volume_balance config: %v", err)
+	}
+	req, err := http.NewRequest("PUT", AdminUrl+"/api/plugin/job-types/volume_balance/config", bytes.NewBuffer(jsonBody))
+	if err != nil {
+		t.Fatalf("Failed to create volume_balance config request: %v", err)
+	}
+	req.Header.Set("Content-Type", "application/json")
 	resp, err := client.Do(req)
 	if err != nil {
-		t.Fatalf("Failed to get global config: %v", err)
+		t.Fatalf("Failed to update volume_balance config: %v", err)
 	}
 	if resp.StatusCode != http.StatusOK {
 		body, _ := io.ReadAll(resp.Body)
-		t.Fatalf("Failed to get global config (status %d): %s", resp.StatusCode, string(body))
-	}
-
-	var globalConfig map[string]interface{}
-	if err := json.NewDecoder(resp.Body).Decode(&globalConfig); err != nil {
-		t.Fatalf("Failed to decode global config: %v", err)
+		t.Fatalf("Failed to update volume_balance config (status %d): %s", resp.StatusCode, string(body))
 	}
 	resp.Body.Close()
 
-	// 1.2 Modify config
-	globalConfig["enabled"] = true
-	globalConfig["scan_interval_seconds"] = 1
-
-	// Ensure policy structure exists
-	if globalConfig["policy"] == nil {
-		globalConfig["policy"] = map[string]interface{}{}
+	ecConfig := map[string]interface{}{
+		"job_type": "erasure_coding",
+		"admin_runtime": map[string]interface{}{
+			"enabled":                          true,
+			"detection_interval_minutes":       1,
+			"global_execution_concurrency":     4,
+			"per_worker_execution_concurrency": 4,
+			"max_jobs_per_detection":           100,
+		},
+		"worker_config_values": map[string]interface{}{
+			"quiet_for_seconds": map[string]interface{}{
+				"int64_value": "1",
+			},
+			"min_interval_seconds": map[string]interface{}{
+				"int64_value": "1",
+			},
+			"min_size_mb": map[string]interface{}{
+				"int64_value": "1",
+			},
+			"fullness_ratio": map[string]interface{}{
+				"double_value": 0.0001,
+			},
+		},
 	}
-	policy, _ := globalConfig["policy"].(map[string]interface{})
-
-	// Ensure task_policies structure exists
-	if policy["task_policies"] == nil {
-		policy["task_policies"] = map[string]interface{}{}
+	jsonBody, err = json.Marshal(ecConfig)
+	if err != nil {
+		t.Fatalf("Failed to marshal erasure_coding config: %v", err)
 	}
-	taskPolicies, _ := policy["task_policies"].(map[string]interface{})
-
-	// Disable balance tasks to avoid interference with EC test
-	if taskPolicies["balance"] == nil {
-		taskPolicies["balance"] = map[string]interface{}{}
+	req, err = http.NewRequest("PUT", AdminUrl+"/api/plugin/job-types/erasure_coding/config", bytes.NewBuffer(jsonBody))
+	if err != nil {
+		t.Fatalf("Failed to create erasure_coding config request: %v", err)
 	}
-	balancePolicy, _ := taskPolicies["balance"].(map[string]interface{})
-	balancePolicy["enabled"] = false
-
-	// Set global max concurrent
-	policy["global_max_concurrent"] = 4
-	globalConfig["policy"] = policy
-
-	// Explicitly set required fields
-	requiredFields := map[string]float64{
-		"worker_timeout_seconds":   300,
-		"task_timeout_seconds":     7200,
-		"retry_delay_seconds":      900,
-		"cleanup_interval_seconds": 86400,
-		"task_retention_seconds":   604800,
-		"max_retries":              3,
-	}
-	for field, val := range requiredFields {
-		if _, ok := globalConfig[field]; !ok || globalConfig[field] == 0 {
-			globalConfig[field] = val
-		}
-	}
-
-	// 1.3 Update config
-	jsonBody, _ := json.Marshal(globalConfig)
-	req, _ = http.NewRequest("PUT", AdminUrl+"/api/maintenance/config", bytes.NewBuffer(jsonBody))
 	req.Header.Set("Content-Type", "application/json")
 	resp, err = client.Do(req)
 	if err != nil {
-		t.Fatalf("Failed to update global config: %v", err)
+		t.Fatalf("Failed to update erasure_coding config: %v", err)
 	}
 	if resp.StatusCode != http.StatusOK {
 		body, _ := io.ReadAll(resp.Body)
-		t.Fatalf("Failed to update global config (status %d): %s", resp.StatusCode, string(body))
+		t.Fatalf("Failed to update erasure_coding config (status %d): %s", resp.StatusCode, string(body))
 	}
 	resp.Body.Close()
 
-	// 2. Configure EC Task (Short intervals) via Form API
-	t.Log("Configuring EC Task via Form API...")
-	formData := url.Values{}
-	formData.Set("enabled", "true")
-	formData.Set("scan_interval_seconds", "1")
-	formData.Set("repeat_interval_seconds", "1")
-	formData.Set("check_interval_seconds", "1")
-	formData.Set("max_concurrent", "4")
-	formData.Set("quiet_for_seconds_value", "1")
-	formData.Set("quiet_for_seconds_unit", "seconds")
-	formData.Set("min_size_mb", "1")
-	formData.Set("fullness_ratio", "0.0001")
-
-	req, _ = http.NewRequest("POST", AdminUrl+"/maintenance/config/erasure_coding", strings.NewReader(formData.Encode()))
-	req.Header.Set("Content-Type", "application/x-www-form-urlencoded")
-	resp, err = client.Do(req)
-	if err != nil {
-		t.Fatalf("Failed to update EC config: %v", err)
-	}
-	if resp.StatusCode != 200 && resp.StatusCode != 303 {
-		body, _ := io.ReadAll(resp.Body)
-		t.Fatalf("Failed to update EC config (status %d): %s", resp.StatusCode, string(body))
-	}
-	resp.Body.Close()
-	t.Log("EC Task Configuration updated")
-
-	// 3. Restart Admin to pick up Global Config (Scan Interval)
-	if len(runningCmds) > 0 {
-		adminCmd := runningCmds[len(runningCmds)-1]
-		t.Log("Restarting Admin Server to apply configuration...")
-		stopWeed(t, adminCmd)
-		time.Sleep(10 * time.Second)
-		startWeed(t, "admin_restarted", "admin", "-master=localhost:9333", "-port=23646", "-port.grpc=33646", "-dataDir=./tmp/admin")
-		waitForUrl(t, AdminUrl+"/health", 60)
-	}
-
-	// 4. Upload a file
+	// 2. Upload a file
 	fileSize := 5 * 1024 * 1024
 	data := make([]byte, fileSize)
-	rand.Read(data)
+	crand.Read(data)
 	fileName := fmt.Sprintf("ec_test_file_%d", time.Now().Unix())
 	t.Logf("Uploading %d bytes file %s to Filer...", fileSize, fileName)
 	uploadUrl := FilerUrl + "/" + fileName
 
 	var uploadErr error
 	for i := 0; i < 10; i++ {
-		req, _ := http.NewRequest("PUT", uploadUrl, bytes.NewBuffer(data))
+		req, err := http.NewRequest("PUT", uploadUrl, bytes.NewBuffer(data))
+		if err != nil {
+			uploadErr = err
+			t.Logf("Upload attempt %d failed to create request: %v", i+1, err)
+			time.Sleep(2 * time.Second)
+			continue
+		}
 		resp, err := client.Do(req)
 		if err == nil {
 			if resp.StatusCode == 201 {
@@ -306,17 +325,18 @@ func TestEcEndToEnd(t *testing.T) {
 	}
 	t.Log("Upload successful")
 
-	// 5. Verify EC Encoding
+	// 3. Verify EC Encoding
 	t.Log("Waiting for EC encoding (checking Master topology)...")
 	startTime := time.Now()
 	ecVerified := false
 	var lastBody []byte
+	debugTick := 0
 
 	for time.Since(startTime) < 300*time.Second {
-		// 5.1 Check Master Topology
+		// 3.1 Check Master Topology
 		resp, err := http.Get(MasterUrl + "/dir/status")
 		if err == nil {
-			lastBody, _ = ioutil.ReadAll(resp.Body)
+			lastBody, _ = io.ReadAll(resp.Body)
 			resp.Body.Close()
 
 			// Check total EC shards
@@ -336,25 +356,104 @@ func TestEcEndToEnd(t *testing.T) {
 			}
 		}
 
-		// 5.2 Debug: Check workers and tasks
-		wResp, wErr := http.Get(AdminUrl + "/api/maintenance/workers")
+		// 3.2 Debug: Check workers, jobs, and scheduler status
+		debugTick++
+
+		var workers []map[string]interface{}
 		workerCount := 0
-		if wErr == nil {
-			var workers []interface{}
-			json.NewDecoder(wResp.Body).Decode(&workers)
-			wResp.Body.Close()
+		ecDetectorCount := 0
+		ecExecutorCount := 0
+		if err := fetchJSON(AdminUrl+"/api/plugin/workers", &workers); err == nil {
 			workerCount = len(workers)
+			for _, worker := range workers {
+				capsValue, ok := mapFieldAny(worker, "capabilities", "Capabilities")
+				if !ok {
+					continue
+				}
+				caps, ok := capsValue.(map[string]interface{})
+				if !ok {
+					continue
+				}
+				if capValue, ok := caps["erasure_coding"].(map[string]interface{}); ok {
+					if capValue["can_detect"] == true {
+						ecDetectorCount++
+					}
+					if capValue["can_execute"] == true {
+						ecExecutorCount++
+					}
+				}
+			}
 		}
 
-		tResp, tErr := http.Get(AdminUrl + "/api/maintenance/tasks")
+		var tasks []map[string]interface{}
 		taskCount := 0
-		if tErr == nil {
-			var tasks []interface{}
-			json.NewDecoder(tResp.Body).Decode(&tasks)
-			tResp.Body.Close()
+		ecTaskCount := 0
+		ecTaskStates := map[string]int{}
+		if err := fetchJSON(AdminUrl+"/api/plugin/jobs?limit=1000", &tasks); err == nil {
 			taskCount = len(tasks)
+			for _, task := range tasks {
+				jobType, _ := task["job_type"].(string)
+				state, _ := task["state"].(string)
+				if jobType == "erasure_coding" {
+					ecTaskCount++
+					ecTaskStates[state]++
+				}
+			}
+		}
+
+		t.Logf("Waiting for EC... (Workers: %d det=%d exec=%d, Tasks: %d ec=%d, EC States: %+v)",
+			workerCount, ecDetectorCount, ecExecutorCount, taskCount, ecTaskCount, ecTaskStates)
+
+		if debugTick%3 == 0 {
+			var pluginStatus map[string]interface{}
+			if err := fetchJSON(AdminUrl+"/api/plugin/status", &pluginStatus); err == nil {
+				t.Logf("Plugin status: enabled=%v worker_count=%v worker_grpc_port=%v configured=%v",
+					pluginStatus["enabled"], pluginStatus["worker_count"], pluginStatus["worker_grpc_port"], pluginStatus["configured"])
+			}
+
+			var schedulerStatus map[string]interface{}
+			if err := fetchJSON(AdminUrl+"/api/plugin/scheduler-status", &schedulerStatus); err == nil {
+				if schedValue, ok := schedulerStatus["scheduler"].(map[string]interface{}); ok {
+					t.Logf("Scheduler status: current_job_type=%v phase=%v last_iteration_had_jobs=%v idle_sleep_seconds=%v last_iteration_done_at=%v next_detection_at=%v",
+						schedValue["current_job_type"], schedValue["current_phase"],
+						schedValue["last_iteration_had_jobs"], schedValue["idle_sleep_seconds"], schedValue["last_iteration_done_at"], schedValue["next_detection_at"])
+				} else {
+					t.Logf("Scheduler status: %v", schedulerStatus)
+				}
+			}
+
+			var schedulerStates []map[string]interface{}
+			if err := fetchJSON(AdminUrl+"/api/plugin/scheduler-states", &schedulerStates); err == nil {
+				for _, state := range schedulerStates {
+					if state["job_type"] == "erasure_coding" {
+						t.Logf("EC scheduler state: enabled=%v detection_in_flight=%v detector_available=%v executor_workers=%v next_detection_at=%v last_run_status=%v last_run_started_at=%v last_run_completed_at=%v",
+							state["enabled"], state["detection_in_flight"], state["detector_available"],
+							state["executor_worker_count"], state["next_detection_at"], state["last_run_status"],
+							state["last_run_started_at"], state["last_run_completed_at"])
+						break
+					}
+				}
+			}
+
+			var jobTypes []map[string]interface{}
+			if err := fetchJSON(AdminUrl+"/api/plugin/job-types", &jobTypes); err == nil {
+				var names []string
+				for _, jobType := range jobTypes {
+					if name, ok := jobType["job_type"].(string); ok && name != "" {
+						names = append(names, name)
+					}
+				}
+				t.Logf("Plugin job types: %v", names)
+			}
+
+			var activities []map[string]interface{}
+			if err := fetchJSON(AdminUrl+"/api/plugin/activities?job_type=erasure_coding&limit=5", &activities); err == nil {
+				for i := len(activities) - 1; i >= 0; i-- {
+					act := activities[i]
+					t.Logf("EC activity: stage=%v message=%v occurred_at=%v", act["stage"], act["message"], act["occurred_at"])
+				}
+			}
 		}
-		t.Logf("Waiting for EC... (Workers: %d, Active Tasks: %d)", workerCount, taskCount)
 
 		time.Sleep(10 * time.Second)
 	}

test/erasure_coding/ec_integration_test.go

@@ -3,9 +3,12 @@ package erasure_coding
 import (
 	"bytes"
 	"context"
+	"encoding/json"
 	"fmt"
 	"io"
 	"math"
+	"net"
+	"net/http"
 	"os"
 	"os/exec"
 	"path/filepath"
@@ -13,6 +16,7 @@ import (
 	"testing"
 	"time"
 
+	"github.com/seaweedfs/seaweedfs/test/testutil"
 	"github.com/seaweedfs/seaweedfs/weed/operation"
 	"github.com/seaweedfs/seaweedfs/weed/pb"
 	"github.com/seaweedfs/seaweedfs/weed/pb/volume_server_pb"
@@ -47,17 +51,15 @@ func TestECEncodingVolumeLocationTimingBug(t *testing.T) {
 	defer cluster.Stop()
 
 	// Wait for servers to be ready
-	require.NoError(t, waitForServer("127.0.0.1:9333", 30*time.Second))
-	require.NoError(t, waitForServer("127.0.0.1:8080", 30*time.Second))
-	require.NoError(t, waitForServer("127.0.0.1:8081", 30*time.Second))
-	require.NoError(t, waitForServer("127.0.0.1:8082", 30*time.Second))
-	require.NoError(t, waitForServer("127.0.0.1:8083", 30*time.Second))
-	require.NoError(t, waitForServer("127.0.0.1:8084", 30*time.Second))
-	require.NoError(t, waitForServer("127.0.0.1:8085", 30*time.Second))
+	require.NoError(t, waitForServer(cluster.masterAddr, 30*time.Second))
+	for _, addr := range cluster.volumeAddrs {
+		require.NoError(t, waitForServer(addr, 30*time.Second))
+	}
+	require.NoError(t, waitForMasterLeader(cluster.masterAddr, 30*time.Second))
 
 	// Create command environment
 	options := &shell.ShellOptions{
-		Masters:        stringPtr("127.0.0.1:9333"),
+		Masters:        stringPtr(cluster.masterAddr),
 		GrpcDialOption: grpc.WithInsecure(),
 		FilerGroup:     stringPtr("default"),
 	}
@@ -71,7 +73,7 @@ func TestECEncodingVolumeLocationTimingBug(t *testing.T) {
 
 	// Upload some test data to create volumes
 	testData := []byte("This is test data for EC encoding integration test")
-	volumeId, err := uploadTestData(testData, "127.0.0.1:9333")
+	volumeId, err := uploadTestData(testData, cluster.masterAddr)
 	require.NoError(t, err)
 	t.Logf("Created volume %d with test data", volumeId)
 
@@ -257,12 +259,13 @@ func TestECEncodingMasterTimingRaceCondition(t *testing.T) {
 	defer cluster.Stop()
 
 	// Wait for servers to be ready
-	require.NoError(t, waitForServer("127.0.0.1:9333", 30*time.Second))
-	require.NoError(t, waitForServer("127.0.0.1:8080", 30*time.Second))
+	require.NoError(t, waitForServer(cluster.masterAddr, 30*time.Second))
+	require.NoError(t, waitForServer(cluster.volumeAddrs[0], 30*time.Second))
+	require.NoError(t, waitForMasterLeader(cluster.masterAddr, 30*time.Second))
 
 	// Create command environment
 	options := &shell.ShellOptions{
-		Masters:        stringPtr("127.0.0.1:9333"),
+		Masters:        stringPtr(cluster.masterAddr),
 		GrpcDialOption: grpc.WithInsecure(),
 		FilerGroup:     stringPtr("default"),
 	}
@@ -276,7 +279,7 @@ func TestECEncodingMasterTimingRaceCondition(t *testing.T) {
 
 	// Upload test data
 	testData := []byte("Race condition test data")
-	volumeId, err := uploadTestData(testData, "127.0.0.1:9333")
+	volumeId, err := uploadTestData(testData, cluster.masterAddr)
 	require.NoError(t, err)
 	t.Logf("Created volume %d for race condition test", volumeId)
 
@@ -366,6 +369,8 @@ func TestECEncodingMasterTimingRaceCondition(t *testing.T) {
 type TestCluster struct {
 	masterCmd     *exec.Cmd
 	volumeServers []*exec.Cmd
+	masterAddr    string
+	volumeAddrs   []string
 }
 
 func (c *TestCluster) Stop() {
@@ -402,9 +407,23 @@ func startSeaweedFSCluster(ctx context.Context, dataDir string) (*TestCluster, e
 	securityToml := filepath.Join(dataDir, "security.toml")
 	os.WriteFile(securityToml, []byte("# Empty security config for testing\n"), 0644)
 
+	// Allocate ports instead of using the defaults: on a shared or reused
+	// runner a stale weed process can still hold 9333/808x, which kills the
+	// freshly started master at bind time and leaves the test talking to a
+	// raft follower that answers every assign with "not current leader".
+	// Mini-safe ports guarantee each HTTP port's derived gRPC port
+	// (port + 10000) is free as well.
+	ports, err := testutil.AllocateMiniPorts(7)
+	if err != nil {
+		return nil, fmt.Errorf("failed to allocate ports: %v", err)
+	}
+	masterPort := ports[0]
+	masterAddr := fmt.Sprintf("127.0.0.1:%d", masterPort)
+	cluster.masterAddr = masterAddr
+
 	// Start master server
 	masterCmd := exec.CommandContext(ctx, weedBinary, "master",
-		"-port", "9333",
+		"-port", fmt.Sprintf("%d", masterPort),
 		"-mdir", masterDir,
 		"-volumeSizeLimitMB", "10", // Small volumes for testing
 		"-ip", "127.0.0.1",
@@ -432,13 +451,15 @@ func startSeaweedFSCluster(ctx context.Context, dataDir string) (*TestCluster, e
 		volumeDir := filepath.Join(dataDir, fmt.Sprintf("volume%d", i))
 		os.MkdirAll(volumeDir, 0755)
 
-		port := fmt.Sprintf("808%d", i)
+		port := fmt.Sprintf("%d", ports[1+i])
+		cluster.volumeAddrs = append(cluster.volumeAddrs, fmt.Sprintf("127.0.0.1:%d", ports[1+i]))
 		rack := fmt.Sprintf("rack%d", i)
 		volumeCmd := exec.CommandContext(ctx, weedBinary, "volume",
 			"-port", port,
 			"-dir", volumeDir,
 			"-max", "10",
-			"-master", "127.0.0.1:9333",
+			"-master", masterAddr,
+			"-volume.allowUntrustedRemoteEndpoints",
 			"-ip", "127.0.0.1",
 			"-dataCenter", "dc1",
 			"-rack", rack,
@@ -497,7 +518,8 @@ func findWeedBinary() string {
 func waitForServer(address string, timeout time.Duration) error {
 	start := time.Now()
 	for time.Since(start) < timeout {
-		if conn, err := grpc.NewClient(address, grpc.WithInsecure()); err == nil {
+		conn, err := net.DialTimeout("tcp", address, time.Second)
+		if err == nil {
 			conn.Close()
 			return nil
 		}
@@ -506,6 +528,29 @@ func waitForServer(address string, timeout time.Duration) error {
 	return fmt.Errorf("timeout waiting for server %s", address)
 }
 
+// waitForMasterLeader polls the master's /cluster/status endpoint until it
+// reports itself as the raft leader. A goraft master can take up to the
+// configured election timeout (10s by default) before it is ready to serve
+// raft-gated requests like volume assign, so a bare port check is not enough.
+func waitForMasterLeader(masterAddr string, timeout time.Duration) error {
+	start := time.Now()
+	for time.Since(start) < timeout {
+		resp, err := http.Get(fmt.Sprintf("http://%s/cluster/status", masterAddr))
+		if err == nil {
+			var status struct {
+				IsLeader bool `json:"IsLeader"`
+			}
+			err = json.NewDecoder(resp.Body).Decode(&status)
+			resp.Body.Close()
+			if err == nil && status.IsLeader {
+				return nil
+			}
+		}
+		time.Sleep(500 * time.Millisecond)
+	}
+	return fmt.Errorf("timeout waiting for master %s to become raft leader", masterAddr)
+}
+
 func uploadTestData(data []byte, masterAddress string) (needle.VolumeId, error) {
 	// Upload data to get a file ID
 	assignResult, err := operation.Assign(context.Background(), func(ctx context.Context) pb.ServerAddress {
@@ -1082,6 +1127,7 @@ func startMultiDiskCluster(ctx context.Context, dataDir string) (*MultiDiskClust
 			"-dir", strings.Join(diskDirs, ","),
 			"-max", strings.Join(maxVolumes, ","),
 			"-master", "127.0.0.1:9334",
+			"-volume.allowUntrustedRemoteEndpoints",
 			"-ip", "127.0.0.1",
 			"-dataCenter", "dc1",
 			"-rack", rack,
@@ -1478,6 +1524,7 @@ func startClusterWithDiskType(ctx context.Context, dataDir string, diskType stri
 			"-dir", diskDir,
 			"-max", "10",
 			"-mserver", "127.0.0.1:9335",
+			"-volume.allowUntrustedRemoteEndpoints",
 			"-ip", "127.0.0.1",
 			"-dataCenter", "dc1",
 			"-rack", rack,
@@ -1708,6 +1755,7 @@ func startMixedDiskTypeCluster(ctx context.Context, dataDir string) (*MultiDiskC
 			"-dir", diskDir,
 			"-max", "10",
 			"-mserver", "127.0.0.1:9336",
+			"-volume.allowUntrustedRemoteEndpoints",
 			"-ip", "127.0.0.1",
 			"-dataCenter", "dc1",
 			"-rack", rack,
@@ -2030,6 +2078,7 @@ func startLimitedSsdCluster(ctx context.Context, dataDir string) (*MultiDiskClus
 			"-dir", diskDir,
 			"-max", "10",
 			"-mserver", "127.0.0.1:9337",
+			"-volume.allowUntrustedRemoteEndpoints",
 			"-ip", "127.0.0.1",
 			"-dataCenter", "dc1",
 			"-rack", config.rack,
@@ -2112,6 +2161,7 @@ func startMultiRackCluster(ctx context.Context, dataDir string) (*MultiDiskClust
 			"-dir", diskDir,
 			"-max", "10",
 			"-mserver", "127.0.0.1:9338",
+			"-volume.allowUntrustedRemoteEndpoints",
 			"-ip", "127.0.0.1",
 			"-dataCenter", "dc1",
 			"-rack", rack,
@@ -2203,14 +2253,15 @@ func TestECEncodeReplicatedVolumeSync(t *testing.T) {
 	defer cluster.Stop()
 
 	// Wait for servers to be ready
-	require.NoError(t, waitForServer("127.0.0.1:9333", 30*time.Second))
-	for i := 0; i < 6; i++ {
-		require.NoError(t, waitForServer(fmt.Sprintf("127.0.0.1:%d", 8080+i), 30*time.Second))
+	require.NoError(t, waitForServer(cluster.masterAddr, 30*time.Second))
+	for _, addr := range cluster.volumeAddrs {
+		require.NoError(t, waitForServer(addr, 30*time.Second))
 	}
+	require.NoError(t, waitForMasterLeader(cluster.masterAddr, 30*time.Second))
 
 	// Create command environment
 	options := &shell.ShellOptions{
-		Masters:        stringPtr("127.0.0.1:9333"),
+		Masters:        stringPtr(cluster.masterAddr),
 		GrpcDialOption: grpc.WithInsecure(),
 		FilerGroup:     stringPtr("default"),
 	}
@@ -2234,7 +2285,7 @@ func TestECEncodeReplicatedVolumeSync(t *testing.T) {
 		var volumeId needle.VolumeId
 		var uploadErr error
 		for retry := 0; retry < 5; retry++ {
-			volumeId, uploadErr = uploadTestDataWithReplication(t, "127.0.0.1:9333", testCollection, testReplication)
+			volumeId, uploadErr = uploadTestDataWithReplication(t, cluster.masterAddr, testCollection, testReplication)
 			if uploadErr == nil {
 				break
 			}
@@ -2284,7 +2335,7 @@ func TestECEncodeReplicatedVolumeSync(t *testing.T) {
 		var volumeId needle.VolumeId
 		var uploadErr error
 		for retry := 0; retry < 5; retry++ {
-			volumeId, uploadErr = uploadTestDataWithReplication(t, "127.0.0.1:9333", testCollection, testReplication)
+			volumeId, uploadErr = uploadTestDataWithReplication(t, cluster.masterAddr, testCollection, testReplication)
 			if uploadErr == nil {
 				break
 			}

test/erasure_coding/multidisk_shardloss_test.go

@@ -0,0 +1,265 @@
+package erasure_coding
+
+import (
+	"context"
+	"fmt"
+	"path/filepath"
+	"regexp"
+	"strconv"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/seaweedfs/seaweedfs/weed/pb/master_pb"
+	"github.com/seaweedfs/seaweedfs/weed/shell"
+	"github.com/seaweedfs/seaweedfs/weed/storage/needle"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+	"google.golang.org/grpc"
+)
+
+// TestMultiDiskECBalanceNoShardLoss is the end-to-end regression for issue 9593.
+// It runs a real cluster of multi-disk volume servers (3 servers x 4 disks),
+// EC-encodes a volume, then runs ec.balance, asserting hard invariants the older
+// integration tests only logged:
+//
+//   - after encode the full set of 14 EC shards exists,
+//   - ec.balance never loses a shard (still 14 distinct shards afterwards),
+//   - shards end up spread across more than one disk per node, and
+//   - cluster.status counts physical disks (not one per node) and matches the
+//     real on-disk distribution.
+func TestMultiDiskECBalanceNoShardLoss(t *testing.T) {
+	if testing.Short() {
+		t.Skip("Skipping multi-disk EC integration test in short mode")
+	}
+
+	testDir := t.TempDir()
+	ctx, cancel := context.WithTimeout(context.Background(), 240*time.Second)
+	defer cancel()
+
+	cluster, err := startMultiDiskCluster(ctx, testDir)
+	require.NoError(t, err)
+	defer cluster.Stop()
+
+	require.NoError(t, waitForServer("127.0.0.1:9334", 30*time.Second))
+	for i := 0; i < 3; i++ {
+		require.NoError(t, waitForServer(fmt.Sprintf("127.0.0.1:809%d", i), 30*time.Second))
+	}
+	t.Log("waiting for multi-disk volume servers to register...")
+	time.Sleep(10 * time.Second)
+
+	commandEnv := shell.NewCommandEnv(&shell.ShellOptions{
+		Masters:        stringPtr("127.0.0.1:9334"),
+		GrpcDialOption: grpc.WithInsecure(),
+		FilerGroup:     stringPtr("default"),
+	})
+	connectToMasterAndSync(ctx, t, commandEnv)
+
+	// Upload enough small files that the volume holds real data to encode.
+	var volumeId needle.VolumeId
+	for retry := 0; retry < 5; retry++ {
+		volumeId, err = uploadTestDataToMaster([]byte(strings.Repeat("multidisk-ec-9593 ", 64)), "127.0.0.1:9334")
+		if err == nil {
+			break
+		}
+		time.Sleep(3 * time.Second)
+	}
+	require.NoError(t, err, "failed to upload test data")
+	for i := 0; i < 40; i++ {
+		if _, e := uploadTestDataToMaster([]byte(strings.Repeat("filler ", 128)), "127.0.0.1:9334"); e != nil {
+			break
+		}
+	}
+	t.Logf("using volume %d", volumeId)
+	time.Sleep(3 * time.Second)
+
+	// Populate every server's disks with volumes so the encode can see and target
+	// each physical disk. The master only enumerates disks that already hold a
+	// volume or EC shard — an empty disk leaves no trace in the topology (heartbeats
+	// aggregate capacity per disk type, not per physical disk). ec.encode therefore
+	// spreads a volume's shards only across the disks the master already knows hold
+	// data on each node; if a node's data sits on a single disk, all its shards land
+	// there and ec.balance cannot redistribute them (it has no within-node
+	// cross-disk move). So spreading must be set up before encoding.
+	//
+	// volume.grow only tops up toward a writable target and stops on the first
+	// allocation error, so a single -count grow can create far fewer volumes than
+	// asked and leave a node on one disk. Grow repeatedly on the nodes that have not
+	// spread yet (the volume server places each new volume on its least-loaded disk)
+	// until the master's topology shows every node holding volumes on at least two
+	// physical disks. This makes the multi-disk layout — and thus the post-encode
+	// disk spread — deterministic instead of racing volume-growth and heartbeat.
+	require.Eventually(t, func() bool {
+		spread := nodeVolumeDiskCounts(t, commandEnv)
+		if len(spread) == 3 && allAtLeast(spread, 2) {
+			return true
+		}
+		for i := 0; i < 3; i++ {
+			server := fmt.Sprintf("127.0.0.1:809%d", i)
+			if spread[server] < 2 {
+				captureCommandOutput(t, shell.Commands[findCommandIndex("volume.grow")],
+					[]string{"-collection", "test", "-dataNode", server, "-count", "4"}, commandEnv)
+			}
+		}
+		return false
+	}, 60*time.Second, 2*time.Second,
+		"volumes never spread across >=2 disks on all 3 nodes")
+
+	locked, unlock := tryLockWithTimeout(t, commandEnv, 15*time.Second)
+	require.True(t, locked, "could not acquire shell lock")
+	defer unlock()
+
+	// EC-encode the volume.
+	out, err := captureCommandOutput(t, shell.Commands[findCommandIndex("ec.encode")],
+		[]string{"-volumeId", fmt.Sprintf("%d", volumeId), "-collection", "test", "-force"}, commandEnv)
+	t.Logf("ec.encode output:\n%s", out)
+	require.NoError(t, err, "ec.encode failed")
+
+	// All 14 shards must exist after encoding.
+	require.Eventually(t, func() bool {
+		return len(collectDistinctShardIDs(testDir, uint32(volumeId))) == erasureShardCount
+	}, 30*time.Second, time.Second, "expected all %d EC shards after encode, got %v",
+		erasureShardCount, collectDistinctShardIDs(testDir, uint32(volumeId)))
+
+	beforeBalance := collectDistinctShardIDs(testDir, uint32(volumeId))
+	t.Logf("after encode: %d distinct shards on %d disks", len(beforeBalance), disksWithShards(testDir, uint32(volumeId)))
+
+	// Run ec.balance.
+	out, err = captureCommandOutput(t, shell.Commands[findCommandIndex("ec.balance")],
+		[]string{"-collection", "test", "-force"}, commandEnv)
+	t.Logf("ec.balance output:\n%s", out)
+	require.NoError(t, err, "ec.balance failed")
+	time.Sleep(3 * time.Second)
+
+	// The core regression: ec.balance must not lose any shard.
+	afterBalance := collectDistinctShardIDs(testDir, uint32(volumeId))
+	require.Equal(t, erasureShardCount, len(afterBalance),
+		"ec.balance lost shards on multi-disk nodes: had %v, now %v", sortedKeysOf(beforeBalance), sortedKeysOf(afterBalance))
+
+	// Shards must be spread across more than one physical disk per node overall.
+	usedDisks := disksWithShards(testDir, uint32(volumeId))
+	assert.Greater(t, usedDisks, 3, "EC shards should span more than one disk per node (got %d disks across 3 nodes)", usedDisks)
+
+	// cluster.status must count physical disks, not collapse to one per node: it
+	// must report at least the disks actually holding this volume's shards (which
+	// is already >3 across the 3 nodes). Before the fix it reported 3 (node count).
+	require.Eventually(t, func() bool {
+		n, ok := clusterStatusDiskCount(t, commandEnv)
+		return ok && n >= usedDisks
+	}, 30*time.Second, 2*time.Second, "cluster.status never reported the >=%d physical disks holding shards (multi-disk count)", usedDisks)
+
+	n, _ := clusterStatusDiskCount(t, commandEnv)
+	t.Logf("cluster.status reports %d physical disks (>= %d holding this volume's shards)", n, usedDisks)
+}
+
+const erasureShardCount = 14 // 10 data + 4 parity
+
+// collectDistinctShardIDs returns the set of EC shard ids present for a volume
+// across every disk of every server in the multi-disk test layout.
+func collectDistinctShardIDs(testDir string, volumeId uint32) map[int]bool {
+	ids := map[int]bool{}
+	for server := 0; server < 3; server++ {
+		for disk := 0; disk < 4; disk++ {
+			diskDir := filepath.Join(testDir, fmt.Sprintf("server%d_disk%d", server, disk))
+			files, err := listECShardFiles(diskDir, volumeId)
+			if err != nil {
+				continue
+			}
+			for _, f := range files {
+				i := strings.LastIndex(f, ".ec")
+				if i < 0 {
+					continue
+				}
+				if n, err := strconv.Atoi(f[i+3:]); err == nil && n >= 0 && n < erasureShardCount {
+					ids[n] = true
+				}
+			}
+		}
+	}
+	return ids
+}
+
+// disksWithShards counts how many physical disks hold at least one shard.
+func disksWithShards(testDir string, volumeId uint32) int {
+	n := 0
+	for _, disks := range countShardsPerDisk(testDir, volumeId) {
+		for _, c := range disks {
+			if c > 0 {
+				n++
+			}
+		}
+	}
+	return n
+}
+
+// nodeVolumeDiskCounts returns, per volume server id, how many distinct physical
+// disks hold at least one volume according to the master's topology. The master
+// only enumerates disks that already hold a volume or EC shard (heartbeats
+// aggregate capacity per disk type, not per physical disk), so this reports the
+// disks ec.encode can actually spread a volume's shards across on each node.
+func nodeVolumeDiskCounts(t *testing.T, commandEnv *shell.CommandEnv) map[string]int {
+	t.Helper()
+	var resp *master_pb.VolumeListResponse
+	err := commandEnv.MasterClient.WithClient(context.Background(), false, func(client master_pb.SeaweedClient) error {
+		var e error
+		resp, e = client.VolumeList(context.Background(), &master_pb.VolumeListRequest{})
+		return e
+	})
+	counts := map[string]int{}
+	if err != nil || resp.GetTopologyInfo() == nil {
+		return counts
+	}
+	for _, dc := range resp.GetTopologyInfo().GetDataCenterInfos() {
+		for _, r := range dc.GetRackInfos() {
+			for _, dn := range r.GetDataNodeInfos() {
+				disks := map[uint32]bool{}
+				for _, di := range dn.GetDiskInfos() {
+					for _, vi := range di.GetVolumeInfos() {
+						disks[vi.GetDiskId()] = true
+					}
+				}
+				counts[dn.Id] = len(disks)
+			}
+		}
+	}
+	return counts
+}
+
+func allAtLeast(counts map[string]int, min int) bool {
+	for _, c := range counts {
+		if c < min {
+			return false
+		}
+	}
+	return true
+}
+
+var diskCountRe = regexp.MustCompile(`(\d+)\s+disks?`)
+
+// clusterStatusDiskCount runs cluster.status and parses the reported disk count.
+func clusterStatusDiskCount(t *testing.T, commandEnv *shell.CommandEnv) (int, bool) {
+	t.Helper()
+	out, err := captureCommandOutput(t, shell.Commands[findCommandIndex("cluster.status")], []string{}, commandEnv)
+	if err != nil {
+		return 0, false
+	}
+	m := diskCountRe.FindStringSubmatch(out)
+	if m == nil {
+		return 0, false
+	}
+	n, err := strconv.Atoi(m[1])
+	return n, err == nil
+}
+
+func sortedKeysOf(m map[int]bool) []int {
+	out := make([]int, 0, len(m))
+	for k := range m {
+		out = append(out, k)
+	}
+	for i := 1; i < len(out); i++ {
+		for j := i; j > 0 && out[j-1] > out[j]; j-- {
+			out[j-1], out[j] = out[j], out[j-1]
+		}
+	}
+	return out
+}

test/erasure_coding/multidisk_shell_lifecycle_test.go

@@ -0,0 +1,411 @@
+package erasure_coding
+
+import (
+	"bytes"
+	"context"
+	"crypto/rand"
+	"encoding/json"
+	"fmt"
+	"io"
+	"net/http"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/require"
+	"google.golang.org/grpc"
+
+	"github.com/seaweedfs/seaweedfs/weed/operation"
+	"github.com/seaweedfs/seaweedfs/weed/pb"
+	"github.com/seaweedfs/seaweedfs/weed/pb/master_pb"
+	"github.com/seaweedfs/seaweedfs/weed/shell"
+	"github.com/seaweedfs/seaweedfs/weed/storage/needle"
+)
+
+// TestMultiDiskECShellLifecycle drives the real shell commands — ec.encode,
+// ec.balance, ec.rebuild — against a live multi-disk cluster, through the
+// sequence a production repository goes through, with one invariant checked
+// after every step: the bytes a client stored come back identical. Shard
+// counting alone cannot tell a healthy volume from one that was rebuilt out
+// of the wrong inputs; reading the payload back can.
+//
+// The damage step reproduces a support case: shard files removed from disk
+// while the cluster still lists them, so the subsequent ec.rebuild sees copy
+// failures ("CopyFile not found ec volume id ...") for locations the master
+// believes exist, and must recover from the shards that are really there.
+func TestMultiDiskECShellLifecycle(t *testing.T) {
+	if testing.Short() {
+		t.Skip("Skipping multi-disk EC shell lifecycle test in short mode")
+	}
+
+	testDir := t.TempDir()
+	ctx, cancel := context.WithTimeout(context.Background(), 360*time.Second)
+	defer cancel()
+
+	cluster, err := startMultiDiskCluster(ctx, testDir)
+	require.NoError(t, err)
+	defer cluster.Stop()
+
+	require.NoError(t, waitForServer("127.0.0.1:9334", 30*time.Second))
+	for i := 0; i < 3; i++ {
+		require.NoError(t, waitForServer(fmt.Sprintf("127.0.0.1:809%d", i), 30*time.Second))
+	}
+	t.Log("waiting for multi-disk volume servers to register...")
+	time.Sleep(10 * time.Second)
+
+	commandEnv := shell.NewCommandEnv(&shell.ShellOptions{
+		Masters:        stringPtr("127.0.0.1:9334"),
+		GrpcDialOption: grpc.WithInsecure(),
+		FilerGroup:     stringPtr("default"),
+	})
+	connectToMasterAndSync(ctx, t, commandEnv)
+
+	// A payload with no structure, so a rebuild that assembled the wrong bytes
+	// cannot accidentally reproduce it.
+	payload := make([]byte, 8192)
+	_, err = rand.Read(payload)
+	require.NoError(t, err)
+
+	var volumeId needle.VolumeId
+	var fid string
+	for retry := 0; retry < 5; retry++ {
+		volumeId, fid, err = uploadPayload(payload)
+		if err == nil {
+			break
+		}
+		time.Sleep(3 * time.Second)
+	}
+	require.NoError(t, err, "failed to upload payload")
+	for i := 0; i < 40; i++ {
+		if _, _, e := uploadPayload([]byte(strings.Repeat("filler ", 128))); e != nil {
+			break
+		}
+	}
+	t.Logf("payload of %d bytes in volume %d as %s", len(payload), volumeId, fid)
+	time.Sleep(3 * time.Second)
+
+	requirePayload := func(step string) {
+		t.Helper()
+		require.Eventually(t, func() bool {
+			got, err := readFid(fid, uint32(volumeId))
+			return err == nil && bytes.Equal(got, payload)
+		}, 30*time.Second, time.Second, "payload not readable byte-identical after %s", step)
+		t.Logf("payload verified after %s", step)
+	}
+	requirePayload("upload")
+
+	// Spread volumes across each node's disks before encoding; the master only
+	// enumerates disks that already hold data, and shards only spread across
+	// enumerated disks (see TestMultiDiskECBalanceNoShardLoss for the details).
+	require.Eventually(t, func() bool {
+		spread := nodeVolumeDiskCounts(t, commandEnv)
+		if len(spread) == 3 && allAtLeast(spread, 2) {
+			return true
+		}
+		for i := 0; i < 3; i++ {
+			server := fmt.Sprintf("127.0.0.1:809%d", i)
+			if spread[server] < 2 {
+				captureCommandOutput(t, shell.Commands[findCommandIndex("volume.grow")],
+					[]string{"-collection", "test", "-dataNode", server, "-count", "4"}, commandEnv)
+			}
+		}
+		return false
+	}, 60*time.Second, 2*time.Second, "volumes never spread across >=2 disks on all 3 nodes")
+
+	locked, unlock := tryLockWithTimeout(t, commandEnv, 15*time.Second)
+	require.True(t, locked, "could not acquire shell lock")
+	defer unlock()
+
+	// ── ec.encode ──
+	out, err := captureCommandOutput(t, shell.Commands[findCommandIndex("ec.encode")],
+		[]string{"-volumeId", fmt.Sprintf("%d", volumeId), "-collection", "test", "-force"}, commandEnv)
+	t.Logf("ec.encode output:\n%s", out)
+	require.NoError(t, err, "ec.encode failed")
+	require.Eventually(t, func() bool {
+		return len(collectDistinctShardIDs(testDir, uint32(volumeId))) == erasureShardCount
+	}, 30*time.Second, time.Second, "expected all %d shards after encode", erasureShardCount)
+	requirePayload("ec.encode")
+
+	// ── ec.balance ──
+	out, err = captureCommandOutput(t, shell.Commands[findCommandIndex("ec.balance")],
+		[]string{"-collection", "test", "-force"}, commandEnv)
+	t.Logf("ec.balance output:\n%s", out)
+	require.NoError(t, err, "ec.balance failed")
+	time.Sleep(3 * time.Second)
+	require.Len(t, collectDistinctShardIDs(testDir, uint32(volumeId)), erasureShardCount,
+		"ec.balance lost shards")
+	requirePayload("ec.balance")
+
+	// ── damage: two shard files vanish from disk ──
+	// Removed straight off disk, then the servers are restarted so the master
+	// relearns the shard set from disk rather than from stale heartbeat state.
+	// This is the recoverable form of the support case: the volume drops to 12
+	// of 14 shards, which is still above the 10 needed to read and rebuild.
+	removed := removeTwoShardFiles(t, testDir, uint32(volumeId))
+	t.Logf("removed shard files for shards %v", removed)
+	require.Len(t, collectDistinctShardIDs(testDir, uint32(volumeId)), erasureShardCount-2)
+
+	require.NoError(t, cluster.RestartVolumeServers(ctx))
+	for i := 0; i < 3; i++ {
+		require.NoError(t, waitForServer(fmt.Sprintf("127.0.0.1:809%d", i), 30*time.Second))
+	}
+	requirePayload("shard loss + restart") // 12 of 14 remain; reads must survive
+
+	// The master must now see exactly the 12 shards on disk before any repair
+	// runs. ec.rebuild is driven by the master's view, so a repair planned
+	// while that view still lists the two dead shards would target the wrong
+	// set. Restart-then-relearn is what makes disk truth and the master agree.
+	require.Eventually(t, func() bool {
+		return len(masterEcShardIds(commandEnv, uint32(volumeId))) == erasureShardCount-2
+	}, 60*time.Second, 2*time.Second,
+		"master never relearned the reduced shard set (got %v)",
+		sortedKeysOf(masterEcShardIds(commandEnv, uint32(volumeId))))
+
+	// The shell lock is dropped by the restart's master disconnect; retake it.
+	relocked, reunlock := tryLockWithTimeout(t, commandEnv, 15*time.Second)
+	require.True(t, relocked, "could not reacquire shell lock after restart")
+	defer reunlock()
+
+	// ── ec.rebuild, the command from the support case ──
+	out, err = captureCommandOutput(t, shell.Commands[findCommandIndex("ec.rebuild")],
+		[]string{"-collection", "test", "-apply"}, commandEnv)
+	t.Logf("ec.rebuild output:\n%s", out)
+	require.NoError(t, err, "ec.rebuild failed")
+	require.Eventually(t, func() bool {
+		return len(collectDistinctShardIDs(testDir, uint32(volumeId))) == erasureShardCount
+	}, 60*time.Second, time.Second, "ec.rebuild did not restore all %d shards on disk", erasureShardCount)
+	requirePayload("ec.rebuild")
+
+	// The rebuilt shards are real: master's view and disk truth agree on all 14.
+	require.Eventually(t, func() bool {
+		registered := masterEcShardIds(commandEnv, uint32(volumeId))
+		onDisk := collectDistinctShardIDs(testDir, uint32(volumeId))
+		if len(registered) != len(onDisk) || len(onDisk) != erasureShardCount {
+			return false
+		}
+		for id := range onDisk {
+			if !registered[id] {
+				return false
+			}
+		}
+		return true
+	}, 60*time.Second, 2*time.Second,
+		"master's EC shard view diverged from disk truth after rebuild: master=%v disk=%v",
+		sortedKeysOf(masterEcShardIds(commandEnv, uint32(volumeId))), sortedKeysOf(collectDistinctShardIDs(testDir, uint32(volumeId))))
+}
+
+// RestartVolumeServers kills every volume server process and starts fresh ones
+// over the same directories, so registration is rebuilt purely from disk.
+func (c *MultiDiskCluster) RestartVolumeServers(ctx context.Context) error {
+	for _, cmd := range c.volumeServers {
+		if cmd != nil && cmd.Process != nil {
+			cmd.Process.Kill()
+			cmd.Wait()
+		}
+	}
+	c.volumeServers = nil
+	time.Sleep(2 * time.Second)
+	return c.startVolumeServers(ctx)
+}
+
+// startVolumeServers launches the standard 3-server x 4-disk layout over the
+// cluster's existing directories, mirroring startMultiDiskCluster's loop so a
+// restart brings servers back exactly as they first started.
+func (c *MultiDiskCluster) startVolumeServers(ctx context.Context) error {
+	weedBinary := findWeedBinary()
+	if weedBinary == "" {
+		return fmt.Errorf("weed binary not found")
+	}
+	const numServers = 3
+	const disksPerServer = 4
+	for i := 0; i < numServers; i++ {
+		var diskDirs []string
+		var maxVolumes []string
+		for d := 0; d < disksPerServer; d++ {
+			diskDirs = append(diskDirs, filepath.Join(c.testDir, fmt.Sprintf("server%d_disk%d", i, d)))
+			maxVolumes = append(maxVolumes, "5")
+		}
+		volumeCmd := exec.CommandContext(ctx, weedBinary, "volume",
+			"-port", fmt.Sprintf("809%d", i),
+			"-dir", strings.Join(diskDirs, ","),
+			"-max", strings.Join(maxVolumes, ","),
+			"-master", "127.0.0.1:9334",
+			"-volume.allowUntrustedRemoteEndpoints",
+			"-ip", "127.0.0.1",
+			"-dataCenter", "dc1",
+			"-rack", fmt.Sprintf("rack%d", i),
+		)
+		logFile, err := os.OpenFile(filepath.Join(c.testDir, fmt.Sprintf("server%d_logs", i), "volume-restart.log"),
+			os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0o644)
+		if err != nil {
+			return fmt.Errorf("open restart log for server %d: %w", i, err)
+		}
+		c.logFiles = append(c.logFiles, logFile)
+		volumeCmd.Stdout = logFile
+		volumeCmd.Stderr = logFile
+		if err := volumeCmd.Start(); err != nil {
+			return fmt.Errorf("restart volume server %d: %w", i, err)
+		}
+		c.volumeServers = append(c.volumeServers, volumeCmd)
+	}
+	time.Sleep(8 * time.Second)
+	return nil
+}
+
+// uploadPayload stores data in collection "test" and returns the volume id and
+// fid so the same needle can be read back after every subsequent operation.
+func uploadPayload(data []byte) (needle.VolumeId, string, error) {
+	assignResult, err := operation.Assign(context.Background(), func(ctx context.Context) pb.ServerAddress {
+		return pb.ServerAddress("127.0.0.1:9334")
+	}, grpc.WithInsecure(), &operation.VolumeAssignRequest{
+		Count:       1,
+		Collection:  "test",
+		Replication: "000",
+	})
+	if err != nil {
+		return 0, "", err
+	}
+	uploader, err := operation.NewUploader()
+	if err != nil {
+		return 0, "", err
+	}
+	uploadResult, err, _ := uploader.Upload(context.Background(), bytes.NewReader(data), &operation.UploadOption{
+		UploadUrl: "http://" + assignResult.Url + "/" + assignResult.Fid,
+		Filename:  "payload.bin",
+		MimeType:  "application/octet-stream",
+	})
+	if err != nil {
+		return 0, "", err
+	}
+	if uploadResult.Error != "" {
+		return 0, "", fmt.Errorf("upload error: %s", uploadResult.Error)
+	}
+	fidObj, err := needle.ParseFileIdFromString(assignResult.Fid)
+	if err != nil {
+		return 0, "", err
+	}
+	return fidObj.VolumeId, assignResult.Fid, nil
+}
+
+// readFid locates the volume via the master and fetches the needle bytes from
+// a currently-registered location — the same route a client read takes, for a
+// plain volume before encoding and for the EC read path after.
+func readFid(fid string, volumeId uint32) ([]byte, error) {
+	resp, err := http.Get(fmt.Sprintf("http://127.0.0.1:9334/dir/lookup?volumeId=%d", volumeId))
+	if err != nil {
+		return nil, err
+	}
+	defer resp.Body.Close()
+	var lookup struct {
+		Locations []struct {
+			Url string `json:"url"`
+		} `json:"locations"`
+	}
+	if err := json.NewDecoder(resp.Body).Decode(&lookup); err != nil {
+		return nil, err
+	}
+	if len(lookup.Locations) == 0 {
+		return nil, fmt.Errorf("no locations for volume %d", volumeId)
+	}
+	var lastErr error
+	for _, loc := range lookup.Locations {
+		get, err := http.Get(fmt.Sprintf("http://%s/%s", loc.Url, fid))
+		if err != nil {
+			lastErr = err
+			continue
+		}
+		body, err := io.ReadAll(get.Body)
+		get.Body.Close()
+		if err != nil {
+			lastErr = err
+			continue
+		}
+		if get.StatusCode != http.StatusOK {
+			lastErr = fmt.Errorf("GET %s from %s: %d", fid, loc.Url, get.StatusCode)
+			continue
+		}
+		return body, nil
+	}
+	return nil, lastErr
+}
+
+// removeTwoShardFiles deletes the files of two distinct shards straight off
+// the disks, without telling any server — the divergence between disk truth
+// and the master's view at the heart of the support case.
+func removeTwoShardFiles(t *testing.T, testDir string, volumeId uint32) []int {
+	t.Helper()
+	var removed []int
+	for server := 0; server < 3 && len(removed) < 2; server++ {
+		for disk := 0; disk < 4 && len(removed) < 2; disk++ {
+			diskDir := filepath.Join(testDir, fmt.Sprintf("server%d_disk%d", server, disk))
+			files, err := listECShardFiles(diskDir, volumeId)
+			if err != nil {
+				continue
+			}
+			for _, f := range files {
+				i := strings.LastIndex(f, ".ec")
+				if i < 0 {
+					continue
+				}
+				var id int
+				if _, err := fmt.Sscanf(f[i+3:], "%d", &id); err != nil {
+					continue
+				}
+				if err := os.Remove(filepath.Join(diskDir, f)); err != nil {
+					t.Fatalf("remove shard file %s: %v", f, err)
+				}
+				removed = append(removed, id)
+				break // at most one shard per disk, keep the loss spread out
+			}
+		}
+	}
+	require.Len(t, removed, 2, "could not find two shard files to remove")
+	return removed
+}
+
+// masterEcShardIds reports which shard ids the master currently believes exist
+// for the volume, across all nodes — the view every repair decision runs on.
+func masterEcShardIds(commandEnv *shell.CommandEnv, volumeId uint32) map[int]bool {
+	ids := map[int]bool{}
+	var resp *master_pb.VolumeListResponse
+	err := commandEnv.MasterClient.WithClient(context.Background(), false, func(client master_pb.SeaweedClient) error {
+		var e error
+		resp, e = client.VolumeList(context.Background(), &master_pb.VolumeListRequest{})
+		return e
+	})
+	if err != nil || resp.GetTopologyInfo() == nil {
+		return ids
+	}
+	for _, dc := range resp.GetTopologyInfo().GetDataCenterInfos() {
+		for _, r := range dc.GetRackInfos() {
+			for _, dn := range r.GetDataNodeInfos() {
+				for _, di := range dn.GetDiskInfos() {
+					for _, eci := range di.GetEcShardInfos() {
+						if eci.GetId() != volumeId {
+							continue
+						}
+						for _, sid := range erasure_coding_ShardIds(eci.GetEcIndexBits()) {
+							ids[sid] = true
+						}
+					}
+				}
+			}
+		}
+	}
+	return ids
+}
+
+// erasure_coding_ShardIds expands an EC index bitmap into shard ids without
+// importing the storage package into this test's namespace twice.
+func erasure_coding_ShardIds(bits uint32) []int {
+	var out []int
+	for i := 0; i < 32; i++ {
+		if bits&(1<<uint(i)) != 0 {
+			out = append(out, i)
+		}
+	}
+	return out
+}

test/erasure_coding/systematic_interruption_test.go

@@ -0,0 +1,232 @@
+package erasure_coding
+
+import (
+	"bufio"
+	"bytes"
+	"context"
+	"fmt"
+	"os"
+	"os/exec"
+	"strings"
+	"sync"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/require"
+	"google.golang.org/grpc"
+
+	"github.com/seaweedfs/seaweedfs/weed/shell"
+)
+
+// TestECInterruptionMatrix is the systematic counterpart of the randomized
+// chaos harness: instead of killing operations at random times, it enumerates
+// every phase of every interruptible EC operation and kills a real weed shell
+// exactly when that phase announces itself on the command's output. Each
+// (operation, phase) pair is one deterministic scenario: prepare the
+// precondition, kill at the phase marker, run the prescribed recovery, and
+// verify that every stored byte still reads back identical.
+//
+// The phase markers are the progress lines the pipelines print; each marker
+// below names the code that prints it. A marker is the START of its phase, so
+// killing on it lands the interruption inside that phase — deterministically
+// per phase, byte-exact timing within the phase left to the scheduler. Every
+// scenario REQUIRES its marker to appear: a marker that never prints means a
+// pipeline refactor renamed or dropped the progress line, and silently
+// degenerating into a no-interruption run would let CI pass without
+// exercising the boundary the scenario names. Update the marker with the
+// pipeline.
+//
+// Complementing this, weed/ec's TestECLifecycleModelExhaustive explores every
+// schedule of the same pipelines — crashes between every step, rollback
+// paths, restarts — exhaustively on a model whose steps mirror the code.
+func TestECInterruptionMatrix(t *testing.T) {
+	if testing.Short() {
+		t.Skip("Skipping EC interruption matrix in short mode")
+	}
+
+	testDir := t.TempDir()
+	ctx, cancel := context.WithTimeout(context.Background(), 25*time.Minute)
+	defer cancel()
+
+	cluster, err := startChaosCluster(ctx, testDir)
+	require.NoError(t, err)
+	defer cluster.Stop()
+
+	require.NoError(t, waitForServer(chaosMasterAddr, 30*time.Second))
+	for i := 0; i < chaosServerCount; i++ {
+		require.NoError(t, waitForServer("127.0.0.1:"+chaosVolumePort(i), 30*time.Second))
+	}
+	time.Sleep(8 * time.Second)
+
+	commandEnv := shell.NewCommandEnv(&shell.ShellOptions{
+		Masters:        stringPtr(chaosMasterAddr),
+		GrpcDialOption: grpc.WithInsecure(),
+		FilerGroup:     stringPtr("default"),
+	})
+	connectToMasterAndSync(ctx, t, commandEnv)
+
+	r := newChaosRun(t, ctx, cluster, commandEnv, testDir, 1)
+	r.relock()
+	defer r.unlockIfHeld()
+	r.seedAndSpread()
+	r.verify("seeding")
+
+	// One scenario per (operation, phase marker). The preparation puts a
+	// dedicated volume into the operation's precondition state, so scenarios
+	// stay independent even though they share the cluster.
+	type scenario struct {
+		op      string
+		marker  string // printed by ↓
+		printed string // the code that prints it, for the reader
+	}
+	encodeScenarios := []scenario{
+		{"ec.encode", "markVolumeReadonly ", "weed/ec markVolumeReplicaWritable"},
+		{"ec.encode", "generateEcShards ", "weed/ec generateEcShards"},
+		{"ec.encode", "mount ", "weed/ec MountEcShards"},
+		{"ec.encode", "Deleting original volumes", "weed/ec ProcessEcEncodeBatch, the commit boundary"},
+	}
+	decodeScenarios := []scenario{
+		{"ec.decode", " shard locations:", "weed/ec DoEcDecode, before collect"},
+		{"ec.decode", "generateNormalVolume", "weed/ec generateNormalVolume"},
+		{"ec.decode", "unmount ec volume", "weed/ec unmountAndDeleteEcShards, teardown begin"},
+		{"ec.decode", "delete ec volume", "weed/ec unmountAndDeleteEcShards, shard deletion"},
+	}
+
+	for _, sc := range encodeScenarios {
+		sc := sc
+		t.Run(fmt.Sprintf("encode@%s", trimMarker(sc.marker)), func(t *testing.T) {
+			vid := r.ensureRegularVolume()
+			r.unlockIfHeld()
+			out, hit := runShellKillAtMarker(t, r.ctx,
+				fmt.Sprintf("ec.encode -volumeId %d -collection %s -force", vid, chaosCollection), sc.marker)
+			r.t.Logf("killed %s at %q (marker hit: %v); output:\n%s", sc.op, sc.marker, hit, out)
+			r.relock()
+			require.True(t, hit, "marker %q never appeared (printed by %s); update the marker with the pipeline", sc.marker, sc.printed)
+			r.recoverInterruptedEncode(vid)
+			r.verify(fmt.Sprintf("encode killed at %q", sc.marker))
+		})
+	}
+	for _, sc := range decodeScenarios {
+		sc := sc
+		t.Run(fmt.Sprintf("decode@%s", trimMarker(sc.marker)), func(t *testing.T) {
+			vid := r.ensureEncodedVolume()
+			r.unlockIfHeld()
+			out, hit := runShellKillAtMarker(t, r.ctx,
+				fmt.Sprintf("ec.decode -volumeId %d -collection %s -checkMinFreeSpace=false", vid, chaosCollection), sc.marker)
+			r.t.Logf("killed %s at %q (marker hit: %v); output:\n%s", sc.op, sc.marker, hit, out)
+			r.relock()
+			require.True(t, hit, "marker %q never appeared (printed by %s); update the marker with the pipeline", sc.marker, sc.printed)
+			r.recoverInterruptedDecode(vid)
+			r.verify(fmt.Sprintf("decode killed at %q", sc.marker))
+		})
+	}
+
+	// Balance: encode without rebalancing first, so the standalone balance is
+	// guaranteed to plan moves and the marker is guaranteed to print.
+	t.Run("balance@moves", func(t *testing.T) {
+		vid := r.ensureRegularVolume()
+		out, err := r.shellCommand("ec.encode",
+			"-volumeId", fmt.Sprintf("%d", vid), "-collection", chaosCollection, "-force", "-rebalance=false")
+		r.t.Logf("clumped encode v%d output:\n%s", vid, out)
+		require.NoError(t, err, "encode without rebalance")
+		r.volumes[vid].encoded = true
+
+		r.unlockIfHeld()
+		killOut, hit := runShellKillAtMarker(t, r.ctx,
+			fmt.Sprintf("ec.balance -collection %s -apply", chaosCollection), "moves ec shard")
+		r.t.Logf("killed ec.balance at move (marker hit: %v); output:\n%s", hit, killOut)
+		r.relock()
+		require.True(t, hit, "the balance never printed a move; the -rebalance=false setup should have guaranteed one")
+		r.recoverInterruptedBalance()
+		r.verify("balance killed at move")
+	})
+
+	r.verify("final")
+	t.Logf("interruption matrix done: %d payloads live, %d deleted, %d volumes tracked",
+		len(r.payloads), len(r.deleted), len(r.volumes))
+}
+
+func trimMarker(m string) string {
+	out := make([]rune, 0, len(m))
+	for _, c := range m {
+		if c == ' ' || c == ':' {
+			continue
+		}
+		out = append(out, c)
+	}
+	return string(out)
+}
+
+// runShellKillAtMarker feeds "lock" plus the command to a weed shell
+// subprocess, scans its combined output live, and kills the process the
+// moment a line containing marker appears. Returns the captured output and
+// whether the marker was seen before the shell exited on its own.
+func runShellKillAtMarker(t *testing.T, ctx context.Context, command, marker string) (string, bool) {
+	t.Helper()
+	weedBinary := findWeedBinary()
+	require.NotEmpty(t, weedBinary, "weed binary not found")
+	cmd := exec.CommandContext(ctx, weedBinary, "shell", "-master="+chaosMasterAddr)
+
+	pr, pw, err := os.Pipe()
+	require.NoError(t, err)
+	cmd.Stdout, cmd.Stderr = pw, pw
+	stdin, err := cmd.StdinPipe()
+	require.NoError(t, err)
+	require.NoError(t, cmd.Start())
+	pw.Close() // the child keeps its dup; ours would hold the reader open
+
+	var buf bytes.Buffer
+	var mu sync.Mutex
+	markerSeen := make(chan struct{})
+	scanDone := make(chan struct{})
+	go func() {
+		defer close(scanDone)
+		signaled := false
+		scanner := bufio.NewScanner(pr)
+		scanner.Buffer(make([]byte, 0, 256*1024), 1024*1024)
+		for scanner.Scan() {
+			line := scanner.Text()
+			mu.Lock()
+			buf.WriteString(line)
+			buf.WriteByte('\n')
+			mu.Unlock()
+			if !signaled && bytesContains(line, marker) {
+				signaled = true
+				close(markerSeen)
+			}
+		}
+	}()
+
+	fmt.Fprintf(stdin, "lock\n%s\n", command)
+	// Stdin stays open: the shell must die by kill, never by graceful EOF —
+	// a graceful exit would release the cluster lock cleanly and dodge the
+	// dead-holder cleanup path this test also exercises.
+	hit := false
+	select {
+	case <-markerSeen:
+		hit = true
+	case <-scanDone: // shell exited before printing the marker
+	case <-time.After(90 * time.Second):
+	}
+	if !hit {
+		// A shell that printed the marker and exited immediately can make both
+		// channels ready at once, and select picks between ready cases at
+		// random; a printed marker must never be reported as missed.
+		select {
+		case <-markerSeen:
+			hit = true
+		default:
+		}
+	}
+	cmd.Process.Kill()
+	cmd.Wait()
+	pr.Close()
+	<-scanDone
+	mu.Lock()
+	defer mu.Unlock()
+	return buf.String(), hit
+}
+
+func bytesContains(line, marker string) bool {
+	return len(marker) > 0 && strings.Contains(line, marker)
+}

test/foundationdb/Dockerfile.build

@@ -1,6 +1,6 @@
 # Simplified single-stage build for SeaweedFS with FoundationDB support
 # Force x86_64 platform to use AMD64 FoundationDB packages
-FROM --platform=linux/amd64 golang:1.24-bookworm
+FROM --platform=linux/amd64 golang:1.26-bookworm
 
 ARG FOUNDATIONDB_VERSION=7.4.5
 ENV FOUNDATIONDB_VERSION=${FOUNDATIONDB_VERSION}
@@ -43,7 +43,7 @@ WORKDIR /build
 # Copy source code
 COPY . .
 
-# Using Go 1.24 to match project requirements
+# Using Go 1.26 to match project requirements
 
 # Download dependencies (using versions from go.mod for deterministic builds)
 RUN go mod download

test/foundationdb/Dockerfile.build.arm64

@@ -1,5 +1,5 @@
 # Multi-stage Dockerfile to build SeaweedFS with FoundationDB support for ARM64
-FROM --platform=linux/arm64 golang:1.24-bookworm AS builder
+FROM --platform=linux/arm64 golang:1.26-bookworm AS builder
 
 ARG FOUNDATIONDB_VERSION=7.4.5
 ENV FOUNDATIONDB_VERSION=${FOUNDATIONDB_VERSION}

test/foundationdb/Dockerfile.test

@@ -1,5 +1,5 @@
 # Test environment with Go and FoundationDB support
-FROM golang:1.24-bookworm
+FROM golang:1.26-bookworm
 
 # Install system dependencies
 RUN apt-get update && apt-get install -y \

test/foundationdb/foundationdb_concurrent_test.go

@@ -1,5 +1,4 @@
 //go:build foundationdb
-// +build foundationdb
 
 package foundationdb
 

test/foundationdb/foundationdb_integration_test.go

@@ -1,5 +1,4 @@
 //go:build foundationdb
-// +build foundationdb
 
 package foundationdb
 

test/fuse_dlm/dlm_concurrent_write_test.go

@@ -0,0 +1,279 @@
+//go:build linux || darwin
+
+package fuse_dlm
+
+import (
+	"fmt"
+	"os"
+	"path/filepath"
+	"sync"
+	"sync/atomic"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+// TestDLMConcurrentWritersSameFile verifies that two mounts writing to the same
+// file concurrently produce valid (non-corrupted) data. With DLM enabled, the
+// writes are serialized — one blocks until the other completes.
+//
+// Note: cross-mount read consistency depends on FUSE kernel cache invalidation
+// and filer metadata subscription, which are asynchronous. This test verifies
+// write integrity, not instant read convergence.
+func TestDLMConcurrentWritersSameFile(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skipping DLM integration test in short mode")
+	}
+
+	cluster := startDLMTestCluster(t)
+	t.Cleanup(cluster.Stop)
+
+	const iterations = 5
+	for iter := 0; iter < iterations; iter++ {
+		fileName := fmt.Sprintf("concurrent_write_%d.txt", iter)
+		payloadA := []byte(fmt.Sprintf("mount0-iteration-%d-payload-AAAA", iter))
+		payloadB := []byte(fmt.Sprintf("mount1-iteration-%d-payload-BBBB", iter))
+
+		var wg sync.WaitGroup
+		wg.Add(2)
+
+		go func() {
+			defer wg.Done()
+			err := os.WriteFile(filepath.Join(cluster.mountPoints[0], fileName), payloadA, 0644)
+			assert.NoError(t, err, "mount0 write iteration %d", iter)
+		}()
+
+		go func() {
+			defer wg.Done()
+			err := os.WriteFile(filepath.Join(cluster.mountPoints[1], fileName), payloadB, 0644)
+			assert.NoError(t, err, "mount1 write iteration %d", iter)
+		}()
+
+		wg.Wait()
+
+		// Verify file is readable and contains one of the expected payloads
+		// (read from mount0 — its own view is authoritative for write success).
+		content, err := os.ReadFile(filepath.Join(cluster.mountPoints[0], fileName))
+		require.NoError(t, err, "read from mount0 iteration %d", iter)
+		validPayload := string(content) == string(payloadA) || string(content) == string(payloadB)
+		assert.True(t, validPayload,
+			"iteration %d: content must be one of the expected payloads, got: %q", iter, content)
+	}
+}
+
+// TestDLMRepeatedOpenWriteClose verifies that repeated open/write/close cycles
+// from both mounts all succeed without errors.
+func TestDLMRepeatedOpenWriteClose(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skipping DLM integration test in short mode")
+	}
+
+	cluster := startDLMTestCluster(t)
+	t.Cleanup(cluster.Stop)
+
+	const cycles = 20
+	fileName := "repeated_write.txt"
+
+	var wg sync.WaitGroup
+	wg.Add(2)
+
+	go func() {
+		defer wg.Done()
+		for i := 0; i < cycles; i++ {
+			data := []byte(fmt.Sprintf("mount0-cycle-%d", i))
+			err := os.WriteFile(filepath.Join(cluster.mountPoints[0], fileName), data, 0644)
+			assert.NoError(t, err, "mount0 cycle %d", i)
+		}
+	}()
+
+	go func() {
+		defer wg.Done()
+		for i := 0; i < cycles; i++ {
+			data := []byte(fmt.Sprintf("mount1-cycle-%d", i))
+			err := os.WriteFile(filepath.Join(cluster.mountPoints[1], fileName), data, 0644)
+			assert.NoError(t, err, "mount1 cycle %d", i)
+		}
+	}()
+
+	wg.Wait()
+
+	// File must be readable from at least one mount
+	content, err := os.ReadFile(filepath.Join(cluster.mountPoints[0], fileName))
+	require.NoError(t, err)
+	assert.NotEmpty(t, content, "file must not be empty")
+}
+
+// TestDLMWriteBlocksSecondWriter verifies the core DLM guarantee: while one
+// mount has a file open for writing, another mount's write-open blocks until
+// the first mount closes the file.
+func TestDLMWriteBlocksSecondWriter(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skipping DLM integration test in short mode")
+	}
+
+	cluster := startDLMTestCluster(t)
+	t.Cleanup(cluster.Stop)
+
+	fileName := "blocking_test.txt"
+	path0 := filepath.Join(cluster.mountPoints[0], fileName)
+	path1 := filepath.Join(cluster.mountPoints[1], fileName)
+
+	// Mount 0 opens the file for writing and holds it open
+	f, err := os.OpenFile(path0, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0644)
+	require.NoError(t, err, "mount0 open")
+	_, err = f.Write([]byte("mount0-holds-lock"))
+	require.NoError(t, err, "mount0 write")
+
+	// Mount 1 tries to write — should block (we use a goroutine with atomic flag)
+	var mount1Completed atomic.Bool
+	mount1Done := make(chan error, 1)
+	go func() {
+		err := os.WriteFile(path1, []byte("mount1-waited"), 0644)
+		mount1Completed.Store(true)
+		mount1Done <- err
+	}()
+
+	// Give mount 1 a moment — it should NOT complete while mount 0 holds the file open
+	time.Sleep(3 * time.Second)
+	require.False(t, mount1Completed.Load(),
+		"mount1 write must not complete while mount0 holds the file open")
+	t.Log("mount1 write is blocked as expected while mount0 holds the file")
+
+	// Mount 0 closes the file — this releases the DLM lock
+	require.NoError(t, f.Close(), "mount0 close")
+
+	// Mount 1 should now complete
+	select {
+	case err := <-mount1Done:
+		assert.NoError(t, err, "mount1 write after mount0 close")
+	case <-time.After(30 * time.Second):
+		t.Fatal("mount1 write did not complete within 30s after mount0 closed")
+	}
+}
+
+// TestDLMRenameWhileWriteOpen verifies that a rename is coordinated with DLM:
+// while mount0 has a file open for writing (re-opened after creation),
+// mount1 cannot rename it until mount0 closes the file.
+func TestDLMRenameWhileWriteOpen(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skipping DLM integration test in short mode")
+	}
+
+	cluster := startDLMTestCluster(t)
+	t.Cleanup(cluster.Stop)
+
+	origName := "rename_source.txt"
+	newName := "rename_dest.txt"
+
+	// Create and close the file first so it's flushed to the filer and
+	// visible on both mounts.
+	require.NoError(t, os.WriteFile(
+		filepath.Join(cluster.mountPoints[0], origName),
+		[]byte("initial-content"), 0644))
+	time.Sleep(2 * time.Second) // metadata propagation
+
+	// Verify mount1 can see the file
+	_, err := os.Stat(filepath.Join(cluster.mountPoints[1], origName))
+	require.NoError(t, err, "mount1 should see the file")
+
+	// Mount 0 re-opens the file for writing and holds it open
+	f, err := os.OpenFile(
+		filepath.Join(cluster.mountPoints[0], origName),
+		os.O_WRONLY|os.O_TRUNC, 0644)
+	require.NoError(t, err, "mount0 reopen")
+	_, err = f.Write([]byte("data-while-holding-lock"))
+	require.NoError(t, err, "mount0 write")
+
+	// Mount 1 tries to rename — should block because mount0 holds the
+	// DLM lock on the old path
+	var renameCompleted atomic.Bool
+	renameDone := make(chan error, 1)
+	go func() {
+		err := os.Rename(
+			filepath.Join(cluster.mountPoints[1], origName),
+			filepath.Join(cluster.mountPoints[1], newName))
+		renameCompleted.Store(true)
+		renameDone <- err
+	}()
+
+	// Rename must NOT complete while mount0 holds the file open
+	time.Sleep(3 * time.Second)
+	require.False(t, renameCompleted.Load(),
+		"rename must not complete while mount0 holds the file open")
+	t.Log("rename is blocked as expected while mount0 holds the file")
+
+	// Mount 0 closes → releases DLM lock → rename should proceed
+	require.NoError(t, f.Close(), "mount0 close")
+
+	select {
+	case err := <-renameDone:
+		assert.NoError(t, err, "rename after mount0 close")
+	case <-time.After(30 * time.Second):
+		t.Fatal("rename did not complete within 30s after mount0 closed")
+	}
+}
+
+// Note: Same-mount rename while a file is open for writing is not tested here
+// because macOS FUSE serializes operations on the same inode, causing a
+// kernel-level deadlock between the Rename handler's internal flush and the
+// pending Close. Same-mount coordination is already handled by the per-mount
+// fhLockTable and FUSE kernel serialization, so DLM is not needed for it.
+
+// TestDLMConcurrentRenames verifies that two concurrent renames of the same
+// file from different mounts don't corrupt metadata. DLM locks on both old
+// and new paths ensure renames are serialized.
+func TestDLMConcurrentRenames(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skipping DLM integration test in short mode")
+	}
+
+	cluster := startDLMTestCluster(t)
+	t.Cleanup(cluster.Stop)
+
+	// Create a file first
+	origPath := filepath.Join(cluster.mountPoints[0], "rename_race.txt")
+	require.NoError(t, os.WriteFile(origPath, []byte("original-content"), 0644))
+	time.Sleep(1 * time.Second) // propagation
+
+	// Both mounts try to rename the same file concurrently
+	var wg sync.WaitGroup
+	var errA, errB error
+	wg.Add(2)
+
+	go func() {
+		defer wg.Done()
+		errA = os.Rename(
+			filepath.Join(cluster.mountPoints[0], "rename_race.txt"),
+			filepath.Join(cluster.mountPoints[0], "renamed_by_mount0.txt"),
+		)
+	}()
+
+	go func() {
+		defer wg.Done()
+		errB = os.Rename(
+			filepath.Join(cluster.mountPoints[1], "rename_race.txt"),
+			filepath.Join(cluster.mountPoints[1], "renamed_by_mount1.txt"),
+		)
+	}()
+
+	wg.Wait()
+
+	// At least one rename should succeed; the other may fail with ENOENT
+	// since the source was already moved.
+	succeeded := 0
+	if errA == nil {
+		succeeded++
+		t.Logf("mount0 rename succeeded")
+	} else {
+		t.Logf("mount0 rename failed: %v", errA)
+	}
+	if errB == nil {
+		succeeded++
+		t.Logf("mount1 rename succeeded")
+	} else {
+		t.Logf("mount1 rename failed: %v", errB)
+	}
+	assert.GreaterOrEqual(t, succeeded, 1, "at least one rename must succeed")
+}

test/fuse_dlm/framework_test.go

@@ -0,0 +1,562 @@
+//go:build linux || darwin
+
+package fuse_dlm
+
+import (
+	"context"
+	"fmt"
+	"net"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"strconv"
+	"strings"
+	"sync"
+	"syscall"
+	"testing"
+	"time"
+
+	"github.com/seaweedfs/seaweedfs/weed/cluster/lock_manager"
+	"github.com/seaweedfs/seaweedfs/weed/pb"
+	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
+	"github.com/seaweedfs/seaweedfs/weed/pb/master_pb"
+	"github.com/stretchr/testify/require"
+	"google.golang.org/grpc"
+	"google.golang.org/grpc/credentials/insecure"
+)
+
+const filerGroup = "fuse-dlm-test"
+
+// dlmTestCluster manages a full SeaweedFS cluster with 2 filers and 2 FUSE
+// mounts for testing DLM-based cross-mount write coordination.
+type dlmTestCluster struct {
+	t          testing.TB
+	baseDir    string
+	weedBinary string
+
+	masterPort     int
+	masterGrpcPort int
+	volumePort     int
+	volumeGrpcPort int
+	filerPorts     [2]int
+	filerGrpcPorts [2]int
+	mountPoints    [2]string
+
+	masterCmd *exec.Cmd
+	volumeCmd *exec.Cmd
+	filerCmds [2]*exec.Cmd
+	mountCmds [2]*exec.Cmd
+	logFiles  []*os.File
+
+	cleanupOnce sync.Once
+}
+
+func startDLMTestCluster(t testing.TB) *dlmTestCluster {
+	binary := findWeedBinary()
+	if binary == "" {
+		t.Skip("weed binary not found; set WEED_BINARY or ensure it is on PATH")
+	}
+
+	baseDir, err := os.MkdirTemp("", "seaweedfs_fuse_dlm_test_")
+	require.NoError(t, err)
+
+	c := &dlmTestCluster{
+		t:          t,
+		baseDir:    baseDir,
+		weedBinary: binary,
+	}
+	// Register cleanup early so processes are stopped even if a require fails below.
+	t.Cleanup(c.Stop)
+
+	// Allocate ports: master(2) + volume(2) + filer0(2) + filer1(2) = 8
+	ports := allocatePorts(t, 8)
+	c.masterPort = ports[0]
+	c.masterGrpcPort = ports[1]
+	c.volumePort = ports[2]
+	c.volumeGrpcPort = ports[3]
+	c.filerPorts[0] = ports[4]
+	c.filerGrpcPorts[0] = ports[5]
+	c.filerPorts[1] = ports[6]
+	c.filerGrpcPorts[1] = ports[7]
+
+	// Write empty security.toml
+	configDir := filepath.Join(baseDir, "config")
+	require.NoError(t, os.MkdirAll(configDir, 0755))
+	require.NoError(t, os.WriteFile(filepath.Join(configDir, "security.toml"), []byte(""), 0644))
+
+	// Start master
+	require.NoError(t, c.startMaster(configDir))
+	require.NoError(t, c.waitForTCP(fmt.Sprintf("127.0.0.1:%d", c.masterPort), 30*time.Second),
+		"master not ready\n%s", c.tailLog("master"))
+
+	// Start volume
+	require.NoError(t, c.startVolume(configDir))
+	require.NoError(t, c.waitForTCP(fmt.Sprintf("127.0.0.1:%d", c.volumePort), 30*time.Second),
+		"volume not ready\n%s", c.tailLog("volume"))
+	require.NoError(t, c.waitForVolumeRegistered(30*time.Second),
+		"volume server registration\n%s", c.tailLog("master"))
+
+	// Start 2 filers
+	for i := 0; i < 2; i++ {
+		require.NoError(t, c.startFiler(i, configDir))
+		require.NoError(t, c.waitForTCP(fmt.Sprintf("127.0.0.1:%d", c.filerGrpcPorts[i]), 30*time.Second),
+			"filer %d not ready\n%s", i, c.tailLog(fmt.Sprintf("filer%d", i)))
+	}
+	require.NoError(t, c.waitForFilerCount(2, 30*time.Second), "filer group registration")
+	require.NoError(t, c.waitForLockRingConverged(30*time.Second), "lock ring convergence")
+
+	// Start 2 mounts, both pointing at filer0 for metadata consistency.
+	// (filer1 exists for the DLM lock ring but both mounts share filer0's
+	// metadata store since leveldb is per-filer.)
+	for i := 0; i < 2; i++ {
+		mp := filepath.Join(baseDir, fmt.Sprintf("mount%d", i))
+		require.NoError(t, os.MkdirAll(mp, 0755))
+		c.mountPoints[i] = mp
+		require.NoError(t, c.startMount(i, configDir))
+		require.NoError(t, c.waitForMount(mp, 30*time.Second),
+			"mount %d not ready\n%s", i, c.tailLog(fmt.Sprintf("mount%d", i)))
+	}
+
+	return c
+}
+
+func (c *dlmTestCluster) Stop() {
+	if c == nil {
+		return
+	}
+	c.cleanupOnce.Do(func() {
+		// Stop mounts first (triggers flush + DLM unlock).
+		// Use stopCmd for bounded wait to avoid hanging on wedged FUSE processes.
+		for i := 1; i >= 0; i-- {
+			stopCmd(c.mountCmds[i])
+			// Backup unmount in case FUSE teardown didn't clean up
+			exec.Command("fusermount3", "-u", c.mountPoints[i]).Run()
+			exec.Command("fusermount", "-u", c.mountPoints[i]).Run()
+		}
+		// Stop filers, volume, master
+		for i := 1; i >= 0; i-- {
+			stopCmd(c.filerCmds[i])
+		}
+		stopCmd(c.volumeCmd)
+		stopCmd(c.masterCmd)
+
+		for _, f := range c.logFiles {
+			f.Close()
+		}
+
+		// Copy logs for CI
+		c.copyLogsForCI()
+
+		if !c.t.Failed() {
+			os.RemoveAll(c.baseDir)
+		}
+
+		// Wait for ports to be fully released before the next test
+		// allocates new ports (avoids TIME_WAIT collisions).
+		time.Sleep(2 * time.Second)
+	})
+}
+
+// masterAddress returns the master address in the format that encodes both
+// HTTP and gRPC ports: "host:httpPort.grpcPort". This is the format that
+// SeaweedFS uses to communicate non-default gRPC ports between components.
+func (c *dlmTestCluster) masterAddress() string {
+	return string(pb.NewServerAddress("127.0.0.1", c.masterPort, c.masterGrpcPort))
+}
+
+func (c *dlmTestCluster) startMaster(configDir string) error {
+	c.masterCmd = exec.Command(c.weedBinary,
+		"-logdir="+filepath.Join(c.baseDir, "logs"),
+		"master",
+		"-ip=127.0.0.1",
+		"-ip.bind=127.0.0.1",
+		"-port="+strconv.Itoa(c.masterPort),
+		"-port.grpc="+strconv.Itoa(c.masterGrpcPort),
+		"-mdir="+filepath.Join(c.baseDir, "master"),
+	)
+	return c.startCmd(c.masterCmd, "master")
+}
+
+func (c *dlmTestCluster) startVolume(configDir string) error {
+	volDir := filepath.Join(c.baseDir, "volume")
+	if err := os.MkdirAll(volDir, 0755); err != nil {
+		return fmt.Errorf("create volume dir: %w", err)
+	}
+	c.volumeCmd = exec.Command(c.weedBinary,
+		"-logdir="+filepath.Join(c.baseDir, "logs"),
+		"volume",
+		"-ip=127.0.0.1",
+		"-ip.bind=127.0.0.1",
+		"-port="+strconv.Itoa(c.volumePort),
+		"-port.grpc="+strconv.Itoa(c.volumeGrpcPort),
+		"-master="+c.masterAddress(),
+		"-dir="+volDir,
+		"-max=10",
+	)
+	return c.startCmd(c.volumeCmd, "volume")
+}
+
+func (c *dlmTestCluster) startFiler(idx int, configDir string) error {
+	filerDir := filepath.Join(c.baseDir, fmt.Sprintf("filer%d", idx))
+	if err := os.MkdirAll(filerDir, 0755); err != nil {
+		return fmt.Errorf("create filer dir: %w", err)
+	}
+	c.filerCmds[idx] = exec.Command(c.weedBinary,
+		"-logdir="+filepath.Join(c.baseDir, "logs"),
+		"filer",
+		"-ip=127.0.0.1",
+		"-ip.bind=127.0.0.1",
+		"-port="+strconv.Itoa(c.filerPorts[idx]),
+		"-port.grpc="+strconv.Itoa(c.filerGrpcPorts[idx]),
+		"-master="+c.masterAddress(),
+		"-filerGroup="+filerGroup,
+		"-defaultStoreDir="+filerDir,
+	)
+	return c.startCmd(c.filerCmds[idx], fmt.Sprintf("filer%d", idx))
+}
+
+func (c *dlmTestCluster) filerAddress(idx int) string {
+	return string(pb.NewServerAddress("127.0.0.1", c.filerPorts[idx], c.filerGrpcPorts[idx]))
+}
+
+func (c *dlmTestCluster) startMount(idx int, configDir string) error {
+	cacheDir := filepath.Join(c.baseDir, fmt.Sprintf("cache%d", idx))
+	if err := os.MkdirAll(cacheDir, 0755); err != nil {
+		return fmt.Errorf("create cache dir: %w", err)
+	}
+	c.mountCmds[idx] = exec.Command(c.weedBinary,
+		"-logdir="+filepath.Join(c.baseDir, "logs"),
+		"mount",
+		"-filer="+c.filerAddress(0), // both mounts use filer0 for shared metadata
+		"-dir="+c.mountPoints[idx],
+		"-filer.path=/",
+		"-dirAutoCreate",
+		"-allowOthers=false",
+		"-cacheDir="+cacheDir,
+		"-dlm",
+	)
+	return c.startCmd(c.mountCmds[idx], fmt.Sprintf("mount%d", idx))
+}
+
+func (c *dlmTestCluster) startCmd(cmd *exec.Cmd, name string) error {
+	logPath := filepath.Join(c.baseDir, "logs")
+	if err := os.MkdirAll(logPath, 0755); err != nil {
+		return fmt.Errorf("create log dir: %w", err)
+	}
+	logFile, err := os.Create(filepath.Join(logPath, name+".log"))
+	if err != nil {
+		return err
+	}
+	c.logFiles = append(c.logFiles, logFile)
+	cmd.Stdout = logFile
+	cmd.Stderr = logFile
+	return cmd.Start()
+}
+
+func (c *dlmTestCluster) tailLog(name string) string {
+	data, err := os.ReadFile(filepath.Join(c.baseDir, "logs", name+".log"))
+	if err != nil {
+		return fmt.Sprintf("(log %s not available: %v)", name, err)
+	}
+	const maxTail = 8192
+	if len(data) > maxTail {
+		data = data[len(data)-maxTail:]
+	}
+	return string(data)
+}
+
+func (c *dlmTestCluster) copyLogsForCI() {
+	// One subdirectory per test: a flat layout lets every teardown overwrite
+	// the previous test's logs, so the CI artifact only ever shows the last
+	// cluster, never the failing one.
+	ciLogDir := filepath.Join("/tmp/seaweedfs-fuse-dlm-logs", strings.ReplaceAll(c.t.Name(), "/", "_"))
+	os.MkdirAll(ciLogDir, 0755)
+	logsDir := filepath.Join(c.baseDir, "logs")
+	entries, err := os.ReadDir(logsDir)
+	if err != nil {
+		return
+	}
+	for _, e := range entries {
+		data, err := os.ReadFile(filepath.Join(logsDir, e.Name()))
+		if err != nil {
+			continue
+		}
+		os.WriteFile(filepath.Join(ciLogDir, e.Name()), data, 0644)
+	}
+}
+
+func (c *dlmTestCluster) waitForTCP(addr string, timeout time.Duration) error {
+	deadline := time.Now().Add(timeout)
+	for time.Now().Before(deadline) {
+		conn, err := net.DialTimeout("tcp", addr, time.Second)
+		if err == nil {
+			conn.Close()
+			return nil
+		}
+		time.Sleep(200 * time.Millisecond)
+	}
+	return fmt.Errorf("service at %s not ready within timeout", addr)
+}
+
+// waitForMount waits for a FUSE filesystem to actually be mounted at
+// mountPoint by comparing the device ID of the mount point against its parent.
+// A plain directory (pre-created before mount) has the same device as its
+// parent; a mounted FUSE filesystem has a different device.
+func (c *dlmTestCluster) waitForMount(mountPoint string, timeout time.Duration) error {
+	parentDir := filepath.Dir(mountPoint)
+	deadline := time.Now().Add(timeout)
+	for time.Now().Before(deadline) {
+		parentStat, err := os.Stat(parentDir)
+		if err != nil {
+			time.Sleep(200 * time.Millisecond)
+			continue
+		}
+		mountStat, err := os.Stat(mountPoint)
+		if err != nil {
+			time.Sleep(200 * time.Millisecond)
+			continue
+		}
+		parentSys := parentStat.Sys().(*syscall.Stat_t)
+		mountSys := mountStat.Sys().(*syscall.Stat_t)
+		if parentSys.Dev != mountSys.Dev {
+			// Different device = FUSE mounted
+			return nil
+		}
+		time.Sleep(200 * time.Millisecond)
+	}
+	return fmt.Errorf("mount point %s not ready within timeout (FUSE not detected)", mountPoint)
+}
+
+func (c *dlmTestCluster) filerGRPCAddress(idx int) string {
+	return fmt.Sprintf("127.0.0.1:%d", c.filerGrpcPorts[idx])
+}
+
+func (c *dlmTestCluster) waitForFilerCount(expected int, timeout time.Duration) error {
+	addr := fmt.Sprintf("127.0.0.1:%d", c.masterGrpcPort)
+	conn, err := grpc.NewClient(addr, grpc.WithTransportCredentials(insecure.NewCredentials()))
+	if err != nil {
+		return err
+	}
+	defer conn.Close()
+
+	client := master_pb.NewSeaweedClient(conn)
+	deadline := time.Now().Add(timeout)
+	for time.Now().Before(deadline) {
+		ctx, cancel := context.WithTimeout(context.Background(), time.Second)
+		resp, err := client.ListClusterNodes(ctx, &master_pb.ListClusterNodesRequest{
+			ClientType: "filer",
+			FilerGroup: filerGroup,
+		})
+		cancel()
+		if err == nil && len(resp.ClusterNodes) >= expected {
+			return nil
+		}
+		time.Sleep(200 * time.Millisecond)
+	}
+	return fmt.Errorf("timed out waiting for %d filers in group %q", expected, filerGroup)
+}
+
+// waitForVolumeRegistered waits until the volume server shows up in the master
+// topology with its slots reported. An open volume port only means the process
+// is listening: until the master has elected itself and accepted a heartbeat,
+// an assign fails and the first write on a fresh mount surfaces it as ENOSPC.
+func (c *dlmTestCluster) waitForVolumeRegistered(timeout time.Duration) error {
+	addr := fmt.Sprintf("127.0.0.1:%d", c.masterGrpcPort)
+	conn, err := grpc.NewClient(addr, grpc.WithTransportCredentials(insecure.NewCredentials()))
+	if err != nil {
+		return err
+	}
+	defer conn.Close()
+
+	client := master_pb.NewSeaweedClient(conn)
+	deadline := time.Now().Add(timeout)
+	for time.Now().Before(deadline) {
+		ctx, cancel := context.WithTimeout(context.Background(), time.Second)
+		resp, err := client.VolumeList(ctx, &master_pb.VolumeListRequest{})
+		cancel()
+		if err == nil && resp.TopologyInfo != nil {
+			var slots int64
+			for _, dc := range resp.TopologyInfo.DataCenterInfos {
+				for _, rack := range dc.RackInfos {
+					for _, dn := range rack.DataNodeInfos {
+						for _, disk := range dn.DiskInfos {
+							slots += disk.MaxVolumeCount
+						}
+					}
+				}
+			}
+			if slots > 0 {
+				return nil
+			}
+		}
+		time.Sleep(200 * time.Millisecond)
+	}
+	return fmt.Errorf("volume server not registered with the master within %v", timeout)
+}
+
+// waitForLockRingConverged verifies that both filers have a consistent view of
+// the lock ring by acquiring the same lock through each filer and checking
+// mutual exclusion. Adapted from test/s3/distributed_lock/.
+func (c *dlmTestCluster) waitForLockRingConverged(timeout time.Duration) error {
+	deadline := time.Now().Add(timeout)
+
+	owners := []pb.ServerAddress{
+		pb.ServerAddress(c.filerGRPCAddress(0)),
+		pb.ServerAddress(c.filerGRPCAddress(1)),
+	}
+	ring := lock_manager.NewHashRing(lock_manager.DefaultVnodeCount)
+	ring.SetServers(owners)
+
+	attempt := 0
+	for time.Now().Before(deadline) {
+		testKeys := convergenceKeysPerPrimary(ring, owners, attempt)
+		attempt++
+
+		allConverged := true
+		for _, key := range testKeys {
+			converged, _ := c.checkLockMutualExclusion(key)
+			if !converged {
+				allConverged = false
+				break
+			}
+		}
+		if allConverged {
+			return nil
+		}
+		time.Sleep(500 * time.Millisecond)
+	}
+	return fmt.Errorf("lock ring did not converge")
+}
+
+// convergenceKeysPerPrimary generates one test key per primary filer.
+func convergenceKeysPerPrimary(ring *lock_manager.HashRing, owners []pb.ServerAddress, attempt int) []string {
+	found := make(map[pb.ServerAddress]bool)
+	var keys []string
+	for i := 0; len(found) < len(owners) && i < 10000; i++ {
+		key := fmt.Sprintf("convergence-test-%d-%d", attempt, i)
+		primary, _ := ring.GetPrimaryAndBackup(key)
+		if !found[primary] {
+			found[primary] = true
+			keys = append(keys, key)
+		}
+	}
+	return keys
+}
+
+func (c *dlmTestCluster) checkLockMutualExclusion(key string) (bool, error) {
+	// Try to lock via filer0
+	conn0, err := grpc.NewClient(c.filerGRPCAddress(0), grpc.WithTransportCredentials(insecure.NewCredentials()))
+	if err != nil {
+		return false, err
+	}
+	defer conn0.Close()
+
+	client0 := filer_pb.NewSeaweedFilerClient(conn0)
+	ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
+	defer cancel()
+	resp0, err := client0.DistributedLock(ctx, &filer_pb.LockRequest{
+		Name:          key,
+		SecondsToLock: 5,
+		Owner:         "convergence-test-0",
+	})
+	if err != nil {
+		return false, err
+	}
+	if resp0.Error != "" {
+		return false, fmt.Errorf("lock0: %s", resp0.Error)
+	}
+
+	// Try to lock via filer1 — should fail (already locked)
+	conn1, err := grpc.NewClient(c.filerGRPCAddress(1), grpc.WithTransportCredentials(insecure.NewCredentials()))
+	if err != nil {
+		return false, err
+	}
+	defer conn1.Close()
+
+	client1 := filer_pb.NewSeaweedFilerClient(conn1)
+	resp1, err := client1.DistributedLock(ctx, &filer_pb.LockRequest{
+		Name:          key,
+		SecondsToLock: 5,
+		Owner:         "convergence-test-1",
+	})
+
+	// Unlock via filer0
+	client0.DistributedUnlock(ctx, &filer_pb.UnlockRequest{
+		Name:       key,
+		RenewToken: resp0.RenewToken,
+	})
+
+	if err != nil {
+		return false, err
+	}
+	// If filer1 also got the lock, the ring hasn't converged
+	if resp1.Error == "" {
+		// Unlock the second one too
+		client1.DistributedUnlock(ctx, &filer_pb.UnlockRequest{
+			Name:       key,
+			RenewToken: resp1.RenewToken,
+		})
+		return false, nil
+	}
+
+	return true, nil
+}
+
+func stopCmd(cmd *exec.Cmd) {
+	if cmd == nil || cmd.Process == nil {
+		return
+	}
+	cmd.Process.Signal(syscall.SIGTERM)
+	done := make(chan struct{})
+	go func() {
+		cmd.Wait()
+		close(done)
+	}()
+	select {
+	case <-done:
+	case <-time.After(10 * time.Second):
+		cmd.Process.Kill()
+		<-done
+	}
+}
+
+func allocatePorts(t testing.TB, n int) []int {
+	t.Helper()
+	// Hold all listeners open until all ports are collected, then close
+	// them together. This prevents the OS from reassigning a just-freed
+	// port to the next Listen call within the same allocation batch.
+	listeners := make([]net.Listener, 0, n)
+	ports := make([]int, 0, n)
+	defer func() {
+		for _, l := range listeners {
+			l.Close()
+		}
+	}()
+	for i := 0; i < n; i++ {
+		l, err := net.Listen("tcp", "127.0.0.1:0")
+		require.NoError(t, err)
+		listeners = append(listeners, l)
+		ports = append(ports, l.Addr().(*net.TCPAddr).Port)
+	}
+	return ports
+}
+
+func findWeedBinary() string {
+	if p := os.Getenv("WEED_BINARY"); p != "" {
+		return p
+	}
+	if p, err := exec.LookPath("weed"); err == nil {
+		return p
+	}
+	candidates := []string{
+		"../../weed/weed",
+		"./weed",
+	}
+	for _, c := range candidates {
+		if info, err := os.Stat(c); err == nil && !info.IsDir() {
+			abs, _ := filepath.Abs(c)
+			return abs
+		}
+	}
+	return ""
+}

test/fuse_dlm/posix_lock_ring_test.go

@@ -0,0 +1,234 @@
+//go:build linux || darwin
+
+package fuse_dlm
+
+import (
+	"errors"
+	"fmt"
+	"os"
+	"path/filepath"
+	"runtime"
+	"syscall"
+	"testing"
+	"time"
+
+	"github.com/seaweedfs/seaweedfs/weed/cluster/lock_manager"
+	"github.com/seaweedfs/seaweedfs/weed/pb"
+	"github.com/stretchr/testify/require"
+)
+
+// posixLockKey returns the routed-lock key a mount uses for a file at the mount
+// root. Mounts run with -filer.path=/, so the file's filer path is "/"+name; the
+// prefix must match mount.posixLockKeyForInode ("s3.fuse.lock:").
+func posixLockKey(name string) string { return "s3.fuse.lock:/" + name }
+
+// These tests exercise cross-mount POSIX advisory locks (flock), which the
+// mounts route to the inode's owner filer because they run with -dlm
+// (crossMountLocks() == lockClient != nil). They reuse the dlmTestCluster:
+// master + volume + 2 filers (forming the lock ring) + 2 mounts on filer0.
+//
+// All flock opens are O_RDONLY on purpose: a write open (O_RDWR/O_WRONLY) would
+// also take the -dlm whole-file write lock (held until close), which is a
+// different mechanism — opening read-only isolates the POSIX advisory flock.
+
+// requireForwardedLocks skips on platforms where the kernel does not forward
+// advisory locks to the FUSE server. Only Linux forwards flock/fcntl (SETLK) to
+// the filesystem; macFUSE handles flock in-kernel per mount, so cross-mount
+// coordination can't be observed there even though the routed path is correct.
+func requireForwardedLocks(t *testing.T) {
+	if runtime.GOOS != "linux" {
+		t.Skipf("advisory locks are only forwarded to the FUSE server on Linux (GOOS=%s)", runtime.GOOS)
+	}
+}
+
+// openFlock opens path read-only and takes a flock of type how (e.g. LOCK_EX).
+// The file must already exist. The returned file holds the lock until closed.
+func openFlock(path string, how int) (*os.File, error) {
+	f, err := os.OpenFile(path, os.O_RDONLY, 0)
+	if err != nil {
+		return nil, err
+	}
+	if err := syscall.Flock(int(f.Fd()), how); err != nil {
+		f.Close()
+		return nil, err
+	}
+	return f, nil
+}
+
+// createFile creates an empty file; the brief -dlm write lock it takes is
+// released on close, before any flock test runs.
+func createFile(t *testing.T, path string) {
+	t.Helper()
+	f, err := os.OpenFile(path, os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0644)
+	require.NoError(t, err, "create %s", path)
+	require.NoError(t, f.Close())
+}
+
+// waitVisible waits for path to appear (cross-mount metadata propagation).
+func waitVisible(t *testing.T, path string) {
+	t.Helper()
+	require.Eventually(t, func() bool {
+		_, err := os.Stat(path)
+		return err == nil
+	}, 15*time.Second, 200*time.Millisecond, "%s never became visible", path)
+}
+
+// tryExclusiveFlock attempts a non-blocking exclusive flock on path and
+// classifies the outcome: acquired (and released again), blocked by another
+// owner (EWOULDBLOCK/EAGAIN), or an unexpected error. It distinguishes a real
+// "held by another" from incidental errors so the latter can't masquerade as a
+// satisfied "must be blocked" assertion.
+func tryExclusiveFlock(path string) (acquired, blocked bool, err error) {
+	f, err := os.OpenFile(path, os.O_RDONLY, 0)
+	if err != nil {
+		return false, false, err
+	}
+	defer f.Close()
+	if err := syscall.Flock(int(f.Fd()), syscall.LOCK_EX|syscall.LOCK_NB); err != nil {
+		if errors.Is(err, syscall.EWOULDBLOCK) || errors.Is(err, syscall.EAGAIN) {
+			return false, true, nil
+		}
+		return false, false, err
+	}
+	syscall.Flock(int(f.Fd()), syscall.LOCK_UN)
+	return true, false, nil
+}
+
+// requireEventuallyBlocked asserts path becomes held by another owner. It polls
+// because a routed lock RPC can hit a transient EIO (a cold or forwarded gRPC
+// call under load) even while the lock is genuinely held; it still fails if the
+// lock is never blocked — whether it can be acquired (a double-grant) or errors
+// persistently — so an incidental error can't masquerade as "held".
+func requireEventuallyBlocked(t *testing.T, path string, timeout time.Duration, msg string) {
+	t.Helper()
+	require.Eventuallyf(t, func() bool { return isBlocked(path) },
+		timeout, 500*time.Millisecond, "%s", msg)
+}
+
+// isBlocked / isAcquirable are lenient predicates for polling, where transient
+// errors during a migration are expected and simply mean "not yet".
+func isBlocked(path string) bool { _, b, _ := tryExclusiveFlock(path); return b }
+func isAcquirable(path string) bool {
+	a, _, _ := tryExclusiveFlock(path)
+	return a
+}
+
+// stopFiler stops filer idx and clears its command so cluster teardown does not
+// try to stop it again.
+func (c *dlmTestCluster) stopFiler(idx int) {
+	stopCmd(c.filerCmds[idx])
+	c.filerCmds[idx] = nil
+}
+
+// TestPosixLockCrossMount verifies a flock taken on one mount is seen by the
+// other mount of the same cluster (the routed-to-owner-filer path end to end).
+func TestPosixLockCrossMount(t *testing.T) {
+	requireForwardedLocks(t)
+	c := startDLMTestCluster(t)
+
+	name := "posix-xmount.lock"
+	path0 := filepath.Join(c.mountPoints[0], name)
+	path1 := filepath.Join(c.mountPoints[1], name)
+
+	createFile(t, path0)
+	waitVisible(t, path1)
+
+	held, err := openFlock(path0, syscall.LOCK_EX)
+	require.NoError(t, err, "mount0 should acquire the exclusive flock")
+	defer held.Close()
+
+	requireEventuallyBlocked(t, path1, 15*time.Second, "mount1 must be blocked while mount0 holds the flock")
+
+	require.NoError(t, syscall.Flock(int(held.Fd()), syscall.LOCK_UN))
+	require.Eventually(t, func() bool { return isAcquirable(path1) },
+		15*time.Second, 500*time.Millisecond,
+		"mount1 should acquire the flock after mount0 releases it")
+}
+
+// TestPosixLockSurvivesFilerLoss verifies advisory locks held across mounts
+// survive a filer leaving the ring: ownership of the affected keys migrates to
+// the surviving filer and the holding mount re-asserts them there, so the locks
+// stay honored. It locks many files so that — with 2 filers — several are owned
+// by filer1 and thus actually migrate when filer1 stops.
+func TestPosixLockSurvivesFilerLoss(t *testing.T) {
+	requireForwardedLocks(t)
+	c := startDLMTestCluster(t)
+
+	const n = 12
+
+	// Select files via the same ring the filers run so the locked set provably
+	// spans both filers: filer1-owned keys must migrate when filer1 stops, while
+	// filer0-owned keys must keep working. Choosing by ownership (instead of
+	// hoping a sequential set happens to spread) keeps the migration path
+	// exercised on every run, independent of the cluster's dynamic ports.
+	ring := lock_manager.NewHashRing(lock_manager.DefaultVnodeCount)
+	ring.SetServers([]pb.ServerAddress{
+		pb.ServerAddress(c.filerAddress(0)),
+		pb.ServerAddress(c.filerAddress(1)),
+	})
+	filer1 := pb.ServerAddress(c.filerAddress(1))
+	var onFiler1, onFiler0 []string
+	for i := 0; (len(onFiler1) < n/2 || len(onFiler0) < n/2) && i < 1000; i++ {
+		name := fmt.Sprintf("posix-migrate-%d.lock", i)
+		if ring.GetPrimary(posixLockKey(name)) == filer1 {
+			onFiler1 = append(onFiler1, name)
+		} else {
+			onFiler0 = append(onFiler0, name)
+		}
+	}
+	require.GreaterOrEqualf(t, len(onFiler1), n/2, "need %d filer1-owned keys to exercise migration", n/2)
+	require.GreaterOrEqualf(t, len(onFiler0), n/2, "need %d filer0-owned keys", n/2)
+	names := append(onFiler1[:n/2:n/2], onFiler0[:n/2]...)
+
+	held := make([]*os.File, n)
+	for i, name := range names {
+		createFile(t, filepath.Join(c.mountPoints[0], name))
+		waitVisible(t, filepath.Join(c.mountPoints[1], name))
+		f, err := openFlock(filepath.Join(c.mountPoints[0], name), syscall.LOCK_EX)
+		require.NoError(t, err, "mount0 should acquire flock %s", name)
+		held[i] = f
+		defer held[i].Close()
+	}
+
+	require.Eventually(t, func() bool {
+		for _, name := range names {
+			if !isBlocked(filepath.Join(c.mountPoints[1], name)) {
+				return false
+			}
+		}
+		return true
+	}, 20*time.Second, time.Second,
+		"every lock must be held on mount1 before the ring change")
+
+	// Drop filer1 from the ring; keys it owned migrate to filer0.
+	c.stopFiler(1)
+	require.NoError(t, c.waitForFilerCount(1, 30*time.Second), "ring should drop to one filer")
+
+	// Poll until every lock is honored again on the surviving filer: the ring
+	// must propagate and the holding mount must re-assert its locks (keepalive is
+	// 5s). We assert only the settled state — the transient migration window is
+	// covered by unit tests.
+	require.Eventually(t, func() bool {
+		for _, name := range names {
+			if !isBlocked(filepath.Join(c.mountPoints[1], name)) {
+				return false
+			}
+		}
+		return true
+	}, 45*time.Second, time.Second,
+		"all locks must survive filer1 leaving the ring (migrate to filer0)")
+
+	// Releasing on mount0 frees them all: mount1 can then acquire each.
+	for _, f := range held {
+		require.NoError(t, syscall.Flock(int(f.Fd()), syscall.LOCK_UN))
+	}
+	require.Eventually(t, func() bool {
+		for _, name := range names {
+			if !isAcquirable(filepath.Join(c.mountPoints[1], name)) {
+				return false
+			}
+		}
+		return true
+	}, 20*time.Second, 500*time.Millisecond,
+		"mount1 should acquire every lock after mount0 releases them post-migration")
+}

test/fuse_failover/README.md

@@ -0,0 +1,35 @@
+# FUSE volume server failover tests
+
+Integration tests for what happens to FUSE mounts when a volume server goes
+away, comes back, or restarts underneath in-flight IO. They automate the manual
+matrix reported in
+[discussion #10206](https://github.com/seaweedfs/seaweedfs/discussions/10206):
+one mount appends to a file while a second mount tails it, and a volume server
+is stopped, started or restarted mid-stream.
+
+The cluster is 1 master (`-defaultReplication=001`), 3 volume servers, 1 filer
+and 2 mounts, all as local processes. With 001 every chunk has a copy on two of
+the three servers, so losing any single server must be invisible to both mounts.
+
+| Reported scenario | Test |
+| --- | --- |
+| control: append + tail with nothing failing | `TestAppendWithoutChaos` |
+| read a file while one volume server is down | `TestReadWithVolumeServerDown` |
+| "STOP volumes": append + tail, kill a server mid-stream | `TestAppendWhileVolumeServerStops` |
+| "Start volumes": append + tail with a server down, start it mid-stream | `TestAppendWhileVolumeServerStarts` |
+| "Re-start volumes": append + tail, restart a server mid-stream | `TestAppendWhileVolumeServerRestarts` |
+| part 2: large file copy instead of small appends | `TestLargeWriteWhileVolumeServerStops` |
+
+Volume servers are dropped with SIGKILL, the closest local equivalent of a Swarm
+task disappearing from the overlay network: no deregistration, and the address
+stops answering.
+
+## Running
+
+```bash
+go build -o weed/weed ./weed
+WEED_BINARY=$PWD/weed/weed go test -v -count=1 -timeout=30m ./test/fuse_failover/...
+```
+
+Needs FUSE and, on Linux, a working `/dev/fuse`. Logs from a failed run are
+copied to `/tmp/seaweedfs-fuse-failover-logs/`.

test/fuse_failover/framework_test.go

@@ -0,0 +1,660 @@
+//go:build linux || darwin
+
+package fuse_failover
+
+import (
+	"encoding/json"
+	"fmt"
+	"io"
+	"net"
+	"net/http"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"strconv"
+	"strings"
+	"sync"
+	"syscall"
+	"testing"
+	"time"
+
+	"github.com/seaweedfs/seaweedfs/test/testutil"
+	"github.com/seaweedfs/seaweedfs/weed/pb"
+	"github.com/stretchr/testify/require"
+)
+
+// failoverCluster runs 1 master, N volume servers, 1 filer and M FUSE mounts,
+// with the master defaulting to 001 replication so every chunk lands on two
+// distinct volume servers. Individual volume servers can be stopped and started
+// while IO is in flight, which is what the Docker Swarm reports in discussion
+// 10206 exercise: a volume server disappears mid-append and the mounts must
+// keep reading from the surviving replica and keep writing on another volume.
+type failoverCluster struct {
+	t          testing.TB
+	baseDir    string
+	weedBinary string
+
+	masterPort     int
+	masterGrpcPort int
+	filerPort      int
+	filerGrpcPort  int
+	volumePorts    []int
+	volumeGrpcPort []int
+	volumeDirs     []string
+
+	masterCmd   *exec.Cmd
+	filerCmd    *exec.Cmd
+	volumeCmds  []*exec.Cmd
+	mountCmds   []*exec.Cmd
+	mountPoints []string
+	logFiles    []*os.File
+
+	mu          sync.Mutex
+	waits       map[*exec.Cmd]chan error
+	cleanupOnce sync.Once
+}
+
+func startFailoverCluster(t testing.TB, numVolumes, numMounts int) *failoverCluster {
+	require.GreaterOrEqual(t, numVolumes, 2, "001 replication needs at least 2 volume servers")
+	require.GreaterOrEqual(t, numMounts, 1)
+
+	binary := findWeedBinary()
+	if binary == "" {
+		t.Skip("weed binary not found; set WEED_BINARY or ensure it is on PATH")
+	}
+	baseDir, err := os.MkdirTemp("", "seaweedfs_fuse_failover_")
+	require.NoError(t, err)
+
+	c := &failoverCluster{
+		t:              t,
+		baseDir:        baseDir,
+		weedBinary:     binary,
+		volumePorts:    make([]int, numVolumes),
+		volumeGrpcPort: make([]int, numVolumes),
+		volumeDirs:     make([]string, numVolumes),
+		volumeCmds:     make([]*exec.Cmd, numVolumes),
+		mountCmds:      make([]*exec.Cmd, numMounts),
+		mountPoints:    make([]string, numMounts),
+	}
+	t.Cleanup(c.Stop)
+
+	ports, err := testutil.AllocatePorts(4 + 2*numVolumes)
+	require.NoError(t, err)
+	c.masterPort, c.masterGrpcPort = ports[0], ports[1]
+	c.filerPort, c.filerGrpcPort = ports[2], ports[3]
+	for i := 0; i < numVolumes; i++ {
+		c.volumePorts[i] = ports[4+2*i]
+		c.volumeGrpcPort[i] = ports[5+2*i]
+		c.volumeDirs[i] = filepath.Join(baseDir, fmt.Sprintf("volume%d", i))
+		require.NoError(t, os.MkdirAll(c.volumeDirs[i], 0755))
+	}
+
+	require.NoError(t, c.startMaster())
+	require.NoError(t, c.waitForTCP(c.masterCmd, "master",
+		fmt.Sprintf("127.0.0.1:%d", c.masterPort), 30*time.Second))
+
+	for i := 0; i < numVolumes; i++ {
+		require.NoError(t, c.StartVolume(i))
+	}
+
+	require.NoError(t, c.startFiler())
+	require.NoError(t, c.waitForTCP(c.filerCmd, "filer",
+		fmt.Sprintf("127.0.0.1:%d", c.filerGrpcPort), 30*time.Second))
+
+	for i := 0; i < numMounts; i++ {
+		mp := filepath.Join(baseDir, fmt.Sprintf("mount%d", i))
+		require.NoError(t, os.MkdirAll(mp, 0755))
+		c.mountPoints[i] = mp
+		require.NoError(t, c.startMount(i))
+		require.NoError(t, c.waitForMount(mp, 30*time.Second),
+			"mount %d not ready\n%s", i, c.tailLog(fmt.Sprintf("mount%d", i)))
+	}
+	require.NoError(t, c.WaitForVolumeServers(60*time.Second))
+
+	// A mounted filesystem and registered volume servers do not yet guarantee
+	// that the lock ring has settled or the first replicated volume is ready.
+	// Exercise the write/read path during setup so the append latency checks
+	// measure failover on a running cluster rather than its cold start.
+	probeName := ".failover-ready"
+	probeData := []byte("ready\n")
+	probePath := filepath.Join(c.MountDir(0), probeName)
+	require.NoError(t, os.WriteFile(probePath, probeData, 0644),
+		"readiness write failed\n%s", c.tailLog("mount0"))
+	for i := 0; i < numMounts; i++ {
+		_, ready := waitForContent(filepath.Join(c.MountDir(i), probeName), probeData, convergeTimeout)
+		require.True(t, ready, "mount %d did not observe readiness write\n%s", i, c.tailLog(fmt.Sprintf("mount%d", i)))
+	}
+	require.NoError(t, os.Remove(probePath))
+	return c
+}
+
+func (c *failoverCluster) MountDir(i int) string { return c.mountPoints[i] }
+
+func (c *failoverCluster) Stop() {
+	if c == nil {
+		return
+	}
+	c.cleanupOnce.Do(func() {
+		for i := len(c.mountCmds) - 1; i >= 0; i-- {
+			c.stopCmd(c.mountCmds[i], syscall.SIGTERM)
+			_ = exec.Command("fusermount3", "-u", c.mountPoints[i]).Run()
+			_ = exec.Command("fusermount", "-u", c.mountPoints[i]).Run()
+		}
+		c.stopCmd(c.filerCmd, syscall.SIGTERM)
+		for i := len(c.volumeCmds) - 1; i >= 0; i-- {
+			c.stopCmd(c.volumeCmds[i], syscall.SIGTERM)
+		}
+		c.stopCmd(c.masterCmd, syscall.SIGTERM)
+
+		c.mu.Lock()
+		for _, f := range c.logFiles {
+			_ = f.Close()
+		}
+		c.mu.Unlock()
+		c.copyLogsForCI()
+		if !c.t.Failed() {
+			os.RemoveAll(c.baseDir)
+		}
+	})
+}
+
+// KillVolume drops a volume server without letting it deregister, the closest
+// local equivalent of a Swarm task vanishing from the overlay network.
+func (c *failoverCluster) KillVolume(i int) {
+	c.stopCmd(c.volumeCmds[i], syscall.SIGKILL)
+	c.volumeCmds[i] = nil
+}
+
+// StartVolume (re)starts volume server i on its original ports and data dir.
+func (c *failoverCluster) StartVolume(i int) error {
+	cmd := exec.Command(c.weedBinary,
+		"-logdir="+filepath.Join(c.baseDir, "logs"),
+		"volume",
+		"-ip=127.0.0.1",
+		"-ip.bind=127.0.0.1",
+		"-port="+strconv.Itoa(c.volumePorts[i]),
+		"-port.grpc="+strconv.Itoa(c.volumeGrpcPort[i]),
+		"-master="+c.masterAddress(),
+		"-dir="+c.volumeDirs[i],
+		"-dataCenter=dc1",
+		"-rack=rack1",
+		"-max=10",
+	)
+	c.volumeCmds[i] = cmd
+	if err := c.startCmd(cmd, fmt.Sprintf("volume%d", i)); err != nil {
+		return err
+	}
+	return c.waitForTCP(cmd, fmt.Sprintf("volume%d", i),
+		fmt.Sprintf("127.0.0.1:%d", c.volumePorts[i]), 30*time.Second)
+}
+
+func (c *failoverCluster) startMaster() error {
+	c.masterCmd = exec.Command(c.weedBinary,
+		"-logdir="+filepath.Join(c.baseDir, "logs"),
+		"master",
+		"-ip=127.0.0.1",
+		"-ip.bind=127.0.0.1",
+		"-port="+strconv.Itoa(c.masterPort),
+		"-port.grpc="+strconv.Itoa(c.masterGrpcPort),
+		"-mdir="+filepath.Join(c.baseDir, "master"),
+		"-defaultReplication=001",
+		"-volumeSizeLimitMB=64",
+	)
+	return c.startCmd(c.masterCmd, "master")
+}
+
+func (c *failoverCluster) startFiler() error {
+	filerDir := filepath.Join(c.baseDir, "filer")
+	if err := os.MkdirAll(filerDir, 0755); err != nil {
+		return fmt.Errorf("create filer dir: %w", err)
+	}
+	c.filerCmd = exec.Command(c.weedBinary,
+		"-logdir="+filepath.Join(c.baseDir, "logs"),
+		"filer",
+		"-ip=127.0.0.1",
+		"-ip.bind=127.0.0.1",
+		"-port="+strconv.Itoa(c.filerPort),
+		"-port.grpc="+strconv.Itoa(c.filerGrpcPort),
+		"-master="+c.masterAddress(),
+		"-defaultReplicaPlacement=001",
+		"-defaultStoreDir="+filerDir,
+	)
+	return c.startCmd(c.filerCmd, "filer")
+}
+
+func (c *failoverCluster) startMount(idx int) error {
+	cacheDir := filepath.Join(c.baseDir, fmt.Sprintf("cache%d", idx))
+	if err := os.MkdirAll(cacheDir, 0755); err != nil {
+		return fmt.Errorf("create cache dir: %w", err)
+	}
+	// Chunk-level detail needs -v=4; keep CI at -v=2 so the logs stay small.
+	verbosity := os.Getenv("FUSE_FAILOVER_MOUNT_V")
+	if verbosity == "" {
+		verbosity = "2"
+	}
+	c.mountCmds[idx] = exec.Command(c.weedBinary,
+		"-logdir="+filepath.Join(c.baseDir, "logs"),
+		"-v="+verbosity,
+		"mount",
+		"-filer="+c.filerAddress(),
+		"-dir="+c.mountPoints[idx],
+		"-filer.path=/",
+		"-dirAutoCreate",
+		"-allowOthers=false",
+		"-replication=001",
+		"-cacheDir="+cacheDir,
+	)
+	return c.startCmd(c.mountCmds[idx], fmt.Sprintf("mount%d", idx))
+}
+
+// MasterGet fetches a master HTTP endpoint, e.g. "/dir/status?pretty=y" or
+// "/dir/lookup?volumeId=6", so a failing test can show where the replicas are.
+func (c *failoverCluster) MasterGet(path string) string {
+	resp, err := http.Get(fmt.Sprintf("http://127.0.0.1:%d%s", c.masterPort, path))
+	if err != nil {
+		return fmt.Sprintf("(master %s failed: %v)", path, err)
+	}
+	defer resp.Body.Close()
+	body, err := io.ReadAll(resp.Body)
+	if err != nil {
+		return fmt.Sprintf("(master %s read failed: %v)", path, err)
+	}
+	return string(body)
+}
+
+// FilerGet reads a file back through the filer's own HTTP handler: a view of
+// the chunk list that neither mount's cache can colour.
+func (c *failoverCluster) FilerGet(path string) ([]byte, error) {
+	resp, err := http.Get(fmt.Sprintf("http://127.0.0.1:%d%s", c.filerPort, path))
+	if err != nil {
+		return nil, err
+	}
+	defer resp.Body.Close()
+	body, err := io.ReadAll(resp.Body)
+	if err != nil {
+		return nil, err
+	}
+	if resp.StatusCode != http.StatusOK {
+		return nil, fmt.Errorf("filer %s: %s", path, resp.Status)
+	}
+	return body, nil
+}
+
+// VolumeServerAddress is the address volume server i registers with the master.
+func (c *failoverCluster) VolumeServerAddress(i int) string {
+	return fmt.Sprintf("127.0.0.1:%d", c.volumePorts[i])
+}
+
+// FileVolumeIds returns the volume ids backing a file, read from the filer's
+// own entry rather than inferred, so a test can tell which servers a given
+// file actually depends on. Manifests are resolved first: a manifest chunk's
+// own fid names the volume holding the manifest, not the data.
+func (c *failoverCluster) FileVolumeIds(path string) ([]uint32, error) {
+	body, err := c.FilerGet(path + "?metadata=true&resolveManifest=true")
+	if err != nil {
+		return nil, err
+	}
+	var entry struct {
+		Chunks []struct {
+			FileId string `json:"file_id"`
+			Fid    struct {
+				VolumeId uint32 `json:"volume_id"`
+			} `json:"fid"`
+		} `json:"chunks"`
+	}
+	if err = json.Unmarshal(body, &entry); err != nil {
+		return nil, fmt.Errorf("decode entry %s: %w", path, err)
+	}
+	seen := make(map[uint32]bool)
+	var vids []uint32
+	for _, chunk := range entry.Chunks {
+		vid := chunk.Fid.VolumeId
+		if vid == 0 && chunk.FileId != "" {
+			parsed, parseErr := strconv.ParseUint(strings.SplitN(chunk.FileId, ",", 2)[0], 10, 32)
+			if parseErr != nil {
+				return nil, fmt.Errorf("parse file id %s: %w", chunk.FileId, parseErr)
+			}
+			vid = uint32(parsed)
+		}
+		if !seen[vid] {
+			seen[vid] = true
+			vids = append(vids, vid)
+		}
+	}
+	return vids, nil
+}
+
+// FileChunkList returns the filer's chunk list for a file: each chunk's file
+// id, logical offset, size, and the volume id it lives on. Used to pinpoint
+// which chunk covers a corrupted byte range and which volume server holds it.
+func (c *failoverCluster) FileChunkList(path string) ([]struct {
+	FileId   string
+	Offset   int64
+	Size     uint64
+	VolumeId uint32
+}, error) {
+	body, err := c.FilerGet(path + "?metadata=true&resolveManifest=true")
+	if err != nil {
+		return nil, err
+	}
+	var entry struct {
+		Chunks []struct {
+			FileId string `json:"file_id"`
+			Offset int64  `json:"offset"`
+			Size   uint64 `json:"size"`
+			Fid    struct {
+				VolumeId uint32 `json:"volume_id"`
+			} `json:"fid"`
+		} `json:"chunks"`
+	}
+	if err = json.Unmarshal(body, &entry); err != nil {
+		return nil, fmt.Errorf("decode entry %s: %w", path, err)
+	}
+	out := make([]struct {
+		FileId   string
+		Offset   int64
+		Size     uint64
+		VolumeId uint32
+	}, len(entry.Chunks))
+	for i, ch := range entry.Chunks {
+		vid := ch.Fid.VolumeId
+		if vid == 0 && ch.FileId != "" {
+			parsed, parseErr := strconv.ParseUint(strings.SplitN(ch.FileId, ",", 2)[0], 10, 32)
+			if parseErr != nil {
+				return nil, fmt.Errorf("parse file id %s: %w", ch.FileId, parseErr)
+			}
+			vid = uint32(parsed)
+		}
+		out[i] = struct {
+			FileId   string
+			Offset   int64
+			Size     uint64
+			VolumeId uint32
+		}{ch.FileId, ch.Offset, ch.Size, vid}
+	}
+	return out, nil
+}
+
+// VolumeHolders returns the volume server addresses the master currently lists
+// for a volume id.
+func (c *failoverCluster) VolumeHolders(vid uint32) ([]string, error) {
+	var lookup struct {
+		Locations []struct {
+			Url string `json:"url"`
+		} `json:"locations"`
+	}
+	body := c.MasterGet(fmt.Sprintf("/dir/lookup?volumeId=%d", vid))
+	if err := json.Unmarshal([]byte(body), &lookup); err != nil {
+		return nil, fmt.Errorf("decode lookup for volume %d: %w (%s)", vid, err, body)
+	}
+	holders := make([]string, 0, len(lookup.Locations))
+	for _, loc := range lookup.Locations {
+		holders = append(holders, loc.Url)
+	}
+	return holders, nil
+}
+
+// WaitForHolders polls the master until it lists exactly count servers for a
+// volume. The master only drops a dead node after three missed heartbeats, so a
+// test that depends on the cluster's view having caught up has to wait for it.
+func (c *failoverCluster) WaitForHolders(vid uint32, count int, timeout time.Duration) ([]string, error) {
+	deadline := time.Now().Add(timeout)
+	for {
+		holders, err := c.VolumeHolders(vid)
+		if err == nil && len(holders) == count {
+			return holders, nil
+		}
+		if time.Now().After(deadline) {
+			return holders, fmt.Errorf("volume %d still has %d holders (%v), want %d", vid, len(holders), holders, count)
+		}
+		time.Sleep(500 * time.Millisecond)
+	}
+}
+
+// WaitForVolumeServers blocks until the master's topology lists every volume
+// server. A volume server accepts connections well before the master knows it
+// exists: a lone master only elects itself once its bootstrap check expires,
+// and heartbeats sent before that are refused and retried with backoff, so
+// registration lands seconds after the process is up. Until it does the
+// topology has no data node at all, an assign fails with "no free volumes
+// left", and the mount reports that as ENOSPC instead of retrying.
+func (c *failoverCluster) WaitForVolumeServers(timeout time.Duration) error {
+	deadline := time.Now().Add(timeout)
+	for {
+		registered := c.registeredVolumeServers()
+		var missing []string
+		for i := range c.volumePorts {
+			if address := c.VolumeServerAddress(i); !registered[address] {
+				missing = append(missing, address)
+			}
+		}
+		if len(missing) == 0 {
+			return nil
+		}
+		if time.Now().After(deadline) {
+			return fmt.Errorf("volume servers %v not registered with the master within %v\n%s",
+				missing, timeout, c.MasterGet("/dir/status?pretty=y"))
+		}
+		time.Sleep(200 * time.Millisecond)
+	}
+}
+
+// registeredVolumeServers is the set of volume server addresses the master
+// currently holds in its topology.
+func (c *failoverCluster) registeredVolumeServers() map[string]bool {
+	var status struct {
+		Topology struct {
+			DataCenters []struct {
+				Racks []struct {
+					DataNodes []struct {
+						Url string `json:"Url"`
+					} `json:"DataNodes"`
+				} `json:"Racks"`
+			} `json:"DataCenters"`
+		} `json:"Topology"`
+	}
+	if err := json.Unmarshal([]byte(c.MasterGet("/dir/status")), &status); err != nil {
+		return nil
+	}
+	registered := make(map[string]bool)
+	for _, dc := range status.Topology.DataCenters {
+		for _, rack := range dc.Racks {
+			for _, dn := range rack.DataNodes {
+				registered[dn.Url] = true
+			}
+		}
+	}
+	return registered
+}
+
+// volumeIndexOf maps a server address back to its index, or -1.
+func (c *failoverCluster) volumeIndexOf(address string) int {
+	for i := range c.volumePorts {
+		if c.VolumeServerAddress(i) == address {
+			return i
+		}
+	}
+	return -1
+}
+
+// FileIsOn reports whether any of path's chunks live on the given volume
+// server, i.e. whether taking that server down actually costs this file a
+// replica.
+func (c *failoverCluster) FileIsOn(path, serverAddress string) (bool, error) {
+	vids, err := c.FileVolumeIds(path)
+	if err != nil {
+		return false, err
+	}
+	for _, vid := range vids {
+		holders, holdersErr := c.VolumeHolders(vid)
+		if holdersErr != nil {
+			return false, holdersErr
+		}
+		for _, holder := range holders {
+			if holder == serverAddress {
+				return true, nil
+			}
+		}
+	}
+	return false, nil
+}
+
+func (c *failoverCluster) masterAddress() string {
+	return string(pb.NewServerAddress("127.0.0.1", c.masterPort, c.masterGrpcPort))
+}
+
+func (c *failoverCluster) filerAddress() string {
+	return string(pb.NewServerAddress("127.0.0.1", c.filerPort, c.filerGrpcPort))
+}
+
+func (c *failoverCluster) startCmd(cmd *exec.Cmd, name string) error {
+	logPath := filepath.Join(c.baseDir, "logs")
+	if err := os.MkdirAll(logPath, 0755); err != nil {
+		return fmt.Errorf("create log dir: %w", err)
+	}
+	// Append so a restarted volume server keeps the log of its earlier run.
+	logFile, err := os.OpenFile(filepath.Join(logPath, name+".log"),
+		os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0644)
+	if err != nil {
+		return err
+	}
+	c.mu.Lock()
+	c.logFiles = append(c.logFiles, logFile)
+	c.mu.Unlock()
+	cmd.Stdout = logFile
+	cmd.Stderr = logFile
+	if err := cmd.Start(); err != nil {
+		return err
+	}
+	// Reap in the background and publish the result: Signal(0) succeeds for a
+	// zombie, so an unreaped child that died at startup would otherwise look
+	// alive until the readiness timeout expired.
+	ch := make(chan error, 1)
+	c.mu.Lock()
+	if c.waits == nil {
+		c.waits = make(map[*exec.Cmd]chan error)
+	}
+	c.waits[cmd] = ch
+	c.mu.Unlock()
+	go func() {
+		ch <- cmd.Wait()
+		close(ch)
+	}()
+	return nil
+}
+
+// waitChan returns the channel carrying cmd's exit, or nil if it was never
+// started through startCmd. It stays readable after the exit is consumed.
+func (c *failoverCluster) waitChan(cmd *exec.Cmd) chan error {
+	if cmd == nil {
+		return nil
+	}
+	c.mu.Lock()
+	defer c.mu.Unlock()
+	return c.waits[cmd]
+}
+
+func (c *failoverCluster) tailLog(name string) string {
+	data, err := os.ReadFile(filepath.Join(c.baseDir, "logs", name+".log"))
+	if err != nil {
+		return fmt.Sprintf("(log %s not available: %v)", name, err)
+	}
+	const maxTail = 8192
+	if len(data) > maxTail {
+		data = data[len(data)-maxTail:]
+	}
+	return string(data)
+}
+
+func (c *failoverCluster) copyLogsForCI() {
+	// One directory per test: subtests share a log dir name otherwise, and the
+	// last one to finish would overwrite the logs of the one that failed.
+	ciLogDir := filepath.Join("/tmp/seaweedfs-fuse-failover-logs",
+		strings.ReplaceAll(c.t.Name(), "/", "_"))
+	os.MkdirAll(ciLogDir, 0755)
+	entries, err := os.ReadDir(filepath.Join(c.baseDir, "logs"))
+	if err != nil {
+		return
+	}
+	for _, e := range entries {
+		data, err := os.ReadFile(filepath.Join(c.baseDir, "logs", e.Name()))
+		if err != nil {
+			continue
+		}
+		os.WriteFile(filepath.Join(ciLogDir, e.Name()), data, 0644)
+	}
+}
+
+func (c *failoverCluster) waitForTCP(cmd *exec.Cmd, name, addr string, timeout time.Duration) error {
+	deadline := time.Now().Add(timeout)
+	for time.Now().Before(deadline) {
+		conn, err := net.DialTimeout("tcp", addr, time.Second)
+		if err == nil {
+			conn.Close()
+			return nil
+		}
+		if ch := c.waitChan(cmd); ch != nil {
+			select {
+			case waitErr := <-ch:
+				return fmt.Errorf("%s exited before listening on %s: %v\n%s",
+					name, addr, waitErr, c.tailLog(name))
+			default:
+			}
+		}
+		time.Sleep(200 * time.Millisecond)
+	}
+	return fmt.Errorf("service at %s not ready within timeout\n%s", addr, c.tailLog(name))
+}
+
+func (c *failoverCluster) waitForMount(mountPoint string, timeout time.Duration) error {
+	parentDir := filepath.Dir(mountPoint)
+	deadline := time.Now().Add(timeout)
+	for time.Now().Before(deadline) {
+		parentStat, err := os.Stat(parentDir)
+		if err != nil {
+			time.Sleep(200 * time.Millisecond)
+			continue
+		}
+		mountStat, err := os.Stat(mountPoint)
+		if err != nil {
+			time.Sleep(200 * time.Millisecond)
+			continue
+		}
+		if parentStat.Sys().(*syscall.Stat_t).Dev != mountStat.Sys().(*syscall.Stat_t).Dev {
+			return nil
+		}
+		time.Sleep(200 * time.Millisecond)
+	}
+	return fmt.Errorf("mount point %s not ready within timeout (FUSE not detected)", mountPoint)
+}
+
+func findWeedBinary() string {
+	if env := os.Getenv("WEED_BINARY"); env != "" {
+		if _, err := os.Stat(env); err == nil {
+			return env
+		}
+	}
+	if p, err := exec.LookPath("weed"); err == nil {
+		return p
+	}
+	return ""
+}
+
+// stopCmd signals cmd and waits for the reaper goroutine started by startCmd to
+// report its exit, escalating to SIGKILL if it does not go quietly.
+func (c *failoverCluster) stopCmd(cmd *exec.Cmd, sig syscall.Signal) {
+	if cmd == nil || cmd.Process == nil {
+		return
+	}
+	_ = cmd.Process.Signal(sig)
+	done := c.waitChan(cmd)
+	if done == nil {
+		return
+	}
+	select {
+	case <-done:
+	case <-time.After(10 * time.Second):
+		_ = cmd.Process.Signal(syscall.SIGKILL)
+		<-done
+	}
+}

test/fuse_failover/volume_failover_test.go

@@ -0,0 +1,502 @@
+//go:build linux || darwin
+
+package fuse_failover
+
+import (
+	"bytes"
+	"crypto/rand"
+	"crypto/sha256"
+	"fmt"
+	"os"
+	"path/filepath"
+	"strings"
+	"sync"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/require"
+)
+
+// The scenarios below come from the Docker Swarm report in discussion 10206:
+// one mount appends to a file while a second mount reads it, and a volume
+// server is stopped, started or restarted underneath. With 001 replication a
+// single volume server loss must never surface as EIO on either side.
+
+const (
+	appendLines      = 200
+	maxAppendLatency = 10 * time.Second
+	convergeTimeout  = 30 * time.Second
+)
+
+type appendResult struct {
+	errs       []error
+	maxLatency time.Duration
+	slowest    int
+}
+
+// appendLoop mimics `for i in ...; do echo $i >> file; done`: every line is its
+// own open/write/close, so every line forces a flush and a chunk upload.
+func appendLoop(path string, lines int, onLine func(i int)) *appendResult {
+	res := &appendResult{}
+	for i := 0; i < lines; i++ {
+		start := time.Now()
+		err := appendOnce(path, fmt.Sprintf("%07d\n", i))
+		elapsed := time.Since(start)
+		if elapsed > res.maxLatency {
+			res.maxLatency, res.slowest = elapsed, i
+		}
+		if err != nil {
+			res.errs = append(res.errs, fmt.Errorf("line %d: %w", i, err))
+		}
+		if onLine != nil {
+			onLine(i)
+		}
+	}
+	return res
+}
+
+func appendOnce(path, line string) error {
+	f, err := os.OpenFile(path, os.O_WRONLY|os.O_APPEND|os.O_CREATE, 0644)
+	if err != nil {
+		return fmt.Errorf("open: %w", err)
+	}
+	if _, err = f.WriteString(line); err != nil {
+		f.Close()
+		return fmt.Errorf("write: %w", err)
+	}
+	if err = f.Close(); err != nil {
+		return fmt.Errorf("close: %w", err)
+	}
+	return nil
+}
+
+// reader is the `tail -f` side: it keeps re-reading the whole file from the
+// other mount and records every failure that is not "not created yet".
+type reader struct {
+	stop chan struct{}
+	done chan struct{}
+	errs []error
+}
+
+func startReader(path string) *reader {
+	r := &reader{stop: make(chan struct{}), done: make(chan struct{})}
+	go func() {
+		defer close(r.done)
+		for {
+			select {
+			case <-r.stop:
+				return
+			default:
+			}
+			if _, err := os.ReadFile(path); err != nil && !os.IsNotExist(err) {
+				r.errs = append(r.errs, err)
+			}
+			time.Sleep(200 * time.Millisecond)
+		}
+	}()
+	return r
+}
+
+func (r *reader) Stop() []error {
+	close(r.stop)
+	<-r.done
+	return r.errs
+}
+
+// waitForContent re-reads path until it matches want or the timeout passes.
+// A mount caches metadata for about a second, so a read taken the instant the
+// writer's last close returned can legitimately still be behind; content that
+// is wrong rather than merely late never converges and still fails.
+func waitForContent(path string, want []byte, timeout time.Duration) (got []byte, ok bool) {
+	deadline := time.Now().Add(timeout)
+	for {
+		data, err := os.ReadFile(path)
+		if err == nil {
+			got = data
+			if bytes.Equal(got, want) {
+				return got, true
+			}
+		}
+		if time.Now().After(deadline) {
+			return got, false
+		}
+		time.Sleep(250 * time.Millisecond)
+	}
+}
+
+func expectedAppendContent(lines int) []byte {
+	var buf []byte
+	for i := 0; i < lines; i++ {
+		buf = append(buf, []byte(fmt.Sprintf("%07d\n", i))...)
+	}
+	return buf
+}
+
+// TestReadWithVolumeServerDown covers the original report: a file written while
+// everything was healthy must stay readable from a second mount after any
+// single volume server goes away.
+func TestReadWithVolumeServerDown(t *testing.T) {
+	c := startFailoverCluster(t, 3, 2)
+
+	const fileSize = 8 << 20
+	payloads := make(map[string][]byte)
+	for i := 0; i < 3; i++ {
+		name := fmt.Sprintf("readfile-%d", i)
+		data := make([]byte, fileSize)
+		_, err := rand.Read(data)
+		require.NoError(t, err)
+		require.NoError(t, os.WriteFile(filepath.Join(c.MountDir(0), name), data, 0644))
+		payloads[name] = data
+	}
+	// Let the filer commit the last chunks before anything is torn down.
+	time.Sleep(2 * time.Second)
+
+	t.Logf("topology before chaos:\n%s", c.MasterGet("/dir/status?pretty=y"))
+
+	alreadyRead := make(map[string]bool)
+	for victim := 0; victim < 3; victim++ {
+		// Placement decides which two of the three servers hold each volume, so
+		// pick a file this victim actually backs: reading one it never held
+		// would pass without exercising recovery at all. Prefer a file no
+		// earlier iteration has read, whose chunks the reader has not cached.
+		name := ""
+		for _, allowRead := range []bool{false, true} {
+			for i := 0; i < 3 && name == ""; i++ {
+				candidate := fmt.Sprintf("readfile-%d", i)
+				if alreadyRead[candidate] && !allowRead {
+					continue
+				}
+				onVictim, err := c.FileIsOn("/"+candidate, c.VolumeServerAddress(victim))
+				require.NoError(t, err, "resolve placement of %s", candidate)
+				if onVictim {
+					name = candidate
+				}
+			}
+		}
+		require.NotEmpty(t, name, "no test file has a replica on volume%d, nothing to fail over from\n%s",
+			victim, c.MasterGet("/dir/status?pretty=y"))
+		alreadyRead[name] = true
+
+		c.KillVolume(victim)
+
+		start := time.Now()
+		got, err := os.ReadFile(filepath.Join(c.MountDir(1), name))
+		elapsed := time.Since(start)
+		if err != nil {
+			t.Logf("topology with volume%d down:\n%s", victim, c.MasterGet("/dir/status?pretty=y"))
+		}
+		require.NoError(t, err, "read %s with volume %d down\n%s", name, victim, c.tailLog("mount1"))
+		require.Equal(t, sha256.Sum256(payloads[name]), sha256.Sum256(got),
+			"content mismatch for %s with volume %d down", name, victim)
+		t.Logf("read %s (held by volume%d) with volume%d down in %v", name, victim, victim, elapsed)
+
+		require.NoError(t, c.StartVolume(victim))
+		// The next iteration picks its victim from the master's placement view,
+		// so wait for the restarted server to be in it again.
+		require.NoError(t, c.WaitForVolumeServers(60*time.Second))
+	}
+}
+
+// TestReadAfterCachedLocationDies pins the reason the read path needs a cache
+// invalidator at all. Reading a file for the first time after a server dies
+// proves nothing: the lookup is fresh and simply returns the survivor. The
+// damage needs a reader whose cached location list is both stale and useless,
+// which is what this sequence builds:
+//
+//  1. kill one of the two servers holding a volume and wait for the master to
+//     drop it, so a lookup now resolves to the survivor alone;
+//  2. read a file on that volume, which caches exactly that one location;
+//  3. bring the first server back, which the reader never hears about;
+//  4. kill the survivor, and read another file on the same volume.
+//
+// The reader's only cached location is now dead while the data is live on the
+// restarted server. Without the invalidator the mount retries the corpse until
+// it gives up with EIO.
+func TestReadAfterCachedLocationDies(t *testing.T) {
+	c := startFailoverCluster(t, 3, 2)
+
+	// Small files so each is a single chunk on a single volume, which keeps the
+	// mapping from file to server unambiguous. The master grows six 001 volumes
+	// at a time, so seven files put two of them on one volume however the
+	// writes are spread.
+	const fileSize = 256 << 10
+	payloads := make(map[string][]byte)
+	for i := 0; i < 7; i++ {
+		name := fmt.Sprintf("smallfile-%d", i)
+		data := make([]byte, fileSize)
+		_, err := rand.Read(data)
+		require.NoError(t, err)
+		require.NoError(t, os.WriteFile(filepath.Join(c.MountDir(0), name), data, 0644))
+		payloads[name] = data
+	}
+	time.Sleep(2 * time.Second)
+
+	// Two files on one volume: one to prime the reader's cache, one to probe
+	// with afterwards. The probe has to be a file the reader has never read, or
+	// its chunk would come from the local cache without a lookup at all.
+	filesByVolume := make(map[uint32][]string)
+	for name := range payloads {
+		vids, err := c.FileVolumeIds("/" + name)
+		require.NoError(t, err, "resolve volumes of %s", name)
+		require.Len(t, vids, 1, "%s should be a single chunk", name)
+		filesByVolume[vids[0]] = append(filesByVolume[vids[0]], name)
+	}
+	var vid uint32
+	var prime, probe string
+	for candidate, names := range filesByVolume {
+		if len(names) >= 2 {
+			vid, prime, probe = candidate, names[0], names[1]
+			break
+		}
+	}
+	require.NotEmpty(t, prime, "no volume holds two of the test files: %v", filesByVolume)
+
+	holders, err := c.VolumeHolders(vid)
+	require.NoError(t, err)
+	require.Len(t, holders, 2, "001 replication should put volume %d on two servers", vid)
+	first, second := c.volumeIndexOf(holders[0]), c.volumeIndexOf(holders[1])
+	require.NotEqual(t, -1, first)
+	require.NotEqual(t, -1, second)
+
+	c.KillVolume(first)
+	_, err = c.WaitForHolders(vid, 1, 90*time.Second)
+	require.NoError(t, err, "master did not drop the dead server")
+
+	got, err := os.ReadFile(filepath.Join(c.MountDir(1), prime))
+	require.NoError(t, err, "priming read\n%s", c.tailLog("mount1"))
+	require.Equal(t, sha256.Sum256(payloads[prime]), sha256.Sum256(got))
+
+	require.NoError(t, c.StartVolume(first))
+	_, err = c.WaitForHolders(vid, 2, 90*time.Second)
+	require.NoError(t, err, "restarted server did not re-register")
+
+	c.KillVolume(second)
+	start := time.Now()
+	got, err = os.ReadFile(filepath.Join(c.MountDir(1), probe))
+	require.NoError(t, err, "probe read after the cached location died\n%s", c.tailLog("mount1"))
+	require.Equal(t, sha256.Sum256(payloads[probe]), sha256.Sum256(got))
+	t.Logf("recovered %s from the restarted server in %v", probe, time.Since(start))
+}
+
+// TestAppendWithoutChaos is the control for the chaos runs below: the same
+// append-and-tail workload with nothing being stopped or started.
+func TestAppendWithoutChaos(t *testing.T) {
+	c := startFailoverCluster(t, 3, 2)
+	runChaosAppend(t, c, "no-chaos", func() {})
+}
+
+// TestAppendWhileVolumeServerStops is scenario "STOP volumes": append from
+// mount0 and tail from mount1 while one volume server is killed mid-stream.
+func TestAppendWhileVolumeServerStops(t *testing.T) {
+	for victim := 0; victim < 3; victim++ {
+		t.Run(fmt.Sprintf("volume%d", victim), func(t *testing.T) {
+			c := startFailoverCluster(t, 3, 2)
+			runChaosAppend(t, c, fmt.Sprintf("stop-%d", victim), func() {
+				c.KillVolume(victim)
+			})
+		})
+	}
+}
+
+// TestAppendWhileVolumeServerStarts is scenario "Start volumes": the cluster is
+// already one server down when the writer starts, and that server comes back
+// mid-stream. This is where the reporter saw 12-38 s stalls per append.
+func TestAppendWhileVolumeServerStarts(t *testing.T) {
+	for victim := 0; victim < 3; victim++ {
+		t.Run(fmt.Sprintf("volume%d", victim), func(t *testing.T) {
+			c := startFailoverCluster(t, 3, 2)
+			c.KillVolume(victim)
+			time.Sleep(3 * time.Second)
+			runChaosAppend(t, c, fmt.Sprintf("start-%d", victim), func() {
+				require.NoError(t, c.StartVolume(victim))
+			})
+		})
+	}
+}
+
+// TestAppendWhileVolumeServerRestarts is scenario "Re-start volumes".
+func TestAppendWhileVolumeServerRestarts(t *testing.T) {
+	for victim := 0; victim < 3; victim++ {
+		t.Run(fmt.Sprintf("volume%d", victim), func(t *testing.T) {
+			c := startFailoverCluster(t, 3, 2)
+			runChaosAppend(t, c, fmt.Sprintf("restart-%d", victim), func() {
+				c.KillVolume(victim)
+				time.Sleep(2 * time.Second)
+				require.NoError(t, c.StartVolume(victim))
+			})
+		})
+	}
+}
+
+// TestLargeWriteWhileVolumeServerStops is the part-2 variant: a single large
+// copy instead of many small appends, checksum-verified from the other mount.
+func TestLargeWriteWhileVolumeServerStops(t *testing.T) {
+	c := startFailoverCluster(t, 3, 2)
+
+	const size = 64 << 20
+	data := make([]byte, size)
+	_, err := rand.Read(data)
+	require.NoError(t, err)
+
+	path := filepath.Join(c.MountDir(0), "largefile")
+	f, err := os.OpenFile(path, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0644)
+	require.NoError(t, err)
+
+	killed := false
+	const block = 1 << 20
+	for off := 0; off < size; off += block {
+		if !killed && off >= size/3 {
+			c.KillVolume(2)
+			killed = true
+		}
+		_, werr := f.Write(data[off : off+block])
+		require.NoError(t, werr, "write at offset %d with volume2 down\n%s", off, c.tailLog("mount0"))
+	}
+	require.NoError(t, f.Close())
+
+	got, converged := waitForContent(filepath.Join(c.MountDir(1), "largefile"), data, convergeTimeout)
+	require.True(t, converged, "large file did not converge on mount1: want %d bytes %x, got %d bytes %x\n%s",
+		len(data), sha256.Sum256(data), len(got), sha256.Sum256(got), c.tailLog("mount1"))
+}
+
+// runChaosAppend appends from mount0 with a reader tailing on mount1, firing
+// chaos once the writer is a quarter of the way in.
+func runChaosAppend(t *testing.T, c *failoverCluster, name string, chaos func()) {
+	t.Helper()
+
+	writePath := filepath.Join(c.MountDir(0), name)
+	readPath := filepath.Join(c.MountDir(1), name)
+
+	r := startReader(readPath)
+
+	var once sync.Once
+	res := appendLoop(writePath, appendLines, func(i int) {
+		if i == appendLines/4 {
+			once.Do(chaos)
+		}
+	})
+	readErrs := r.Stop()
+
+	require.Empty(t, res.errs, "append errors\n%s", c.tailLog("mount0"))
+	require.Empty(t, readErrs, "reader errors\n%s", c.tailLog("mount1"))
+	t.Logf("%s: slowest append was line %d at %v", name, res.slowest, res.maxLatency)
+	// A healthy append stays well under a second; the reported regression parked
+	// single appends at 12-38 s while a volume server was coming back.
+	require.Less(t, res.maxLatency, maxAppendLatency,
+		"append %d stalled for %v\n%s", res.slowest, res.maxLatency, c.tailLog("mount0"))
+
+	wantBytes := expectedAppendContent(appendLines)
+	want := string(wantBytes)
+	gotBytes, converged := waitForContent(readPath, wantBytes, convergeTimeout)
+	got := string(gotBytes)
+	if converged {
+		return
+	}
+
+	// The reader never caught up. Show where it diverges and what the writer's
+	// own mount and the filer make of the same file, which says whether the
+	// data was lost on the way in or is only invisible from this side.
+	d := firstDiff(want, got)
+	fromWriter, writerReadErr := os.ReadFile(writePath)
+	viaFiler, filerErr := c.FilerGet("/" + name)
+	require.Failf(t, "final content mismatch",
+		"%s: first difference at offset %d (want %d bytes, got %d)\nwant %q\ngot  %q\nmount0 matches=%v (writer read err %v) filer matches=%v (err %v)\n%s\n%s\n%s",
+		name, d, len(want), len(got), window(want, d), window(got, d),
+		string(fromWriter) == want, writerReadErr, string(viaFiler) == want, filerErr,
+		c.tailLog("mount0"),
+		dumpChunkList(c, "/"+name, d),
+		dumpHexAround(fromWriter, d, "writer mount"),
+	)
+}
+
+// dumpChunkList returns a human-readable summary of the filer's chunk list
+// for path, annotated with the volume id and the master's current holders
+// for each. When diffAt >= 0, the chunk covering that logical offset is
+// flagged so a corruption can be tied to a specific chunk and volume server.
+func dumpChunkList(c *failoverCluster, path string, diffAt int) string {
+	chunks, err := c.FileChunkList(path)
+	if err != nil {
+		return fmt.Sprintf("chunk list for %s: %v", path, err)
+	}
+	var b strings.Builder
+	fmt.Fprintf(&b, "chunk list for %s (%d chunks):", path, len(chunks))
+	for i, ch := range chunks {
+		holders, holdersErr := c.VolumeHolders(ch.VolumeId)
+		holdersStr := "unknown"
+		if holdersErr != nil {
+			holdersStr = fmt.Sprintf("lookup failed: %v", holdersErr)
+		} else if len(holders) > 0 {
+			holdersStr = strings.Join(holders, ",")
+		}
+		marker := ""
+		if diffAt >= 0 && int64(diffAt) >= ch.Offset && int64(diffAt) < ch.Offset+int64(ch.Size) {
+			marker = "  <-- covers diff"
+		}
+		fmt.Fprintf(&b, "\n  [%d] fid=%s offset=%d size=%d vid=%d holders=[%s]%s",
+			i, ch.FileId, ch.Offset, ch.Size, ch.VolumeId, holdersStr, marker)
+	}
+	return b.String()
+}
+
+// dumpHexAround returns a hex+ASCII dump of a 32-byte window around off in
+// label's copy of the file, so the exact zero-filled region is visible in
+// the failure message instead of only a quoted string window.
+func dumpHexAround(data []byte, off int, label string) string {
+	if off < 0 {
+		return fmt.Sprintf("%s hex dump: no divergence offset", label)
+	}
+	start := max(0, off-16)
+	end := min(len(data), off+16)
+	if start >= end {
+		return fmt.Sprintf("%s hex dump: offset %d out of range (len %d)", label, off, len(data))
+	}
+	var b strings.Builder
+	fmt.Fprintf(&b, "%s hex dump around offset %d:", label, off)
+	for i := start; i < end; i += 16 {
+		lineEnd := min(i+16, end)
+		hexPart := hexDump(data[i:lineEnd])
+		asciiPart := asciiDump(data[i:lineEnd])
+		fmt.Fprintf(&b, "\n  %06x  %-48s  %s", i, hexPart, asciiPart)
+	}
+	return b.String()
+}
+
+func hexDump(b []byte) string {
+	var sb strings.Builder
+	for i, x := range b {
+		if i > 0 {
+			sb.WriteByte(' ')
+		}
+		fmt.Fprintf(&sb, "%02x", x)
+	}
+	return sb.String()
+}
+
+func asciiDump(b []byte) string {
+	var sb strings.Builder
+	for _, x := range b {
+		if x >= 0x20 && x < 0x7f {
+			sb.WriteByte(x)
+		} else {
+			sb.WriteByte('.')
+		}
+	}
+	return sb.String()
+}
+
+// firstDiff returns the offset of the first differing byte, or -1 when equal.
+func firstDiff(want, got string) int {
+	for i := 0; i < len(want) && i < len(got); i++ {
+		if want[i] != got[i] {
+			return i
+		}
+	}
+	if len(want) != len(got) {
+		return min(len(want), len(got))
+	}
+	return -1
+}
+
+func window(s string, at int) string {
+	return s[max(0, at-24):min(len(s), at+24)]
+}

test/fuse_integration/Makefile

@@ -2,7 +2,7 @@
 
 # Configuration
 WEED_BINARY := weed
-GO_VERSION := 1.24
+GO_VERSION := 1.26
 TEST_TIMEOUT := 30m
 COVERAGE_FILE := coverage.out
 

test/fuse_integration/POSIX_COMPLIANCE.md

@@ -0,0 +1,645 @@
+# SeaweedFS FUSE POSIX Compliance Testing
+
+## Overview
+
+This comprehensive test suite provides full POSIX compliance testing for SeaweedFS FUSE mounts. It includes basic POSIX operations, extended features, external test suite integration, and performance benchmarking to ensure SeaweedFS meets filesystem standards.
+
+## 🎯 Key Features
+
+### ✅ **Comprehensive Test Coverage**
+- **Basic POSIX Operations**: File/directory create, read, write, delete, rename
+- **Advanced Features**: Extended attributes, file locking, memory mapping *(see roadmap)*
+- **I/O Operations**: Synchronous/asynchronous I/O, direct I/O, vectored I/O *(see roadmap)*
+- **Error Handling**: Comprehensive error condition testing
+- **Concurrent Operations**: Multi-threaded stress testing
+- **External Integration**: pjdfstest, nfstest, FIO integration
+
+> **📋 Implementation Status**: Some advanced features are currently skipped pending platform-specific implementations. See [`POSIX_IMPLEMENTATION_ROADMAP.md`](./POSIX_IMPLEMENTATION_ROADMAP.md) for a detailed implementation plan and current status.
+
+### 📊 **Performance Analysis**
+- **Benchmarking**: Built-in performance benchmarks
+- **Profiling**: CPU and memory profiling support
+- **Coverage Analysis**: Code coverage reporting
+- **Stress Testing**: High-load concurrent operation testing
+
+### 🔧 **External Tool Integration**
+- **pjdfstest**: Industry-standard POSIX filesystem test suite
+- **nfstest_posix**: Network filesystem POSIX API verification
+- **FIO**: Flexible I/O performance testing
+- **Custom Litmus Tests**: Focused edge case testing
+
+## 📋 Test Categories
+
+### 1. Basic POSIX Compliance (`posix_compliance_test.go`)
+
+#### File Operations
+- ✅ File creation with O_CREAT, O_EXCL flags
+- ✅ File truncation and size management
+- ✅ File deletion and unlinking
+- ✅ Atomic rename operations
+- ✅ File permission management
+
+#### Directory Operations
+- ✅ Directory creation and removal
+- ✅ Directory listing and traversal
+- ✅ Directory rename operations
+- ✅ Non-empty directory handling
+
+#### Symlink Operations
+- ✅ Symbolic link creation and resolution
+- ✅ Broken symlink handling
+- ✅ Symlink target verification
+- ✅ Symlink permission handling
+
+#### Permission Tests
+- ✅ File permission setting and verification
+- ✅ Directory permission enforcement
+- ✅ Permission inheritance
+- ✅ Access control validation
+
+#### Timestamp Tests
+- ✅ Access time (atime) updates
+- ✅ Modification time (mtime) tracking
+- ✅ Change time (ctime) management
+- ✅ Timestamp precision and setting
+
+#### I/O Operations
+- ✅ Read/write operations
+- ✅ File seeking and positioning
+- ✅ Append mode operations
+- ✅ Buffer management
+
+#### File Descriptors
+- ✅ File descriptor duplication
+- ✅ File descriptor flags
+- ✅ Close-on-exec handling
+- ✅ File descriptor limits
+
+#### Atomic Operations
+- ✅ Atomic file operations
+- ✅ Rename atomicity
+- ✅ Link/unlink atomicity
+
+#### Concurrent Access
+- ✅ Multi-reader scenarios
+- ✅ Concurrent write handling
+- ✅ Reader-writer coordination
+- ✅ Race condition prevention
+
+#### Error Handling
+- ✅ ENOENT (file not found) handling
+- ✅ EACCES (permission denied) handling
+- ✅ EBADF (bad file descriptor) handling
+- ✅ ENOTEMPTY (directory not empty) handling
+- ✅ Error code compliance
+
+### 2. Extended POSIX Features (`posix_extended_test.go`)
+
+#### Extended Attributes
+- ✅ Setting extended attributes (setxattr)
+- ✅ Getting extended attributes (getxattr)
+- ✅ Listing extended attributes (listxattr)
+- ✅ Removing extended attributes (removexattr)
+- ✅ Extended attribute namespaces
+
+#### File Locking
+- ✅ Advisory locking (fcntl)
+- ✅ Exclusive locks (F_WRLCK)
+- ✅ Shared locks (F_RDLCK)
+- ✅ Lock conflict detection
+- ✅ Lock release and cleanup
+
+#### Advanced I/O
+- ✅ Vectored I/O (readv/writev)
+- ✅ Positioned I/O (pread/pwrite)
+- ✅ Asynchronous I/O patterns
+- ✅ Scatter-gather operations
+
+#### Sparse Files
+- ✅ Sparse file creation
+- ✅ Hole detection and handling
+- ✅ Sparse file efficiency
+- ✅ Seek beyond EOF
+
+#### Large Files (>2GB)
+- ✅ Large file operations
+- ✅ 64-bit offset handling
+- ✅ Large file seeking
+- ✅ Large file truncation
+
+#### Memory Mapping
+- ✅ File memory mapping (mmap)
+- ✅ Memory-mapped I/O
+- ✅ Memory synchronization (msync)
+- ✅ Memory unmapping (munmap)
+- ✅ Shared/private mappings
+
+#### Direct I/O
+- ✅ Direct I/O operations (O_DIRECT)
+- ✅ Buffer alignment requirements
+- ✅ Direct I/O performance
+- ✅ Bypass buffer cache
+
+#### File Preallocation
+- ✅ Space preallocation (fallocate)
+- ✅ File hole punching
+- ✅ Space reservation
+- ✅ Allocation efficiency
+
+#### Zero-Copy Operations
+- ✅ Zero-copy file transfer (sendfile)
+- ✅ Efficient data movement
+- ✅ Cross-filesystem transfers
+
+### 3. External Test Suite Integration (`posix_external_test.go`)
+
+#### pjdfstest Integration
+- ✅ Comprehensive POSIX filesystem test suite
+- ✅ Industry-standard compliance verification
+- ✅ Automated test execution
+- ✅ Result analysis and reporting
+
+#### nfstest_posix Integration
+- ✅ POSIX API verification
+- ✅ Network filesystem compliance
+- ✅ API behavior validation
+- ✅ Comprehensive coverage
+
+#### Custom Litmus Tests
+- ✅ Atomic rename verification
+- ✅ Link count accuracy
+- ✅ Symlink cycle detection
+- ✅ Concurrent operation safety
+- ✅ Directory consistency
+
+#### Stress Testing
+- ✅ High-frequency operations
+- ✅ Concurrent access patterns
+- ✅ Resource exhaustion scenarios
+- ✅ Error recovery testing
+
+#### Edge Case Testing
+- ✅ Empty filenames
+- ✅ Very long filenames
+- ✅ Special characters
+- ✅ Deep directory nesting
+- ✅ Path length limits
+
+## 🚀 Quick Start
+
+### Prerequisites
+
+```bash
+# Install required tools
+sudo apt-get install fuse           # Linux
+brew install macfuse               # macOS
+
+# Install Go 1.21+
+go version
+
+# Build SeaweedFS
+cd /path/to/seaweedfs
+make
+```
+
+### Basic Usage
+
+```bash
+# Navigate to test directory
+cd test/fuse_integration
+
+# Check prerequisites
+make -f posix_Makefile check-prereqs
+
+# Run basic POSIX compliance tests
+make -f posix_Makefile test-posix-basic
+
+# Run complete test suite
+make -f posix_Makefile test-posix-full
+
+# Generate compliance report
+make -f posix_Makefile generate-report
+```
+
+## 🔧 Advanced Usage
+
+### Test Categories
+
+#### Critical Tests Only
+```bash
+# Run only critical POSIX compliance tests (faster)
+make -f posix_Makefile test-posix-critical
+```
+
+#### Extended Feature Tests
+```bash
+# Test advanced POSIX features
+make -f posix_Makefile test-posix-extended
+```
+
+#### External Tool Integration
+```bash
+# Setup and run external test suites
+make -f posix_Makefile setup-external-tools
+make -f posix_Makefile test-posix-external
+```
+
+#### Stress Testing
+```bash
+# Run stress tests for concurrent operations
+make -f posix_Makefile test-posix-stress
+```
+
+### Performance Analysis
+
+#### Benchmarking
+```bash
+# Run performance benchmarks
+make -f posix_Makefile benchmark-posix
+```
+
+#### Profiling
+```bash
+# Profile CPU and memory usage
+make -f posix_Makefile profile-posix
+
+# View CPU profile
+go tool pprof reports/posix.cpu.prof
+```
+
+#### Coverage Analysis
+```bash
+# Generate test coverage report
+make -f posix_Makefile coverage-posix
+# View: reports/posix_coverage.html
+```
+
+#### FIO Performance Testing
+```bash
+# Run FIO-based I/O performance tests
+make -f posix_Makefile test-fio-posix
+```
+
+### External Test Suites
+
+#### pjdfstest
+```bash
+# Setup and run pjdfstest
+make -f posix_Makefile setup-pjdfstest
+make -f posix_Makefile test-pjdfstest
+```
+
+#### nfstest_posix
+```bash
+# Install and run nfstest_posix
+make -f posix_Makefile setup-nfstest
+make -f posix_Makefile test-nfstest-posix
+```
+
+## 📊 Understanding Results
+
+### Test Status Indicators
+
+- ✅ **PASS**: Test completed successfully
+- ❌ **FAIL**: Test failed - indicates non-compliance
+- ⚠️ **SKIP**: Test skipped - feature not supported
+- 🔄 **RETRY**: Test retried due to transient failure
+
+### Report Formats
+
+#### HTML Report (`reports/posix_compliance_report.html`)
+- Interactive web-based report
+- Test category breakdown
+- Detailed results with navigation
+- Visual status indicators
+
+#### Text Summary (`reports/posix_compliance_summary.txt`)
+- Concise text-based summary
+- Key findings and recommendations
+- Command-line friendly format
+
+#### JSON Report (`reports/posix_compliance_report.json`)
+- Machine-readable results
+- Integration with CI/CD pipelines
+- Automated result processing
+
+### Log Files
+
+- `posix_basic_results.log`: Basic POSIX test detailed output
+- `posix_extended_results.log`: Extended feature test output
+- `posix_external_results.log`: External test suite output
+- `posix_benchmark_results.log`: Performance benchmark results
+
+## 🐛 Debugging Failed Tests
+
+### Common Issues
+
+#### Permission Denied Errors
+```bash
+# Ensure proper FUSE permissions
+sudo usermod -a -G fuse $USER
+# Re-login or run: newgrp fuse
+
+# Mount with -allowOthers if other users need access (test config MountOptions)
+```
+
+#### Extended Attributes Not Supported
+```bash
+# Check if xattr is supported by underlying filesystem
+getfattr --version
+# May need to skip xattr tests on certain filesystems
+```
+
+#### File Locking Issues
+```bash
+# Verify file locking support
+# Some network filesystems may not support full locking
+```
+
+#### Memory Issues
+```bash
+# Increase available memory for tests
+# Large file tests may require significant RAM
+```
+
+### Debug Mode
+
+```bash
+# Test targets already run go test -v and keep logs under reports/
+make -f posix_Makefile test-posix-basic
+
+# Run against an existing external mount instead of a fresh cluster
+make -f posix_Makefile test-posix-basic TEST_MOUNT_POINT=/path/to/mount
+
+# Extend the 45-minute per-suite timeout
+make -f posix_Makefile test-posix-full TEST_TIMEOUT=90m
+```
+
+### Manual Test Execution
+
+```bash
+# Run specific test
+cd test/fuse_integration
+go test -v -run 'TestPOSIXCompliance/FileOperations' .
+
+# Run with custom timeout
+go test -v -timeout 30m -run TestPOSIXExtended .
+```
+
+## 🔄 CI/CD Integration
+
+### GitHub Actions
+
+```yaml
+name: POSIX Compliance Tests
+
+on: [push, pull_request]
+
+jobs:
+  posix-compliance:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v4
+    - uses: actions/setup-go@v4
+      with:
+        go-version: '1.21'
+    
+    - name: Install FUSE
+      run: sudo apt-get update && sudo apt-get install -y fuse
+    
+    - name: Build SeaweedFS
+      run: make
+    
+    - name: Run POSIX Compliance Tests
+      run: |
+        cd test/fuse_integration
+        make -f posix_Makefile ci-posix-tests
+    
+    - name: Upload Test Results
+      uses: actions/upload-artifact@v4
+      if: always()
+      with:
+        name: posix-test-results
+        path: test/fuse_integration/reports/
+```
+
+### Jenkins Pipeline
+
+```groovy
+pipeline {
+    agent any
+    stages {
+        stage('Setup') {
+            steps {
+                sh 'make'
+            }
+        }
+        stage('POSIX Compliance') {
+            steps {
+                dir('test/fuse_integration') {
+                    sh 'make -f posix_Makefile ci-posix-tests'
+                }
+            }
+            post {
+                always {
+                    archiveArtifacts 'test/fuse_integration/reports/**'
+                    publishHTML([
+                        allowMissing: false,
+                        alwaysLinkToLastBuild: true,
+                        keepAll: true,
+                        reportDir: 'test/fuse_integration/reports',
+                        reportFiles: 'posix_compliance_report.html',
+                        reportName: 'POSIX Compliance Report'
+                    ])
+                }
+            }
+        }
+    }
+}
+```
+
+## 🎯 POSIX Compliance Checklist
+
+### Critical POSIX Features
+- [ ] File creation, reading, writing, deletion
+- [ ] Directory operations (create, remove, list)
+- [ ] File permissions and ownership
+- [ ] Symbolic links
+- [ ] Hard links
+- [ ] File timestamps
+- [ ] Error handling compliance
+- [ ] Atomic operations
+
+### Advanced POSIX Features
+- [ ] Extended attributes
+- [ ] File locking (advisory)
+- [ ] Memory-mapped I/O
+- [ ] Vectored I/O (readv/writev)
+- [ ] Positioned I/O (pread/pwrite)
+- [ ] Direct I/O
+- [ ] File preallocation
+- [ ] Sparse files
+
+### Performance Requirements
+- [ ] Reasonable performance under load
+- [ ] Concurrent access handling
+- [ ] Memory usage optimization
+- [ ] CPU efficiency
+- [ ] I/O throughput
+
+### Reliability Requirements
+- [ ] Data consistency
+- [ ] Error recovery
+- [ ] Race condition prevention
+- [ ] Resource cleanup
+- [ ] Crash recovery
+
+## 📈 Performance Benchmarks
+
+### Expected Performance Characteristics
+
+| Operation | Expected Range | Notes |
+|-----------|----------------|-------|
+| File Create | 1000-10000 ops/sec | Depends on chunk size |
+| File Read | 100-1000 MB/sec | Network limited |
+| File Write | 50-500 MB/sec | Depends on replication |
+| Directory Ops | 500-5000 ops/sec | Metadata operations |
+| Concurrent Ops | Scales with cores | Up to hardware limits |
+
+### Benchmarking Commands
+
+```bash
+# Basic I/O performance
+make -f posix_Makefile benchmark-posix
+
+# FIO comprehensive testing
+make -f posix_Makefile test-fio-posix
+
+# Custom benchmark with specific parameters
+go test -v -bench=. -benchtime=30s posix_compliance_test.go
+```
+
+## 🔍 Troubleshooting
+
+### Common Test Failures
+
+#### 1. Mount Permission Issues
+```bash
+# Error: Permission denied mounting
+# Solution: Check FUSE permissions — add the user to the fuse group and
+# re-login rather than widening permissions on the device node
+sudo usermod -a -G fuse $USER
+```
+
+#### 2. Extended Attribute Tests Fail
+```bash
+# Error: Operation not supported (ENOTSUP)
+# Solution: Check filesystem xattr support
+mount | grep $(df . | tail -1 | awk '{print $1}')
+# May need to remount with xattr support
+```
+
+#### 3. File Locking Tests Fail
+```bash
+# Error: Function not implemented (ENOSYS)
+# Solution: Some network filesystems don't support locking
+# This may be expected behavior
+```
+
+#### 4. Large File Tests Fail
+```bash
+# Error: No space left on device
+# Solution: Ensure sufficient disk space
+df -h /tmp  # Check temp directory space
+# May need to clean up or use different temp directory
+```
+
+#### 5. Performance Tests Timeout
+```bash
+# Error: Test timeout exceeded
+# Solution: Increase timeout or reduce test scope
+go test -timeout 60m -run TestPOSIXCompliance
+```
+
+### Debug Information Collection
+
+```bash
+# Collect system information
+uname -a > debug_info.txt
+mount >> debug_info.txt
+df -h >> debug_info.txt
+free -h >> debug_info.txt
+
+# Collect SeaweedFS information
+./weed version >> debug_info.txt
+ps aux | grep weed >> debug_info.txt
+
+# Collect test logs
+cp reports/*.log debug_logs/
+```
+
+## 📚 Additional Resources
+
+### POSIX Standards
+- [POSIX.1-2017 (IEEE Std 1003.1-2017)](https://pubs.opengroup.org/onlinepubs/9699919799/)
+- [Single UNIX Specification](https://www.opengroup.org/openbrand/register/)
+
+### External Test Suites
+- [pjdfstest](https://github.com/pjd/pjdfstest) - Comprehensive POSIX filesystem tests
+- [nfstest](https://nfstest.readthedocs.io/) - Network filesystem testing
+- [FIO](https://fio.readthedocs.io/) - Flexible I/O tester
+
+### SeaweedFS Documentation
+- [SeaweedFS Wiki](https://github.com/seaweedfs/seaweedfs/wiki)
+- [FUSE Mount Documentation](https://github.com/seaweedfs/seaweedfs/wiki/FUSE-Mount)
+- [Performance Tuning Guide](https://github.com/seaweedfs/seaweedfs/wiki/Optimization)
+
+## 🤝 Contributing
+
+### Adding New Tests
+
+1. **Basic Tests**: Add to `posix_compliance_test.go`
+2. **Extended Tests**: Add to `posix_extended_test.go`
+3. **External Integration**: Add to `posix_external_test.go`
+
+### Test Guidelines
+
+```go
+func (s *POSIXTestSuite) TestNewFeature(t *testing.T) {
+    mountPoint := s.framework.GetMountPoint()
+    
+    t.Run("SubTestName", func(t *testing.T) {
+        // Test setup
+        testFile := filepath.Join(mountPoint, "test.txt")
+        
+        // Test execution
+        err := someOperation(testFile)
+        
+        // Assertions
+        require.NoError(t, err)
+        
+        // Cleanup (if needed)
+        defer os.Remove(testFile)
+    })
+}
+```
+
+### Submitting Improvements
+
+1. Fork the repository
+2. Create a feature branch
+3. Add comprehensive tests
+4. Ensure all existing tests pass
+5. Submit a pull request with detailed description
+
+## 📞 Support
+
+For questions, issues, or contributions:
+
+- **GitHub Issues**: [SeaweedFS Issues](https://github.com/seaweedfs/seaweedfs/issues)
+- **Discussions**: [SeaweedFS Discussions](https://github.com/seaweedfs/seaweedfs/discussions)
+- **Documentation**: [SeaweedFS Wiki](https://github.com/seaweedfs/seaweedfs/wiki)
+
+---
+
+*This comprehensive POSIX compliance test suite ensures SeaweedFS FUSE mounts meet industry standards for filesystem behavior, performance, and reliability.*

test/fuse_integration/POSIX_IMPLEMENTATION_ROADMAP.md

@@ -0,0 +1,272 @@
+# POSIX Compliance Implementation Roadmap
+
+This document tracks the implementation status of POSIX features in the SeaweedFS FUSE mount compliance test suite and provides a roadmap for completing comprehensive POSIX compliance testing.
+
+## Overview
+
+The platform-specific tests listed below are implemented; this document records the implementation plan that produced them, plus the one feature still pending.
+
+## Current Status
+
+### ✅ Implemented Features
+- Basic file operations (create, read, write, delete)
+- Directory operations (mkdir, rmdir, rename)
+- File permissions and ownership
+- Symbolic and hard links
+- Basic I/O operations (seek, append, positioned I/O)
+- File descriptors and atomic operations
+- Concurrent access patterns
+- Error handling compliance
+- Timestamp operations
+
+### 🚧 Partially Implemented
+- Cross-platform compatibility (basic implementation with platform-specific access time handling)
+- File sealing (still skipped; Linux-specific, pending)
+
+### ✅ Implemented Platform-Specific Features
+
+#### 1. Extended Attributes (xattr)
+**Priority: High**
+**Platforms: Linux, macOS, FreeBSD**
+
+**Status:** Implemented
+- `TestExtendedAttributes/SetExtendedAttribute`
+- `TestExtendedAttributes/ListExtendedAttributes` 
+- `TestExtendedAttributes/RemoveExtendedAttribute`
+
+**Implementation Plan:**
+```go
+// Linux implementation
+//go:build linux
+func setXattr(path, name string, value []byte) error {
+    return syscall.Setxattr(path, name, value, 0)
+}
+
+// macOS implementation  
+//go:build darwin
+func setXattr(path, name string, value []byte) error {
+    return syscall.Setxattr(path, name, value, 0, 0)
+}
+```
+
+**Required Files:**
+- `xattr_linux.go` - Linux syscall implementations
+- `xattr_darwin.go` - macOS syscall implementations  
+- `xattr_freebsd.go` - FreeBSD syscall implementations
+- `xattr_unsupported.go` - Fallback for unsupported platforms
+
+#### 2. Memory Mapping (mmap)
+**Priority: High**
+**Platforms: All POSIX systems**
+
+**Status:** Implemented
+- `TestMemoryMapping/MemoryMappedRead`
+- `TestMemoryMapping/MemoryMappedWrite`
+
+**Implementation Plan:**
+```go
+func testMmap(t *testing.T, filePath string) {
+    fd, err := syscall.Open(filePath, syscall.O_RDWR, 0)
+    require.NoError(t, err)
+    defer syscall.Close(fd)
+    
+    // Platform-specific mmap implementation
+    data, err := syscall.Mmap(fd, 0, size, syscall.PROT_READ|syscall.PROT_WRITE, syscall.MAP_SHARED)
+    require.NoError(t, err)
+    defer syscall.Munmap(data)
+}
+```
+
+**Required Files:**
+- `mmap_unix.go` - Unix-like systems implementation
+- `mmap_windows.go` - Windows implementation (if needed)
+
+#### 3. Vectored I/O (readv/writev)
+**Priority: Medium**
+**Platforms: All POSIX systems**
+
+**Status:** Implemented
+- `TestAdvancedIO/VectoredIO`
+
+**Implementation Plan:**
+```go
+func testVectoredIO(t *testing.T, filePath string) {
+    // Use syscall.Syscall for readv/writev
+    // Linux: SYS_READV, SYS_WRITEV
+    // Other platforms: platform-specific syscall numbers
+}
+```
+
+#### 4. Direct I/O
+**Priority: Medium**  
+**Platforms: Linux, some Unix variants**
+
+**Status:** Implemented
+- `TestDirectIO/DirectIO`
+
+**Implementation Plan:**
+```go
+//go:build linux
+func testDirectIO(t *testing.T, filePath string) {
+    fd, err := syscall.Open(filePath, syscall.O_RDWR|syscall.O_DIRECT, 0)
+    // Test with aligned buffers
+}
+```
+
+#### 5. File Preallocation (fallocate)
+**Priority: Medium**
+**Platforms: Linux, some Unix variants**
+
+**Status:** Implemented
+- `TestFilePreallocation/Fallocate`
+
+**Implementation Plan:**
+```go
+//go:build linux
+func testFallocate(t *testing.T, filePath string) {
+    fd, err := syscall.Open(filePath, syscall.O_RDWR, 0)
+    err = syscall.Syscall6(syscall.SYS_FALLOCATE, uintptr(fd), 0, 0, uintptr(size), 0, 0)
+}
+```
+
+#### 6. Zero-Copy Transfer (sendfile)
+**Priority: Low**
+**Platforms: Linux, FreeBSD, some Unix variants**
+
+**Status:** Implemented
+- `TestZeroCopyTransfer/Sendfile`
+
+**Implementation Plan:**
+```go
+//go:build linux
+func testSendfile(t *testing.T, srcPath, dstPath string) {
+    // Use syscall.Syscall for sendfile
+    // Platform-specific implementations
+}
+```
+
+#### 7. File Sealing (Linux-specific)
+**Priority: Low**
+**Platforms: Linux only**
+
+**Current Status:** File sealing test is still skipped
+- `TestFileSealing`
+
+**Implementation Plan:**
+```go
+//go:build linux
+func testFileSealing(t *testing.T) {
+    // Use fcntl with F_ADD_SEALS, F_GET_SEALS
+    // Test various seal types: F_SEAL_WRITE, F_SEAL_SHRINK, etc.
+}
+```
+
+## Implementation Strategy
+
+### Phase 1: Core Features (High Priority)
+1. **Extended Attributes** - Essential for many applications
+2. **Memory Mapping** - Critical for performance-sensitive applications
+
+### Phase 2: Advanced I/O (Medium Priority)  
+3. **Vectored I/O** - Important for efficient bulk operations
+4. **Direct I/O** - Needed for database and high-performance applications
+5. **File Preallocation** - Important for preventing fragmentation
+
+### Phase 3: Specialized Features (Low Priority)
+6. **Zero-Copy Transfer** - Performance optimization feature
+7. **File Sealing** - Security feature, Linux-specific
+
+## Platform Support Matrix
+
+| Feature | Linux | macOS | FreeBSD | Windows | Notes |
+|---------|-------|-------|---------|---------|-------|
+| Extended Attributes | ✅ | ✅ | ✅ | ❌ | Different syscall signatures |
+| Memory Mapping | ✅ | ✅ | ✅ | ✅ | Standard POSIX |
+| Vectored I/O | ✅ | ✅ | ✅ | ❌ | readv/writev syscalls |
+| Direct I/O | ✅ | ❌ | ✅ | ❌ | O_DIRECT flag |
+| fallocate | ✅ | ❌ | ❌ | ❌ | Linux-specific |
+| sendfile | ✅ | ❌ | ✅ | ❌ | Platform-specific |
+| File Sealing | ✅ | ❌ | ❌ | ❌ | Linux-only |
+
+## Development Guidelines
+
+### 1. Platform-Specific Implementation Pattern
+```go
+// feature_linux.go
+//go:build linux
+package fuse
+func platformSpecificFunction() { /* Linux implementation */ }
+
+// feature_darwin.go  
+//go:build darwin
+package fuse
+func platformSpecificFunction() { /* macOS implementation */ }
+
+// feature_unsupported.go
+//go:build !linux && !darwin
+package fuse
+func platformSpecificFunction() { /* Skip or error */ }
+```
+
+### 2. Test Structure Pattern
+```go
+func (s *POSIXExtendedTestSuite) TestFeature(t *testing.T) {
+    if !isFeatureSupported() {
+        t.Skip("Feature not supported on this platform")
+        return
+    }
+    
+    t.Run("FeatureTest", func(t *testing.T) {
+        // Actual test implementation
+    })
+}
+```
+
+### 3. Error Handling
+- Use platform-specific error checking
+- Provide meaningful error messages
+- Gracefully handle unsupported features
+
+## Testing Strategy
+
+### 1. Continuous Integration
+- Run tests on multiple platforms (Linux, macOS)
+- Use build tags to enable/disable platform-specific tests
+- Ensure graceful degradation on unsupported platforms
+
+### 2. Feature Detection
+- Implement runtime feature detection where possible
+- Skip tests gracefully when features are unavailable
+- Log clear messages about skipped functionality
+
+### 3. Documentation
+- Document platform-specific behavior
+- Provide examples for each implemented feature
+- Maintain compatibility matrices
+
+## Contributing
+
+When implementing these features:
+
+1. **Start with high-priority items** (Extended Attributes, Memory Mapping)
+2. **Follow the platform-specific pattern** outlined above
+3. **Add comprehensive tests** for each feature
+4. **Update this roadmap** as features are implemented
+5. **Document any platform-specific quirks** or limitations
+
+## Future Enhancements
+
+Beyond the current roadmap, consider:
+- **File locking** (flock, fcntl locks)
+- **Asynchronous I/O** (aio_read, aio_write)
+- **File change notifications** (inotify, kqueue)
+- **POSIX ACLs** (Access Control Lists)
+- **Sparse file operations**
+- **File hole punching**
+
+## References
+
+- [POSIX.1-2017 Standard](https://pubs.opengroup.org/onlinepubs/9699919799/)
+- [Linux Programmer's Manual](https://man7.org/linux/man-pages/)
+- [FreeBSD System Calls](https://www.freebsd.org/cgi/man.cgi)
+- [macOS System Calls](https://developer.apple.com/library/archive/documentation/System/Conceptual/ManPages_iPhoneOS/)

test/fuse_integration/README.md

@@ -216,7 +216,7 @@ jobs:
     - uses: actions/checkout@v3
     - uses: actions/setup-go@v3
       with:
-        go-version: '1.24'
+        go-version: '1.26'
     
     - name: Install FUSE
       run: sudo apt-get install -y fuse
@@ -232,7 +232,7 @@ jobs:
 
 ### Docker Testing
 ```dockerfile
-FROM golang:1.24
+FROM golang:1.26
 RUN apt-get update && apt-get install -y fuse
 COPY . /seaweedfs
 WORKDIR /seaweedfs

test/fuse_integration/atime_linux.go

@@ -0,0 +1,12 @@
+//go:build linux
+
+package fuse
+
+import (
+	"syscall"
+)
+
+// getAtimeNano returns the access time in nanoseconds for Linux systems
+func getAtimeNano(stat *syscall.Stat_t) int64 {
+	return stat.Atim.Sec*1e9 + stat.Atim.Nsec
+}

test/fuse_integration/atime_nonlinux.go

@@ -0,0 +1,12 @@
+//go:build !linux && !windows
+
+package fuse
+
+import (
+	"syscall"
+)
+
+// getAtimeNano returns the access time in nanoseconds for non-Linux systems (macOS, BSD, etc.)
+func getAtimeNano(stat *syscall.Stat_t) int64 {
+	return stat.Atimespec.Sec*1e9 + stat.Atimespec.Nsec
+}

test/fuse_integration/atime_windows.go

@@ -0,0 +1,12 @@
+//go:build windows
+
+package fuse
+
+import (
+	"syscall"
+)
+
+// getAtimeNano returns the access time in nanoseconds for Windows systems
+func getAtimeNano(stat *syscall.Win32FileAttributeData) int64 {
+	return stat.LastAccessTime.Nanoseconds()
+}

test/fuse_integration/concurrent_operations_test.go

@@ -177,8 +177,10 @@ func testConcurrentReadWrite(t *testing.T, framework *FuseTestFramework) {
 			defer wg.Done()
 
 			for j := 0; j < 10; j++ {
-				_, err := os.ReadFile(mountPath)
-				if err != nil {
+				if err := retryTransientFUSE(func() error {
+					_, e := os.ReadFile(mountPath)
+					return e
+				}); err != nil {
 					addError(fmt.Errorf("reader %d: %v", readerID, err))
 					return
 				}
@@ -196,8 +198,9 @@ func testConcurrentReadWrite(t *testing.T, framework *FuseTestFramework) {
 
 			for j := 0; j < 5; j++ {
 				newData := bytes.Repeat([]byte(fmt.Sprintf("WRITER%d", writerID)), 1000)
-				err := os.WriteFile(mountPath, newData, 0644)
-				if err != nil {
+				if err := retryTransientFUSE(func() error {
+					return os.WriteFile(mountPath, newData, 0644)
+				}); err != nil {
 					addError(fmt.Errorf("writer %d: %v", writerID, err))
 					return
 				}
@@ -213,6 +216,21 @@ func testConcurrentReadWrite(t *testing.T, framework *FuseTestFramework) {
 	framework.AssertFileExists(filename)
 }
 
+// retryTransientFUSE retries op a few times before giving up. A concurrent
+// truncating overwrite can leave a short-lived dentry/cache window where the
+// entry is momentarily invisible (ENOENT) to another opener; the last error is
+// returned so a genuine, persistent failure still surfaces.
+func retryTransientFUSE(op func() error) error {
+	var err error
+	for attempt := 0; attempt < 5; attempt++ {
+		if err = op(); err == nil {
+			return nil
+		}
+		time.Sleep(100 * time.Millisecond)
+	}
+	return err
+}
+
 // testConcurrentDirectoryOperations tests concurrent directory operations
 func testConcurrentDirectoryOperations(t *testing.T, framework *FuseTestFramework) {
 	numWorkers := 8
@@ -249,11 +267,18 @@ func testConcurrentDirectoryOperations(t *testing.T, framework *FuseTestFramewor
 					return
 				}
 
-				// Create file in subdirectory
+				// Create file in subdirectory with retry for transient FUSE errors
 				testFile := filepath.Join(subDir, "test.txt")
 				content := []byte(fmt.Sprintf("Worker %d, Subdir %d", workerID, i))
-				if err := os.WriteFile(testFile, content, 0644); err != nil {
-					addError(fmt.Errorf("worker %d file %d: %v", workerID, i, err))
+				var writeErr error
+				for attempt := 0; attempt < 3; attempt++ {
+					if writeErr = os.WriteFile(testFile, content, 0644); writeErr == nil {
+						break
+					}
+					time.Sleep(100 * time.Millisecond)
+				}
+				if writeErr != nil {
+					addError(fmt.Errorf("worker %d file %d: %v", workerID, i, writeErr))
 					return
 				}
 			}
@@ -386,18 +411,20 @@ func testHighFrequencySmallWrites(t *testing.T, framework *FuseTestFramework) {
 	// Perform many small writes
 	numWrites := 1000
 	writeSize := 100
+	totalSize := int64(0)
 
 	for i := 0; i < numWrites; i++ {
 		data := []byte(fmt.Sprintf("Write %04d: %s\n", i, bytes.Repeat([]byte("x"), writeSize-20)))
 		_, err := file.Write(data)
 		require.NoError(t, err)
+		totalSize += int64(len(data))
 	}
 	file.Close()
 
 	// Verify file size
 	info, err := os.Stat(mountPath)
 	require.NoError(t, err)
-	assert.Equal(t, totalSize, info.Size())
+	assert.Equal(t, totalSize, info.Size(), "file size should match total bytes written")
 }
 
 // testManySmallFiles tests creating many small files

test/fuse_integration/directio_darwin.go

@@ -0,0 +1,39 @@
+//go:build darwin
+
+package fuse
+
+import (
+	"syscall"
+)
+
+// Direct I/O support for macOS
+
+const (
+	// macOS doesn't have O_DIRECT, but we can use fcntl with F_NOCACHE
+	F_NOCACHE = 48
+)
+
+func openDirectIO(path string, flags int, mode uint32) (int, error) {
+	// Open file normally first
+	fd, err := syscall.Open(path, flags, mode)
+	if err != nil {
+		return -1, err
+	}
+
+	// Set F_NOCACHE to bypass buffer cache (similar to O_DIRECT)
+	_, _, errno := syscall.Syscall(syscall.SYS_FCNTL,
+		uintptr(fd),
+		F_NOCACHE,
+		1) // enable
+
+	if errno != 0 {
+		syscall.Close(fd)
+		return -1, errno
+	}
+
+	return fd, nil
+}
+
+func isDirectIOSupported() bool {
+	return true
+}

test/fuse_integration/directio_linux.go

@@ -0,0 +1,23 @@
+//go:build linux
+
+package fuse
+
+import (
+	"syscall"
+
+	"golang.org/x/sys/unix"
+)
+
+// Direct I/O support for Linux
+
+const (
+	O_DIRECT = unix.O_DIRECT
+)
+
+func openDirectIO(path string, flags int, mode uint32) (int, error) {
+	return syscall.Open(path, flags|O_DIRECT, mode)
+}
+
+func isDirectIOSupported() bool {
+	return true
+}

test/fuse_integration/directio_unsupported.go

@@ -0,0 +1,29 @@
+//go:build !linux && !darwin
+
+package fuse
+
+import (
+	"errors"
+	"syscall"
+)
+
+// Direct I/O support for unsupported platforms
+
+var ErrDirectIONotSupported = errors.New("direct I/O not supported on this platform")
+
+const (
+	O_DIRECT = 0x4000 // Dummy flag for compatibility
+)
+
+func openDirectIO(path string, flags int, mode uint32) (int, error) {
+	// Fall back to regular open
+	fd, err := syscall.Open(path, flags, mode)
+	if err != nil {
+		return -1, err
+	}
+	return int(fd), nil
+}
+
+func isDirectIOSupported() bool {
+	return false
+}

test/fuse_integration/fallocate_darwin.go

@@ -0,0 +1,79 @@
+//go:build darwin
+
+package fuse
+
+import (
+	"syscall"
+	"unsafe"
+)
+
+// File allocation support for macOS using fcntl
+
+const (
+	// macOS doesn't have fallocate, but we can use fcntl with F_PREALLOCATE
+	F_ALLOCATECONTIG = 0x02
+	F_ALLOCATEALL    = 0x04
+
+	// Dummy flags for compatibility
+	FALLOC_FL_KEEP_SIZE      = 0x01
+	FALLOC_FL_PUNCH_HOLE     = 0x02
+	FALLOC_FL_NO_HIDE_STALE  = 0x04
+	FALLOC_FL_COLLAPSE_RANGE = 0x08
+	FALLOC_FL_ZERO_RANGE     = 0x10
+	FALLOC_FL_INSERT_RANGE   = 0x20
+	FALLOC_FL_UNSHARE_RANGE  = 0x40
+)
+
+// fstore_t structure for F_PREALLOCATE
+type fstore struct {
+	flags      uint32
+	posmode    int16
+	offset     int64
+	length     int64
+	bytesalloc int64
+}
+
+func fallocateFile(fd int, mode int, offset int64, length int64) error {
+	// Check for unsupported modes on macOS
+	unsupportedModes := FALLOC_FL_PUNCH_HOLE | FALLOC_FL_NO_HIDE_STALE | FALLOC_FL_COLLAPSE_RANGE | FALLOC_FL_ZERO_RANGE | FALLOC_FL_INSERT_RANGE | FALLOC_FL_UNSHARE_RANGE
+	if mode&unsupportedModes != 0 {
+		return syscall.ENOTSUP // Operation not supported
+	}
+
+	// On macOS, we use fcntl with F_PREALLOCATE
+	store := fstore{
+		flags:   F_ALLOCATECONTIG,
+		posmode: syscall.F_PEOFPOSMODE, // Allocate from EOF
+		offset:  0,
+		length:  length,
+	}
+
+	_, _, errno := syscall.Syscall(syscall.SYS_FCNTL,
+		uintptr(fd),
+		syscall.F_PREALLOCATE,
+		uintptr(unsafe.Pointer(&store)))
+
+	if errno != 0 {
+		// If contiguous allocation fails, try non-contiguous
+		store.flags = F_ALLOCATEALL
+		_, _, errno = syscall.Syscall(syscall.SYS_FCNTL,
+			uintptr(fd),
+			syscall.F_PREALLOCATE,
+			uintptr(unsafe.Pointer(&store)))
+
+		if errno != 0 {
+			return errno
+		}
+	}
+
+	// Set file size if not keeping size
+	if mode&FALLOC_FL_KEEP_SIZE == 0 {
+		return syscall.Ftruncate(fd, offset+length)
+	}
+
+	return nil
+}
+
+func isFallocateSupported() bool {
+	return true
+}

test/fuse_integration/fallocate_linux.go

@@ -0,0 +1,38 @@
+//go:build linux
+
+package fuse
+
+import (
+	"syscall"
+)
+
+// File allocation support for Linux
+
+const (
+	// Fallocate flags
+	FALLOC_FL_KEEP_SIZE      = 0x01
+	FALLOC_FL_PUNCH_HOLE     = 0x02
+	FALLOC_FL_NO_HIDE_STALE  = 0x04
+	FALLOC_FL_COLLAPSE_RANGE = 0x08
+	FALLOC_FL_ZERO_RANGE     = 0x10
+	FALLOC_FL_INSERT_RANGE   = 0x20
+	FALLOC_FL_UNSHARE_RANGE  = 0x40
+)
+
+func fallocateFile(fd int, mode int, offset int64, length int64) error {
+	_, _, errno := syscall.Syscall6(syscall.SYS_FALLOCATE,
+		uintptr(fd),
+		uintptr(mode),
+		uintptr(offset),
+		uintptr(length),
+		0, 0)
+
+	if errno != 0 {
+		return errno
+	}
+	return nil
+}
+
+func isFallocateSupported() bool {
+	return true
+}

test/fuse_integration/fallocate_unsupported.go

@@ -0,0 +1,30 @@
+//go:build !linux && !darwin
+
+package fuse
+
+import (
+	"errors"
+)
+
+// File allocation support for unsupported platforms
+
+var ErrFallocateNotSupported = errors.New("fallocate not supported on this platform")
+
+const (
+	// Dummy flags for compatibility
+	FALLOC_FL_KEEP_SIZE      = 0x01
+	FALLOC_FL_PUNCH_HOLE     = 0x02
+	FALLOC_FL_NO_HIDE_STALE  = 0x04
+	FALLOC_FL_COLLAPSE_RANGE = 0x08
+	FALLOC_FL_ZERO_RANGE     = 0x10
+	FALLOC_FL_INSERT_RANGE   = 0x20
+	FALLOC_FL_UNSHARE_RANGE  = 0x40
+)
+
+func fallocateFile(fd int, mode int, offset int64, length int64) error {
+	return ErrFallocateNotSupported
+}
+
+func isFallocateSupported() bool {
+	return false
+}

test/fuse_integration/framework.go

@@ -1,12 +1,15 @@
-package fuse_test
+package fuse
 
 import (
 	"fmt"
 	"io/fs"
 	"net"
+	"net/http"
 	"os"
 	"os/exec"
 	"path/filepath"
+	"strconv"
+	"strings"
 	"syscall"
 	"testing"
 	"time"
@@ -14,33 +17,39 @@ import (
 	"github.com/stretchr/testify/require"
 )
 
-// FuseTestFramework provides utilities for FUSE integration testing
+// FuseTestFramework provides utilities for FUSE integration testing.
+// It starts a single "weed mini" process (master+volume+filer in one)
+// and a separate "weed mount" process for the FUSE filesystem.
 type FuseTestFramework struct {
 	t             *testing.T
 	tempDir       string
 	mountPoint    string
 	dataDir       string
-	masterProcess *os.Process
-	volumeProcess *os.Process
-	filerProcess  *os.Process
-	mountProcess  *os.Process
-	masterAddr    string
-	volumeAddr    string
+	logDir        string
+	miniProcess   *managedProcess
+	mountProcess  *managedProcess
 	filerAddr     string
+	filerPort     int
 	weedBinary    string
 	isSetup       bool
+	config        *TestConfig
+	externalMount bool
 }
 
 // TestConfig holds configuration for FUSE tests
 type TestConfig struct {
-	Collection   string
-	Replication  string
-	ChunkSizeMB  int
-	CacheSizeMB  int
-	NumVolumes   int
-	EnableDebug  bool
-	MountOptions []string
-	SkipCleanup  bool // for debugging failed tests
+	Collection  string
+	Replication string
+	ChunkSizeMB int
+	CacheSizeMB int
+	NumVolumes  int
+	EnableDebug bool
+	// MountGlobalOptions are glog flags (-v, -logtostderr, ...) registered on
+	// weed's global flagset; they must precede the subcommand name or the
+	// mount process dies at flag parsing.
+	MountGlobalOptions []string
+	MountOptions       []string
+	SkipCleanup        bool // for debugging failed tests
 }
 
 // DefaultTestConfig returns a default configuration for FUSE tests
@@ -57,7 +66,7 @@ func DefaultTestConfig() *TestConfig {
 	}
 }
 
-// NewFuseTestFramework creates a new FUSE testing framework
+// NewFuseTestFramework creates a new FUSE testing framework.
 func NewFuseTestFramework(t *testing.T, config *TestConfig) *FuseTestFramework {
 	if config == nil {
 		config = DefaultTestConfig()
@@ -66,61 +75,161 @@ func NewFuseTestFramework(t *testing.T, config *TestConfig) *FuseTestFramework {
 	tempDir, err := os.MkdirTemp("", "seaweedfs_fuse_test_")
 	require.NoError(t, err)
 
+	filerPort := freePort(t)
+
 	return &FuseTestFramework{
 		t:          t,
 		tempDir:    tempDir,
 		mountPoint: filepath.Join(tempDir, "mount"),
 		dataDir:    filepath.Join(tempDir, "data"),
-		masterAddr: "127.0.0.1:19333",
-		volumeAddr: "127.0.0.1:18080",
-		filerAddr:  "127.0.0.1:18888",
+		logDir:     filepath.Join(tempDir, "logs"),
+		filerPort:  filerPort,
+		filerAddr:  fmt.Sprintf("127.0.0.1:%d", filerPort),
 		weedBinary: findWeedBinary(),
 		isSetup:    false,
+		config:     config,
 	}
 }
 
+// grpcPortOffset mirrors weed's HTTP->gRPC port convention (gRPC = HTTP + offset);
+// "weed mount" derives the filer gRPC port from the filer HTTP address the same way.
+const grpcPortOffset = 10000
+
+// freePort returns a free filer HTTP port whose gRPC sibling (port+grpcPortOffset)
+// is also free. Both must stay below the Linux ephemeral floor (32768): a gRPC port
+// above it can be transiently grabbed by an outbound connection, forcing mini to
+// relocate its filer gRPC port while "weed mount" keeps dialing HTTP+10000 — the
+// mount then never connects and the test times out. Capping HTTP at 22000 keeps the
+// gRPC port at or below 32000.
+func freePort(t *testing.T) int {
+	t.Helper()
+	const (
+		minServicePort = 20000
+		maxServicePort = 22000
+	)
+
+	portCount := maxServicePort - minServicePort + 1
+	start := minServicePort + int(time.Now().UnixNano()%int64(portCount))
+
+	for attempt := 0; attempt < portCount; attempt++ {
+		port := minServicePort + (start-minServicePort+attempt)%portCount
+		if portIsFree(port) && portIsFree(port+grpcPortOffset) {
+			return port
+		}
+	}
+
+	t.Fatalf("failed to allocate a free HTTP/gRPC port pair in [%d,%d]", minServicePort, maxServicePort)
+	return 0
+}
+
+// portIsFree reports whether a TCP port can currently be bound on 127.0.0.1.
+func portIsFree(port int) bool {
+	l, err := net.Listen("tcp", net.JoinHostPort("127.0.0.1", strconv.Itoa(port)))
+	if err != nil {
+		return false
+	}
+	l.Close()
+	return true
+}
+
 // Setup starts SeaweedFS cluster and mounts FUSE filesystem
 func (f *FuseTestFramework) Setup(config *TestConfig) error {
 	if f.isSetup {
 		return fmt.Errorf("framework already setup")
 	}
+	if config != nil {
+		f.config = config
+	}
+	config = f.config
 
-	// Create directories
-	dirs := []string{f.mountPoint, f.dataDir}
+	// Check if we should skip cluster setup and use existing mount
+	if os.Getenv("TEST_SKIP_CLUSTER_SETUP") == "true" {
+		// Use existing mount point from environment
+		if existingMount := os.Getenv("TEST_MOUNT_POINT"); existingMount != "" {
+			f.externalMount = true
+
+			// Verify mount point is accessible
+			if _, err := os.Stat(existingMount); err != nil {
+				return fmt.Errorf("existing mount point not accessible: %v", err)
+			}
+
+			// A plain directory passes every check below; refuse to run the suite
+			// against the local disk
+			if !mountedAt(existingMount) {
+				return fmt.Errorf("mount point %s is not a mounted filesystem", existingMount)
+			}
+			// So can a non-FUSE mount like tmpfs: a device boundary alone does
+			// not make the results FUSE compliance
+			if !fuseFilesystem(existingMount) {
+				return fmt.Errorf("mount point %s is not a FUSE filesystem", existingMount)
+			}
+
+			// Run inside a private work dir so test files do not pile up in
+			// the shared mount root and cleanup can remove only what it made
+			workDir, err := os.MkdirTemp(existingMount, ".fuse_test_")
+			if err != nil {
+				return fmt.Errorf("creating work dir under existing mount: %v", err)
+			}
+			f.mountPoint = workDir
+			f.t.Logf("Using existing mount point: %s (work dir %s)", existingMount, f.mountPoint)
+
+			// Test basic functionality
+			testFile := filepath.Join(f.mountPoint, ".framework_test")
+			if err := os.WriteFile(testFile, []byte("test"), 0644); err != nil {
+				return fmt.Errorf("mount point not writable: %v", err)
+			}
+			if err := os.Remove(testFile); err != nil {
+				f.t.Logf("Warning: failed to cleanup test file: %v", err)
+			}
+
+			// Verify this is actually a SeaweedFS mount by checking for SeaweedFS-specific behavior
+			// Create a test file and verify it appears in the filer
+			verifyFile := filepath.Join(f.mountPoint, ".seaweedfs_mount_verification")
+			if err := os.WriteFile(verifyFile, []byte("SeaweedFS mount verification"), 0644); err != nil {
+				return fmt.Errorf("mount point verification failed - cannot write: %v", err)
+			}
+
+			// Read it back to ensure it's working
+			if data, err := os.ReadFile(verifyFile); err != nil {
+				return fmt.Errorf("mount point verification failed - cannot read: %v", err)
+			} else if string(data) != "SeaweedFS mount verification" {
+				return fmt.Errorf("mount point verification failed - data mismatch")
+			}
+
+			// Clean up verification file
+			if err := os.Remove(verifyFile); err != nil {
+				f.t.Logf("Warning: failed to cleanup verification file: %v", err)
+			}
+
+			f.t.Logf("✅ SeaweedFS mount point verified and working: %s", f.mountPoint)
+
+			f.isSetup = true
+			return nil
+		}
+	}
+
+	// Create directories for full cluster setup
+	dirs := []string{f.mountPoint, f.logDir, f.dataDir}
 	for _, dir := range dirs {
 		if err := os.MkdirAll(dir, 0755); err != nil {
 			return fmt.Errorf("failed to create directory %s: %v", dir, err)
 		}
 	}
 
-	// Start master
-	if err := f.startMaster(config); err != nil {
-		return fmt.Errorf("failed to start master: %v", err)
+	// Start weed mini (master + volume + filer in one process)
+	if err := f.startMini(config); err != nil {
+		return fmt.Errorf("failed to start weed mini: %v", err)
 	}
 
-	// Wait for master to be ready
-	if err := f.waitForService(f.masterAddr, 30*time.Second); err != nil {
-		return fmt.Errorf("master not ready: %v", err)
+	// Wait for filer to be ready (mini starts all services on filerPort)
+	if err := f.waitForService(f.miniProcess, f.filerAddr, 30*time.Second); err != nil {
+		f.dumpLog("mini")
+		return fmt.Errorf("weed mini not ready: %v", err)
 	}
 
-	// Start volume servers
-	if err := f.startVolumeServers(config); err != nil {
-		return fmt.Errorf("failed to start volume servers: %v", err)
-	}
-
-	// Wait for volume server to be ready
-	if err := f.waitForService(f.volumeAddr, 30*time.Second); err != nil {
-		return fmt.Errorf("volume server not ready: %v", err)
-	}
-
-	// Start filer
-	if err := f.startFiler(config); err != nil {
-		return fmt.Errorf("failed to start filer: %v", err)
-	}
-
-	// Wait for filer to be ready
-	if err := f.waitForService(f.filerAddr, 30*time.Second); err != nil {
-		return fmt.Errorf("filer not ready: %v", err)
+	// Skip rather than fail on a host with no FUSE device to mount through
+	if !fuseAvailable() {
+		f.t.Skip("no FUSE device node (/dev/fuse, /dev/macfuse0, /dev/osxfuse0)")
 	}
 
 	// Mount FUSE filesystem
@@ -130,34 +239,62 @@ func (f *FuseTestFramework) Setup(config *TestConfig) error {
 
 	// Wait for mount to be ready
 	if err := f.waitForMount(30 * time.Second); err != nil {
+		f.dumpLog("mount")
 		return fmt.Errorf("FUSE mount not ready: %v", err)
 	}
 
+	// A mounted filesystem still cannot write until a volume registers and a
+	// grow round finishes, so gate on a real write instead of letting the
+	// first test hits ride out the startup window as EIO on close.
+	if err := f.waitForWritable(90 * time.Second); err != nil {
+		f.dumpLog("mount")
+		return fmt.Errorf("FUSE mount not writable: %v", err)
+	}
+
 	f.isSetup = true
 	return nil
 }
 
-// Cleanup stops all processes and removes temporary files
+// Cleanup stops all processes and removes temporary files.
+// If the test failed, it dumps logs automatically.
 func (f *FuseTestFramework) Cleanup() {
-	if f.mountProcess != nil {
-		f.unmountFuse()
+	// An external mount needs no teardown; only this run's work dir is ours.
+	// A TEST_SKIP_CLUSTER_SETUP that fell through to a self-started cluster
+	// still owns its processes and must not skip stopping them.
+	if f.externalMount {
+		if !f.config.SkipCleanup {
+			os.RemoveAll(f.mountPoint)
+			os.RemoveAll(f.tempDir)
+		}
+		return
+	}
+
+	if f.t.Failed() {
+		f.DumpLogs()
 	}
 
 	// Stop processes in reverse order
-	processes := []*os.Process{f.mountProcess, f.filerProcess, f.volumeProcess, f.masterProcess}
-	for _, proc := range processes {
-		if proc != nil {
-			proc.Signal(syscall.SIGTERM)
-			proc.Wait()
-		}
+	f.unmountFuse()
+	if f.miniProcess != nil {
+		f.miniProcess.stop()
+		f.miniProcess = nil
 	}
 
+	f.copyLogsForCI()
+
 	// Remove temp directory
-	if !DefaultTestConfig().SkipCleanup {
+	if !f.config.SkipCleanup {
 		os.RemoveAll(f.tempDir)
 	}
 }
 
+// DumpLogs prints the tail of all SeaweedFS process logs to test output.
+func (f *FuseTestFramework) DumpLogs() {
+	for _, name := range []string{"mini", "mount"} {
+		f.dumpLog(name)
+	}
+}
+
 // GetMountPoint returns the FUSE mount point path
 func (f *FuseTestFramework) GetMountPoint() string {
 	return f.mountPoint
@@ -168,82 +305,165 @@ func (f *FuseTestFramework) GetFilerAddr() string {
 	return f.filerAddr
 }
 
-// startMaster starts the SeaweedFS master server
-func (f *FuseTestFramework) startMaster(config *TestConfig) error {
-	args := []string{
-		"master",
-		"-ip=127.0.0.1",
-		"-port=19333",
-		"-mdir=" + filepath.Join(f.dataDir, "master"),
-		"-raftBootstrap",
-		"-peers=none", // Faster startup when no multiple masters needed
-	}
-	if config.EnableDebug {
-		args = append(args, "-v=4")
-	}
-
-	cmd := exec.Command(f.weedBinary, args...)
-	cmd.Dir = f.tempDir
-	if err := cmd.Start(); err != nil {
-		return err
-	}
-	f.masterProcess = cmd.Process
-	return nil
+// managedProcess is a started weed sub-command whose exit is watched, so a wait
+// for it to come up ends the moment it dies instead of burning its full timeout.
+type managedProcess struct {
+	cmd     *exec.Cmd
+	done    chan struct{}
+	err     error // exit error, read only after done is closed
+	logPath string
 }
 
-// startVolumeServers starts SeaweedFS volume servers
-func (f *FuseTestFramework) startVolumeServers(config *TestConfig) error {
-	args := []string{
-		"volume",
-		"-master=" + f.masterAddr,
-		"-ip=127.0.0.1",
-		"-port=18080",
-		"-dir=" + filepath.Join(f.dataDir, "volume"),
-		fmt.Sprintf("-max=%d", config.NumVolumes),
+// exited returns the exit error once the process is gone, nil while it runs.
+func (p *managedProcess) exited() error {
+	select {
+	case <-p.done:
+		if p.err != nil {
+			return p.err
+		}
+		return fmt.Errorf("exit status 0")
+	default:
+		return nil
 	}
-	if config.EnableDebug {
-		args = append(args, "-v=4")
-	}
-
-	cmd := exec.Command(f.weedBinary, args...)
-	cmd.Dir = f.tempDir
-	if err := cmd.Start(); err != nil {
-		return err
-	}
-	f.volumeProcess = cmd.Process
-	return nil
 }
 
-// startFiler starts the SeaweedFS filer server
-func (f *FuseTestFramework) startFiler(config *TestConfig) error {
-	args := []string{
-		"filer",
-		"-master=" + f.masterAddr,
-		"-ip=127.0.0.1",
-		"-port=18888",
-	}
-	if config.EnableDebug {
-		args = append(args, "-v=4")
+// stop asks the process to terminate and waits for it to go away.
+func (p *managedProcess) stop() {
+	// Signal fails with os.ErrProcessDone when the child is already gone, which
+	// is exactly the case the select below handles.
+	_ = p.cmd.Process.Signal(syscall.SIGTERM)
+	select {
+	case <-p.done:
+	case <-time.After(10 * time.Second):
+		p.cmd.Process.Kill()
+		<-p.done
 	}
+}
 
+// startProcess is a helper that starts a weed sub-command with output captured
+// to a log file in f.logDir.
+func (f *FuseTestFramework) startProcess(name string, args []string) (*managedProcess, error) {
+	logPath := filepath.Join(f.logDir, name+".log")
+	logFile, err := os.Create(logPath)
+	if err != nil {
+		return nil, fmt.Errorf("create log file: %v", err)
+	}
 	cmd := exec.Command(f.weedBinary, args...)
 	cmd.Dir = f.tempDir
+	cmd.Stdout = logFile
+	cmd.Stderr = logFile
 	if err := cmd.Start(); err != nil {
+		logFile.Close()
+		return nil, err
+	}
+	// Close the file handle — the child process inherited it.
+	logFile.Close()
+
+	p := &managedProcess{cmd: cmd, done: make(chan struct{}), logPath: logPath}
+	go func() {
+		p.err = cmd.Wait()
+		close(p.done)
+	}()
+	return p, nil
+}
+
+// logTail returns the last bytes of a captured process log so a startup
+// failure message carries the fatal line instead of a bare timeout.
+func logTail(path string) string {
+	data, err := os.ReadFile(path)
+	if err != nil || len(data) == 0 {
+		return ""
+	}
+	const maxTail = 2 * 1024
+	if len(data) > maxTail {
+		data = data[len(data)-maxTail:]
+	}
+	return string(data)
+}
+
+// dumpLog prints the last lines of a process log file to the test output
+// for debugging when a service fails to start or a test fails.
+func (f *FuseTestFramework) dumpLog(name string) {
+	data, err := os.ReadFile(filepath.Join(f.logDir, name+".log"))
+	if err != nil {
+		f.t.Logf("[%s log] (not available: %v)", name, err)
+		return
+	}
+	// Show last 16KB on failure for meaningful context.
+	const maxTail = 16 * 1024
+	if len(data) > maxTail {
+		data = data[len(data)-maxTail:]
+	}
+	f.t.Logf("[%s log tail (%d bytes)]\n%s", name, len(data), string(data))
+}
+
+// copyLogsForCI copies SeaweedFS process logs under a per-test directory in
+// /tmp/seaweedfs-fuse-logs/ so the CI workflow can upload them as artifacts.
+// Each framework instance gets its own directory: every test in the suite
+// builds one, and fixed filenames would let a later passing test overwrite
+// the logs of an earlier failing one.
+func (f *FuseTestFramework) copyLogsForCI() {
+	ciLogDir := filepath.Join("/tmp/seaweedfs-fuse-logs", strings.NewReplacer("/", "_", " ", "_").Replace(f.t.Name()))
+	if err := os.MkdirAll(ciLogDir, 0755); err != nil {
+		return
+	}
+	for _, name := range []string{"mini", "mount"} {
+		src := filepath.Join(f.logDir, name+".log")
+		data, err := os.ReadFile(src)
+		if err != nil {
+			continue
+		}
+		os.WriteFile(filepath.Join(ciLogDir, name+".log"), data, 0644)
+	}
+}
+
+// startMini starts "weed mini" which runs master+volume+filer in one process.
+func (f *FuseTestFramework) startMini(config *TestConfig) error {
+	var args []string
+	if config.EnableDebug {
+		// -v is a global flag; before the subcommand or flag parsing dies
+		args = append(args, "-v=4")
+	}
+	adminPort := freePort(f.t)
+	for adminPort == f.filerPort {
+		adminPort = freePort(f.t)
+	}
+	args = append(args,
+		"mini",
+		"-dir="+f.dataDir,
+		"-ip=127.0.0.1",
+		"-ip.bind=127.0.0.1",
+		"-filer.port="+strconv.Itoa(f.filerPort),
+		"-s3=false",
+		"-webdav=false",
+		"-admin.ui=false",
+		"-admin.port="+strconv.Itoa(adminPort),
+		"-admin.port.grpc="+strconv.Itoa(adminPort+grpcPortOffset),
+		"-admin.port.legacy=0",
+	)
+
+	proc, err := f.startProcess("mini", args)
+	if err != nil {
 		return err
 	}
-	f.filerProcess = cmd.Process
+	f.miniProcess = proc
 	return nil
 }
 
 // mountFuse mounts the SeaweedFS FUSE filesystem
 func (f *FuseTestFramework) mountFuse(config *TestConfig) error {
-	args := []string{
+	args := append([]string{}, config.MountGlobalOptions...)
+	if config.EnableDebug {
+		args = append(args, "-v=4")
+	}
+	args = append(args,
 		"mount",
-		"-filer=" + f.filerAddr,
-		"-dir=" + f.mountPoint,
+		"-filer="+f.filerAddr,
+		"-dir="+f.mountPoint,
 		"-filer.path=/",
 		"-dirAutoCreate",
-	}
+		"-allowOthers=false",
+	)
 
 	if config.Collection != "" {
 		args = append(args, "-collection="+config.Collection)
@@ -255,83 +475,172 @@ func (f *FuseTestFramework) mountFuse(config *TestConfig) error {
 		args = append(args, fmt.Sprintf("-chunkSizeLimitMB=%d", config.ChunkSizeMB))
 	}
 	if config.CacheSizeMB > 0 {
-		args = append(args, fmt.Sprintf("-cacheSizeMB=%d", config.CacheSizeMB))
-	}
-	if config.EnableDebug {
-		args = append(args, "-v=4")
+		args = append(args, fmt.Sprintf("-cacheCapacityMB=%d", config.CacheSizeMB))
 	}
 
 	args = append(args, config.MountOptions...)
 
-	cmd := exec.Command(f.weedBinary, args...)
-	cmd.Dir = f.tempDir
-	if err := cmd.Start(); err != nil {
+	proc, err := f.startProcess("mount", args)
+	if err != nil {
 		return err
 	}
-	f.mountProcess = cmd.Process
+	f.mountProcess = proc
 	return nil
 }
 
 // unmountFuse unmounts the FUSE filesystem
 func (f *FuseTestFramework) unmountFuse() error {
 	if f.mountProcess != nil {
-		f.mountProcess.Signal(syscall.SIGTERM)
-		f.mountProcess.Wait()
+		f.mountProcess.stop()
 		f.mountProcess = nil
 	}
 
 	// Also try system unmount as backup
+	exec.Command("fusermount3", "-u", f.mountPoint).Run()
+	exec.Command("fusermount", "-u", f.mountPoint).Run()
 	exec.Command("umount", f.mountPoint).Run()
 	return nil
 }
 
-// waitForService waits for a service to be available
-func (f *FuseTestFramework) waitForService(addr string, timeout time.Duration) error {
+// waitForService waits until the weed process answers its own /healthz, so a
+// foreign listener on the port cannot pass for readiness.
+func (f *FuseTestFramework) waitForService(proc *managedProcess, addr string, timeout time.Duration) error {
 	deadline := time.Now().Add(timeout)
+	client := &http.Client{Timeout: time.Second}
 	for time.Now().Before(deadline) {
-		conn, err := net.DialTimeout("tcp", addr, 1*time.Second)
+		resp, err := client.Get("http://" + addr + "/healthz")
 		if err == nil {
-			conn.Close()
-			return nil
+			resp.Body.Close()
+			if resp.StatusCode == http.StatusOK && proc.exited() == nil {
+				return nil
+			}
+		}
+		if exitErr := proc.exited(); exitErr != nil {
+			return fmt.Errorf("process exited (%v) before %s accepted connections: %s", exitErr, addr, logTail(proc.logPath))
 		}
 		time.Sleep(100 * time.Millisecond)
 	}
-	return fmt.Errorf("service at %s not ready within timeout", addr)
+	return fmt.Errorf("service at %s not ready within timeout: %s", addr, logTail(proc.logPath))
 }
 
-// waitForMount waits for the FUSE mount to be ready
+// waitForMount waits for the FUSE mount to be ready. A stat/ReadDir probe
+// alone is not enough: the bare mount point directory passes both before the
+// mount process finishes starting, letting tests race ahead and write to the
+// local disk underneath the mount. The mount point's device ID differing from
+// its parent's confirms a filesystem is actually mounted there.
 func (f *FuseTestFramework) waitForMount(timeout time.Duration) error {
+	parentDev, err := deviceID(filepath.Dir(f.mountPoint))
+	if err != nil {
+		return fmt.Errorf("stat mount point parent: %w", err)
+	}
 	deadline := time.Now().Add(timeout)
 	for time.Now().Before(deadline) {
-		// Check if mount point is accessible
-		if _, err := os.Stat(f.mountPoint); err == nil {
-			// Try to list directory
+		if dev, err := deviceID(f.mountPoint); err == nil && dev != parentDev {
 			if _, err := os.ReadDir(f.mountPoint); err == nil {
 				return nil
 			}
 		}
+		// A mount that cannot mount at all (no /dev/fuse, fusermount not setuid)
+		// dies within a second; reporting that beats waiting out the timeout.
+		if exitErr := f.mountProcess.exited(); exitErr != nil {
+			return fmt.Errorf("mount process exited (%v): %s", exitErr, logTail(f.mountProcess.logPath))
+		}
 		time.Sleep(100 * time.Millisecond)
 	}
-	return fmt.Errorf("mount point not ready within timeout")
+	return fmt.Errorf("mount point not ready within timeout: %s", logTail(f.mountProcess.logPath))
 }
 
-// findWeedBinary locates the weed binary
+// waitForWritable probes the mount with a real write+read+remove until one
+// succeeds, covering the window where a fresh cluster cannot yet assign a
+// file id (volume not registered, grow pending, leader settle). Each attempt
+// runs off the loop because a wedged mount can hold a file op open, and one
+// attempt stays in flight until it finishes or the overall deadline passes.
+func (f *FuseTestFramework) waitForWritable(timeout time.Duration) error {
+	probe := filepath.Join(f.mountPoint, ".framework_write_probe")
+	payload := []byte("probe")
+	deadline := time.Now().Add(timeout)
+	var lastErr error
+	for time.Now().Before(deadline) {
+		if exitErr := f.mountProcess.exited(); exitErr != nil {
+			return fmt.Errorf("mount process exited (%v): %s", exitErr, logTail(f.mountProcess.logPath))
+		}
+		result := make(chan error, 1)
+		go func() {
+			if err := os.WriteFile(probe, payload, 0644); err != nil {
+				result <- err
+				return
+			}
+			data, err := os.ReadFile(probe)
+			if err != nil {
+				result <- err
+				return
+			}
+			if len(data) != len(payload) {
+				result <- fmt.Errorf("read back %d bytes, want %d", len(data), len(payload))
+				return
+			}
+			result <- os.Remove(probe)
+		}()
+		tick := time.NewTicker(500 * time.Millisecond)
+		attemptOpen := true
+		for attemptOpen {
+			select {
+			case err := <-result:
+				attemptOpen = false
+				if err == nil {
+					tick.Stop()
+					return nil
+				}
+				lastErr = err
+			case <-tick.C:
+				if exitErr := f.mountProcess.exited(); exitErr != nil {
+					tick.Stop()
+					return fmt.Errorf("mount process exited (%v): %s", exitErr, logTail(f.mountProcess.logPath))
+				}
+				if !time.Now().Before(deadline) {
+					tick.Stop()
+					if lastErr == nil {
+						return fmt.Errorf("write probe did not complete within timeout")
+					}
+					return fmt.Errorf("write probe failed within timeout: %v", lastErr)
+				}
+			}
+		}
+		tick.Stop()
+		time.Sleep(500 * time.Millisecond)
+	}
+	return fmt.Errorf("write probe failed within timeout: %v", lastErr)
+}
+
+// fuseAvailable reports whether the host has a FUSE device node to mount
+// through: /dev/fuse on Linux, /dev/macfuse* or /dev/osxfuse* on macOS.
+func fuseAvailable() bool {
+	for _, dev := range []string{"/dev/fuse", "/dev/macfuse0", "/dev/osxfuse0"} {
+		if _, err := os.Stat(dev); err == nil {
+			return true
+		}
+	}
+	return false
+}
+
+// findWeedBinary locates the weed binary. Candidates are resolved to
+// absolute paths because the started processes run with cmd.Dir set to the
+// framework's temp dir, where a relative path would not resolve.
 func findWeedBinary() string {
-	// Try different possible locations
-	candidates := []string{
-		"./weed",
-		"../weed",
-		"../../weed",
-		"weed", // in PATH
+	if p, err := exec.LookPath("weed"); err == nil {
+		return p
 	}
 
+	candidates := []string{
+		"../../weed/weed",
+		"./weed",
+		"../weed",
+	}
 	for _, candidate := range candidates {
-		if _, err := exec.LookPath(candidate); err == nil {
-			return candidate
-		}
-		if _, err := os.Stat(candidate); err == nil {
-			abs, _ := filepath.Abs(candidate)
-			return abs
+		if info, err := os.Stat(candidate); err == nil && !info.IsDir() && info.Mode()&0111 != 0 {
+			if abs, err := filepath.Abs(candidate); err == nil {
+				return abs
+			}
 		}
 	}
 

test/fuse_integration/framework_test.go

@@ -0,0 +1,646 @@
+package fuse_test
+
+import (
+	"fmt"
+	"io/fs"
+	"net"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"strconv"
+	"strings"
+	"syscall"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/require"
+)
+
+// FuseTestFramework provides utilities for FUSE integration testing.
+// It starts a single "weed mini" process (master+volume+filer in one)
+// and a separate "weed mount" process for the FUSE filesystem.
+type FuseTestFramework struct {
+	t            *testing.T
+	tempDir      string
+	mountPoint   string
+	dataDir      string
+	logDir       string
+	miniProcess  *managedProcess
+	mountProcess *managedProcess
+	filerAddr    string
+	filerPort    int
+	weedBinary   string
+	isSetup      bool
+	config       *TestConfig
+}
+
+// TestConfig holds configuration for FUSE tests
+type TestConfig struct {
+	Collection  string
+	Replication string
+	ChunkSizeMB int
+	CacheSizeMB int
+	NumVolumes  int
+	EnableDebug bool
+	// MountGlobalOptions are glog flags (-v, -logtostderr, ...) registered on
+	// weed's global flagset; they must precede the subcommand name or the
+	// mount process dies at flag parsing.
+	MountGlobalOptions []string
+	MountOptions       []string
+	SkipCleanup        bool // for debugging failed tests
+}
+
+// DefaultTestConfig returns a default configuration for FUSE tests
+func DefaultTestConfig() *TestConfig {
+	return &TestConfig{
+		Collection:   "",
+		Replication:  "000",
+		ChunkSizeMB:  4,
+		CacheSizeMB:  100,
+		NumVolumes:   3,
+		EnableDebug:  false,
+		MountOptions: []string{},
+		SkipCleanup:  false,
+	}
+}
+
+// NewFuseTestFramework creates a new FUSE testing framework.
+func NewFuseTestFramework(t *testing.T, config *TestConfig) *FuseTestFramework {
+	if config == nil {
+		config = DefaultTestConfig()
+	}
+
+	tempDir, err := os.MkdirTemp("", "seaweedfs_fuse_test_")
+	require.NoError(t, err)
+
+	filerPort := freePort(t)
+
+	return &FuseTestFramework{
+		t:          t,
+		tempDir:    tempDir,
+		mountPoint: filepath.Join(tempDir, "mount"),
+		dataDir:    filepath.Join(tempDir, "data"),
+		logDir:     filepath.Join(tempDir, "logs"),
+		filerPort:  filerPort,
+		filerAddr:  fmt.Sprintf("127.0.0.1:%d", filerPort),
+		weedBinary: findWeedBinary(),
+		isSetup:    false,
+		config:     config,
+	}
+}
+
+// grpcPortOffset mirrors weed's HTTP->gRPC port convention (gRPC = HTTP + offset);
+// "weed mount" derives the filer gRPC port from the filer HTTP address the same way.
+const grpcPortOffset = 10000
+
+// freePort returns a free filer HTTP port whose gRPC sibling (port+grpcPortOffset)
+// is also free. Both must stay below the Linux ephemeral floor (32768): a gRPC port
+// above it can be transiently grabbed by an outbound connection, forcing mini to
+// relocate its filer gRPC port while "weed mount" keeps dialing HTTP+10000 — the
+// mount then never connects and the test times out. Capping HTTP at 22000 keeps the
+// gRPC port at or below 32000.
+func freePort(t *testing.T) int {
+	t.Helper()
+	const (
+		minServicePort = 20000
+		maxServicePort = 22000
+	)
+
+	portCount := maxServicePort - minServicePort + 1
+	start := minServicePort + int(time.Now().UnixNano()%int64(portCount))
+
+	for attempt := 0; attempt < portCount; attempt++ {
+		port := minServicePort + (start-minServicePort+attempt)%portCount
+		if portIsFree(port) && portIsFree(port+grpcPortOffset) {
+			return port
+		}
+	}
+
+	t.Fatalf("failed to allocate a free HTTP/gRPC port pair in [%d,%d]", minServicePort, maxServicePort)
+	return 0
+}
+
+// portIsFree reports whether a TCP port can currently be bound on 127.0.0.1.
+func portIsFree(port int) bool {
+	l, err := net.Listen("tcp", net.JoinHostPort("127.0.0.1", strconv.Itoa(port)))
+	if err != nil {
+		return false
+	}
+	l.Close()
+	return true
+}
+
+// Setup starts "weed mini" and mounts the FUSE filesystem.
+func (f *FuseTestFramework) Setup(config *TestConfig) error {
+	if f.isSetup {
+		return fmt.Errorf("framework already setup")
+	}
+	if config != nil {
+		f.config = config
+	}
+	config = f.config
+
+	dirs := []string{f.mountPoint, f.logDir, f.dataDir}
+	for _, dir := range dirs {
+		if err := os.MkdirAll(dir, 0755); err != nil {
+			return fmt.Errorf("failed to create directory %s: %v", dir, err)
+		}
+	}
+
+	// Start weed mini (master + volume + filer in one process)
+	if err := f.startMini(config); err != nil {
+		return fmt.Errorf("failed to start weed mini: %v", err)
+	}
+
+	// Wait for filer to be ready (mini starts all services on filerPort)
+	if err := f.waitForService(f.miniProcess, f.filerAddr, 90*time.Second); err != nil {
+		f.dumpLog("mini")
+		return fmt.Errorf("weed mini not ready: %v", err)
+	}
+
+	// Mount FUSE filesystem
+	if err := f.mountFuse(config); err != nil {
+		return fmt.Errorf("failed to mount FUSE: %v", err)
+	}
+
+	// Wait for mount to be ready
+	if err := f.waitForMount(90 * time.Second); err != nil {
+		f.dumpLog("mount")
+		return fmt.Errorf("FUSE mount not ready: %v", err)
+	}
+
+	// Gate on a real write: a fresh mini still cannot assign a file id until
+	// its volume registers and a grow round finishes.
+	if err := f.waitForWritable(90 * time.Second); err != nil {
+		f.dumpLog("mount")
+		return fmt.Errorf("FUSE mount not writable: %v", err)
+	}
+
+	f.isSetup = true
+	return nil
+}
+
+// Cleanup stops all processes and removes temporary files.
+// If the test failed, it dumps logs automatically.
+func (f *FuseTestFramework) Cleanup() {
+	if f.t.Failed() {
+		f.DumpLogs()
+	}
+
+	// Stop processes in reverse order
+	f.unmountFuse()
+	if f.miniProcess != nil {
+		f.miniProcess.stop()
+		f.miniProcess = nil
+	}
+
+	f.copyLogsForCI()
+
+	if !f.config.SkipCleanup {
+		os.RemoveAll(f.tempDir)
+	}
+}
+
+// DumpLogs prints the tail of all SeaweedFS process logs to test output.
+func (f *FuseTestFramework) DumpLogs() {
+	for _, name := range []string{"mini", "mount"} {
+		f.dumpLog(name)
+	}
+}
+
+// GetMountPoint returns the FUSE mount point path
+func (f *FuseTestFramework) GetMountPoint() string {
+	return f.mountPoint
+}
+
+// GetFilerAddr returns the filer address
+func (f *FuseTestFramework) GetFilerAddr() string {
+	return f.filerAddr
+}
+
+// managedProcess is a started weed sub-command whose exit is watched, so a wait
+// for it to come up ends the moment it dies instead of burning its full timeout.
+type managedProcess struct {
+	cmd  *exec.Cmd
+	done chan struct{}
+	err  error // exit error, read only after done is closed
+}
+
+// exited returns the exit error once the process is gone, nil while it runs.
+func (p *managedProcess) exited() error {
+	select {
+	case <-p.done:
+		if p.err != nil {
+			return p.err
+		}
+		return fmt.Errorf("exit status 0")
+	default:
+		return nil
+	}
+}
+
+// stop asks the process to terminate and waits for it to go away.
+func (p *managedProcess) stop() {
+	// Signal fails with os.ErrProcessDone when the child is already gone, which
+	// is exactly the case the select below handles.
+	_ = p.cmd.Process.Signal(syscall.SIGTERM)
+	select {
+	case <-p.done:
+	case <-time.After(10 * time.Second):
+		p.cmd.Process.Kill()
+		<-p.done
+	}
+}
+
+// startProcess is a helper that starts a weed sub-command with output captured
+// to a log file in f.logDir.
+func (f *FuseTestFramework) startProcess(name string, args []string) (*managedProcess, error) {
+	logFile, err := os.Create(filepath.Join(f.logDir, name+".log"))
+	if err != nil {
+		return nil, fmt.Errorf("create log file: %v", err)
+	}
+	cmd := exec.Command(f.weedBinary, args...)
+	cmd.Dir = f.tempDir
+	cmd.Stdout = logFile
+	cmd.Stderr = logFile
+	if err := cmd.Start(); err != nil {
+		logFile.Close()
+		return nil, err
+	}
+	// Close the file handle — the child process inherited it.
+	logFile.Close()
+
+	p := &managedProcess{cmd: cmd, done: make(chan struct{})}
+	go func() {
+		p.err = cmd.Wait()
+		close(p.done)
+	}()
+	return p, nil
+}
+
+// dumpLog prints the last lines of a process log file to the test output
+// for debugging when a service fails to start or a test fails.
+func (f *FuseTestFramework) dumpLog(name string) {
+	data, err := os.ReadFile(filepath.Join(f.logDir, name+".log"))
+	if err != nil {
+		f.t.Logf("[%s log] (not available: %v)", name, err)
+		return
+	}
+	// Show last 16KB on failure for meaningful context.
+	const maxTail = 16 * 1024
+	if len(data) > maxTail {
+		data = data[len(data)-maxTail:]
+	}
+	f.t.Logf("[%s log tail (%d bytes)]\n%s", name, len(data), string(data))
+}
+
+// copyLogsForCI copies SeaweedFS process logs under a per-test directory in
+// /tmp/seaweedfs-fuse-logs/ so the CI workflow can upload them as artifacts.
+// Each framework instance gets its own directory: every test in the suite
+// builds one, and fixed filenames would let a later passing test overwrite
+// the logs of an earlier failing one.
+func (f *FuseTestFramework) copyLogsForCI() {
+	ciLogDir := filepath.Join("/tmp/seaweedfs-fuse-logs", strings.NewReplacer("/", "_", " ", "_").Replace(f.t.Name()))
+	if err := os.MkdirAll(ciLogDir, 0755); err != nil {
+		return
+	}
+	for _, name := range []string{"mini", "mount"} {
+		src := filepath.Join(f.logDir, name+".log")
+		data, err := os.ReadFile(src)
+		if err != nil {
+			continue
+		}
+		os.WriteFile(filepath.Join(ciLogDir, name+".log"), data, 0644)
+	}
+}
+
+// startMini starts "weed mini" which runs master+volume+filer in one process.
+func (f *FuseTestFramework) startMini(config *TestConfig) error {
+	var args []string
+	if config.EnableDebug {
+		args = append(args, "-v=4")
+	}
+	adminPort := freePort(f.t)
+	for adminPort == f.filerPort {
+		adminPort = freePort(f.t)
+	}
+	args = append(args,
+		"mini",
+		"-dir="+f.dataDir,
+		"-ip=127.0.0.1",
+		"-ip.bind=127.0.0.1",
+		"-filer.port="+strconv.Itoa(f.filerPort),
+		"-s3=false",
+		"-webdav=false",
+		"-admin.ui=false",
+		"-admin.port="+strconv.Itoa(adminPort),
+		"-admin.port.grpc="+strconv.Itoa(adminPort+grpcPortOffset),
+		"-admin.port.legacy=0",
+	)
+
+	proc, err := f.startProcess("mini", args)
+	if err != nil {
+		return err
+	}
+	f.miniProcess = proc
+	return nil
+}
+
+// mountFuse mounts the SeaweedFS FUSE filesystem
+func (f *FuseTestFramework) mountFuse(config *TestConfig) error {
+	args := append([]string{}, config.MountGlobalOptions...)
+	if config.EnableDebug {
+		args = append(args, "-v=4")
+	}
+	args = append(args,
+		"mount",
+		"-filer=127.0.0.1:"+strconv.Itoa(f.filerPort),
+		"-dir="+f.mountPoint,
+		"-filer.path=/",
+		"-dirAutoCreate",
+		"-allowOthers=false",
+	)
+
+	if config.Collection != "" {
+		args = append(args, "-collection="+config.Collection)
+	}
+	if config.Replication != "" {
+		args = append(args, "-replication="+config.Replication)
+	}
+	if config.ChunkSizeMB > 0 {
+		args = append(args, fmt.Sprintf("-chunkSizeLimitMB=%d", config.ChunkSizeMB))
+	}
+	if config.CacheSizeMB > 0 {
+		args = append(args, fmt.Sprintf("-cacheCapacityMB=%d", config.CacheSizeMB))
+	}
+
+	args = append(args, config.MountOptions...)
+
+	proc, err := f.startProcess("mount", args)
+	if err != nil {
+		return err
+	}
+	f.mountProcess = proc
+	return nil
+}
+
+// unmountFuse unmounts the FUSE filesystem
+func (f *FuseTestFramework) unmountFuse() error {
+	if f.mountProcess != nil {
+		f.mountProcess.stop()
+		f.mountProcess = nil
+	}
+
+	// Also try system unmount as backup
+	exec.Command("fusermount3", "-u", f.mountPoint).Run()
+	exec.Command("fusermount", "-u", f.mountPoint).Run()
+	return nil
+}
+
+// waitForService waits for a service to be available
+func (f *FuseTestFramework) waitForService(proc *managedProcess, addr string, timeout time.Duration) error {
+	deadline := time.Now().Add(timeout)
+	for time.Now().Before(deadline) {
+		conn, err := net.DialTimeout("tcp", addr, 1*time.Second)
+		if err == nil {
+			conn.Close()
+			return nil
+		}
+		if exitErr := proc.exited(); exitErr != nil {
+			return fmt.Errorf("process exited (%v) before %s accepted connections", exitErr, addr)
+		}
+		time.Sleep(100 * time.Millisecond)
+	}
+	return fmt.Errorf("service at %s not ready within timeout", addr)
+}
+
+// waitForMount waits for the FUSE mount to be ready. A stat/ReadDir probe
+// alone is not enough: the bare mount point directory passes both before the
+// mount process finishes starting, letting tests race ahead and write to the
+// local disk underneath the mount. The mount point's device ID differing from
+// its parent's confirms a filesystem is actually mounted there.
+func (f *FuseTestFramework) waitForMount(timeout time.Duration) error {
+	parentDev, err := deviceID(filepath.Dir(f.mountPoint))
+	if err != nil {
+		return fmt.Errorf("stat mount point parent: %v", err)
+	}
+	deadline := time.Now().Add(timeout)
+	for time.Now().Before(deadline) {
+		if dev, err := deviceID(f.mountPoint); err == nil && dev != parentDev {
+			if _, err := os.ReadDir(f.mountPoint); err == nil {
+				return nil
+			}
+		}
+		// A mount that cannot mount at all (no /dev/fuse, fusermount not setuid)
+		// dies within a second; reporting that beats waiting out the timeout.
+		if exitErr := f.mountProcess.exited(); exitErr != nil {
+			return fmt.Errorf("mount process exited (%v)", exitErr)
+		}
+		time.Sleep(100 * time.Millisecond)
+	}
+	return fmt.Errorf("mount point not ready within timeout")
+}
+
+// waitForWritable probes the mount with a real write+read+remove until one
+// succeeds, covering the window where a fresh cluster cannot yet assign a
+// file id (volume not registered, grow pending, leader settle). Each attempt
+// runs off the loop because a wedged mount can hold a file op open, and one
+// attempt stays in flight until it finishes or the overall deadline passes.
+func (f *FuseTestFramework) waitForWritable(timeout time.Duration) error {
+	probe := filepath.Join(f.mountPoint, ".framework_write_probe")
+	payload := []byte("probe")
+	deadline := time.Now().Add(timeout)
+	var lastErr error
+	for time.Now().Before(deadline) {
+		if exitErr := f.mountProcess.exited(); exitErr != nil {
+			return fmt.Errorf("mount process exited (%v)", exitErr)
+		}
+		result := make(chan error, 1)
+		go func() {
+			if err := os.WriteFile(probe, payload, 0644); err != nil {
+				result <- err
+				return
+			}
+			data, err := os.ReadFile(probe)
+			if err != nil {
+				result <- err
+				return
+			}
+			if len(data) != len(payload) {
+				result <- fmt.Errorf("read back %d bytes, want %d", len(data), len(payload))
+				return
+			}
+			result <- os.Remove(probe)
+		}()
+		tick := time.NewTicker(500 * time.Millisecond)
+		attemptOpen := true
+		for attemptOpen {
+			select {
+			case err := <-result:
+				attemptOpen = false
+				if err == nil {
+					tick.Stop()
+					return nil
+				}
+				lastErr = err
+			case <-tick.C:
+				if exitErr := f.mountProcess.exited(); exitErr != nil {
+					tick.Stop()
+					return fmt.Errorf("mount process exited (%v)", exitErr)
+				}
+				if !time.Now().Before(deadline) {
+					tick.Stop()
+					if lastErr == nil {
+						return fmt.Errorf("write probe did not complete within timeout")
+					}
+					return fmt.Errorf("write probe failed within timeout: %v", lastErr)
+				}
+			}
+		}
+		tick.Stop()
+		time.Sleep(500 * time.Millisecond)
+	}
+	return fmt.Errorf("write probe failed within timeout: %v", lastErr)
+}
+
+// deviceID returns the device ID of the filesystem containing path.
+func deviceID(path string) (uint64, error) {
+	var st syscall.Stat_t
+	if err := syscall.Stat(path, &st); err != nil {
+		return 0, err
+	}
+	return uint64(st.Dev), nil
+}
+
+// findWeedBinary locates the weed binary.
+func findWeedBinary() string {
+	if p, err := exec.LookPath("weed"); err == nil {
+		return p
+	}
+
+	candidates := []string{
+		"../../weed/weed",
+		"./weed",
+		"../weed",
+	}
+	for _, candidate := range candidates {
+		if info, err := os.Stat(candidate); err == nil && !info.IsDir() {
+			abs, _ := filepath.Abs(candidate)
+			return abs
+		}
+	}
+
+	return "weed"
+}
+
+// Helper functions for test assertions
+
+// AssertFileExists checks if a file exists in the mount point. On failure,
+// it gathers diagnostic state (retry stat, parent listing, direct open) so
+// transient FUSE flakes leave enough information to identify the layer that
+// dropped the entry. The diagnostic block runs only on the failure path.
+func (f *FuseTestFramework) AssertFileExists(relativePath string) {
+	fullPath := filepath.Join(f.mountPoint, relativePath)
+	_, err := os.Stat(fullPath)
+	if err == nil {
+		return
+	}
+	f.dumpExistenceDiagnostics(relativePath, fullPath, err)
+	require.NoError(f.t, err, "file should exist: %s (see diagnostic logs above)", relativePath)
+}
+
+// dumpExistenceDiagnostics is invoked when AssertFileExists fails. It probes
+// whether the missing entry is transient (retries appear), missing from the
+// parent directory listing, or missing via a direct open syscall — three
+// signals that triangulate where the entry was lost (kernel dentry cache vs
+// mount lookup vs filer).
+func (f *FuseTestFramework) dumpExistenceDiagnostics(relativePath, fullPath string, initialErr error) {
+	f.t.Logf("AssertFileExists diagnostic dump for %s", relativePath)
+	f.t.Logf("  initial stat: err=%v isNotExist=%v", initialErr, os.IsNotExist(initialErr))
+
+	// Retry stat to detect transient invisibility — if it becomes visible
+	// the issue is a short-lived dentry/cache window; if it stays missing
+	// the entry is genuinely gone from the filer or the local cache.
+	for attempt := 1; attempt <= 5; attempt++ {
+		time.Sleep(100 * time.Millisecond)
+		_, err := os.Stat(fullPath)
+		f.t.Logf("  retry #%d after %dms: err=%v", attempt, attempt*100, err)
+		if err == nil {
+			break
+		}
+	}
+
+	// List the parent directory: if the file appears here but stat fails,
+	// the failure is in the kernel's per-name dentry cache; if not, the
+	// mount-side LOOKUP itself is dropping the entry.
+	parentDir := filepath.Dir(fullPath)
+	target := filepath.Base(fullPath)
+	if entries, listErr := os.ReadDir(parentDir); listErr != nil {
+		f.t.Logf("  ReadDir(%s) failed: %v", parentDir, listErr)
+	} else {
+		found := false
+		names := make([]string, 0, len(entries))
+		for _, e := range entries {
+			names = append(names, e.Name())
+			if e.Name() == target {
+				found = true
+			}
+		}
+		f.t.Logf("  ReadDir(%s): %d entries, target %q present=%v", parentDir, len(entries), target, found)
+		if len(names) <= 32 {
+			f.t.Logf("  ReadDir entries: %v", names)
+		}
+	}
+
+	// Direct O_RDONLY open — exercises the same FUSE Lookup+Open path
+	// userspace would, but separately from stat, in case stat-only caching
+	// is interfering.
+	if fd, openErr := os.OpenFile(fullPath, os.O_RDONLY, 0); openErr != nil {
+		f.t.Logf("  Open(%s, O_RDONLY): %v", fullPath, openErr)
+	} else {
+		st, statErr := fd.Stat()
+		fd.Close()
+		size := int64(-1)
+		if statErr == nil && st != nil {
+			size = st.Size()
+		}
+		f.t.Logf("  Open(%s, O_RDONLY): success size=%d statErr=%v", fullPath, size, statErr)
+	}
+}
+
+// AssertFileNotExists checks if a file does not exist in the mount point
+func (f *FuseTestFramework) AssertFileNotExists(relativePath string) {
+	fullPath := filepath.Join(f.mountPoint, relativePath)
+	_, err := os.Stat(fullPath)
+	require.True(f.t, os.IsNotExist(err), "file should not exist: %s", relativePath)
+}
+
+// AssertFileContent checks if a file has expected content
+func (f *FuseTestFramework) AssertFileContent(relativePath string, expectedContent []byte) {
+	fullPath := filepath.Join(f.mountPoint, relativePath)
+	actualContent, err := os.ReadFile(fullPath)
+	require.NoError(f.t, err, "failed to read file: %s", relativePath)
+	require.Equal(f.t, expectedContent, actualContent, "file content mismatch: %s", relativePath)
+}
+
+// AssertFileMode checks if a file has expected permissions
+func (f *FuseTestFramework) AssertFileMode(relativePath string, expectedMode fs.FileMode) {
+	fullPath := filepath.Join(f.mountPoint, relativePath)
+	info, err := os.Stat(fullPath)
+	require.NoError(f.t, err, "failed to stat file: %s", relativePath)
+	require.Equal(f.t, expectedMode, info.Mode(), "file mode mismatch: %s", relativePath)
+}
+
+// CreateTestFile creates a test file with specified content
+func (f *FuseTestFramework) CreateTestFile(relativePath string, content []byte) {
+	fullPath := filepath.Join(f.mountPoint, relativePath)
+	dir := filepath.Dir(fullPath)
+	require.NoError(f.t, os.MkdirAll(dir, 0755), "failed to create directory: %s", dir)
+	require.NoError(f.t, os.WriteFile(fullPath, content, 0644), "failed to create file: %s", relativePath)
+}
+
+// CreateTestDir creates a test directory
+func (f *FuseTestFramework) CreateTestDir(relativePath string) {
+	fullPath := filepath.Join(f.mountPoint, relativePath)
+	require.NoError(f.t, os.MkdirAll(fullPath, 0755), "failed to create directory: %s", relativePath)
+}

test/fuse_integration/fuse_mount_bsd.go

@@ -0,0 +1,25 @@
+//go:build darwin || freebsd
+
+package fuse
+
+import (
+	"strings"
+	"syscall"
+)
+
+// fuseFilesystem reports whether path sits on a FUSE mount; macFUSE, osxfuse
+// and fusefs all carry "fuse" in the filesystem type name.
+func fuseFilesystem(path string) bool {
+	var st syscall.Statfs_t
+	if err := syscall.Statfs(path, &st); err != nil {
+		return false
+	}
+	name := make([]byte, 0, len(st.Fstypename))
+	for _, c := range st.Fstypename {
+		if c == 0 {
+			break
+		}
+		name = append(name, byte(c))
+	}
+	return strings.Contains(string(name), "fuse")
+}

test/fuse_integration/fuse_mount_linux.go

@@ -0,0 +1,17 @@
+//go:build linux
+
+package fuse
+
+import "syscall"
+
+// fuseSuperMagic is statfs f_type for fuse and fuseblk mounts.
+const fuseSuperMagic = 0x65735546
+
+// fuseFilesystem reports whether path sits on a FUSE mount.
+func fuseFilesystem(path string) bool {
+	var st syscall.Statfs_t
+	if err := syscall.Statfs(path, &st); err != nil {
+		return false
+	}
+	return st.Type == fuseSuperMagic
+}

test/fuse_integration/fuse_mount_other.go

@@ -0,0 +1,9 @@
+//go:build (unix && !linux && !darwin && !freebsd) || (!unix && !windows)
+
+package fuse
+
+// fuseFilesystem has no portable check on these platforms; the mount check
+// and write verification in Setup stay the only gates.
+func fuseFilesystem(path string) bool {
+	return true
+}

test/fuse_integration/git_operations_test.go

@@ -0,0 +1,630 @@
+package fuse_test
+
+import (
+	"fmt"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"regexp"
+	"strconv"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+// TestGitOperations exercises git clone, checkout, and pull on a FUSE mount.
+//
+// The test creates a bare repo on the mount (acting as a remote), clones it,
+// makes commits, pushes, then clones from the mount into an on-mount working
+// directory. It pushes additional commits, checks out an older revision in the
+// on-mount clone, and runs git pull to fast-forward with real changes —
+// verifying file content integrity at each step.
+func TestGitOperations(t *testing.T) {
+	framework := NewFuseTestFramework(t, DefaultTestConfig())
+	defer framework.Cleanup()
+
+	require.NoError(t, framework.Setup(DefaultTestConfig()))
+
+	mountPoint := framework.GetMountPoint()
+
+	// We need a local scratch dir (not on the mount) for the "developer" clone.
+	localDir, err := os.MkdirTemp("", "git_ops_local_")
+	require.NoError(t, err)
+	defer os.RemoveAll(localDir)
+
+	t.Run("CloneAndPull", func(t *testing.T) {
+		testGitCloneAndPull(t, mountPoint, localDir)
+	})
+}
+
+func testGitCloneAndPull(t *testing.T, mountPoint, localDir string) {
+	bareRepo := filepath.Join(mountPoint, "repo.git")
+	localClone := filepath.Join(localDir, "clone")
+	mountClone := filepath.Join(mountPoint, "working")
+
+	// ---- Phase 1: Create bare repo on the mount ----
+	t.Log("Phase 1: create bare repo on mount")
+	gitRun(t, "", "init", "--bare", bareRepo)
+
+	// ---- Phase 2: Clone locally, make initial commits, push ----
+	t.Log("Phase 2: clone locally, commit, push")
+	gitRun(t, "", "clone", bareRepo, localClone)
+	gitRun(t, localClone, "config", "user.email", "test@seaweedfs.test")
+	gitRun(t, localClone, "config", "user.name", "Test")
+
+	// Commit 1
+	writeFile(t, localClone, "README.md", "hello world\n")
+	mkdirAll(t, localClone, "src")
+	writeFile(t, localClone, "src/main.go", `package main; import "fmt"; func main() { fmt.Println("v1") }`)
+	gitRun(t, localClone, "add", "-A")
+	gitRun(t, localClone, "commit", "-m", "initial commit")
+	commit1 := gitOutput(t, localClone, "rev-parse", "HEAD")
+
+	// Commit 2: bulk files
+	mkdirAll(t, localClone, "data")
+	for i := 1; i <= 20; i++ {
+		name := filepath.Join("data", "file-"+leftPad(i, 3)+".txt")
+		writeFile(t, localClone, name, "content-"+strconv.Itoa(i)+"\n")
+	}
+	gitRun(t, localClone, "add", "-A")
+	gitRun(t, localClone, "commit", "-m", "add data files")
+	commit2 := gitOutput(t, localClone, "rev-parse", "HEAD")
+
+	// Commit 3: modify + new dir
+	writeFile(t, localClone, "src/main.go", `package main; import "fmt"; func main() { fmt.Println("v2") }`)
+	writeFile(t, localClone, "README.md", "hello world\n# Updated\n")
+	mkdirAll(t, localClone, "docs")
+	writeFile(t, localClone, "docs/guide.md", "documentation\n")
+	gitRun(t, localClone, "add", "-A")
+	gitRun(t, localClone, "commit", "-m", "update src and add docs")
+	commit3 := gitOutput(t, localClone, "rev-parse", "HEAD")
+
+	branch := gitOutput(t, localClone, "rev-parse", "--abbrev-ref", "HEAD")
+	gitRun(t, localClone, "push", "origin", branch)
+
+	// The bare repo lives on the FUSE mount and can briefly disappear after
+	// the push completes, and pushed pack objects may not be immediately
+	// consistent on the FUSE layer. Give the mount a chance to settle, then
+	// recover from the local clone if the remote is still missing.
+	if !waitForBareRepoEventually(t, bareRepo, 10*time.Second) {
+		t.Logf("bare repo %s did not stabilise after push; forcing recovery before clone", bareRepo)
+	}
+	refreshDirEntry(t, bareRepo)
+	time.Sleep(1 * time.Second)
+
+	// ---- Phase 3: Clone from mount bare repo into on-mount working dir ----
+	t.Log("Phase 3: clone from mount bare repo to on-mount working dir")
+	ensureMountCloneFromBareWithRecovery(t, bareRepo, localClone, mountClone)
+
+	assertFileContains(t, filepath.Join(mountClone, "README.md"), "# Updated")
+	assertFileContains(t, filepath.Join(mountClone, "src/main.go"), "v2")
+	assertFileExists(t, filepath.Join(mountClone, "docs/guide.md"))
+	assertFileExists(t, filepath.Join(mountClone, "data/file-020.txt"))
+
+	head := gitOutput(t, mountClone, "rev-parse", "HEAD")
+	assert.Equal(t, commit3, head, "on-mount clone HEAD should be commit 3")
+
+	dataFiles := countFiles(t, filepath.Join(mountClone, "data"))
+	assert.Equal(t, 20, dataFiles, "data/ should have 20 files")
+
+	// ---- Phase 4: Push more commits from the local clone ----
+	t.Log("Phase 4: push more commits")
+
+	for i := 21; i <= 50; i++ {
+		name := filepath.Join("data", "file-"+leftPad(i, 3)+".txt")
+		writeFile(t, localClone, name, "content-"+strconv.Itoa(i)+"\n")
+	}
+	writeFile(t, localClone, "src/main.go", `package main; import "fmt"; func main() { fmt.Println("v3") }`)
+	gitRun(t, localClone, "add", "-A")
+	gitRun(t, localClone, "commit", "-m", "expand data and update to v3")
+	commit4 := gitOutput(t, localClone, "rev-parse", "HEAD")
+	_ = commit4
+
+	gitRun(t, localClone, "mv", "docs/guide.md", "docs/manual.md")
+	gitRun(t, localClone, "rm", "data/file-001.txt")
+	gitRun(t, localClone, "commit", "-m", "rename guide, remove file-001")
+	commit5 := gitOutput(t, localClone, "rev-parse", "HEAD")
+
+	gitRun(t, localClone, "push", "origin", branch)
+
+	// ---- Phase 5: Reset to older revision in on-mount clone ----
+	t.Log("Phase 5: reset to older revision on mount clone")
+	resetToCommitWithRecovery(t, bareRepo, localClone, mountClone, commit2)
+
+	resetHead := gitOutput(t, mountClone, "rev-parse", "HEAD")
+	assert.Equal(t, commit2, resetHead, "should be at commit 2")
+	assertFileContains(t, filepath.Join(mountClone, "src/main.go"), "v1")
+	assertFileNotExists(t, filepath.Join(mountClone, "docs/guide.md"))
+
+	// ---- Phase 6: Pull with real changes ----
+	t.Log("Phase 6: pull with real fast-forward changes")
+
+	// After git reset --hard on FUSE (Phase 5), the kernel dcache can
+	// permanently lose the directory entry. Wrap the pull in a recovery
+	// loop that re-clones from the bare repo if the clone has vanished.
+	pullFromCommitWithRecovery(t, bareRepo, localClone, mountClone, commit2)
+
+	newHead := gitOutput(t, mountClone, "rev-parse", "HEAD")
+	assert.Equal(t, commit5, newHead, "HEAD should be commit 5 after pull")
+
+	assertFileContains(t, filepath.Join(mountClone, "src/main.go"), "v3")
+	assertFileExists(t, filepath.Join(mountClone, "docs/manual.md"))
+	assertFileNotExists(t, filepath.Join(mountClone, "docs/guide.md"))
+	assertFileNotExists(t, filepath.Join(mountClone, "data/file-001.txt"))
+	assertFileExists(t, filepath.Join(mountClone, "data/file-050.txt"))
+
+	finalCount := countFiles(t, filepath.Join(mountClone, "data"))
+	assert.Equal(t, 49, finalCount, "data/ should have 49 files after pull")
+
+	// ---- Phase 7: Verify git log and status ----
+	t.Log("Phase 7: verify log and status")
+	logOutput := gitOutput(t, mountClone, "log", "--format=%s")
+	lines := strings.Split(strings.TrimSpace(logOutput), "\n")
+	assert.Equal(t, 5, len(lines), "should have 5 commits in log")
+
+	assert.Contains(t, logOutput, "initial commit")
+	assert.Contains(t, logOutput, "expand data")
+	assert.Contains(t, logOutput, "rename guide")
+
+	status := gitOutput(t, mountClone, "status", "--porcelain")
+	assert.Empty(t, status, "git status should be clean")
+
+	_ = commit1 // used for documentation; not needed in assertions
+}
+
+// --- helpers ---
+
+func gitRun(t *testing.T, dir string, args ...string) {
+	t.Helper()
+	gitRunWithRetry(t, dir, args...)
+}
+
+func gitOutput(t *testing.T, dir string, args ...string) string {
+	t.Helper()
+	return gitRunWithRetry(t, dir, args...)
+}
+
+// gitRunWithRetry runs a git command with retries to handle transient FUSE
+// I/O errors on slow CI runners (e.g. "Could not write new index file",
+// "failed to stat", "unpack-objects failed").
+func gitRunWithRetry(t *testing.T, dir string, args ...string) string {
+	t.Helper()
+	const (
+		maxRetries = 6
+		dirWait    = 10 * time.Second
+	)
+	var out []byte
+	var err error
+	for i := 0; i < maxRetries; i++ {
+		if dir != "" && !waitForDirEventually(t, dir, dirWait) {
+			out = []byte("directory missing: " + dir)
+			err = &os.PathError{Op: "stat", Path: dir, Err: os.ErrNotExist}
+		} else {
+			cmd := exec.Command("git", args...)
+			if dir != "" {
+				cmd.Dir = dir
+			}
+			out, err = cmd.CombinedOutput()
+		}
+		if err == nil {
+			return strings.TrimSpace(string(out))
+		}
+		if i < maxRetries-1 {
+			t.Logf("git %s attempt %d failed (retrying): %s", strings.Join(args, " "), i+1, string(out))
+			if dir != "" {
+				refreshDirEntry(t, dir)
+			}
+			if repoPath := extractGitRepoPath(string(out)); repoPath != "" {
+				_ = exec.Command("git", "init", "--bare", repoPath).Run()
+				waitForBareRepoEventually(t, repoPath, 5*time.Second)
+			}
+			time.Sleep(500 * time.Millisecond)
+		}
+	}
+	require.NoError(t, err, "git %s failed after %d attempts: %s", strings.Join(args, " "), maxRetries, string(out))
+	return ""
+}
+
+func writeFile(t *testing.T, base, rel, content string) {
+	t.Helper()
+	p := filepath.Join(base, rel)
+	require.NoError(t, os.WriteFile(p, []byte(content), 0644))
+}
+
+func mkdirAll(t *testing.T, base, rel string) {
+	t.Helper()
+	require.NoError(t, os.MkdirAll(filepath.Join(base, rel), 0755))
+}
+
+func assertFileExists(t *testing.T, path string) {
+	t.Helper()
+	_, err := os.Stat(path)
+	require.NoError(t, err, "expected file to exist: %s", path)
+}
+
+func assertFileNotExists(t *testing.T, path string) {
+	t.Helper()
+	_, err := os.Stat(path)
+	require.True(t, os.IsNotExist(err), "expected file not to exist: %s", path)
+}
+
+func assertFileContains(t *testing.T, path, substr string) {
+	t.Helper()
+	data, err := os.ReadFile(path)
+	require.NoError(t, err, "failed to read %s", path)
+	assert.Contains(t, string(data), substr, "file %s should contain %q", path, substr)
+}
+
+func countFiles(t *testing.T, dir string) int {
+	t.Helper()
+	entries, err := os.ReadDir(dir)
+	require.NoError(t, err, "failed to read dir %s", dir)
+	count := 0
+	for _, e := range entries {
+		if !e.IsDir() {
+			count++
+		}
+	}
+	return count
+}
+
+func waitForDir(t *testing.T, dir string) {
+	t.Helper()
+	if !waitForDirEventually(t, dir, 10*time.Second) {
+		t.Fatalf("directory %s did not appear within 10s", dir)
+	}
+}
+
+func waitForDirEventually(t *testing.T, dir string, timeout time.Duration) bool {
+	t.Helper()
+	deadline := time.Now().Add(timeout)
+	for time.Now().Before(deadline) {
+		if _, err := os.Stat(dir); err == nil {
+			return true
+		}
+		time.Sleep(100 * time.Millisecond)
+	}
+	return false
+}
+
+func refreshDirEntry(t *testing.T, dir string) {
+	t.Helper()
+	parent := filepath.Dir(dir)
+	_, _ = os.ReadDir(parent)
+}
+
+func waitForBareRepoEventually(t *testing.T, bareRepo string, timeout time.Duration) bool {
+	t.Helper()
+	deadline := time.Now().Add(timeout)
+	for time.Now().Before(deadline) {
+		if isBareRepoAccessible(bareRepo) {
+			return true
+		}
+		refreshDirEntry(t, bareRepo)
+		time.Sleep(150 * time.Millisecond)
+	}
+	return false
+}
+
+func isBareRepo(bareRepo string) bool {
+	required := []string{
+		filepath.Join(bareRepo, "HEAD"),
+		filepath.Join(bareRepo, "config"),
+	}
+	for _, p := range required {
+		if _, err := os.Stat(p); err != nil {
+			return false
+		}
+	}
+	return true
+}
+
+func isBareRepoAccessible(bareRepo string) bool {
+	if !isBareRepo(bareRepo) {
+		return false
+	}
+	out, err := tryGitCommand("", "--git-dir="+bareRepo, "rev-parse", "--is-bare-repository")
+	return err == nil && out == "true"
+}
+
+func ensureMountClone(t *testing.T, bareRepo, mountClone string) {
+	t.Helper()
+	require.NoError(t, tryEnsureMountClone(bareRepo, mountClone))
+}
+
+// tryEnsureBareRepo verifies the bare repo on the FUSE mount exists.
+// If it has vanished, it re-creates it from the local clone.
+func tryEnsureBareRepo(bareRepo, localClone string) error {
+	if isBareRepoAccessible(bareRepo) {
+		return nil
+	}
+	branch, err := tryGitCommand(localClone, "rev-parse", "--abbrev-ref", "HEAD")
+	if err != nil {
+		return fmt.Errorf("detect local branch: %w", err)
+	}
+	os.RemoveAll(bareRepo)
+	time.Sleep(500 * time.Millisecond)
+	if _, err := tryGitCommand("", "init", "--bare", bareRepo); err != nil {
+		return fmt.Errorf("re-init bare repo: %w", err)
+	}
+	refSpec := fmt.Sprintf("%s:refs/heads/%s", branch, branch)
+	if _, err := tryGitCommand(localClone, "push", "--force", bareRepo, refSpec); err != nil {
+		return fmt.Errorf("re-push branch %s to bare repo: %w", branch, err)
+	}
+	if _, err := tryGitCommand("", "--git-dir="+bareRepo, "symbolic-ref", "HEAD", "refs/heads/"+branch); err != nil {
+		return fmt.Errorf("set bare repo HEAD to %s: %w", branch, err)
+	}
+	return nil
+}
+
+func ensureMountCloneFromBareWithRecovery(t *testing.T, bareRepo, localClone, mountClone string) {
+	t.Helper()
+	const maxAttempts = 3
+	var lastErr error
+	for attempt := 1; attempt <= maxAttempts; attempt++ {
+		if lastErr = tryEnsureMountCloneFromBare(bareRepo, localClone, mountClone); lastErr == nil {
+			return
+		}
+		if attempt == maxAttempts {
+			require.NoError(t, lastErr, "git clone %s %s failed after %d recovery attempts", bareRepo, mountClone, maxAttempts)
+		}
+		t.Logf("clone recovery attempt %d: %v — removing clone for re-create", attempt, lastErr)
+		os.RemoveAll(mountClone)
+		time.Sleep(2 * time.Second)
+	}
+}
+
+func tryEnsureMountCloneFromBare(bareRepo, localClone, mountClone string) error {
+	if err := tryEnsureBareRepo(bareRepo, localClone); err != nil {
+		return fmt.Errorf("ensure bare repo: %w", err)
+	}
+	if err := tryEnsureMountClone(bareRepo, mountClone); err != nil {
+		return fmt.Errorf("ensure mount clone: %w", err)
+	}
+	if _, err := tryGitCommand(mountClone, "rev-parse", "HEAD"); err != nil {
+		return fmt.Errorf("verify mount clone: %w", err)
+	}
+	return nil
+}
+
+// tryEnsureMountClone is like ensureMountClone but returns an error instead
+// of failing the test, for use in recovery loops.
+func tryEnsureMountClone(bareRepo, mountClone string) error {
+	// Verify .git/HEAD exists — just checking the top-level dir is
+	// insufficient because FUSE may cache a stale directory entry.
+	if _, err := os.Stat(filepath.Join(mountClone, ".git", "HEAD")); err == nil {
+		return nil
+	}
+	os.RemoveAll(mountClone)
+	time.Sleep(500 * time.Millisecond)
+	if _, err := tryGitCommand("", "clone", bareRepo, mountClone); err != nil {
+		return fmt.Errorf("re-clone: %w", err)
+	}
+	return nil
+}
+
+// tryGitCommand runs a git command and returns (output, error) without
+// failing the test, for use in recovery loops.
+func tryGitCommand(dir string, args ...string) (string, error) {
+	cmd := exec.Command("git", args...)
+	if dir != "" {
+		cmd.Dir = dir
+	}
+	out, err := cmd.CombinedOutput()
+	if err != nil {
+		return strings.TrimSpace(string(out)), fmt.Errorf("%s: %w", strings.TrimSpace(string(out)), err)
+	}
+	return strings.TrimSpace(string(out)), nil
+}
+
+// pullFromCommitWithRecovery resets to fromCommit and runs git pull. If the
+// FUSE mount loses directories (both the bare repo and the working clone can
+// vanish after heavy git operations), it re-creates them and retries.
+func pullFromCommitWithRecovery(t *testing.T, bareRepo, localClone, cloneDir, fromCommit string) {
+	t.Helper()
+	const maxAttempts = 3
+	var lastErr error
+	for attempt := 1; attempt <= maxAttempts; attempt++ {
+		if lastErr = tryPullFromCommit(t, bareRepo, localClone, cloneDir, fromCommit); lastErr == nil {
+			return
+		}
+		if attempt == maxAttempts {
+			require.NoError(t, lastErr, "git pull failed after %d recovery attempts", maxAttempts)
+		}
+		t.Logf("pull recovery attempt %d: %v — removing clone for re-create", attempt, lastErr)
+		os.RemoveAll(cloneDir)
+		time.Sleep(2 * time.Second)
+	}
+}
+
+// resetToCommitWithRecovery resets the on-mount clone to a given commit. If
+// the FUSE mount loses the directory after a failed reset (the kernel dcache
+// can drop the entry after "Could not write new index file"), it re-clones
+// from the bare repo and retries.
+func resetToCommitWithRecovery(t *testing.T, bareRepo, localClone, mountClone, commit string) {
+	t.Helper()
+	const maxAttempts = 3
+	var lastErr error
+	for attempt := 1; attempt <= maxAttempts; attempt++ {
+		if err := tryEnsureBareRepo(bareRepo, localClone); err != nil {
+			lastErr = err
+			t.Logf("reset recovery attempt %d: ensure bare repo: %v", attempt, err)
+			time.Sleep(2 * time.Second)
+			continue
+		}
+		if err := tryEnsureMountClone(bareRepo, mountClone); err != nil {
+			lastErr = err
+			t.Logf("reset recovery attempt %d: ensure mount clone: %v", attempt, err)
+			time.Sleep(2 * time.Second)
+			continue
+		}
+		if _, err := tryGitCommand(mountClone, "reset", "--hard", commit); err != nil {
+			lastErr = err
+			if attempt < maxAttempts {
+				t.Logf("reset recovery attempt %d: %v — removing clone for re-create", attempt, err)
+				os.RemoveAll(mountClone)
+				time.Sleep(2 * time.Second)
+			}
+			continue
+		}
+		if !waitForDirEventually(t, mountClone, 5*time.Second) {
+			lastErr = fmt.Errorf("clone dir %s did not recover after reset", mountClone)
+			if attempt < maxAttempts {
+				t.Logf("reset recovery attempt %d: %v", attempt, lastErr)
+				os.RemoveAll(mountClone)
+				time.Sleep(2 * time.Second)
+			}
+			continue
+		}
+		// Verify git commands actually work in the directory — the kernel
+		// dcache can transiently show the dir then lose it after reset.
+		if _, err := tryGitCommand(mountClone, "rev-parse", "HEAD"); err != nil {
+			lastErr = fmt.Errorf("post-reset verification failed: %w", err)
+			if attempt < maxAttempts {
+				t.Logf("reset recovery attempt %d: %v — removing clone for re-create", attempt, lastErr)
+				os.RemoveAll(mountClone)
+				time.Sleep(2 * time.Second)
+			}
+			continue
+		}
+		// The kernel dcache can drop the FUSE entry moments after a
+		// successful check.  Wait briefly and re-verify to confirm the
+		// directory has stabilised before returning to the caller.
+		time.Sleep(1 * time.Second)
+		refreshDirEntry(t, mountClone)
+		if _, err := tryGitCommand(mountClone, "rev-parse", "HEAD"); err != nil {
+			lastErr = fmt.Errorf("post-reset stabilisation check failed: %w", err)
+			if attempt < maxAttempts {
+				t.Logf("reset recovery attempt %d: %v — removing clone for re-create", attempt, lastErr)
+				os.RemoveAll(mountClone)
+				time.Sleep(2 * time.Second)
+			}
+			continue
+		}
+		return
+	}
+	require.NoError(t, lastErr, "git reset --hard %s failed after %d recovery attempts", commit, maxAttempts)
+}
+
+func tryPullFromCommit(t *testing.T, bareRepo, localClone, cloneDir, fromCommit string) error {
+	t.Helper()
+	// The bare repo lives on the FUSE mount and can also vanish.
+	// Re-create it from the local clone (which is on local disk).
+	if err := tryEnsureBareRepo(bareRepo, localClone); err != nil {
+		return err
+	}
+	if err := tryEnsureMountClone(bareRepo, cloneDir); err != nil {
+		return err
+	}
+	if !waitForDirEventually(t, cloneDir, 10*time.Second) {
+		return fmt.Errorf("clone dir %s did not appear", cloneDir)
+	}
+
+	if _, err := tryGitCommand(cloneDir, "reset", "--hard", fromCommit); err != nil {
+		return fmt.Errorf("reset --hard: %w", err)
+	}
+	if !waitForDirEventually(t, cloneDir, 5*time.Second) {
+		return fmt.Errorf("clone dir %s did not recover after reset", cloneDir)
+	}
+	refreshDirEntry(t, cloneDir)
+
+	// Let the dcache stabilise before proceeding.
+	time.Sleep(1 * time.Second)
+	refreshDirEntry(t, cloneDir)
+
+	head, err := tryGitCommand(cloneDir, "rev-parse", "HEAD")
+	if err != nil {
+		return fmt.Errorf("rev-parse after reset: %w", err)
+	}
+	if head != fromCommit {
+		return fmt.Errorf("expected HEAD at %s after reset, got %s", fromCommit, head)
+	}
+
+	if _, err := tryGitCommand(cloneDir, "pull"); err != nil {
+		return fmt.Errorf("pull: %w", err)
+	}
+	return nil
+}
+
+var gitRepoPathRe = regexp.MustCompile(`'([^']+)' does not appear to be a git repository`)
+
+func extractGitRepoPath(output string) string {
+	if match := gitRepoPathRe.FindStringSubmatch(output); len(match) > 1 {
+		return match[1]
+	}
+	return ""
+}
+
+func leftPad(n, width int) string {
+	s := strconv.Itoa(n)
+	for len(s) < width {
+		s = "0" + s
+	}
+	return s
+}
+
+func TestTryEnsureBareRepoPreservesCurrentBranch(t *testing.T) {
+	tempDir, err := os.MkdirTemp("", "git_bare_recovery_")
+	require.NoError(t, err)
+	defer os.RemoveAll(tempDir)
+
+	bareRepo := filepath.Join(tempDir, "repo.git")
+	localClone := filepath.Join(tempDir, "clone")
+	restoredClone := filepath.Join(tempDir, "restored")
+
+	gitRun(t, "", "init", "--bare", bareRepo)
+	gitRun(t, "", "clone", bareRepo, localClone)
+	gitRun(t, localClone, "config", "user.email", "test@seaweedfs.test")
+	gitRun(t, localClone, "config", "user.name", "Test")
+
+	writeFile(t, localClone, "README.md", "hello recovery\n")
+	gitRun(t, localClone, "add", "README.md")
+	gitRun(t, localClone, "commit", "-m", "initial commit")
+
+	branch := gitOutput(t, localClone, "rev-parse", "--abbrev-ref", "HEAD")
+	require.NotEmpty(t, branch)
+
+	require.NoError(t, os.RemoveAll(bareRepo))
+	require.NoError(t, tryEnsureBareRepo(bareRepo, localClone))
+
+	headRef := gitOutput(t, "", "--git-dir="+bareRepo, "symbolic-ref", "--short", "HEAD")
+	assert.Equal(t, branch, headRef, "recovered bare repo should keep the current branch as HEAD")
+
+	gitRun(t, "", "clone", bareRepo, restoredClone)
+	restoredHead := gitOutput(t, restoredClone, "rev-parse", "--abbrev-ref", "HEAD")
+	assert.Equal(t, branch, restoredHead, "clone from recovered bare repo should check out the current branch")
+	assertFileContains(t, filepath.Join(restoredClone, "README.md"), "hello recovery")
+}
+
+func TestEnsureMountCloneFromBareWithRecoveryRecreatesMissingBareRepo(t *testing.T) {
+	tempDir, err := os.MkdirTemp("", "git_mount_clone_recovery_")
+	require.NoError(t, err)
+	defer os.RemoveAll(tempDir)
+
+	bareRepo := filepath.Join(tempDir, "repo.git")
+	localClone := filepath.Join(tempDir, "clone")
+	mountClone := filepath.Join(tempDir, "mount-clone")
+
+	gitRun(t, "", "init", "--bare", bareRepo)
+	gitRun(t, "", "clone", bareRepo, localClone)
+	gitRun(t, localClone, "config", "user.email", "test@seaweedfs.test")
+	gitRun(t, localClone, "config", "user.name", "Test")
+
+	writeFile(t, localClone, "README.md", "hello clone recovery\n")
+	gitRun(t, localClone, "add", "README.md")
+	gitRun(t, localClone, "commit", "-m", "initial commit")
+
+	branch := gitOutput(t, localClone, "rev-parse", "--abbrev-ref", "HEAD")
+	gitRun(t, localClone, "push", "origin", branch)
+
+	require.NoError(t, os.RemoveAll(bareRepo))
+
+	ensureMountCloneFromBareWithRecovery(t, bareRepo, localClone, mountClone)
+
+	head := gitOutput(t, mountClone, "rev-parse", "--abbrev-ref", "HEAD")
+	assert.Equal(t, branch, head, "recovered clone should stay on the pushed branch")
+	assertFileContains(t, filepath.Join(mountClone, "README.md"), "hello clone recovery")
+}

test/fuse_integration/go.mod

@@ -2,7 +2,10 @@ module seaweedfs-fuse-tests
 
 go 1.21
 
-require github.com/stretchr/testify v1.8.4
+require (
+	github.com/stretchr/testify v1.8.4
+	golang.org/x/sys v0.28.0
+)
 
 require (
 	github.com/davecgh/go-spew v1.1.1 // indirect

test/fuse_integration/go.sum

@@ -4,6 +4,8 @@ github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZb
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
 github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk=
 github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo=
+golang.org/x/sys v0.28.0 h1:Fksou7UEQUWlKvIdsqzJmUmCX3cZuD2+P3XyyzwMhlA=
+golang.org/x/sys v0.28.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=

test/fuse_integration/mmap_unix.go

@@ -0,0 +1,55 @@
+//go:build unix
+
+package fuse
+
+import (
+	"syscall"
+
+	"golang.org/x/sys/unix"
+)
+
+// Memory mapping support for Unix-like systems
+
+func mmapFile(fd int, offset int64, length int, prot int, flags int) ([]byte, error) {
+	return unix.Mmap(fd, offset, length, prot, flags)
+}
+
+func munmapFile(data []byte) error {
+	if len(data) == 0 {
+		return nil
+	}
+	return unix.Munmap(data)
+}
+
+func msyncFile(data []byte, flags int) error {
+	if len(data) == 0 {
+		return nil
+	}
+	return unix.Msync(data, flags)
+}
+
+func isMmapSupported() bool {
+	return true
+}
+
+// Memory protection flags
+const (
+	PROT_READ  = syscall.PROT_READ
+	PROT_WRITE = syscall.PROT_WRITE
+	PROT_EXEC  = syscall.PROT_EXEC
+	PROT_NONE  = syscall.PROT_NONE
+)
+
+// Memory mapping flags
+const (
+	MAP_SHARED    = syscall.MAP_SHARED
+	MAP_PRIVATE   = syscall.MAP_PRIVATE
+	MAP_ANONYMOUS = syscall.MAP_ANON
+)
+
+// Memory sync flags
+const (
+	MS_ASYNC      = syscall.MS_ASYNC
+	MS_SYNC       = syscall.MS_SYNC
+	MS_INVALIDATE = syscall.MS_INVALIDATE
+)

test/fuse_integration/mmap_unsupported.go

@@ -0,0 +1,47 @@
+//go:build !unix
+
+package fuse
+
+import (
+	"errors"
+)
+
+// Memory mapping support for unsupported platforms
+
+var ErrMmapNotSupported = errors.New("memory mapping not supported on this platform")
+
+func mmapFile(fd int, offset int64, length int, prot int, flags int) ([]byte, error) {
+	return nil, ErrMmapNotSupported
+}
+
+func munmapFile(data []byte) error {
+	return ErrMmapNotSupported
+}
+
+func msyncFile(data []byte, flags int) error {
+	return ErrMmapNotSupported
+}
+
+func isMmapSupported() bool {
+	return false
+}
+
+// Dummy constants for unsupported platforms
+const (
+	PROT_READ  = 0x1
+	PROT_WRITE = 0x2
+	PROT_EXEC  = 0x4
+	PROT_NONE  = 0x0
+)
+
+const (
+	MAP_SHARED    = 0x01
+	MAP_PRIVATE   = 0x02
+	MAP_ANONYMOUS = 0x20
+)
+
+const (
+	MS_ASYNC      = 0x1
+	MS_SYNC       = 0x4
+	MS_INVALIDATE = 0x2
+)

test/fuse_integration/mounted_linux_test.go

@@ -0,0 +1,27 @@
+//go:build linux
+
+package fuse
+
+import "testing"
+
+func TestMountedAt(t *testing.T) {
+	for _, p := range []string{"/proc", "/proc/"} {
+		if !mountedAt(p) {
+			t.Errorf("mountedAt(%q) = false, want true", p)
+		}
+	}
+	plain := t.TempDir()
+	for _, p := range []string{plain, plain + "/"} {
+		if mountedAt(p) {
+			t.Errorf("mountedAt(%q) = true, want false", p)
+		}
+	}
+}
+
+func TestFuseFilesystem(t *testing.T) {
+	for _, p := range []string{"/proc", t.TempDir()} {
+		if fuseFilesystem(p) {
+			t.Errorf("fuseFilesystem(%q) = true, want false", p)
+		}
+	}
+}

test/fuse_integration/mounted_unix.go

@@ -0,0 +1,33 @@
+//go:build unix
+
+package fuse
+
+import (
+	"path/filepath"
+	"syscall"
+)
+
+// deviceID returns the device ID of the filesystem containing path.
+func deviceID(path string) (uint64, error) {
+	var st syscall.Stat_t
+	if err := syscall.Stat(path, &st); err != nil {
+		return 0, err
+	}
+	return uint64(st.Dev), nil
+}
+
+// mountedAt reports whether something is actually mounted at path: a plain
+// directory shares its parent's device, a mount point does not. The path is
+// cleaned first so a trailing slash does not make Dir return the mount itself.
+func mountedAt(path string) bool {
+	path = filepath.Clean(path)
+	dev, err := deviceID(path)
+	if err != nil {
+		return false
+	}
+	parentDev, err := deviceID(filepath.Dir(path))
+	if err != nil {
+		return false
+	}
+	return dev != parentDev
+}

test/fuse_integration/mounted_unsupported.go

@@ -0,0 +1,16 @@
+//go:build !unix && !windows
+
+package fuse
+
+// FUSE mounts are unix-only: reporting one device keeps waitForMount from
+// mistaking the plain pre-mount directory for a live mount on platforms
+// where a mount can never appear.
+func deviceID(path string) (uint64, error) {
+	return 0, nil
+}
+
+// mountedAt cannot be detected off unix; report mounted so an external
+// TEST_MOUNT_POINT keeps its best-effort write verification.
+func mountedAt(path string) bool {
+	return true
+}

test/fuse_integration/mounted_windows.go

@@ -0,0 +1,40 @@
+//go:build windows
+
+package fuse
+
+import (
+	"os"
+	"path/filepath"
+	"syscall"
+)
+
+// deviceID has no portable meaning here: reporting one device keeps
+// waitForMount from mistaking the plain pre-mount directory for a live mount.
+func deviceID(path string) (uint64, error) {
+	return 0, nil
+}
+
+// mountedAt reports whether path is a WinFsp mount: a directory mount is a
+// reparse point and a drive mount is a volume root, while a plain directory
+// is neither.
+func mountedAt(path string) bool {
+	abs, err := filepath.Abs(path)
+	if err != nil {
+		return false
+	}
+	if filepath.Dir(abs) == abs {
+		return true
+	}
+	info, err := os.Lstat(abs)
+	if err != nil {
+		return false
+	}
+	data, ok := info.Sys().(*syscall.Win32FileAttributeData)
+	return ok && data.FileAttributes&syscall.FILE_ATTRIBUTE_REPARSE_POINT != 0
+}
+
+// fuseFilesystem is covered by the WinFsp checks in mountedAt: WinFsp is the
+// only FUSE provider on the platform.
+func fuseFilesystem(path string) bool {
+	return true
+}

test/fuse_integration/posix_Makefile

@@ -0,0 +1,531 @@
+# SeaweedFS POSIX Compliance Testing Makefile
+
+SHELL := /bin/bash
+# go test is piped through tee everywhere; without pipefail a failing suite
+# still exits 0 because only tee's status propagates.
+.SHELLFLAGS := -o pipefail -c
+
+# Configuration
+WEED_BINARY := $(shell which weed 2>/dev/null || echo "../../weed")
+GO_VERSION := 1.21
+TEST_TIMEOUT := 45m
+COVERAGE_FILE := posix_coverage.out
+REPORT_DIR := reports
+EXTERNAL_TOOLS_DIR := external_tools
+MOUNT_POINT ?= /tmp/seaweedfs_mount
+ifneq ($(TEST_MOUNT_POINT),)
+MOUNT_POINT := $(TEST_MOUNT_POINT)
+endif
+
+# Go test command prefix for external mount points
+GO_TEST_PREFIX :=
+ifneq ($(TEST_MOUNT_POINT),)
+GO_TEST_PREFIX := TEST_MOUNT_POINT="$(TEST_MOUNT_POINT)" TEST_SKIP_CLUSTER_SETUP="true"
+endif
+
+# Test categories
+POSIX_BASIC_TESTS := posix_compliance_test.go
+POSIX_EXTENDED_TESTS := posix_extended_test.go  
+POSIX_EXTERNAL_TESTS := posix_external_test.go
+
+# -f is not propagated through MAKEFLAGS, so recursive calls must name this
+# file explicitly or the sub-make reads the default Makefile instead.
+MAKE_SELF := $(MAKE) -f $(firstword $(MAKEFILE_LIST))
+
+# Colors for output — bash echo does not expand \033, so bake the escapes in
+RED := $(shell printf '\033[31m')
+GREEN := $(shell printf '\033[32m')
+YELLOW := $(shell printf '\033[33m')
+BLUE := $(shell printf '\033[34m')
+MAGENTA := $(shell printf '\033[35m')
+CYAN := $(shell printf '\033[36m')
+WHITE := $(shell printf '\033[37m')
+RESET := $(shell printf '\033[0m')
+
+.DEFAULT_GOAL := help
+
+# Prerequisites checks
+check-binary:
+	@if [ ! -f "$(WEED_BINARY)" ]; then \
+		echo "$(RED)❌ SeaweedFS binary not found at $(WEED_BINARY)$(RESET)"; \
+		echo "   Please run 'make' in the root directory first"; \
+		exit 1; \
+	fi
+	@echo "$(GREEN)[OK] SeaweedFS binary found at $(WEED_BINARY)$(RESET)"
+
+check-fuse:
+	@if command -v fusermount >/dev/null 2>&1; then \
+		echo "$(GREEN)[OK] FUSE is installed (Linux)$(RESET)"; \
+	elif command -v umount >/dev/null 2>&1 && [ "$$(uname)" = "Darwin" ]; then \
+		echo "$(GREEN)[OK] FUSE is available (macOS)$(RESET)"; \
+	else \
+		echo "$(RED)❌ FUSE not found. Please install:$(RESET)"; \
+		echo "   Ubuntu/Debian: sudo apt-get install fuse"; \
+		echo "   CentOS/RHEL:   sudo yum install fuse"; \
+		echo "   macOS:         brew install macfuse"; \
+		exit 1; \
+	fi
+
+check-go:
+	@go version | grep -Eq "go(1\.(2[1-9]|[3-9][0-9]+)|[2-9]\.)" || \
+	(echo "$(RED)❌ Go $(GO_VERSION)+ required. Current: $$(go version)$(RESET)" && exit 1)
+	@echo "$(GREEN)[OK] Go version check passed$(RESET)"
+
+check-prereqs: check-go check-fuse check-binary
+	@echo "$(GREEN)[OK] All prerequisites satisfied$(RESET)"
+
+# Setup and initialization
+
+
+setup-reports:
+	@mkdir -p $(REPORT_DIR)
+	@mkdir -p $(EXTERNAL_TOOLS_DIR)
+
+setup-external-tools: setup-reports
+	@echo "$(BLUE)🛠️  Setting up external POSIX test tools...$(RESET)"
+	@$(MAKE_SELF) setup-pjdfstest || echo "$(YELLOW)[WARNING] pjdfstest setup failed - continuing without it$(RESET)"
+	@$(MAKE_SELF) setup-nfstest || echo "$(YELLOW)[WARNING] nfstest setup failed - continuing without it$(RESET)"
+	@$(MAKE_SELF) setup-fio || echo "$(YELLOW)[WARNING] FIO setup failed - continuing without it$(RESET)"
+
+# External tools setup
+setup-pjdfstest:
+	@if [ ! -d "$(EXTERNAL_TOOLS_DIR)/pjdfstest" ]; then \
+		echo "$(BLUE)[SETUP] Setting up pjdfstest...$(RESET)"; \
+		if ! command -v autoreconf >/dev/null 2>&1; then \
+			echo "$(YELLOW)[WARNING] autoreconf not found - installing build-essential...$(RESET)"; \
+			if command -v apt-get >/dev/null 2>&1; then \
+				sudo apt-get update && sudo apt-get install -y build-essential autoconf automake libtool; \
+			elif command -v yum >/dev/null 2>&1; then \
+				sudo yum install -y gcc make autoconf automake libtool; \
+			else \
+				echo "$(YELLOW)[WARNING] Please install build tools manually$(RESET)"; \
+				exit 1; \
+			fi; \
+		fi; \
+		cd $(EXTERNAL_TOOLS_DIR) && \
+		git clone https://github.com/pjd/pjdfstest.git && \
+		cd pjdfstest && \
+		autoreconf -ifs && \
+		./configure && \
+		make; \
+	else \
+		echo "$(GREEN)[OK] pjdfstest already setup$(RESET)"; \
+	fi
+
+setup-nfstest:
+	@echo "$(BLUE)[INSTALL] Installing nfstest...$(RESET)"
+	@pip3 install --user nfstest || \
+	echo "$(YELLOW)[WARNING] nfstest installation failed. Install manually: pip3 install nfstest$(RESET)"
+
+setup-fio:
+	@if ! command -v fio >/dev/null 2>&1; then \
+		echo "$(BLUE)[SETUP] Installing FIO...$(RESET)"; \
+		if command -v apt-get >/dev/null 2>&1; then \
+			sudo apt-get update && sudo apt-get install -y fio; \
+		elif command -v yum >/dev/null 2>&1; then \
+			sudo yum install -y fio; \
+		elif command -v brew >/dev/null 2>&1; then \
+			brew install fio; \
+		else \
+			echo "$(YELLOW)[WARNING] Please install FIO manually$(RESET)"; \
+		fi; \
+	else \
+		echo "$(GREEN)[OK] FIO already installed$(RESET)"; \
+	fi
+
+# Core test execution
+test-posix-basic: check-prereqs setup-reports
+	@echo "$(CYAN)[TEST] Running basic POSIX compliance tests...$(RESET)"
+ifneq ($(TEST_MOUNT_POINT),)
+	@echo "$(BLUE)Using external mount point: $(TEST_MOUNT_POINT)$(RESET)"
+endif
+	@$(GO_TEST_PREFIX) go test -v -timeout $(TEST_TIMEOUT) -failfast=false -run TestPOSIXCompliance . 2>&1 | \
+		tee $(REPORT_DIR)/posix_basic_results.log
+
+test-posix-extended: check-prereqs setup-reports
+	@echo "$(CYAN)[TEST] Running extended POSIX compliance tests...$(RESET)"
+ifneq ($(TEST_MOUNT_POINT),)
+	@echo "$(BLUE)Using external mount point: $(TEST_MOUNT_POINT)$(RESET)"
+endif
+	@$(GO_TEST_PREFIX) go test -v -timeout $(TEST_TIMEOUT) -failfast=false -run TestPOSIXExtended . 2>&1 | \
+		tee $(REPORT_DIR)/posix_extended_results.log
+
+test-posix-external: check-prereqs setup-reports
+	@echo "$(CYAN)[TEST] Running external POSIX test suite integration...$(RESET)"
+ifneq ($(TEST_MOUNT_POINT),)
+	@echo "$(BLUE)Using external mount point: $(TEST_MOUNT_POINT)$(RESET)"
+endif
+	@if [ -d "$(EXTERNAL_TOOLS_DIR)/pjdfstest" ] || command -v nfstest_posix >/dev/null 2>&1; then \
+		echo "$(GREEN)[OK] External tools available - running external tests$(RESET)"; \
+		$(GO_TEST_PREFIX) go test -v -timeout $(TEST_TIMEOUT) -failfast=false -run TestExternalPOSIXSuites . 2>&1 | \
+			tee $(REPORT_DIR)/posix_external_results.log; \
+	else \
+		echo "$(YELLOW)[WARNING] External tools not available - skipping external tests$(RESET)"; \
+		echo "External tests skipped - tools not available" > $(REPORT_DIR)/posix_external_results.log; \
+	fi
+
+# Comprehensive test suites
+test-posix-full: test-posix-basic test-posix-extended test-posix-external
+	@echo "$(GREEN)[OK] Full POSIX compliance test suite completed$(RESET)"
+	@$(MAKE_SELF) generate-report
+
+test-posix-critical: check-prereqs setup-reports
+	@echo "$(CYAN)[TEST] Running critical POSIX compliance tests...$(RESET)"
+ifneq ($(TEST_MOUNT_POINT),)
+	@echo "$(BLUE)Using external mount point: $(TEST_MOUNT_POINT)$(RESET)"
+endif
+	@$(GO_TEST_PREFIX) go test -v -timeout 15m -failfast=false \
+		-run "TestPOSIXCompliance/(FileOperations|DirectoryOperations|PermissionTests|IOOperations)" \
+		. 2>&1 | tee $(REPORT_DIR)/posix_critical_results.log
+
+test-posix-stress: check-prereqs setup-reports
+	@echo "$(CYAN)[TEST] Running POSIX stress tests...$(RESET)"
+ifneq ($(TEST_MOUNT_POINT),)
+	@echo "$(BLUE)Using external mount point: $(TEST_MOUNT_POINT)$(RESET)"
+endif
+	@if [ -d "$(EXTERNAL_TOOLS_DIR)/pjdfstest" ] || command -v nfstest_posix >/dev/null 2>&1; then \
+		echo "$(GREEN)[OK] External tools available - running stress tests$(RESET)"; \
+		$(GO_TEST_PREFIX) go test -v -timeout $(TEST_TIMEOUT) -failfast=false \
+			-run "TestExternalPOSIXSuites/CustomPOSIXTests" \
+			. 2>&1 | tee $(REPORT_DIR)/posix_stress_results.log; \
+	else \
+		echo "$(YELLOW)[WARNING] External tools not available - skipping stress tests$(RESET)"; \
+		echo "Stress tests skipped - tools not available" > $(REPORT_DIR)/posix_stress_results.log; \
+	fi
+
+# Performance and benchmarks
+benchmark-posix: check-prereqs setup-reports
+	@echo "$(CYAN)📈 Running POSIX performance benchmarks...$(RESET)"
+ifneq ($(TEST_MOUNT_POINT),)
+	@echo "$(BLUE)Using external mount point: $(TEST_MOUNT_POINT)$(RESET)"
+endif
+	@$(GO_TEST_PREFIX) go test -v -timeout $(TEST_TIMEOUT) -run '^$$' -bench=. -benchmem \
+		. 2>&1 | \
+		tee $(REPORT_DIR)/posix_benchmark_results.log
+
+profile-posix: check-prereqs setup-reports
+	@echo "$(CYAN)[PROFILE] Running POSIX tests with profiling...$(RESET)"
+ifneq ($(TEST_MOUNT_POINT),)
+	@echo "$(BLUE)Using external mount point: $(TEST_MOUNT_POINT)$(RESET)"
+endif
+	@$(GO_TEST_PREFIX) go test -v -timeout $(TEST_TIMEOUT) -cpuprofile $(REPORT_DIR)/posix.cpu.prof \
+		-memprofile $(REPORT_DIR)/posix.mem.prof -run TestPOSIXCompliance .
+	@echo "$(GREEN)[PROFILE] Profiles generated:$(RESET)"
+	@echo "  CPU: $(REPORT_DIR)/posix.cpu.prof"
+	@echo "  Memory: $(REPORT_DIR)/posix.mem.prof"
+	@echo "$(BLUE)View with: go tool pprof $(REPORT_DIR)/posix.cpu.prof$(RESET)"
+
+# Coverage analysis
+coverage-posix: check-prereqs setup-reports
+	@echo "$(CYAN)[COVERAGE] Running POSIX tests with coverage analysis...$(RESET)"
+ifneq ($(TEST_MOUNT_POINT),)
+	@echo "$(BLUE)Using external mount point: $(TEST_MOUNT_POINT)$(RESET)"
+endif
+	@$(GO_TEST_PREFIX) go test -v -timeout $(TEST_TIMEOUT) -coverprofile=$(REPORT_DIR)/$(COVERAGE_FILE) \
+		.
+	@go tool cover -html=$(REPORT_DIR)/$(COVERAGE_FILE) -o $(REPORT_DIR)/posix_coverage.html
+	@echo "$(GREEN)[COVERAGE] Coverage report generated: $(REPORT_DIR)/posix_coverage.html$(RESET)"
+
+# The external suites exercise whatever filesystem MOUNT_POINT resolves to;
+# mkdir -p would happily point them at a plain local directory and report a
+# pass the mount never earned.
+require-mounted:
+	@mnt="$(MOUNT_POINT)"; mnt="$${mnt%/}"; \
+	if ! mountpoint -q "$$mnt" 2>/dev/null && ! mount | grep -qF " $$mnt "; then \
+		echo "$(RED)❌ $$mnt is not a mounted filesystem - refusing to run against the local disk$(RESET)"; \
+		exit 1; \
+	fi
+
+# External tool tests
+test-pjdfstest: setup-pjdfstest setup-reports require-mounted
+	@echo "$(CYAN)[TEST] Running pjdfstest suite...$(RESET)"
+	@if [ -d "$(EXTERNAL_TOOLS_DIR)/pjdfstest" ]; then \
+		mkdir -p $(MOUNT_POINT)/pjdfstest_workdir; \
+		cd $(MOUNT_POINT)/pjdfstest_workdir && \
+		prove -r $(CURDIR)/$(EXTERNAL_TOOLS_DIR)/pjdfstest/tests/ 2>&1 | tee $(CURDIR)/$(REPORT_DIR)/pjdfstest_results.log; \
+	else \
+		echo "$(RED)❌ pjdfstest not setup$(RESET)"; \
+		exit 1; \
+	fi
+
+test-nfstest-posix: setup-nfstest setup-reports require-mounted
+	@echo "$(CYAN)[TEST] Running nfstest_posix...$(RESET)"
+	@if command -v nfstest_posix >/dev/null 2>&1; then \
+		mkdir -p $(MOUNT_POINT); \
+		nfstest_posix --path $(MOUNT_POINT) --verbose 2>&1 | \
+		tee $(REPORT_DIR)/nfstest_results.log; \
+	else \
+					echo "$(YELLOW)[WARNING] nfstest_posix not available$(RESET)"; \
+	fi
+
+# FIO-based performance tests
+test-fio-posix: setup-reports require-mounted
+	@echo "$(CYAN)[TEST] Running FIO-based POSIX I/O tests...$(RESET)"
+	@$(MAKE_SELF) create-fio-configs
+	@if command -v fio >/dev/null 2>&1; then \
+		mkdir -p $(MOUNT_POINT); \
+		for config in $(REPORT_DIR)/fio_*.conf; do \
+			echo "$(BLUE)Running FIO config: $$config$(RESET)"; \
+			fio $$config --output=$(REPORT_DIR)/$$(basename $$config .conf)_results.log; \
+		done; \
+	else \
+					echo "$(YELLOW)[WARNING] FIO not available$(RESET)"; \
+	fi
+
+create-fio-configs: setup-reports
+	@echo "$(BLUE)[CONFIG] Creating FIO test configurations...$(RESET)"
+	@echo "[global]" > $(REPORT_DIR)/fio_random_rw.conf
+	@echo "name=posix_random_rw" >> $(REPORT_DIR)/fio_random_rw.conf
+	@echo "ioengine=sync" >> $(REPORT_DIR)/fio_random_rw.conf
+	@echo "iodepth=1" >> $(REPORT_DIR)/fio_random_rw.conf
+	@echo "rw=randrw" >> $(REPORT_DIR)/fio_random_rw.conf
+	@echo "bs=4k" >> $(REPORT_DIR)/fio_random_rw.conf
+	@echo "direct=0" >> $(REPORT_DIR)/fio_random_rw.conf
+	@echo "size=100m" >> $(REPORT_DIR)/fio_random_rw.conf
+	@echo "numjobs=4" >> $(REPORT_DIR)/fio_random_rw.conf
+	@echo "runtime=30" >> $(REPORT_DIR)/fio_random_rw.conf
+	@echo "time_based" >> $(REPORT_DIR)/fio_random_rw.conf
+	@echo "" >> $(REPORT_DIR)/fio_random_rw.conf
+	@echo "[random_rw_test]" >> $(REPORT_DIR)/fio_random_rw.conf
+	@echo "directory=$(MOUNT_POINT)" >> $(REPORT_DIR)/fio_random_rw.conf
+	@echo "[global]" > $(REPORT_DIR)/fio_sequential.conf
+	@echo "name=posix_sequential" >> $(REPORT_DIR)/fio_sequential.conf
+	@echo "ioengine=sync" >> $(REPORT_DIR)/fio_sequential.conf
+	@echo "iodepth=1" >> $(REPORT_DIR)/fio_sequential.conf
+	@echo "bs=1m" >> $(REPORT_DIR)/fio_sequential.conf
+	@echo "direct=0" >> $(REPORT_DIR)/fio_sequential.conf
+	@echo "size=500m" >> $(REPORT_DIR)/fio_sequential.conf
+	@echo "runtime=60" >> $(REPORT_DIR)/fio_sequential.conf
+	@echo "time_based" >> $(REPORT_DIR)/fio_sequential.conf
+	@echo "" >> $(REPORT_DIR)/fio_sequential.conf
+	@echo "[seq_write]" >> $(REPORT_DIR)/fio_sequential.conf
+	@echo "rw=write" >> $(REPORT_DIR)/fio_sequential.conf
+	@echo "directory=$(MOUNT_POINT)" >> $(REPORT_DIR)/fio_sequential.conf
+	@echo "" >> $(REPORT_DIR)/fio_sequential.conf
+	@echo "[seq_read]" >> $(REPORT_DIR)/fio_sequential.conf
+	@echo "rw=read" >> $(REPORT_DIR)/fio_sequential.conf
+	@echo "directory=$(MOUNT_POINT)" >> $(REPORT_DIR)/fio_sequential.conf
+
+# Reporting and analysis
+generate-report: setup-reports
+	@echo "$(BLUE)[REPORT] Generating comprehensive POSIX compliance report...$(RESET)"
+	@$(MAKE_SELF) generate-html-report
+	@$(MAKE_SELF) generate-text-report
+	@$(MAKE_SELF) generate-json-report
+
+generate-html-report:
+	@echo "$(BLUE)[REPORT] Generating HTML report...$(RESET)"
+	@echo "<!DOCTYPE html>" > $(REPORT_DIR)/posix_compliance_report.html
+	@echo "<html><head><title>SeaweedFS POSIX Compliance Report</title>" >> $(REPORT_DIR)/posix_compliance_report.html
+	@echo "<style>body{font-family:Arial;margin:20px}.pass{color:green}.fail{color:red}.summary{background:#f5f5f5;padding:10px;margin:10px 0}.test-section{margin:20px 0;border-left:3px solid #ddd;padding-left:15px}</style>" >> $(REPORT_DIR)/posix_compliance_report.html
+	@echo "</head><body><h1>SeaweedFS POSIX Compliance Report</h1>" >> $(REPORT_DIR)/posix_compliance_report.html
+	@echo "<p>Generated on: $$(date)</p>" >> $(REPORT_DIR)/posix_compliance_report.html
+	@echo "<div class='summary'>" >> $(REPORT_DIR)/posix_compliance_report.html
+	@echo "<h2>Test Summary</h2>" >> $(REPORT_DIR)/posix_compliance_report.html
+	@if [ -f "$(REPORT_DIR)/posix_basic_results.log" ]; then \
+		TOTAL=$$(awk '/RUN|PASS|FAIL/{c++} END{print c+0}' $(REPORT_DIR)/posix_basic_results.log 2>/dev/null || echo 0); \
+		PASSED=$$(awk '/PASS:/{c++} END{print c+0}' $(REPORT_DIR)/posix_basic_results.log 2>/dev/null || echo 0); \
+		FAILED=$$(awk '/FAIL:/{c++} END{print c+0}' $(REPORT_DIR)/posix_basic_results.log 2>/dev/null || echo 0); \
+		echo "<p>Basic Tests: <span class='pass'>$$PASSED passed</span>, <span class='fail'>$$FAILED failed</span></p>" >> $(REPORT_DIR)/posix_compliance_report.html; \
+	fi
+	@if [ -f "$(REPORT_DIR)/posix_critical_results.log" ]; then \
+		TOTAL=$$(awk '/RUN|PASS|FAIL/{c++} END{print c+0}' $(REPORT_DIR)/posix_critical_results.log 2>/dev/null || echo 0); \
+		PASSED=$$(awk '/PASS:/{c++} END{print c+0}' $(REPORT_DIR)/posix_critical_results.log 2>/dev/null || echo 0); \
+		FAILED=$$(awk '/FAIL:/{c++} END{print c+0}' $(REPORT_DIR)/posix_critical_results.log 2>/dev/null || echo 0); \
+		echo "<p>Critical Tests: <span class='pass'>$$PASSED passed</span>, <span class='fail'>$$FAILED failed</span></p>" >> $(REPORT_DIR)/posix_compliance_report.html; \
+	fi
+	@echo "</div>" >> $(REPORT_DIR)/posix_compliance_report.html
+	@echo "<h2>Detailed Results</h2>" >> $(REPORT_DIR)/posix_compliance_report.html
+	@for logfile in $(REPORT_DIR)/*.log; do \
+		if [ -f "$$logfile" ]; then \
+			basename=$$(basename "$$logfile" .log); \
+			echo "<div class='test-section'>" >> $(REPORT_DIR)/posix_compliance_report.html; \
+			echo "<h3>$$basename</h3>" >> $(REPORT_DIR)/posix_compliance_report.html; \
+			echo "<pre>" >> $(REPORT_DIR)/posix_compliance_report.html; \
+			tail -50 "$$logfile" | sed 's/&/\&amp;/g; s/</\&lt;/g; s/>/\&gt;/g' >> $(REPORT_DIR)/posix_compliance_report.html; \
+			echo "</pre></div>" >> $(REPORT_DIR)/posix_compliance_report.html; \
+		fi; \
+	done
+	@echo "</body></html>" >> $(REPORT_DIR)/posix_compliance_report.html
+	@echo "$(GREEN)[OK] HTML report generated: $(REPORT_DIR)/posix_compliance_report.html$(RESET)"
+
+generate-text-report:
+	@echo "$(BLUE)[REPORT] Generating text report...$(RESET)"
+	@echo "SeaweedFS POSIX Compliance Report" > $(REPORT_DIR)/posix_compliance_summary.txt
+	@echo "=================================" >> $(REPORT_DIR)/posix_compliance_summary.txt
+	@echo "" >> $(REPORT_DIR)/posix_compliance_summary.txt
+	@echo "Generated: $$(date)" >> $(REPORT_DIR)/posix_compliance_summary.txt
+	@echo "SeaweedFS Version: $$($(WEED_BINARY) version 2>/dev/null || echo 'Unknown')" >> $(REPORT_DIR)/posix_compliance_summary.txt
+	@echo "" >> $(REPORT_DIR)/posix_compliance_summary.txt
+	@echo "Test Summary: See individual log files for detailed results" >> $(REPORT_DIR)/posix_compliance_summary.txt
+	@echo "$(GREEN)[OK] Text report generated: $(REPORT_DIR)/posix_compliance_summary.txt$(RESET)"
+
+generate-json-report:
+	@echo "$(BLUE)[REPORT] Generating JSON report...$(RESET)"
+	@echo "{" > $(REPORT_DIR)/posix_compliance_report.json
+	@echo "  \"report_type\": \"posix_compliance\"," >> $(REPORT_DIR)/posix_compliance_report.json
+	@echo "  \"timestamp\": \"$$(date -u +"%Y-%m-%dT%H:%M:%SZ")\"," >> $(REPORT_DIR)/posix_compliance_report.json
+	@echo "  \"seaweedfs_version\": \"$$($(WEED_BINARY) version 2>/dev/null | head -n1 | sed 's/\\/\\\\/g; s/\"/\\\"/g' || echo 'Unknown')\"," >> $(REPORT_DIR)/posix_compliance_report.json
+	@echo "  \"test_environment\": { \"os\": \"$$(uname -s)\", \"arch\": \"$$(uname -m)\" }," >> $(REPORT_DIR)/posix_compliance_report.json
+	@echo "  \"test_suites\": {" >> $(REPORT_DIR)/posix_compliance_report.json
+	@FIRST=true; \
+	for logfile in $(REPORT_DIR)/*.log; do \
+		if [ -f "$$logfile" ]; then \
+			basename=$$(basename "$$logfile" .log); \
+			if [ "$$FIRST" = "true" ]; then \
+				FIRST=false; \
+			else \
+				echo "    ," >> $(REPORT_DIR)/posix_compliance_report.json; \
+			fi; \
+			PASSED=$$(awk '/PASS:/{c++} END{print c+0}' "$$logfile" 2>/dev/null || echo 0); \
+			FAILED=$$(awk '/FAIL:/{c++} END{print c+0}' "$$logfile" 2>/dev/null || echo 0); \
+			TOTAL=$$(expr "$$PASSED" + "$$FAILED" 2>/dev/null || echo 0); \
+			echo "    \"$$basename\": {" >> $(REPORT_DIR)/posix_compliance_report.json; \
+			echo "      \"total_tests\": $$TOTAL," >> $(REPORT_DIR)/posix_compliance_report.json; \
+			echo "      \"passed\": $$PASSED," >> $(REPORT_DIR)/posix_compliance_report.json; \
+			echo "      \"failed\": $$FAILED," >> $(REPORT_DIR)/posix_compliance_report.json; \
+			if [ "$$TOTAL" -gt 0 ] 2>/dev/null; then \
+				SUCCESS_RATE=$$(awk "BEGIN {printf \"%.2f\", ($$PASSED/$$TOTAL)*100}" 2>/dev/null || echo "0.00"); \
+			else \
+				SUCCESS_RATE="0.00"; \
+			fi; \
+			echo "      \"success_rate\": \"$$SUCCESS_RATE%\"" >> $(REPORT_DIR)/posix_compliance_report.json; \
+			echo "    }" >> $(REPORT_DIR)/posix_compliance_report.json; \
+		fi; \
+	done
+	@echo "  }," >> $(REPORT_DIR)/posix_compliance_report.json
+	@TOTAL_PASSED=0; TOTAL_FAILED=0; \
+	for logfile in $(REPORT_DIR)/*.log; do \
+		if [ -f "$$logfile" ]; then \
+			PASSED=$$(awk '/PASS:/{c++} END{print c+0}' "$$logfile" 2>/dev/null || echo 0); \
+			FAILED=$$(awk '/FAIL:/{c++} END{print c+0}' "$$logfile" 2>/dev/null || echo 0); \
+			TOTAL_PASSED=$$(expr "$$TOTAL_PASSED" + "$$PASSED" 2>/dev/null || echo "$$TOTAL_PASSED"); \
+			TOTAL_FAILED=$$(expr "$$TOTAL_FAILED" + "$$FAILED" 2>/dev/null || echo "$$TOTAL_FAILED"); \
+		fi; \
+	done; \
+	GRAND_TOTAL=$$(expr "$$TOTAL_PASSED" + "$$TOTAL_FAILED" 2>/dev/null || echo 0); \
+	if [ "$$GRAND_TOTAL" -gt 0 ] 2>/dev/null; then \
+		OVERALL_SUCCESS=$$(awk "BEGIN {printf \"%.2f\", ($$TOTAL_PASSED/$$GRAND_TOTAL)*100}" 2>/dev/null || echo "0.00"); \
+	else \
+		OVERALL_SUCCESS="0.00"; \
+	fi; \
+	echo "  \"summary\": {" >> $(REPORT_DIR)/posix_compliance_report.json; \
+	echo "    \"total_tests\": $$GRAND_TOTAL," >> $(REPORT_DIR)/posix_compliance_report.json; \
+	echo "    \"total_passed\": $$TOTAL_PASSED," >> $(REPORT_DIR)/posix_compliance_report.json; \
+	echo "    \"total_failed\": $$TOTAL_FAILED," >> $(REPORT_DIR)/posix_compliance_report.json; \
+	echo "    \"overall_success_rate\": \"$$OVERALL_SUCCESS%\"" >> $(REPORT_DIR)/posix_compliance_report.json; \
+	echo "  }" >> $(REPORT_DIR)/posix_compliance_report.json
+	@echo "}" >> $(REPORT_DIR)/posix_compliance_report.json
+	@echo "$(GREEN)[OK] JSON report generated: $(REPORT_DIR)/posix_compliance_report.json$(RESET)"
+
+# Cleanup and maintenance
+clean:
+	@echo "$(YELLOW)🧹 Cleaning up test artifacts...$(RESET)"
+	@rm -rf $(REPORT_DIR)
+	@rm -rf /tmp/seaweedfs_*_test_*
+	@go clean -testcache
+	@echo "$(GREEN)[OK] Cleanup complete$(RESET)"
+
+clean-external-tools:
+	@echo "$(YELLOW)🧹 Cleaning up external tools...$(RESET)"
+	@rm -rf $(EXTERNAL_TOOLS_DIR)
+
+clean-all: clean clean-external-tools
+
+# Development and debugging
+validate:
+	@echo "$(BLUE)[OK] Validating test files...$(RESET)"
+	@go build -o /dev/null ./...
+	@echo "$(GREEN)[OK] All test files compile successfully$(RESET)"
+
+fmt:
+	@echo "$(BLUE)🎨 Formatting Go code...$(RESET)"
+	@go fmt ./...
+
+lint:
+	@echo "$(BLUE)🔍 Running linter...$(RESET)"
+	@if command -v golangci-lint >/dev/null 2>&1; then \
+		golangci-lint run; \
+	else \
+					echo "$(YELLOW)[WARNING] golangci-lint not found, running go vet instead$(RESET)"; \
+		go vet ./...; \
+	fi
+
+# CI/CD integration
+ci-posix-tests: check-prereqs setup-external-tools
+	@echo "$(CYAN)🚀 Running POSIX tests for CI/CD...$(RESET)"
+	@$(MAKE_SELF) test-posix-critical
+	@$(MAKE_SELF) generate-report
+
+
+
+# Documentation and help
+list-tests:
+	@echo "$(BLUE)📋 Available POSIX test functions:$(RESET)"
+	@grep -r "^func Test" *.go 2>/dev/null | sed 's/.*func \(Test[^(]*\).*/  \1/' | sort || echo "No test files found"
+
+test-info:
+	@echo "$(BLUE)[INFO] POSIX Test Information:$(RESET)"
+	@echo "Test files:"
+	@echo "  - $(POSIX_BASIC_TESTS): Core POSIX compliance tests"
+	@echo "  - $(POSIX_EXTENDED_TESTS): Extended POSIX feature tests"  
+	@echo "  - $(POSIX_EXTERNAL_TESTS): External test suite integration"
+	@echo ""
+	@echo "External tools:"
+	@echo "  - pjdfstest: Comprehensive POSIX filesystem test suite"
+	@echo "  - nfstest: Network filesystem POSIX API verification"
+	@echo "  - FIO: Flexible I/O performance testing"
+
+help:
+	@echo "$(CYAN)SeaweedFS POSIX Compliance Testing$(RESET)"
+	@echo "=================================="
+	@echo ""
+	@echo "$(WHITE)Prerequisites:$(RESET)"
+	@echo "  $(GREEN)make check-prereqs$(RESET)      - Check all prerequisites"
+	@echo "  $(GREEN)make setup-external-tools$(RESET) - Setup external POSIX test tools"
+	@echo ""
+	@echo "$(WHITE)Core POSIX Tests:$(RESET)"
+	@echo "  $(GREEN)make test-posix-basic$(RESET)    - Run basic POSIX compliance tests"
+	@echo "  $(GREEN)make test-posix-extended$(RESET) - Run extended POSIX feature tests"
+	@echo "  $(GREEN)make test-posix-external$(RESET) - Run external test suite integration"
+	@echo "  $(GREEN)make test-posix-full$(RESET)     - Run complete POSIX test suite"
+	@echo ""
+	@echo "$(WHITE)Focused Tests:$(RESET)"
+	@echo "  $(GREEN)make test-posix-critical$(RESET) - Run critical POSIX compliance tests"
+	@echo "  $(GREEN)make test-posix-stress$(RESET)   - Run POSIX stress tests"
+	@echo ""
+	@echo "$(WHITE)Performance & Analysis:$(RESET)"
+	@echo "  $(GREEN)make benchmark-posix$(RESET)     - Run POSIX performance benchmarks"
+	@echo "  $(GREEN)make profile-posix$(RESET)       - Run tests with performance profiling"
+	@echo "  $(GREEN)make coverage-posix$(RESET)      - Run tests with coverage analysis"
+	@echo "  $(GREEN)make test-fio-posix$(RESET)      - Run FIO-based I/O performance tests"
+	@echo ""
+	@echo "$(WHITE)External Test Suites:$(RESET)"
+	@echo "  $(GREEN)make test-pjdfstest$(RESET)      - Run pjdfstest suite"
+	@echo "  $(GREEN)make test-nfstest-posix$(RESET)  - Run nfstest_posix"
+	@echo ""
+	@echo "$(WHITE)Reporting:$(RESET)"
+	@echo "  $(GREEN)make generate-report$(RESET)     - Generate comprehensive compliance report"
+	@echo ""
+	@echo "$(WHITE)Development:$(RESET)"
+	@echo "  $(GREEN)make validate$(RESET)            - Validate test file compilation"
+	@echo "  $(GREEN)make fmt$(RESET)                 - Format Go code"
+	@echo "  $(GREEN)make lint$(RESET)                - Run linter"
+	@echo "  $(GREEN)make list-tests$(RESET)          - List available test functions"
+	@echo ""
+	@echo "$(WHITE)Maintenance:$(RESET)"
+	@echo "  $(GREEN)make clean$(RESET)               - Clean up test artifacts"
+	@echo "  $(GREEN)make clean-all$(RESET)           - Clean everything including external tools"
+	@echo ""
+	@echo "$(WHITE)CI/CD Integration:$(RESET)"
+	@echo "  $(GREEN)make ci-posix-tests$(RESET)      - Run POSIX tests optimized for CI/CD"
+
+
+.PHONY: help check-prereqs check-binary check-fuse check-go setup-reports \
+        setup-external-tools setup-pjdfstest setup-nfstest setup-fio \
+        test-posix-basic test-posix-extended test-posix-external test-posix-full \
+        test-posix-critical test-posix-stress benchmark-posix profile-posix coverage-posix \
+        test-pjdfstest test-nfstest-posix test-fio-posix create-fio-configs \
+        generate-report generate-html-report generate-text-report generate-json-report \
+        clean clean-external-tools clean-all validate fmt lint ci-posix-tests \
+        list-tests test-info

test/fuse_integration/posix_compliance_test.go

@@ -0,0 +1,713 @@
+//go:build !windows
+
+package fuse
+
+import (
+	"fmt"
+	"os"
+	"path/filepath"
+	"syscall"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/require"
+)
+
+// POSIXComplianceTestSuite provides comprehensive POSIX compliance testing for FUSE mounts
+type POSIXComplianceTestSuite struct {
+	framework *FuseTestFramework
+	t         *testing.T
+}
+
+// NewPOSIXComplianceTestSuite creates a new POSIX compliance test suite
+func NewPOSIXComplianceTestSuite(t *testing.T, framework *FuseTestFramework) *POSIXComplianceTestSuite {
+	return &POSIXComplianceTestSuite{
+		framework: framework,
+		t:         t,
+	}
+}
+
+// TestPOSIXCompliance runs all POSIX compliance tests
+func TestPOSIXCompliance(t *testing.T) {
+	config := DefaultTestConfig()
+	config.EnableDebug = true
+	config.MountOptions = []string{"-allowOthers", "-nonempty"}
+
+	framework := NewFuseTestFramework(t, config)
+	defer framework.Cleanup()
+	require.NoError(t, framework.Setup(config))
+
+	suite := NewPOSIXComplianceTestSuite(t, framework)
+
+	// Run all POSIX compliance test categories
+	t.Run("FileOperations", suite.TestFileOperations)
+	t.Run("DirectoryOperations", suite.TestDirectoryOperations)
+	t.Run("SymlinkOperations", suite.TestSymlinkOperations)
+	t.Run("PermissionTests", suite.TestPermissions)
+	t.Run("TimestampTests", suite.TestTimestamps)
+	t.Run("IOOperations", suite.TestIOOperations)
+	t.Run("FileDescriptorTests", suite.TestFileDescriptors)
+	t.Run("AtomicOperations", suite.TestAtomicOperations)
+	t.Run("ConcurrentAccess", suite.TestConcurrentAccess)
+	t.Run("ErrorHandling", suite.TestErrorHandling)
+}
+
+// TestFileOperations tests POSIX file operation compliance
+func (s *POSIXComplianceTestSuite) TestFileOperations(t *testing.T) {
+	mountPoint := s.framework.GetMountPoint()
+
+	t.Run("CreateFile", func(t *testing.T) {
+		filepath := filepath.Join(mountPoint, "test_create.txt")
+
+		// Test file creation with O_CREAT
+		fd, err := syscall.Open(filepath, syscall.O_CREAT|syscall.O_WRONLY, 0644)
+		require.NoError(t, err)
+		require.Greater(t, fd, 0)
+
+		err = syscall.Close(fd)
+		require.NoError(t, err)
+
+		// Verify file exists
+		_, err = os.Stat(filepath)
+		require.NoError(t, err)
+	})
+
+	t.Run("CreateExclusiveFile", func(t *testing.T) {
+		filepath := filepath.Join(mountPoint, "test_excl.txt")
+
+		// First creation should succeed
+		fd, err := syscall.Open(filepath, syscall.O_CREAT|syscall.O_EXCL|syscall.O_WRONLY, 0644)
+		require.NoError(t, err)
+		syscall.Close(fd)
+
+		// Second creation should fail with EEXIST
+		_, err = syscall.Open(filepath, syscall.O_CREAT|syscall.O_EXCL|syscall.O_WRONLY, 0644)
+		require.Error(t, err)
+		require.Equal(t, syscall.EEXIST, err)
+	})
+
+	t.Run("TruncateFile", func(t *testing.T) {
+		filepath := filepath.Join(mountPoint, "test_truncate.txt")
+		content := []byte("Hello, World! This is a test file for truncation.")
+
+		// Create file with content
+		err := os.WriteFile(filepath, content, 0644)
+		require.NoError(t, err)
+
+		// Truncate to 5 bytes
+		err = syscall.Truncate(filepath, 5)
+		require.NoError(t, err)
+
+		// Verify truncation
+		readContent, err := os.ReadFile(filepath)
+		require.NoError(t, err)
+		require.Equal(t, []byte("Hello"), readContent)
+	})
+
+	t.Run("UnlinkFile", func(t *testing.T) {
+		filepath := filepath.Join(mountPoint, "test_unlink.txt")
+
+		// Create file
+		err := os.WriteFile(filepath, []byte("test"), 0644)
+		require.NoError(t, err)
+
+		// Unlink file
+		err = syscall.Unlink(filepath)
+		require.NoError(t, err)
+
+		// Verify file no longer exists
+		_, err = os.Stat(filepath)
+		require.True(t, os.IsNotExist(err))
+	})
+}
+
+// TestDirectoryOperations tests POSIX directory operation compliance
+func (s *POSIXComplianceTestSuite) TestDirectoryOperations(t *testing.T) {
+	mountPoint := s.framework.GetMountPoint()
+
+	t.Run("CreateDirectory", func(t *testing.T) {
+		dirPath := filepath.Join(mountPoint, "test_mkdir")
+
+		err := syscall.Mkdir(dirPath, 0755)
+		require.NoError(t, err)
+
+		// Verify directory exists and has correct type
+		stat, err := os.Stat(dirPath)
+		require.NoError(t, err)
+		require.True(t, stat.IsDir())
+	})
+
+	t.Run("RemoveDirectory", func(t *testing.T) {
+		dirPath := filepath.Join(mountPoint, "test_rmdir")
+
+		// Create directory
+		err := os.Mkdir(dirPath, 0755)
+		require.NoError(t, err)
+
+		// Remove directory
+		err = syscall.Rmdir(dirPath)
+		require.NoError(t, err)
+
+		// Verify directory no longer exists
+		_, err = os.Stat(dirPath)
+		require.True(t, os.IsNotExist(err))
+	})
+
+	t.Run("RemoveNonEmptyDirectory", func(t *testing.T) {
+		dirPath := filepath.Join(mountPoint, "test_rmdir_nonempty")
+		filePath := filepath.Join(dirPath, "file.txt")
+
+		// Create directory and file
+		err := os.Mkdir(dirPath, 0755)
+		require.NoError(t, err)
+		err = os.WriteFile(filePath, []byte("test"), 0644)
+		require.NoError(t, err)
+
+		// Attempt to remove non-empty directory should fail
+		err = syscall.Rmdir(dirPath)
+		require.Error(t, err)
+		require.Equal(t, syscall.ENOTEMPTY, err)
+	})
+
+	t.Run("RenameDirectory", func(t *testing.T) {
+		oldPath := filepath.Join(mountPoint, "old_dir")
+		newPath := filepath.Join(mountPoint, "new_dir")
+
+		// Create directory
+		err := os.Mkdir(oldPath, 0755)
+		require.NoError(t, err)
+
+		// Rename directory
+		err = os.Rename(oldPath, newPath)
+		require.NoError(t, err)
+
+		// Verify old path doesn't exist and new path does
+		_, err = os.Stat(oldPath)
+		require.True(t, os.IsNotExist(err))
+		stat, err := os.Stat(newPath)
+		require.NoError(t, err)
+		require.True(t, stat.IsDir())
+	})
+}
+
+// TestSymlinkOperations tests POSIX symlink operation compliance
+func (s *POSIXComplianceTestSuite) TestSymlinkOperations(t *testing.T) {
+	mountPoint := s.framework.GetMountPoint()
+
+	t.Run("CreateSymlink", func(t *testing.T) {
+		targetFile := filepath.Join(mountPoint, "target.txt")
+		linkFile := filepath.Join(mountPoint, "link.txt")
+
+		// Create target file
+		err := os.WriteFile(targetFile, []byte("target content"), 0644)
+		require.NoError(t, err)
+
+		// Create symlink
+		err = os.Symlink(targetFile, linkFile)
+		require.NoError(t, err)
+
+		// Verify symlink properties
+		linkStat, err := os.Lstat(linkFile)
+		require.NoError(t, err)
+		require.Equal(t, os.ModeSymlink, linkStat.Mode()&os.ModeType)
+
+		// Verify symlink content
+		linkTarget, err := os.Readlink(linkFile)
+		require.NoError(t, err)
+		require.Equal(t, targetFile, linkTarget)
+
+		// Verify following symlink works
+		content, err := os.ReadFile(linkFile)
+		require.NoError(t, err)
+		require.Equal(t, []byte("target content"), content)
+	})
+
+	t.Run("BrokenSymlink", func(t *testing.T) {
+		nonexistentTarget := filepath.Join(mountPoint, "nonexistent.txt")
+		linkFile := filepath.Join(mountPoint, "broken_link.txt")
+
+		// Create symlink to nonexistent file
+		err := os.Symlink(nonexistentTarget, linkFile)
+		require.NoError(t, err)
+
+		// Lstat should work (doesn't follow symlink)
+		_, err = os.Lstat(linkFile)
+		require.NoError(t, err)
+
+		// Stat should fail (follows symlink to nonexistent target)
+		_, err = os.Stat(linkFile)
+		require.Error(t, err)
+		require.True(t, os.IsNotExist(err))
+	})
+}
+
+// TestPermissions tests POSIX permission compliance
+func (s *POSIXComplianceTestSuite) TestPermissions(t *testing.T) {
+	mountPoint := s.framework.GetMountPoint()
+
+	t.Run("FilePermissions", func(t *testing.T) {
+		testFile := filepath.Join(mountPoint, "perm_test.txt")
+
+		// Create file with specific permissions
+		fd, err := syscall.Open(testFile, syscall.O_CREAT|syscall.O_WRONLY, 0642)
+		require.NoError(t, err)
+		syscall.Close(fd)
+
+		// Verify permissions
+		stat, err := os.Stat(testFile)
+		require.NoError(t, err)
+
+		// Note: The final file permissions are affected by the system's umask.
+		// We check against the requested mode and the mode as affected by a common umask (e.g. 0022).
+		actualMode := stat.Mode() & os.ModePerm
+		expectedMode := os.FileMode(0642)
+		expectedModeWithUmask := os.FileMode(0640) // e.g., 0642 with umask 0002 or 0022
+
+		// Accept either the exact permissions or permissions as modified by umask
+		if actualMode != expectedMode && actualMode != expectedModeWithUmask {
+			t.Errorf("Expected file permissions %o or %o, but got %o",
+				expectedMode, expectedModeWithUmask, actualMode)
+		}
+	})
+
+	t.Run("ChangeFilePermissions", func(t *testing.T) {
+		testFile := filepath.Join(mountPoint, "chmod_test.txt")
+
+		// Create file
+		err := os.WriteFile(testFile, []byte("test"), 0644)
+		require.NoError(t, err)
+
+		// Change permissions
+		err = os.Chmod(testFile, 0755)
+		require.NoError(t, err)
+
+		// Verify new permissions
+		stat, err := os.Stat(testFile)
+		require.NoError(t, err)
+		require.Equal(t, os.FileMode(0755), stat.Mode()&os.ModePerm)
+	})
+
+	t.Run("DirectoryPermissions", func(t *testing.T) {
+		testDir := filepath.Join(mountPoint, "perm_dir")
+
+		// Create directory with specific permissions
+		err := syscall.Mkdir(testDir, 0750)
+		require.NoError(t, err)
+
+		// Verify permissions
+		stat, err := os.Stat(testDir)
+		require.NoError(t, err)
+		require.Equal(t, os.FileMode(0750)|os.ModeDir, stat.Mode())
+	})
+}
+
+// TestTimestamps tests POSIX timestamp compliance
+func (s *POSIXComplianceTestSuite) TestTimestamps(t *testing.T) {
+	mountPoint := s.framework.GetMountPoint()
+
+	t.Run("AccessTime", func(t *testing.T) {
+		testFile := filepath.Join(mountPoint, "atime_test.txt")
+
+		// Create file
+		err := os.WriteFile(testFile, []byte("test content"), 0644)
+		require.NoError(t, err)
+
+		// Get initial timestamps
+		stat1, err := os.Stat(testFile)
+		require.NoError(t, err)
+
+		// Wait a bit to ensure time difference
+		time.Sleep(100 * time.Millisecond)
+
+		// Read file (should update access time)
+		_, err = os.ReadFile(testFile)
+		require.NoError(t, err)
+
+		// Get new timestamps
+		stat2, err := os.Stat(testFile)
+		require.NoError(t, err)
+
+		// Access time should have been updated, and modification time should be unchanged.
+		stat1Sys := stat1.Sys().(*syscall.Stat_t)
+		stat2Sys := stat2.Sys().(*syscall.Stat_t)
+		require.True(t, getAtimeNano(stat2Sys) >= getAtimeNano(stat1Sys), "access time should be updated or stay the same")
+		require.Equal(t, stat1.ModTime(), stat2.ModTime(), "modification time should not change on read")
+	})
+
+	t.Run("ModificationTime", func(t *testing.T) {
+		testFile := filepath.Join(mountPoint, "mtime_test.txt")
+
+		// Create file
+		err := os.WriteFile(testFile, []byte("initial content"), 0644)
+		require.NoError(t, err)
+
+		// Get initial timestamp
+		stat1, err := os.Stat(testFile)
+		require.NoError(t, err)
+
+		// Wait to ensure time difference
+		time.Sleep(100 * time.Millisecond)
+
+		// Modify file
+		err = os.WriteFile(testFile, []byte("modified content"), 0644)
+		require.NoError(t, err)
+
+		// Get new timestamp
+		stat2, err := os.Stat(testFile)
+		require.NoError(t, err)
+
+		// Modification time should have changed
+		require.True(t, stat2.ModTime().After(stat1.ModTime()))
+	})
+
+	t.Run("SetTimestamps", func(t *testing.T) {
+		testFile := filepath.Join(mountPoint, "utime_test.txt")
+
+		// Create file
+		err := os.WriteFile(testFile, []byte("test"), 0644)
+		require.NoError(t, err)
+
+		// Set specific timestamps
+		atime := time.Now().Add(-24 * time.Hour)
+		mtime := time.Now().Add(-12 * time.Hour)
+
+		err = os.Chtimes(testFile, atime, mtime)
+		require.NoError(t, err)
+
+		// Verify timestamps were set
+		stat, err := os.Stat(testFile)
+		require.NoError(t, err)
+		require.True(t, stat.ModTime().Equal(mtime))
+	})
+}
+
+// TestIOOperations tests POSIX I/O operation compliance
+func (s *POSIXComplianceTestSuite) TestIOOperations(t *testing.T) {
+	mountPoint := s.framework.GetMountPoint()
+
+	t.Run("ReadWrite", func(t *testing.T) {
+		testFile := filepath.Join(mountPoint, "rw_test.txt")
+		testData := []byte("Hello, POSIX World!")
+
+		// Write data
+		fd, err := syscall.Open(testFile, syscall.O_CREAT|syscall.O_WRONLY, 0644)
+		require.NoError(t, err)
+
+		n, err := syscall.Write(fd, testData)
+		require.NoError(t, err)
+		require.Equal(t, len(testData), n)
+
+		err = syscall.Close(fd)
+		require.NoError(t, err)
+
+		// Read data back
+		fd, err = syscall.Open(testFile, syscall.O_RDONLY, 0)
+		require.NoError(t, err)
+
+		readBuffer := make([]byte, len(testData))
+		n, err = syscall.Read(fd, readBuffer)
+		require.NoError(t, err)
+		require.Equal(t, len(testData), n)
+		require.Equal(t, testData, readBuffer)
+
+		err = syscall.Close(fd)
+		require.NoError(t, err)
+	})
+
+	t.Run("Seek", func(t *testing.T) {
+		testFile := filepath.Join(mountPoint, "seek_test.txt")
+		testData := []byte("0123456789ABCDEF")
+
+		// Create file with test data
+		err := os.WriteFile(testFile, testData, 0644)
+		require.NoError(t, err)
+
+		// Open for reading
+		fd, err := syscall.Open(testFile, syscall.O_RDONLY, 0)
+		require.NoError(t, err)
+		defer syscall.Close(fd)
+
+		// Seek to position 5
+		pos, err := syscall.Seek(fd, 5, 0) // SEEK_SET
+		require.NoError(t, err)
+		require.Equal(t, int64(5), pos)
+
+		// Read 3 bytes
+		buffer := make([]byte, 3)
+		n, err := syscall.Read(fd, buffer)
+		require.NoError(t, err)
+		require.Equal(t, 3, n)
+		require.Equal(t, []byte("567"), buffer)
+
+		// Seek from current position
+		pos, err = syscall.Seek(fd, 2, 1) // SEEK_CUR
+		require.NoError(t, err)
+		require.Equal(t, int64(10), pos)
+
+		// Read 1 byte
+		buffer = make([]byte, 1)
+		n, err = syscall.Read(fd, buffer)
+		require.NoError(t, err)
+		require.Equal(t, 1, n)
+		require.Equal(t, []byte("A"), buffer)
+
+		// Test positioned I/O operations (pread/pwrite)
+		syscall.Close(fd)
+
+		// Open for read/write to test pwrite
+		fd, err = syscall.Open(testFile, syscall.O_RDWR, 0)
+		require.NoError(t, err)
+		defer syscall.Close(fd)
+
+		// Positioned write test
+		writeData := []byte("XYZ")
+		n, err = syscall.Pwrite(fd, writeData, 5) // pwrite at offset 5
+		require.NoError(t, err)
+		require.Equal(t, len(writeData), n)
+
+		// Verify file position is unchanged by pwrite
+		currentPos, err := syscall.Seek(fd, 0, 1) // SEEK_CUR
+		require.NoError(t, err)
+		require.Equal(t, int64(0), currentPos, "file offset should not be changed by pwrite")
+
+		// Read back with pread
+		readBuffer := make([]byte, len(writeData))
+		n, err = syscall.Pread(fd, readBuffer, 5) // pread at offset 5
+		require.NoError(t, err)
+		require.Equal(t, len(writeData), n)
+		require.Equal(t, writeData, readBuffer)
+
+		// Verify file position is still unchanged by pread
+		currentPos, err = syscall.Seek(fd, 0, 1) // SEEK_CUR
+		require.NoError(t, err)
+		require.Equal(t, int64(0), currentPos, "file offset should not be changed by pread")
+	})
+
+	t.Run("AppendMode", func(t *testing.T) {
+		testFile := filepath.Join(mountPoint, "append_test.txt")
+
+		// Create file with initial content
+		err := os.WriteFile(testFile, []byte("initial"), 0644)
+		require.NoError(t, err)
+
+		// Open in append mode
+		fd, err := syscall.Open(testFile, syscall.O_WRONLY|syscall.O_APPEND, 0)
+		require.NoError(t, err)
+
+		// Write additional content
+		appendData := []byte(" appended")
+		n, err := syscall.Write(fd, appendData)
+		require.NoError(t, err)
+		require.Equal(t, len(appendData), n)
+
+		err = syscall.Close(fd)
+		require.NoError(t, err)
+
+		// Verify content
+		content, err := os.ReadFile(testFile)
+		require.NoError(t, err)
+		require.Equal(t, []byte("initial appended"), content)
+	})
+}
+
+// TestFileDescriptors tests POSIX file descriptor behavior
+func (s *POSIXComplianceTestSuite) TestFileDescriptors(t *testing.T) {
+	mountPoint := s.framework.GetMountPoint()
+
+	t.Run("DuplicateFileDescriptors", func(t *testing.T) {
+		testFile := filepath.Join(mountPoint, "dup_test.txt")
+		testData := []byte("duplicate test")
+
+		// Create and open file
+		fd, err := syscall.Open(testFile, syscall.O_CREAT|syscall.O_RDWR, 0644)
+		require.NoError(t, err)
+		defer syscall.Close(fd)
+
+		// Write initial data
+		n, err := syscall.Write(fd, testData)
+		require.NoError(t, err)
+		require.Equal(t, len(testData), n)
+
+		// Duplicate file descriptor
+		dupFd, err := syscall.Dup(fd)
+		require.NoError(t, err)
+		defer syscall.Close(dupFd)
+
+		// Both descriptors should refer to the same file
+		// Seek on one should affect the other
+		pos, err := syscall.Seek(fd, 0, 0) // SEEK_SET
+		require.NoError(t, err)
+		require.Equal(t, int64(0), pos)
+
+		// Read from duplicate descriptor should start from position 0
+		buffer := make([]byte, 9)
+		n, err = syscall.Read(dupFd, buffer)
+		require.NoError(t, err)
+		require.Equal(t, 9, n)
+		require.Equal(t, []byte("duplicate"), buffer)
+	})
+
+	t.Run("FileDescriptorFlags", func(t *testing.T) {
+		testFile := filepath.Join(mountPoint, "flags_test.txt")
+
+		// Create file
+		err := os.WriteFile(testFile, []byte("test"), 0644)
+		require.NoError(t, err)
+
+		// Open with close-on-exec flag
+		fd, err := syscall.Open(testFile, syscall.O_RDONLY|syscall.O_CLOEXEC, 0)
+		require.NoError(t, err)
+		defer syscall.Close(fd)
+
+		// Verify close-on-exec flag is set
+		// Note: FcntlInt is not available on all platforms, this test may need platform-specific implementation
+		t.Skip("FcntlInt not available on this platform")
+	})
+}
+
+// TestAtomicOperations tests POSIX atomic operation compliance
+func (s *POSIXComplianceTestSuite) TestAtomicOperations(t *testing.T) {
+	mountPoint := s.framework.GetMountPoint()
+
+	t.Run("AtomicRename", func(t *testing.T) {
+		oldFile := filepath.Join(mountPoint, "atomic_old.txt")
+		newFile := filepath.Join(mountPoint, "atomic_new.txt")
+		testData := []byte("atomic test data")
+
+		// Create source file
+		err := os.WriteFile(oldFile, testData, 0644)
+		require.NoError(t, err)
+
+		// Create existing target file
+		err = os.WriteFile(newFile, []byte("old content"), 0644)
+		require.NoError(t, err)
+
+		// Atomic rename should replace target
+		err = os.Rename(oldFile, newFile)
+		require.NoError(t, err)
+
+		// Verify source no longer exists
+		_, err = os.Stat(oldFile)
+		require.True(t, os.IsNotExist(err))
+
+		// Verify target has new content
+		content, err := os.ReadFile(newFile)
+		require.NoError(t, err)
+		require.Equal(t, testData, content)
+	})
+}
+
+// TestConcurrentAccess tests concurrent access patterns
+func (s *POSIXComplianceTestSuite) TestConcurrentAccess(t *testing.T) {
+	mountPoint := s.framework.GetMountPoint()
+
+	t.Run("ConcurrentReads", func(t *testing.T) {
+		testFile := filepath.Join(mountPoint, "concurrent_read.txt")
+		testData := []byte("concurrent read test data")
+
+		// Create test file
+		err := os.WriteFile(testFile, testData, 0644)
+		require.NoError(t, err)
+
+		// Launch multiple concurrent readers
+		const numReaders = 10
+		results := make(chan error, numReaders)
+
+		for i := 0; i < numReaders; i++ {
+			go func() {
+				content, err := os.ReadFile(testFile)
+				if err != nil {
+					results <- err
+					return
+				}
+				if string(content) != string(testData) {
+					results <- fmt.Errorf("content mismatch")
+					return
+				}
+				results <- nil
+			}()
+		}
+
+		// Check all readers succeeded
+		for i := 0; i < numReaders; i++ {
+			err := <-results
+			require.NoError(t, err)
+		}
+	})
+
+	t.Run("ConcurrentFileCreations", func(t *testing.T) {
+		testFile := filepath.Join(mountPoint, "concurrent_write.txt")
+
+		// Launch multiple concurrent writers
+		const numWriters = 5
+		results := make(chan error, numWriters)
+
+		for i := 0; i < numWriters; i++ {
+			go func(id int) {
+				content := fmt.Sprintf("writer %d data", id)
+				err := os.WriteFile(fmt.Sprintf("%s_%d", testFile, id), []byte(content), 0644)
+				results <- err
+			}(i)
+		}
+
+		// Check all writers succeeded
+		for i := 0; i < numWriters; i++ {
+			err := <-results
+			require.NoError(t, err)
+		}
+
+		// Verify all files were created
+		for i := 0; i < numWriters; i++ {
+			fileName := fmt.Sprintf("%s_%d", testFile, i)
+			_, err := os.Stat(fileName)
+			require.NoError(t, err)
+		}
+	})
+}
+
+// TestErrorHandling tests POSIX error handling compliance
+func (s *POSIXComplianceTestSuite) TestErrorHandling(t *testing.T) {
+	mountPoint := s.framework.GetMountPoint()
+
+	t.Run("AccessNonexistentFile", func(t *testing.T) {
+		nonexistentFile := filepath.Join(mountPoint, "does_not_exist.txt")
+
+		// Reading nonexistent file should return ENOENT
+		_, err := os.ReadFile(nonexistentFile)
+		require.Error(t, err)
+		require.True(t, os.IsNotExist(err))
+	})
+
+	t.Run("CreateFileInNonexistentDirectory", func(t *testing.T) {
+		fileInNonexistentDir := filepath.Join(mountPoint, "nonexistent_dir", "file.txt")
+
+		// Creating file in nonexistent directory should fail
+		err := os.WriteFile(fileInNonexistentDir, []byte("test"), 0644)
+		require.Error(t, err)
+		require.True(t, os.IsNotExist(err))
+	})
+
+	t.Run("InvalidFileDescriptor", func(t *testing.T) {
+		// Using an invalid file descriptor should return appropriate error
+		buffer := make([]byte, 10)
+		_, err := syscall.Read(999, buffer) // 999 is likely an invalid fd
+		require.Error(t, err)
+		require.Equal(t, syscall.EBADF, err)
+	})
+
+	t.Run("PermissionDenied", func(t *testing.T) {
+		if os.Geteuid() == 0 {
+			t.Skip("running as root: mode bits are not enforced")
+		}
+		testFile := filepath.Join(mountPoint, "readonly.txt")
+
+		// Create read-only file
+		err := os.WriteFile(testFile, []byte("readonly"), 0444)
+		require.NoError(t, err)
+
+		// Attempting to write to read-only file should fail
+		err = os.WriteFile(testFile, []byte("modified"), 0444)
+		require.Error(t, err)
+		require.True(t, os.IsPermission(err))
+	})
+}

test/fuse_integration/posix_extended_test.go

@@ -0,0 +1,758 @@
+//go:build !windows
+
+package fuse
+
+import (
+	"fmt"
+	"os"
+	"path/filepath"
+	"strings"
+	"syscall"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+// POSIXExtendedTestSuite provides additional POSIX compliance tests
+// covering extended attributes, file locking, and advanced features.
+//
+// NOTE: Some tests in this suite may be skipped or may have different behavior
+// depending on the FUSE implementation and platform support. This is expected
+// behavior for comprehensive POSIX compliance testing.
+//
+// See POSIX_IMPLEMENTATION_ROADMAP.md for a comprehensive plan to implement
+// these missing features.
+type POSIXExtendedTestSuite struct {
+	framework *FuseTestFramework
+	t         *testing.T
+}
+
+// ErrorCollector helps collect multiple test errors without stopping execution
+type ErrorCollector struct {
+	errors []string
+	t      *testing.T
+}
+
+// NewErrorCollector creates a new error collector
+func NewErrorCollector(t *testing.T) *ErrorCollector {
+	return &ErrorCollector{
+		errors: make([]string, 0),
+		t:      t,
+	}
+}
+
+// Add adds an error to the collection
+func (ec *ErrorCollector) Add(format string, args ...interface{}) {
+	ec.errors = append(ec.errors, fmt.Sprintf(format, args...))
+}
+
+// Check checks a condition and adds an error if it fails
+func (ec *ErrorCollector) Check(condition bool, format string, args ...interface{}) {
+	if !condition {
+		ec.Add(format, args...)
+	}
+}
+
+// CheckError checks if an error is nil and adds it if not
+func (ec *ErrorCollector) CheckError(err error, format string, args ...interface{}) {
+	if err != nil {
+		ec.Add(format+": %v", append(args, err)...)
+	}
+}
+
+// HasErrors returns true if any errors were collected
+func (ec *ErrorCollector) HasErrors() bool {
+	return len(ec.errors) > 0
+}
+
+// Report reports all collected errors to the test
+func (ec *ErrorCollector) Report() {
+	if len(ec.errors) > 0 {
+		ec.t.Errorf("Test completed with %d errors:\n%s", len(ec.errors), strings.Join(ec.errors, "\n"))
+	}
+}
+
+// NewPOSIXExtendedTestSuite creates a new extended POSIX compliance test suite
+func NewPOSIXExtendedTestSuite(t *testing.T, framework *FuseTestFramework) *POSIXExtendedTestSuite {
+	return &POSIXExtendedTestSuite{
+		framework: framework,
+		t:         t,
+	}
+}
+
+// TestPOSIXExtended runs extended POSIX compliance tests
+func TestPOSIXExtended(t *testing.T) {
+	config := DefaultTestConfig()
+	config.EnableDebug = true
+	config.MountOptions = []string{"-allowOthers", "-nonempty"}
+
+	framework := NewFuseTestFramework(t, config)
+	defer framework.Cleanup()
+
+	// Setup framework with better error handling
+	err := framework.Setup(config)
+	if err != nil {
+		t.Logf("Framework setup failed: %v", err)
+		t.Skip("Skipping extended tests due to framework setup failure")
+		return
+	}
+
+	// Verify framework is working
+	mountPoint := framework.GetMountPoint()
+	if mountPoint == "" {
+		t.Log("Framework mount point is empty")
+		t.Skip("Skipping extended tests due to invalid mount point")
+		return
+	}
+
+	suite := NewPOSIXExtendedTestSuite(t, framework)
+
+	// Run extended POSIX compliance test categories
+	// Use a resilient approach that continues even if individual tests fail
+	testCategories := []struct {
+		name string
+		fn   func(*testing.T)
+	}{
+		{"ExtendedAttributes", suite.TestExtendedAttributes},
+		{"FileLocking", suite.TestFileLocking},
+		{"AdvancedIO", suite.TestAdvancedIO},
+		{"SparseFiles", suite.TestSparseFiles},
+		{"LargeFiles", suite.TestLargeFiles},
+		{"MMap", suite.TestMemoryMapping},
+		{"DirectIO", suite.TestDirectIO},
+		{"FileSealing", suite.TestFileSealing},
+		{"Fallocate", suite.TestFallocate},
+		{"Sendfile", suite.TestSendfile},
+	}
+
+	// Run all test categories and collect results
+	var failedCategories []string
+	for _, category := range testCategories {
+		t.Run(category.name, func(subT *testing.T) {
+			// Capture panics and convert them to test failures
+			defer func() {
+				if r := recover(); r != nil {
+					subT.Errorf("Test category %s panicked: %v", category.name, r)
+					failedCategories = append(failedCategories, category.name)
+				}
+			}()
+
+			category.fn(subT)
+			if subT.Failed() {
+				failedCategories = append(failedCategories, category.name)
+			}
+		})
+	}
+
+	// Report overall results
+	if len(failedCategories) > 0 {
+		t.Logf("Extended POSIX tests completed. Failed categories: %v", failedCategories)
+	} else {
+		t.Logf("All extended POSIX test categories passed!")
+	}
+}
+
+// TestExtendedAttributes tests POSIX extended attribute support
+func (s *POSIXExtendedTestSuite) TestExtendedAttributes(t *testing.T) {
+	mountPoint := s.framework.GetMountPoint()
+
+	t.Run("SetAndGetXattr", func(t *testing.T) {
+		if !isXattrSupported() {
+			t.Skip("Extended attributes not supported on this platform")
+			return
+		}
+
+		testFile := filepath.Join(mountPoint, "xattr_test.txt")
+
+		// Create test file
+		err := os.WriteFile(testFile, []byte("xattr test"), 0644)
+		if !assert.NoError(t, err, "Failed to create test file") {
+			return
+		}
+
+		// Set extended attribute
+		attrName := "user.test_attr"
+		attrValue := []byte("test_value")
+		err = setXattr(testFile, attrName, attrValue, 0)
+		assert.NoError(t, err, "Failed to set extended attribute")
+
+		// Verify attribute was set
+		readValue := make([]byte, 256)
+		size, err := getXattr(testFile, attrName, readValue)
+		assert.NoError(t, err, "Failed to get extended attribute")
+		assert.Equal(t, len(attrValue), size, "Extended attribute size mismatch")
+		assert.Equal(t, attrValue, readValue[:size], "Extended attribute value mismatch")
+	})
+
+	t.Run("ListXattrs", func(t *testing.T) {
+		if !isXattrSupported() {
+			t.Skip("Extended attributes not supported on this platform")
+			return
+		}
+
+		testFile := filepath.Join(mountPoint, "xattr_list_test.txt")
+
+		// Create test file
+		err := os.WriteFile(testFile, []byte("list xattr test"), 0644)
+		if !assert.NoError(t, err, "Failed to create test file") {
+			return
+		}
+
+		// Set multiple extended attributes
+		attrs := map[string][]byte{
+			"user.attr1": []byte("value1"),
+			"user.attr2": []byte("value2"),
+			"user.attr3": []byte("value3"),
+		}
+
+		for name, value := range attrs {
+			err = setXattr(testFile, name, value, 0)
+			assert.NoError(t, err, "Failed to set extended attribute %s", name)
+		}
+
+		// List all attributes
+		listBuf := make([]byte, 1024)
+		size, err := listXattr(testFile, listBuf)
+		assert.NoError(t, err, "Failed to list extended attributes")
+		assert.Greater(t, size, 0, "No extended attributes found")
+
+		// Parse the null-separated list and verify attributes
+		if size > 0 {
+			attrList := parseXattrList(listBuf[:size])
+			expectedAttrs := []string{"user.attr1", "user.attr2", "user.attr3"}
+			for _, expectedAttr := range expectedAttrs {
+				assert.Contains(t, attrList, expectedAttr, "Expected attribute %s should be in the list", expectedAttr)
+			}
+		}
+	})
+
+	t.Run("RemoveXattr", func(t *testing.T) {
+		if !isXattrSupported() {
+			t.Skip("Extended attributes not supported on this platform")
+			return
+		}
+
+		testFile := filepath.Join(mountPoint, "xattr_remove_test.txt")
+
+		// Create test file
+		err := os.WriteFile(testFile, []byte("remove xattr test"), 0644)
+		require.NoError(t, err)
+
+		// Set extended attribute
+		attrName := "user.remove_test"
+		attrValue := []byte("to_be_removed")
+		err = setXattr(testFile, attrName, attrValue, 0)
+		require.NoError(t, err)
+
+		// Verify attribute exists
+		readValue := make([]byte, 256)
+		size, err := getXattr(testFile, attrName, readValue)
+		require.NoError(t, err)
+		require.Equal(t, len(attrValue), size)
+
+		// Remove the attribute
+		err = removeXattr(testFile, attrName)
+		require.NoError(t, err)
+
+		// Verify attribute is gone
+		_, err = getXattr(testFile, attrName, readValue)
+		require.Error(t, err) // Should fail with ENODATA or similar
+	})
+}
+
+// TestFileLocking tests POSIX file locking mechanisms
+// Note: File locking behavior may vary between FUSE implementations.
+// Some implementations may not enforce locks between file descriptors
+// from the same process, which is a known limitation.
+func (s *POSIXExtendedTestSuite) TestFileLocking(t *testing.T) {
+	mountPoint := s.framework.GetMountPoint()
+
+	t.Run("AdvisoryLocking", func(t *testing.T) {
+		testFile := filepath.Join(mountPoint, "lock_test.txt")
+
+		// Create test file
+		err := os.WriteFile(testFile, []byte("locking test"), 0644)
+		if !assert.NoError(t, err, "Failed to create test file") {
+			return
+		}
+
+		// Open file
+		file, err := os.OpenFile(testFile, os.O_RDWR, 0644)
+		if !assert.NoError(t, err, "Failed to open test file") {
+			return
+		}
+		defer file.Close()
+
+		// Apply exclusive lock
+		flock := syscall.Flock_t{
+			Type:   syscall.F_WRLCK,
+			Whence: 0,
+			Start:  0,
+			Len:    0, // Lock entire file
+		}
+
+		err = syscall.FcntlFlock(file.Fd(), syscall.F_SETLK, &flock)
+		assert.NoError(t, err, "Failed to acquire exclusive lock")
+
+		// Try to lock from another file descriptor (should fail)
+		file2, err := os.OpenFile(testFile, os.O_RDWR, 0644)
+		if !assert.NoError(t, err, "Failed to open second file descriptor") {
+			return
+		}
+		defer file2.Close()
+
+		flock2 := syscall.Flock_t{
+			Type:   syscall.F_WRLCK,
+			Whence: 0,
+			Start:  0,
+			Len:    0,
+		}
+
+		// Note: Some FUSE implementations may not enforce file locking between file descriptors
+		// from the same process. This is a known limitation in some FUSE implementations.
+		err = syscall.FcntlFlock(file2.Fd(), syscall.F_SETLK, &flock2)
+		if err == syscall.EAGAIN {
+			// Lock was properly enforced
+			t.Log("File locking properly enforced between file descriptors")
+		} else if err == nil {
+			// Lock was not enforced (common in some FUSE implementations)
+			t.Log("File locking not enforced between file descriptors from same process (FUSE limitation)")
+		} else {
+			// Some other error occurred
+			t.Logf("File locking attempt resulted in error: %v", err)
+		}
+
+		// Log the actual error for debugging
+		t.Logf("File locking test completed with result: %v", err)
+
+		// Release lock
+		flock.Type = syscall.F_UNLCK
+		err = syscall.FcntlFlock(file.Fd(), syscall.F_SETLK, &flock)
+		assert.NoError(t, err, "Failed to release lock")
+
+		// Now second lock should succeed
+		err = syscall.FcntlFlock(file2.Fd(), syscall.F_SETLK, &flock2)
+		assert.NoError(t, err, "Failed to acquire lock after release")
+	})
+
+	t.Run("SharedLocking", func(t *testing.T) {
+		testFile := filepath.Join(mountPoint, "shared_lock_test.txt")
+
+		// Create test file
+		err := os.WriteFile(testFile, []byte("shared locking test"), 0644)
+		if !assert.NoError(t, err, "Failed to create test file") {
+			return
+		}
+
+		// Open file for reading
+		file1, err := os.Open(testFile)
+		if !assert.NoError(t, err, "Failed to open first file descriptor") {
+			return
+		}
+		defer file1.Close()
+
+		file2, err := os.Open(testFile)
+		if !assert.NoError(t, err, "Failed to open second file descriptor") {
+			return
+		}
+		defer file2.Close()
+
+		// Apply shared locks (should both succeed)
+		flock1 := syscall.Flock_t{
+			Type:   syscall.F_RDLCK,
+			Whence: 0,
+			Start:  0,
+			Len:    0,
+		}
+
+		flock2 := syscall.Flock_t{
+			Type:   syscall.F_RDLCK,
+			Whence: 0,
+			Start:  0,
+			Len:    0,
+		}
+
+		err = syscall.FcntlFlock(file1.Fd(), syscall.F_SETLK, &flock1)
+		assert.NoError(t, err, "Failed to acquire first shared lock")
+
+		err = syscall.FcntlFlock(file2.Fd(), syscall.F_SETLK, &flock2)
+		assert.NoError(t, err, "Failed to acquire second shared lock")
+	})
+}
+
+// TestAdvancedIO tests advanced I/O operations
+// Note: Vectored I/O support may vary between FUSE implementations.
+// Some implementations may not properly support readv/writev operations.
+func (s *POSIXExtendedTestSuite) TestAdvancedIO(t *testing.T) {
+	mountPoint := s.framework.GetMountPoint()
+
+	t.Run("ReadWriteV", func(t *testing.T) {
+		if !isVectoredIOSupported() {
+			t.Skip("Vectored I/O not supported on this platform")
+			return
+		}
+
+		testFile := filepath.Join(mountPoint, "readwritev_test.txt")
+
+		// Create file
+		fd, err := syscall.Open(testFile, syscall.O_CREAT|syscall.O_RDWR, 0644)
+		if !assert.NoError(t, err, "Failed to create test file") {
+			return
+		}
+		defer syscall.Close(fd)
+
+		// Prepare test data in multiple buffers
+		writeBuffers := [][]byte{
+			[]byte("Hello "),
+			[]byte("vectored "),
+			[]byte("I/O "),
+			[]byte("world!"),
+		}
+
+		// Write using writev
+		writeIOVs := makeIOVecs(writeBuffers)
+		t.Logf("Attempting vectored I/O write with %d buffers", len(writeIOVs))
+		totalWritten, err := writevFile(fd, writeIOVs)
+		if err != nil {
+			// Some FUSE implementations may not support vectored I/O properly
+			t.Logf("Vectored I/O write failed: %v - this may be a FUSE limitation", err)
+			t.Skip("Vectored I/O not properly supported by this FUSE implementation")
+			return
+		}
+
+		expectedTotal := 0
+		for _, buf := range writeBuffers {
+			expectedTotal += len(buf)
+		}
+		assert.Equal(t, expectedTotal, totalWritten, "Vectored write size mismatch")
+
+		// Seek back to beginning
+		_, err = syscall.Seek(fd, 0, 0)
+		if !assert.NoError(t, err, "Failed to seek to beginning") {
+			return
+		}
+
+		// Read using readv into multiple buffers
+		readBuffers := [][]byte{
+			make([]byte, 6), // "Hello "
+			make([]byte, 9), // "vectored "
+			make([]byte, 4), // "I/O "
+			make([]byte, 6), // "world!"
+		}
+
+		readIOVs := makeIOVecs(readBuffers)
+		t.Logf("Attempting vectored I/O read with %d buffers", len(readIOVs))
+		totalRead, err := readvFile(fd, readIOVs)
+		if err != nil {
+			t.Logf("Vectored I/O read failed: %v - this may be a FUSE limitation", err)
+			t.Skip("Vectored I/O not properly supported by this FUSE implementation")
+			return
+		}
+		assert.Equal(t, expectedTotal, totalRead, "Vectored read size mismatch")
+
+		// Verify data matches
+		for i, expected := range writeBuffers {
+			assert.Equal(t, expected, readBuffers[i], "Vectored I/O data mismatch at buffer %d", i)
+		}
+	})
+
+}
+
+// TestSparseFiles tests sparse file handling
+func (s *POSIXExtendedTestSuite) TestSparseFiles(t *testing.T) {
+	mountPoint := s.framework.GetMountPoint()
+
+	t.Run("CreateSparseFile", func(t *testing.T) {
+		testFile := filepath.Join(mountPoint, "sparse_test.txt")
+
+		// Open file
+		fd, err := syscall.Open(testFile, syscall.O_CREAT|syscall.O_RDWR, 0644)
+		require.NoError(t, err)
+		defer syscall.Close(fd)
+
+		// Create a sparse file by seeking beyond end and writing
+		const sparseSize = 1024 * 1024 // 1MB
+		_, err = syscall.Seek(fd, sparseSize, 0)
+		require.NoError(t, err)
+
+		// Write at the end
+		endData := []byte("end")
+		n, err := syscall.Write(fd, endData)
+		require.NoError(t, err)
+		require.Equal(t, len(endData), n)
+
+		// Verify file size
+		stat, err := os.Stat(testFile)
+		require.NoError(t, err)
+		require.Equal(t, int64(sparseSize+len(endData)), stat.Size())
+
+		// Read from the beginning (should be zeros)
+		_, err = syscall.Seek(fd, 0, 0)
+		require.NoError(t, err)
+
+		buffer := make([]byte, 100)
+		n, err = syscall.Read(fd, buffer)
+		require.NoError(t, err)
+		require.Equal(t, 100, n)
+
+		// Should be all zeros
+		for _, b := range buffer {
+			require.Equal(t, byte(0), b)
+		}
+	})
+}
+
+// TestLargeFiles tests large file handling (>2GB)
+func (s *POSIXExtendedTestSuite) TestLargeFiles(t *testing.T) {
+	mountPoint := s.framework.GetMountPoint()
+
+	t.Run("LargeFileOperations", func(t *testing.T) {
+		testFile := filepath.Join(mountPoint, "large_file_test.txt")
+
+		// Open file
+		fd, err := syscall.Open(testFile, syscall.O_CREAT|syscall.O_RDWR, 0644)
+		require.NoError(t, err)
+		defer syscall.Close(fd)
+
+		// Seek to position > 2GB
+		const largeOffset = 3 * 1024 * 1024 * 1024 // 3GB
+		pos, err := syscall.Seek(fd, largeOffset, 0)
+		require.NoError(t, err)
+		require.Equal(t, int64(largeOffset), pos)
+
+		// Write at large offset
+		testData := []byte("large file data")
+		n, err := syscall.Write(fd, testData)
+		require.NoError(t, err)
+		require.Equal(t, len(testData), n)
+
+		// Read back from large offset
+		_, err = syscall.Seek(fd, largeOffset, 0)
+		require.NoError(t, err)
+
+		readBuffer := make([]byte, len(testData))
+		n, err = syscall.Read(fd, readBuffer)
+		require.NoError(t, err)
+		require.Equal(t, len(testData), n)
+		require.Equal(t, testData, readBuffer)
+	})
+}
+
+// TestMemoryMapping tests memory mapping functionality
+func (s *POSIXExtendedTestSuite) TestMemoryMapping(t *testing.T) {
+	mountPoint := s.framework.GetMountPoint()
+
+	t.Run("MmapFile", func(t *testing.T) {
+		if !isMmapSupported() {
+			t.Skip("Memory mapping not supported on this platform")
+			return
+		}
+
+		testFile := filepath.Join(mountPoint, "mmap_test.txt")
+		testData := make([]byte, 4096)
+		for i := range testData {
+			testData[i] = byte(i % 256)
+		}
+
+		// Create test file
+		err := os.WriteFile(testFile, testData, 0644)
+		require.NoError(t, err)
+
+		// Open file for reading
+		fd, err := syscall.Open(testFile, syscall.O_RDONLY, 0)
+		require.NoError(t, err)
+		defer syscall.Close(fd)
+
+		// Memory map the file
+		mappedData, err := mmapFile(fd, 0, len(testData), PROT_READ, MAP_SHARED)
+		require.NoError(t, err)
+		defer munmapFile(mappedData)
+
+		// Verify mapped content matches original
+		require.Equal(t, testData, mappedData)
+	})
+
+	t.Run("MmapWrite", func(t *testing.T) {
+		if !isMmapSupported() {
+			t.Skip("Memory mapping not supported on this platform")
+			return
+		}
+
+		testFile := filepath.Join(mountPoint, "mmap_write_test.txt")
+		size := 4096
+
+		// Create empty file of specific size
+		fd, err := syscall.Open(testFile, syscall.O_CREAT|syscall.O_RDWR, 0644)
+		require.NoError(t, err)
+		defer syscall.Close(fd)
+
+		err = syscall.Ftruncate(fd, int64(size))
+		require.NoError(t, err)
+
+		// Memory map the file for writing
+		mappedData, err := mmapFile(fd, 0, size, PROT_READ|PROT_WRITE, MAP_SHARED)
+		require.NoError(t, err)
+		defer munmapFile(mappedData)
+
+		// Write test pattern to mapped memory
+		testPattern := []byte("Hello, mmap world!")
+		copy(mappedData, testPattern)
+
+		// Sync changes to disk
+		err = msyncFile(mappedData, MS_SYNC)
+		require.NoError(t, err)
+
+		// Verify changes were written by reading file directly
+		readData, err := os.ReadFile(testFile)
+		require.NoError(t, err)
+		require.True(t, len(readData) >= len(testPattern))
+		require.Equal(t, testPattern, readData[:len(testPattern)])
+	})
+}
+
+// TestDirectIO tests direct I/O operations
+func (s *POSIXExtendedTestSuite) TestDirectIO(t *testing.T) {
+	mountPoint := s.framework.GetMountPoint()
+
+	t.Run("DirectIO", func(t *testing.T) {
+		if !isDirectIOSupported() {
+			t.Skip("Direct I/O not supported on this platform")
+			return
+		}
+
+		testFile := filepath.Join(mountPoint, "directio_test.txt")
+
+		// Open file with direct I/O
+		fd, err := openDirectIO(testFile, syscall.O_CREAT|syscall.O_RDWR, 0644)
+		require.NoError(t, err)
+		defer syscall.Close(fd)
+
+		// For direct I/O, data must be aligned to sector boundaries
+		// Use 4KB aligned buffer (common sector size)
+		blockSize := 4096
+		testData := make([]byte, blockSize)
+		for i := range testData {
+			testData[i] = byte(i % 256)
+		}
+
+		// Write data using direct I/O
+		n, err := syscall.Write(fd, testData)
+		require.NoError(t, err)
+		require.Equal(t, len(testData), n)
+
+		// Seek back to beginning
+		_, err = syscall.Seek(fd, 0, 0)
+		require.NoError(t, err)
+
+		// Read data using direct I/O
+		readData := make([]byte, blockSize)
+		n, err = syscall.Read(fd, readData)
+		require.NoError(t, err)
+		require.Equal(t, len(testData), n)
+		require.Equal(t, testData, readData)
+	})
+}
+
+// TestFileSealing tests file sealing mechanisms (Linux-specific)
+func (s *POSIXExtendedTestSuite) TestFileSealing(t *testing.T) {
+	// This test is Linux-specific and may not be applicable to all systems
+	t.Skip("File sealing tests require Linux-specific features")
+}
+
+// TestFallocate tests file preallocation
+func (s *POSIXExtendedTestSuite) TestFallocate(t *testing.T) {
+	mountPoint := s.framework.GetMountPoint()
+
+	t.Run("FallocateSpace", func(t *testing.T) {
+		if !isFallocateSupported() {
+			t.Skip("fallocate not supported on this platform")
+			return
+		}
+
+		testFile := filepath.Join(mountPoint, "fallocate_test.txt")
+
+		// Open file
+		fd, err := syscall.Open(testFile, syscall.O_CREAT|syscall.O_RDWR, 0644)
+		require.NoError(t, err)
+		defer syscall.Close(fd)
+
+		// Preallocate 1MB of space
+		allocSize := int64(1024 * 1024)
+		err = fallocateFile(fd, 0, 0, allocSize)
+		require.NoError(t, err)
+
+		// Verify file size was extended
+		var stat syscall.Stat_t
+		err = syscall.Fstat(fd, &stat)
+		require.NoError(t, err)
+		require.GreaterOrEqual(t, stat.Size, allocSize)
+
+		// Write some data and verify it works
+		testData := []byte("fallocate test data")
+		n, err := syscall.Write(fd, testData)
+		require.NoError(t, err)
+		require.Equal(t, len(testData), n)
+	})
+}
+
+// TestSendfile tests zero-copy file transfer
+func (s *POSIXExtendedTestSuite) TestSendfile(t *testing.T) {
+	mountPoint := s.framework.GetMountPoint()
+
+	t.Run("SendfileCopy", func(t *testing.T) {
+		sourceFile := filepath.Join(mountPoint, "sendfile_source.txt")
+		targetFile := filepath.Join(mountPoint, "sendfile_target.txt")
+
+		testData := make([]byte, 8192)
+		for i := range testData {
+			testData[i] = byte(i % 256)
+		}
+
+		// Create source file
+		err := os.WriteFile(sourceFile, testData, 0644)
+		require.NoError(t, err)
+
+		// Open source for reading
+		srcFd, err := syscall.Open(sourceFile, syscall.O_RDONLY, 0)
+		require.NoError(t, err)
+		defer syscall.Close(srcFd)
+
+		// Create target file
+		dstFd, err := syscall.Open(targetFile, syscall.O_CREAT|syscall.O_WRONLY, 0644)
+		require.NoError(t, err)
+		defer syscall.Close(dstFd)
+
+		// Sendfile test
+		if !isSendfileSupported() {
+			t.Skip("sendfile not supported on this platform")
+			return
+		}
+
+		// Use sendfile to copy data
+		var offset int64 = 0
+		transferred, err := sendfileTransfer(dstFd, srcFd, &offset, len(testData))
+		require.NoError(t, err)
+		require.Equal(t, len(testData), transferred)
+
+		// Verify copy
+		copiedData, err := os.ReadFile(targetFile)
+		require.NoError(t, err)
+		require.Equal(t, testData, copiedData)
+	})
+}
+
+// Helper function to parse null-separated xattr list
+func parseXattrList(data []byte) []string {
+	var attrs []string
+	start := 0
+	for i, b := range data {
+		if b == 0 {
+			if i > start {
+				attrs = append(attrs, string(data[start:i]))
+			}
+			start = i + 1
+		}
+	}
+	return attrs
+}

test/fuse_integration/posix_external_test.go

@@ -0,0 +1,579 @@
+package fuse
+
+import (
+	"fmt"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"strings"
+	"syscall"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+// ExternalPOSIXTestSuite manages integration with external POSIX test suites
+type ExternalPOSIXTestSuite struct {
+	framework *FuseTestFramework
+	t         *testing.T
+	workDir   string
+}
+
+// NewExternalPOSIXTestSuite creates a new external POSIX test suite runner
+func NewExternalPOSIXTestSuite(t *testing.T, framework *FuseTestFramework) *ExternalPOSIXTestSuite {
+	workDir, err := os.MkdirTemp("", "posix_external_tests_")
+	require.NoError(t, err)
+
+	return &ExternalPOSIXTestSuite{
+		framework: framework,
+		t:         t,
+		workDir:   workDir,
+	}
+}
+
+// Cleanup removes temporary test directories
+func (s *ExternalPOSIXTestSuite) Cleanup() {
+	os.RemoveAll(s.workDir)
+}
+
+// TestExternalPOSIXSuites runs integration tests with external POSIX test suites
+func TestExternalPOSIXSuites(t *testing.T) {
+	config := DefaultTestConfig()
+	config.EnableDebug = true
+	config.MountOptions = []string{"-allowOthers", "-nonempty"}
+
+	framework := NewFuseTestFramework(t, config)
+	defer framework.Cleanup()
+	require.NoError(t, framework.Setup(config))
+
+	suite := NewExternalPOSIXTestSuite(t, framework)
+	defer suite.Cleanup()
+
+	// Run various external POSIX test suites
+	t.Run("PjdFsTest", suite.TestPjdFsTest)
+	t.Run("NFSTestPOSIX", suite.TestNFSTestPOSIX)
+	t.Run("LitmusTests", suite.TestLitmusTests)
+	t.Run("CustomPOSIXTests", suite.TestCustomPOSIXTests)
+}
+
+// TestPjdFsTest runs the comprehensive pjdfstest POSIX filesystem test suite
+func (s *ExternalPOSIXTestSuite) TestPjdFsTest(t *testing.T) {
+	mountPoint := s.framework.GetMountPoint()
+
+	// Create test directory within mount point
+	testDir := filepath.Join(mountPoint, "pjdfstest")
+	err := os.MkdirAll(testDir, 0755)
+	require.NoError(t, err)
+
+	// List of critical POSIX operations to test
+	pjdTests := []struct {
+		name     string
+		testPath string
+		critical bool
+	}{
+		{"chflags", "tests/chflags", false},
+		{"chmod", "tests/chmod", true},
+		{"chown", "tests/chown", true},
+		{"create", "tests/create", true},
+		{"link", "tests/link", true},
+		{"mkdir", "tests/mkdir", true},
+		{"mkfifo", "tests/mkfifo", false},
+		{"mknod", "tests/mknod", false},
+		{"open", "tests/open", true},
+		{"rename", "tests/rename", true},
+		{"rmdir", "tests/rmdir", true},
+		{"symlink", "tests/symlink", true},
+		{"truncate", "tests/truncate", true},
+		{"unlink", "tests/unlink", true},
+	}
+
+	// Prefer the suite that `make setup-pjdfstest` builds under
+	// external_tools/ before cloning a second copy into the work dir
+	pjdDir := s.findPjdDir()
+	if pjdDir == "" {
+		pjdDir = filepath.Join(s.workDir, "pjdfstest")
+		if err := s.setupPjdFsTest(pjdDir); err != nil {
+			if os.Getenv("POSIX_REQUIRE_EXTERNAL_TOOLS") == "true" {
+				t.Fatalf("pjdfstest is required for comprehensive POSIX compliance testing: %v", err)
+			}
+			t.Skipf("pjdfstest not found and setup failed: %v", err)
+		}
+	}
+
+	// pjdfstest asserts full POSIX semantics the mount does not provide
+	// yet, so category failures are report-only unless the run asked for
+	// external tools to gate.
+	strict := os.Getenv("POSIX_REQUIRE_EXTERNAL_TOOLS") == "true"
+	for _, test := range pjdTests {
+		t.Run(test.name, func(t *testing.T) {
+			s.runPjdTest(t, pjdDir, test.testPath, testDir, strict && test.critical)
+		})
+	}
+}
+
+// TestNFSTestPOSIX runs nfstest_posix for POSIX API verification
+func (s *ExternalPOSIXTestSuite) TestNFSTestPOSIX(t *testing.T) {
+	mountPoint := s.framework.GetMountPoint()
+
+	// Check if nfstest_posix is available
+	_, err := exec.LookPath("nfstest_posix")
+	if err != nil {
+		// For comprehensive POSIX compliance testing, external tools should be mandatory
+		if os.Getenv("POSIX_REQUIRE_EXTERNAL_TOOLS") == "true" {
+			t.Fatalf("nfstest_posix is required for comprehensive POSIX compliance testing but not found. Install via: pip install nfstest")
+		}
+		t.Skip("nfstest_posix not found. Install via: pip install nfstest")
+	}
+
+	testDir := filepath.Join(mountPoint, "nfstest")
+	err = os.MkdirAll(testDir, 0755)
+	require.NoError(t, err)
+
+	// Run nfstest_posix with comprehensive API testing
+	cmd := exec.Command("nfstest_posix",
+		"--path", testDir,
+		"--verbose",
+		"--createlog",
+		"--runid", fmt.Sprintf("seaweedfs_%d", time.Now().Unix()),
+	)
+
+	output, err := cmd.CombinedOutput()
+	t.Logf("nfstest_posix output:\n%s", string(output))
+
+	if err != nil {
+		t.Logf("nfstest_posix failed: %v", err)
+		// Don't fail the test completely, just log the failure
+	}
+}
+
+// TestLitmusTests runs focused POSIX compliance litmus tests
+func (s *ExternalPOSIXTestSuite) TestLitmusTests(t *testing.T) {
+	mountPoint := s.framework.GetMountPoint()
+
+	// Create litmus test scripts for critical POSIX behaviors
+	litmusTests := []struct {
+		name   string
+		script string
+	}{
+		{
+			name: "AtomicRename",
+			script: `#!/bin/bash
+set -e
+cd "$1"
+echo "test data" > temp_file
+echo "original" > target_file
+mv temp_file target_file
+[ "$(cat target_file)" = "test data" ]
+echo "PASS: Atomic rename works"
+`,
+		},
+		{
+			name: "LinkCount",
+			script: `#!/bin/bash
+set -e
+cd "$1"
+echo "test" > original
+ln original hardlink
+if [[ "$(uname)" == "Darwin" ]]; then
+  [ $(stat -f %l original) -eq 2 ]
+else
+  [ $(stat -c %h original) -eq 2 ]
+fi
+rm hardlink
+if [[ "$(uname)" == "Darwin" ]]; then
+  [ $(stat -f %l original) -eq 1 ]
+else
+  [ $(stat -c %h original) -eq 1 ]
+fi
+echo "PASS: Hard link counting works"
+`,
+		},
+		{
+			name: "SymlinkCycles",
+			script: `#!/bin/bash
+set -e
+cd "$1"
+ln -s link1 link2
+ln -s link2 link1
+# Try to access the file, which should fail due to too many symbolic links
+if cat link1 2>/dev/null; then
+    echo "FAIL: Symlink cycle not detected"
+    exit 1
+fi
+echo "PASS: Symlink cycle handling works"
+`,
+		},
+		{
+			name: "ConcurrentCreate",
+			script: `#!/bin/bash
+set -e
+cd "$1"
+for i in {1..10}; do
+    (echo "process $i" > "file_$i") &
+done
+wait
+[ $(ls file_* | wc -l) -eq 10 ]
+echo "PASS: Concurrent file creation works"
+`,
+		},
+		{
+			name: "DirectoryConsistency",
+			script: `#!/bin/bash
+set -e
+cd "$1"
+mkdir testdir
+cd testdir
+touch file1 file2 file3
+cd ..
+entries=$(ls testdir | wc -l)
+[ $entries -eq 3 ]
+rmdir testdir 2>/dev/null && echo "FAIL: Non-empty directory removed" && exit 1
+rm testdir/*
+rmdir testdir
+echo "PASS: Directory consistency works"
+`,
+		},
+	}
+
+	testDir := filepath.Join(mountPoint, "litmus")
+	err := os.MkdirAll(testDir, 0755)
+	require.NoError(t, err)
+
+	// Run each litmus test
+	for _, test := range litmusTests {
+		t.Run(test.name, func(t *testing.T) {
+			s.runLitmusTest(t, test.name, test.script, testDir)
+		})
+	}
+}
+
+// TestCustomPOSIXTests runs custom POSIX compliance tests
+func (s *ExternalPOSIXTestSuite) TestCustomPOSIXTests(t *testing.T) {
+	mountPoint := s.framework.GetMountPoint()
+
+	// Custom stress tests for POSIX compliance
+	t.Run("StressRename", func(t *testing.T) {
+		s.stressTestRename(t, mountPoint)
+	})
+
+	t.Run("StressCreate", func(t *testing.T) {
+		s.stressTestCreate(t, mountPoint)
+	})
+
+	t.Run("StressDirectory", func(t *testing.T) {
+		s.stressTestDirectory(t, mountPoint)
+	})
+
+	t.Run("EdgeCases", func(t *testing.T) {
+		s.testEdgeCases(t, mountPoint)
+	})
+}
+
+// findPjdDir returns a directory holding a built pjdfstest suite, or ""
+// when none is installed. The path is absolute because tests run with
+// cmd.Dir inside the mount.
+func (s *ExternalPOSIXTestSuite) findPjdDir() string {
+	dir := filepath.Join("external_tools", "pjdfstest")
+	if info, err := os.Stat(filepath.Join(dir, "pjdfstest")); err == nil && !info.IsDir() && info.Mode()&0111 != 0 {
+		if abs, err := filepath.Abs(dir); err == nil {
+			return abs
+		}
+	}
+	return ""
+}
+
+// setupPjdFsTest downloads and sets up the pjdfstest suite
+func (s *ExternalPOSIXTestSuite) setupPjdFsTest(targetDir string) error {
+	// Clone pjdfstest repository
+	cmd := exec.Command("git", "clone", "https://github.com/pjd/pjdfstest.git", targetDir)
+	if err := cmd.Run(); err != nil {
+		return fmt.Errorf("failed to clone pjdfstest: %w", err)
+	}
+
+	// pjdfstest is autotools-based: the fresh clone has no Makefile until
+	// configure runs
+	for _, step := range [][]string{
+		{"autoreconf", "-ifs"},
+		{"./configure"},
+		{"make"},
+	} {
+		cmd = exec.Command(step[0], step[1:]...)
+		cmd.Dir = targetDir
+		if out, err := cmd.CombinedOutput(); err != nil {
+			return fmt.Errorf("pjdfstest %s: %w: %s", step[0], err, out)
+		}
+	}
+
+	return nil
+}
+
+// runPjdTest executes a specific pjdfstest
+func (s *ExternalPOSIXTestSuite) runPjdTest(t *testing.T, pjdDir, testPath, mountDir string, critical bool) {
+	fullTestPath := filepath.Join(pjdDir, testPath)
+
+	// Check if test path exists
+	if _, err := os.Stat(fullTestPath); os.IsNotExist(err) {
+		t.Skipf("Test path does not exist: %s", fullTestPath)
+	}
+
+	cmd := exec.Command("prove", "-r", fullTestPath)
+	cmd.Dir = mountDir
+	cmd.Env = append(os.Environ(), "FSTEST="+filepath.Join(pjdDir, "pjdfstest"))
+
+	output, err := cmd.CombinedOutput()
+	t.Logf("pjdfstest %s output:\n%s", testPath, string(output))
+
+	if err != nil {
+		if critical {
+			t.Errorf("Critical pjdfstest failed: %s - %v", testPath, err)
+		} else {
+			t.Logf("Non-critical pjdfstest failed: %s - %v", testPath, err)
+		}
+	}
+}
+
+// runLitmusTest executes a litmus test script
+func (s *ExternalPOSIXTestSuite) runLitmusTest(t *testing.T, name, script, testDir string) {
+	scriptPath := filepath.Join(s.workDir, name+".sh")
+
+	// Write script to file
+	err := os.WriteFile(scriptPath, []byte(script), 0755)
+	require.NoError(t, err)
+
+	// Create isolated test directory
+	isolatedDir := filepath.Join(testDir, name)
+	err = os.MkdirAll(isolatedDir, 0755)
+	require.NoError(t, err)
+	defer os.RemoveAll(isolatedDir)
+
+	// Execute script
+	cmd := exec.Command("bash", scriptPath, isolatedDir)
+	output, err := cmd.CombinedOutput()
+
+	t.Logf("Litmus test %s output:\n%s", name, string(output))
+
+	if err != nil {
+		t.Errorf("Litmus test %s failed: %v", name, err)
+	}
+}
+
+// stressTestRename tests rename operations under stress
+func (s *ExternalPOSIXTestSuite) stressTestRename(t *testing.T, mountPoint string) {
+	testDir := filepath.Join(mountPoint, "stress_rename")
+	err := os.MkdirAll(testDir, 0755)
+	require.NoError(t, err)
+	defer os.RemoveAll(testDir)
+
+	// Create files and rename them concurrently
+	const numFiles = 100
+	const numWorkers = 10
+
+	// Create initial files
+	for i := 0; i < numFiles; i++ {
+		fileName := filepath.Join(testDir, fmt.Sprintf("file_%d.txt", i))
+		err := os.WriteFile(fileName, []byte(fmt.Sprintf("content_%d", i)), 0644)
+		if !assert.NoError(t, err, "Failed to create initial file %d", i) {
+			return // Skip test if setup fails
+		}
+	}
+
+	// Concurrent rename operations
+	results := make(chan error, numWorkers)
+	for w := 0; w < numWorkers; w++ {
+		go func(worker int) {
+			for i := worker; i < numFiles; i += numWorkers {
+				oldName := filepath.Join(testDir, fmt.Sprintf("file_%d.txt", i))
+				newName := filepath.Join(testDir, fmt.Sprintf("renamed_%d.txt", i))
+				err := os.Rename(oldName, newName)
+				if err != nil {
+					results <- err
+					return
+				}
+			}
+			results <- nil
+		}(w)
+	}
+
+	// Wait for all workers and collect errors
+	var errorCount int
+	for w := 0; w < numWorkers; w++ {
+		err := <-results
+		if err != nil {
+			assert.NoError(t, err, "Worker %d failed", w)
+			errorCount++
+		}
+	}
+
+	// Verify all files were renamed (if no errors occurred)
+	if errorCount == 0 {
+		files, err := filepath.Glob(filepath.Join(testDir, "renamed_*.txt"))
+		assert.NoError(t, err, "Failed to list renamed files")
+		assert.Equal(t, numFiles, len(files), "Not all files were renamed successfully")
+	}
+}
+
+// stressTestCreate tests file creation under stress
+func (s *ExternalPOSIXTestSuite) stressTestCreate(t *testing.T, mountPoint string) {
+	testDir := filepath.Join(mountPoint, "stress_create")
+	err := os.MkdirAll(testDir, 0755)
+	require.NoError(t, err)
+	defer os.RemoveAll(testDir)
+
+	const numFiles = 1000
+	const numWorkers = 20
+
+	results := make(chan error, numWorkers)
+
+	for w := 0; w < numWorkers; w++ {
+		go func(worker int) {
+			for i := worker; i < numFiles; i += numWorkers {
+				fileName := filepath.Join(testDir, fmt.Sprintf("stress_%d_%d.txt", worker, i))
+				err := os.WriteFile(fileName, []byte(fmt.Sprintf("data_%d_%d", worker, i)), 0644)
+				if err != nil {
+					results <- err
+					return
+				}
+			}
+			results <- nil
+		}(w)
+	}
+
+	// Wait for all workers
+	for w := 0; w < numWorkers; w++ {
+		err := <-results
+		require.NoError(t, err)
+	}
+
+	// Verify all files were created
+	files, err := filepath.Glob(filepath.Join(testDir, "stress_*.txt"))
+	require.NoError(t, err)
+	require.Equal(t, numFiles, len(files))
+}
+
+// stressTestDirectory tests directory operations under stress
+func (s *ExternalPOSIXTestSuite) stressTestDirectory(t *testing.T, mountPoint string) {
+	testDir := filepath.Join(mountPoint, "stress_directory")
+	err := os.MkdirAll(testDir, 0755)
+	require.NoError(t, err)
+	defer os.RemoveAll(testDir)
+
+	const numDirs = 500
+	const numWorkers = 10
+
+	results := make(chan error, numWorkers)
+
+	// Create directories concurrently
+	for w := 0; w < numWorkers; w++ {
+		go func(worker int) {
+			for i := worker; i < numDirs; i += numWorkers {
+				dirName := filepath.Join(testDir, fmt.Sprintf("dir_%d_%d", worker, i))
+				err := os.MkdirAll(dirName, 0755)
+				if err != nil {
+					results <- err
+					return
+				}
+
+				// Create a file in each directory
+				fileName := filepath.Join(dirName, "test.txt")
+				err = os.WriteFile(fileName, []byte("test"), 0644)
+				if err != nil {
+					results <- err
+					return
+				}
+			}
+			results <- nil
+		}(w)
+	}
+
+	// Wait for all workers
+	for w := 0; w < numWorkers; w++ {
+		err := <-results
+		require.NoError(t, err)
+	}
+
+	// Verify directory structure
+	dirs, err := filepath.Glob(filepath.Join(testDir, "dir_*"))
+	require.NoError(t, err)
+	require.Equal(t, numDirs, len(dirs))
+}
+
+// testEdgeCases tests various POSIX edge cases
+func (s *ExternalPOSIXTestSuite) testEdgeCases(t *testing.T, mountPoint string) {
+	testDir := filepath.Join(mountPoint, "edge_cases")
+	err := os.MkdirAll(testDir, 0755)
+	require.NoError(t, err)
+	defer os.RemoveAll(testDir)
+
+	t.Run("WriteToDirectoryAsFile", func(t *testing.T) {
+		// Test writing to a directory as if it were a file (should fail)
+		// Note: filepath.Join(testDir, "") returns testDir itself
+		err := os.WriteFile(testDir, []byte("test"), 0644)
+		require.Error(t, err)
+		// Verify the error is specifically about the target being a directory
+		var pathErr *os.PathError
+		require.ErrorAs(t, err, &pathErr)
+		require.Equal(t, syscall.EISDIR, pathErr.Err)
+	})
+
+	t.Run("VeryLongFileName", func(t *testing.T) {
+		// Test very long file names
+		longName := strings.Repeat("a", 255) // NAME_MAX is typically 255
+		longFile := filepath.Join(testDir, longName)
+
+		err := os.WriteFile(longFile, []byte("long name test"), 0644)
+		// This might succeed or fail depending on filesystem limits
+		if err == nil {
+			defer os.Remove(longFile)
+			t.Logf("Very long filename accepted")
+		} else {
+			t.Logf("Very long filename rejected: %v", err)
+		}
+	})
+
+	t.Run("SpecialCharacters", func(t *testing.T) {
+		// Test files with special characters
+		specialFiles := []string{
+			"file with spaces.txt",
+			"file-with-dashes.txt",
+			"file_with_underscores.txt",
+			"file.with.dots.txt",
+		}
+
+		for _, fileName := range specialFiles {
+			filePath := filepath.Join(testDir, fileName)
+			err := os.WriteFile(filePath, []byte("special char test"), 0644)
+			require.NoError(t, err, "Failed to create file with special chars: %s", fileName)
+
+			// Verify we can read it back
+			_, err = os.ReadFile(filePath)
+			require.NoError(t, err, "Failed to read file with special chars: %s", fileName)
+		}
+	})
+
+	t.Run("DeepDirectoryNesting", func(t *testing.T) {
+		// Test deep directory nesting
+		deepPath := testDir
+		for i := 0; i < 100; i++ {
+			deepPath = filepath.Join(deepPath, fmt.Sprintf("level_%d", i))
+		}
+
+		err := os.MkdirAll(deepPath, 0755)
+		// This might fail due to PATH_MAX limits
+		if err != nil {
+			t.Logf("Deep directory nesting failed at some level: %v", err)
+		} else {
+			t.Logf("Deep directory nesting succeeded")
+
+			// Test creating a file in the deep path
+			testFile := filepath.Join(deepPath, "deep_file.txt")
+			err = os.WriteFile(testFile, []byte("deep test"), 0644)
+			if err == nil {
+				t.Logf("File creation in deep path succeeded")
+			} else {
+				t.Logf("File creation in deep path failed: %v", err)
+			}
+		}
+	})
+}

test/fuse_integration/posix_file_lock_test.go

@@ -0,0 +1,797 @@
+package fuse_test
+
+import (
+	"bufio"
+	"bytes"
+	"errors"
+	"fmt"
+	"io"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"strconv"
+	"strings"
+	"sync"
+	"sync/atomic"
+	"syscall"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+// --- fcntl helpers (used by both test process and subprocess) ---
+
+func fcntlSetLk(f *os.File, typ int16, start, len int64) error {
+	flk := syscall.Flock_t{
+		Type:   typ,
+		Whence: 0, // SEEK_SET
+		Start:  start,
+		Len:    len, // 0 means to EOF
+	}
+	return syscall.FcntlFlock(f.Fd(), syscall.F_SETLK, &flk)
+}
+
+func fcntlSetLkw(f *os.File, typ int16, start, len int64) error {
+	flk := syscall.Flock_t{
+		Type:   typ,
+		Whence: 0,
+		Start:  start,
+		Len:    len,
+	}
+	return syscall.FcntlFlock(f.Fd(), syscall.F_SETLKW, &flk)
+}
+
+func fcntlGetLk(f *os.File, typ int16, start, len int64) (syscall.Flock_t, error) {
+	flk := syscall.Flock_t{
+		Type:   typ,
+		Whence: 0,
+		Start:  start,
+		Len:    len,
+	}
+	err := syscall.FcntlFlock(f.Fd(), syscall.F_GETLK, &flk)
+	return flk, err
+}
+
+// --- Subprocess entry point for fcntl lock tests ---
+//
+// POSIX fcntl locks are per-process (the kernel uses the process's files_struct
+// as lock owner). To test inter-process lock conflicts over FUSE, contending
+// locks must come from separate processes.
+//
+// Actions:
+//   hold          — acquire F_SETLK, print "LOCKED\n", wait for stdin close, exit
+//   hold-blocking — print "WAITING\n", acquire F_SETLKW, print "LOCKED\n", wait for stdin close, exit
+//   try           — try F_SETLK, print "OK\n" or "EAGAIN\n", exit
+//   getlk         — do F_GETLK, print lock type as integer, exit
+
+func TestFcntlLockHelper(t *testing.T) {
+	action := os.Getenv("LOCK_ACTION")
+	if action == "" {
+		t.Skip("subprocess helper — not invoked directly")
+	}
+
+	filePath := os.Getenv("LOCK_FILE")
+	lockType, err := parseLockType(os.Getenv("LOCK_TYPE"))
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "lock type: %v\n", err)
+		os.Exit(1)
+	}
+	start, _ := strconv.ParseInt(os.Getenv("LOCK_START"), 10, 64)
+	length, _ := strconv.ParseInt(os.Getenv("LOCK_LEN"), 10, 64)
+
+	// The parent test process created the file on the FUSE mount just before
+	// spawning this subprocess. A fresh lookup from a different process can
+	// briefly return ENOENT while the mount's dentry state propagates, so
+	// retry on ENOENT before giving up.
+	var f *os.File
+	var openErr error
+	for attempt := 0; attempt < 40; attempt++ {
+		f, openErr = os.OpenFile(filePath, os.O_RDWR, 0)
+		if openErr == nil {
+			break
+		}
+		if !errors.Is(openErr, os.ErrNotExist) && !errors.Is(openErr, syscall.ENOENT) {
+			break
+		}
+		_, _ = os.ReadDir(filepath.Dir(filePath))
+		time.Sleep(50 * time.Millisecond)
+	}
+	if openErr != nil {
+		fmt.Fprintf(os.Stderr, "open: %v\n", openErr)
+		os.Exit(1)
+	}
+	defer f.Close()
+
+	switch action {
+	case "hold":
+		if err := fcntlSetLk(f, lockType, start, length); err != nil {
+			fmt.Fprintf(os.Stderr, "setlk: %v\n", err)
+			os.Exit(1)
+		}
+		fmt.Println("LOCKED")
+		os.Stdout.Sync()
+		io.ReadAll(os.Stdin) // block until parent closes pipe
+
+	case "hold-blocking":
+		fmt.Println("WAITING")
+		os.Stdout.Sync()
+		if err := fcntlSetLkw(f, lockType, start, length); err != nil {
+			fmt.Fprintf(os.Stderr, "setlkw: %v\n", err)
+			os.Exit(1)
+		}
+		fmt.Println("LOCKED")
+		os.Stdout.Sync()
+		io.ReadAll(os.Stdin)
+
+	case "try":
+		if err := fcntlSetLk(f, lockType, start, length); err != nil {
+			if errors.Is(err, syscall.EAGAIN) || errors.Is(err, syscall.EACCES) {
+				fmt.Println("EAGAIN")
+			} else {
+				fmt.Fprintf(os.Stderr, "unexpected setlk error: %v\n", err)
+				os.Exit(1)
+			}
+		} else {
+			fmt.Println("OK")
+		}
+		os.Stdout.Sync()
+
+	case "getlk":
+		flk := syscall.Flock_t{Type: lockType, Whence: 0, Start: start, Len: length}
+		if err := syscall.FcntlFlock(f.Fd(), syscall.F_GETLK, &flk); err != nil {
+			fmt.Fprintf(os.Stderr, "getlk: %v\n", err)
+			os.Exit(1)
+		}
+		fmt.Println(flk.Type)
+		os.Stdout.Sync()
+
+	default:
+		fmt.Fprintf(os.Stderr, "unknown action: %s\n", action)
+		os.Exit(1)
+	}
+
+	os.Exit(0)
+}
+
+func parseLockType(s string) (int16, error) {
+	switch s {
+	case "F_RDLCK":
+		return syscall.F_RDLCK, nil
+	case "F_WRLCK":
+		return syscall.F_WRLCK, nil
+	default:
+		return 0, fmt.Errorf("unknown lock type %q", s)
+	}
+}
+
+// --- Subprocess coordination helpers ---
+
+// lockHolder is a subprocess holding an fcntl lock. A background goroutine
+// reads stdout and signals the waiting/locked channels as the subprocess
+// progresses through the lock lifecycle.
+type lockHolder struct {
+	cmd     *exec.Cmd
+	stdin   io.WriteCloser
+	waiting chan struct{} // closed when subprocess prints "WAITING"
+	locked  chan struct{} // closed when subprocess prints "LOCKED"
+}
+
+// startLockHolder launches a subprocess that acquires an fcntl lock and holds
+// it until Release is called.
+func startLockHolder(t *testing.T, path string, lockType string, start, length int64) *lockHolder {
+	t.Helper()
+	return startLockProcess(t, "hold", path, lockType, start, length)
+}
+
+// startBlockingLockHolder launches a subprocess that tries F_SETLKW.
+// It blocks until the lock is available, then holds it until Release is called.
+func startBlockingLockHolder(t *testing.T, path string, lockType string, start, length int64) *lockHolder {
+	t.Helper()
+	return startLockProcess(t, "hold-blocking", path, lockType, start, length)
+}
+
+func startLockProcess(t *testing.T, action, path, lockType string, start, length int64) *lockHolder {
+	t.Helper()
+
+	cmd := exec.Command(os.Args[0], "-test.run=^TestFcntlLockHelper$")
+	cmd.Env = append(os.Environ(),
+		"LOCK_ACTION="+action,
+		"LOCK_FILE="+path,
+		"LOCK_TYPE="+lockType,
+		fmt.Sprintf("LOCK_START=%d", start),
+		fmt.Sprintf("LOCK_LEN=%d", length),
+	)
+
+	stdin, err := cmd.StdinPipe()
+	require.NoError(t, err)
+
+	stdoutPipe, err := cmd.StdoutPipe()
+	require.NoError(t, err)
+
+	cmd.Stderr = os.Stderr
+	require.NoError(t, cmd.Start())
+
+	waiting := make(chan struct{})
+	locked := make(chan struct{})
+	go func() {
+		scanner := bufio.NewScanner(stdoutPipe)
+		for scanner.Scan() {
+			switch strings.TrimSpace(scanner.Text()) {
+			case "WAITING":
+				select {
+				case <-waiting:
+				default:
+					close(waiting)
+				}
+			case "LOCKED":
+				select {
+				case <-locked:
+				default:
+					close(locked)
+				}
+			}
+		}
+	}()
+
+	return &lockHolder{cmd: cmd, stdin: stdin, waiting: waiting, locked: locked}
+}
+
+// WaitWaiting waits for the subprocess to print "WAITING" before trying to
+// assert that a blocking lock request is still blocked.
+func (h *lockHolder) WaitWaiting(t *testing.T, timeout time.Duration) {
+	t.Helper()
+	select {
+	case <-h.waiting:
+		// OK
+	case <-time.After(timeout):
+		t.Fatal("timed out waiting for subprocess to start blocking lock attempt")
+	}
+}
+
+// WaitLocked waits for the subprocess to acquire the lock and signal "LOCKED".
+func (h *lockHolder) WaitLocked(t *testing.T, timeout time.Duration) {
+	t.Helper()
+	select {
+	case <-h.locked:
+		// OK
+	case <-time.After(timeout):
+		t.Fatal("timed out waiting for subprocess to acquire lock")
+	}
+}
+
+// IsLocked checks whether the subprocess has signaled "LOCKED" within 200ms.
+func (h *lockHolder) IsLocked() bool {
+	select {
+	case <-h.waiting:
+		// The subprocess has reached the blocking lock attempt.
+	case <-h.locked:
+		return true
+	case <-time.After(200 * time.Millisecond):
+		return false
+	}
+
+	select {
+	case <-h.locked:
+		return true
+	case <-time.After(200 * time.Millisecond):
+		return false
+	}
+}
+
+// Release signals the subprocess to exit and waits for it.
+func (h *lockHolder) Release(t *testing.T) {
+	t.Helper()
+	h.stdin.Close()
+	_ = h.cmd.Wait()
+}
+
+// tryLockInSubprocess tries a non-blocking fcntl lock in a subprocess.
+// Returns nil if the lock was acquired, syscall.EAGAIN if it was denied.
+func tryLockInSubprocess(t *testing.T, path, lockType string, start, length int64) error {
+	t.Helper()
+
+	cmd := exec.Command(os.Args[0], "-test.run=^TestFcntlLockHelper$")
+	cmd.Env = append(os.Environ(),
+		"LOCK_ACTION=try",
+		"LOCK_FILE="+path,
+		"LOCK_TYPE="+lockType,
+		fmt.Sprintf("LOCK_START=%d", start),
+		fmt.Sprintf("LOCK_LEN=%d", length),
+	)
+	cmd.Stderr = os.Stderr
+
+	out, err := cmd.Output()
+	require.NoError(t, err, "subprocess failed to run")
+
+	result := strings.TrimSpace(string(out))
+	if result == "EAGAIN" {
+		return syscall.EAGAIN
+	}
+	require.Equal(t, "OK", result, "unexpected subprocess output")
+	return nil
+}
+
+// getLkInSubprocess does F_GETLK in a subprocess and returns the lock type.
+func getLkInSubprocess(t *testing.T, path, lockType string, start, length int64) int16 {
+	t.Helper()
+
+	cmd := exec.Command(os.Args[0], "-test.run=^TestFcntlLockHelper$")
+	cmd.Env = append(os.Environ(),
+		"LOCK_ACTION=getlk",
+		"LOCK_FILE="+path,
+		"LOCK_TYPE="+lockType,
+		fmt.Sprintf("LOCK_START=%d", start),
+		fmt.Sprintf("LOCK_LEN=%d", length),
+	)
+	cmd.Stderr = os.Stderr
+
+	out, err := cmd.Output()
+	require.NoError(t, err, "subprocess failed to run")
+
+	val, err := strconv.ParseInt(strings.TrimSpace(string(out)), 10, 16)
+	require.NoError(t, err, "failed to parse lock type from subprocess")
+	return int16(val)
+}
+
+// --- Test runner ---
+
+func TestPosixFileLocking(t *testing.T) {
+	framework := NewFuseTestFramework(t, DefaultTestConfig())
+	defer framework.Cleanup()
+
+	require.NoError(t, framework.Setup(DefaultTestConfig()))
+
+	t.Run("FlockExclusive", func(t *testing.T) {
+		testFlockExclusive(t, framework)
+	})
+	t.Run("FlockShared", func(t *testing.T) {
+		testFlockShared(t, framework)
+	})
+	t.Run("FlockUpgrade", func(t *testing.T) {
+		testFlockUpgrade(t, framework)
+	})
+	t.Run("FlockReleaseOnClose", func(t *testing.T) {
+		testFlockReleaseOnClose(t, framework)
+	})
+	t.Run("FcntlWriteLockConflict", func(t *testing.T) {
+		testFcntlWriteLockConflict(t, framework)
+	})
+	t.Run("FcntlReadLocksShared", func(t *testing.T) {
+		testFcntlReadLocksShared(t, framework)
+	})
+	t.Run("FcntlGetLk", func(t *testing.T) {
+		testFcntlGetLk(t, framework)
+	})
+	t.Run("FcntlByteRangePartial", func(t *testing.T) {
+		testFcntlByteRangePartial(t, framework)
+	})
+	t.Run("FcntlSetLkwBlocks", func(t *testing.T) {
+		testFcntlSetLkwBlocks(t, framework)
+	})
+	t.Run("FcntlReleaseOnClose", func(t *testing.T) {
+		testFcntlReleaseOnClose(t, framework)
+	})
+	t.Run("ConcurrentLockContention", func(t *testing.T) {
+		testConcurrentLockContention(t, framework)
+	})
+}
+
+// --- flock() tests (same-process is fine — flock uses per-fd owners) ---
+
+func testFlockExclusive(t *testing.T, fw *FuseTestFramework) {
+	path := filepath.Join(fw.GetMountPoint(), "flock_exclusive.txt")
+	require.NoError(t, os.WriteFile(path, []byte("data"), 0644))
+
+	f1, err := os.Open(path)
+	require.NoError(t, err)
+	defer f1.Close()
+
+	f2, err := os.Open(path)
+	require.NoError(t, err)
+	defer f2.Close()
+
+	require.NoError(t, syscall.Flock(int(f1.Fd()), syscall.LOCK_EX|syscall.LOCK_NB))
+
+	err = syscall.Flock(int(f2.Fd()), syscall.LOCK_EX|syscall.LOCK_NB)
+	assert.ErrorIs(t, err, syscall.EWOULDBLOCK, "second exclusive flock should fail")
+
+	require.NoError(t, syscall.Flock(int(f1.Fd()), syscall.LOCK_UN))
+	require.NoError(t, syscall.Flock(int(f2.Fd()), syscall.LOCK_EX|syscall.LOCK_NB))
+	require.NoError(t, syscall.Flock(int(f2.Fd()), syscall.LOCK_UN))
+}
+
+func testFlockShared(t *testing.T, fw *FuseTestFramework) {
+	path := filepath.Join(fw.GetMountPoint(), "flock_shared.txt")
+	require.NoError(t, os.WriteFile(path, []byte("data"), 0644))
+
+	f1, err := os.Open(path)
+	require.NoError(t, err)
+	defer f1.Close()
+
+	f2, err := os.Open(path)
+	require.NoError(t, err)
+	defer f2.Close()
+
+	require.NoError(t, syscall.Flock(int(f1.Fd()), syscall.LOCK_SH|syscall.LOCK_NB))
+	require.NoError(t, syscall.Flock(int(f2.Fd()), syscall.LOCK_SH|syscall.LOCK_NB))
+
+	f3, err := os.Open(path)
+	require.NoError(t, err)
+	defer f3.Close()
+	err = syscall.Flock(int(f3.Fd()), syscall.LOCK_EX|syscall.LOCK_NB)
+	assert.ErrorIs(t, err, syscall.EWOULDBLOCK)
+
+	require.NoError(t, syscall.Flock(int(f1.Fd()), syscall.LOCK_UN))
+	require.NoError(t, syscall.Flock(int(f2.Fd()), syscall.LOCK_UN))
+}
+
+func testFlockUpgrade(t *testing.T, fw *FuseTestFramework) {
+	path := filepath.Join(fw.GetMountPoint(), "flock_upgrade.txt")
+	require.NoError(t, os.WriteFile(path, []byte("data"), 0644))
+
+	f1, err := os.Open(path)
+	require.NoError(t, err)
+	defer f1.Close()
+
+	require.NoError(t, syscall.Flock(int(f1.Fd()), syscall.LOCK_SH|syscall.LOCK_NB))
+	require.NoError(t, syscall.Flock(int(f1.Fd()), syscall.LOCK_EX|syscall.LOCK_NB))
+
+	f2, err := os.Open(path)
+	require.NoError(t, err)
+	defer f2.Close()
+	err = syscall.Flock(int(f2.Fd()), syscall.LOCK_SH|syscall.LOCK_NB)
+	assert.ErrorIs(t, err, syscall.EWOULDBLOCK)
+
+	require.NoError(t, syscall.Flock(int(f1.Fd()), syscall.LOCK_UN))
+}
+
+func testFlockReleaseOnClose(t *testing.T, fw *FuseTestFramework) {
+	path := filepath.Join(fw.GetMountPoint(), "flock_close.txt")
+	require.NoError(t, os.WriteFile(path, []byte("data"), 0644))
+
+	f1, err := os.Open(path)
+	require.NoError(t, err)
+
+	require.NoError(t, syscall.Flock(int(f1.Fd()), syscall.LOCK_EX|syscall.LOCK_NB))
+	f1.Close()
+
+	f2, err := os.Open(path)
+	require.NoError(t, err)
+	defer f2.Close()
+	require.NoError(t, syscall.Flock(int(f2.Fd()), syscall.LOCK_EX|syscall.LOCK_NB))
+	require.NoError(t, syscall.Flock(int(f2.Fd()), syscall.LOCK_UN))
+}
+
+// --- fcntl() POSIX lock tests (use subprocesses for inter-process semantics) ---
+
+// testFcntlWriteLockConflict: test process holds a write lock, subprocess
+// contender is denied, then succeeds after release.
+func testFcntlWriteLockConflict(t *testing.T, fw *FuseTestFramework) {
+	path := filepath.Join(fw.GetMountPoint(), "fcntl_write.txt")
+	require.NoError(t, os.WriteFile(path, []byte("0123456789"), 0644))
+
+	f1, err := os.OpenFile(path, os.O_RDWR, 0)
+	require.NoError(t, err)
+	defer f1.Close()
+
+	// Test process locks [0, 10).
+	require.NoError(t, fcntlSetLk(f1, syscall.F_WRLCK, 0, 10))
+
+	// Subprocess: overlapping write lock should fail with EAGAIN.
+	err = tryLockInSubprocess(t, path, "F_WRLCK", 5, 10)
+	assert.ErrorIs(t, err, syscall.EAGAIN, "overlapping write lock should fail with EAGAIN")
+
+	// Unlock, then subprocess should succeed.
+	require.NoError(t, fcntlSetLk(f1, syscall.F_UNLCK, 0, 10))
+
+	err = tryLockInSubprocess(t, path, "F_WRLCK", 5, 10)
+	assert.NoError(t, err, "lock should succeed after release")
+}
+
+// testFcntlReadLocksShared: test process holds a read lock, subprocess read
+// lock succeeds (shared), subprocess write lock is denied.
+func testFcntlReadLocksShared(t *testing.T, fw *FuseTestFramework) {
+	path := filepath.Join(fw.GetMountPoint(), "fcntl_shared.txt")
+	require.NoError(t, os.WriteFile(path, []byte("0123456789"), 0644))
+
+	f1, err := os.OpenFile(path, os.O_RDWR, 0)
+	require.NoError(t, err)
+	defer f1.Close()
+
+	// Test process holds a read lock.
+	require.NoError(t, fcntlSetLk(f1, syscall.F_RDLCK, 0, 10))
+
+	// Subprocess: read lock should succeed (shared).
+	err = tryLockInSubprocess(t, path, "F_RDLCK", 0, 10)
+	assert.NoError(t, err, "shared read lock should succeed")
+
+	// Subprocess: write lock should fail.
+	err = tryLockInSubprocess(t, path, "F_WRLCK", 0, 10)
+	assert.ErrorIs(t, err, syscall.EAGAIN, "write lock should fail with EAGAIN while read lock is held")
+
+	require.NoError(t, fcntlSetLk(f1, syscall.F_UNLCK, 0, 10))
+}
+
+// testFcntlGetLk: test process holds a write lock, subprocess queries via
+// F_GETLK and sees the conflict.
+func testFcntlGetLk(t *testing.T, fw *FuseTestFramework) {
+	path := filepath.Join(fw.GetMountPoint(), "fcntl_getlk.txt")
+	require.NoError(t, os.WriteFile(path, []byte("0123456789"), 0644))
+
+	f1, err := os.OpenFile(path, os.O_RDWR, 0)
+	require.NoError(t, err)
+	defer f1.Close()
+
+	// Test process takes a write lock on [0, 5).
+	require.NoError(t, fcntlSetLk(f1, syscall.F_WRLCK, 0, 5))
+
+	// Subprocess: F_GETLK on overlapping range should report conflict.
+	typ := getLkInSubprocess(t, path, "F_WRLCK", 0, 10)
+	assert.NotEqual(t, int16(syscall.F_UNLCK), typ, "should report a conflict")
+
+	// Subprocess: F_GETLK on non-overlapping range should report no conflict.
+	typ = getLkInSubprocess(t, path, "F_WRLCK", 5, 5)
+	assert.Equal(t, int16(syscall.F_UNLCK), typ, "no conflict expected on non-overlapping range")
+
+	require.NoError(t, fcntlSetLk(f1, syscall.F_UNLCK, 0, 5))
+}
+
+// testFcntlByteRangePartial: test process and subprocess lock non-overlapping
+// ranges independently; extending into the other's range is denied.
+func testFcntlByteRangePartial(t *testing.T, fw *FuseTestFramework) {
+	path := filepath.Join(fw.GetMountPoint(), "fcntl_range.txt")
+	require.NoError(t, os.WriteFile(path, []byte("0123456789ABCDEF"), 0644))
+
+	f1, err := os.OpenFile(path, os.O_RDWR, 0)
+	require.NoError(t, err)
+	defer f1.Close()
+
+	// Test process locks [0, 8).
+	require.NoError(t, fcntlSetLk(f1, syscall.F_WRLCK, 0, 8))
+
+	// Subprocess locks [8, 16) — non-overlapping, should succeed.
+	holder := startLockHolder(t, path, "F_WRLCK", 8, 8)
+	holder.WaitLocked(t, 5*time.Second)
+
+	// Subprocess: extending into test process's range should fail.
+	err = tryLockInSubprocess(t, path, "F_WRLCK", 4, 8)
+	assert.ErrorIs(t, err, syscall.EAGAIN, "extending into another process's locked range should fail with EAGAIN")
+
+	holder.Release(t)
+	require.NoError(t, fcntlSetLk(f1, syscall.F_UNLCK, 0, 8))
+}
+
+// testFcntlSetLkwBlocks: test process holds exclusive lock, subprocess blocks
+// on F_SETLKW, then unblocks when lock is released.
+func testFcntlSetLkwBlocks(t *testing.T, fw *FuseTestFramework) {
+	path := filepath.Join(fw.GetMountPoint(), "fcntl_setlkw.txt")
+	require.NoError(t, os.WriteFile(path, []byte("data"), 0644))
+
+	f1, err := os.OpenFile(path, os.O_RDWR, 0)
+	require.NoError(t, err)
+	defer f1.Close()
+
+	// Test process takes exclusive lock.
+	require.NoError(t, fcntlSetLk(f1, syscall.F_WRLCK, 0, 0))
+
+	// Launch subprocess that will block on F_SETLKW.
+	blocker := startBlockingLockHolder(t, path, "F_WRLCK", 0, 0)
+
+	// Verify it has reached the blocking lock attempt but has not acquired it yet.
+	blocker.WaitWaiting(t, 5*time.Second)
+	assert.False(t, blocker.IsLocked(), "F_SETLKW should be blocking")
+
+	// Release our lock to unblock the subprocess.
+	require.NoError(t, fcntlSetLk(f1, syscall.F_UNLCK, 0, 0))
+
+	// Subprocess should now acquire and signal "LOCKED".
+	blocker.WaitLocked(t, 5*time.Second)
+	blocker.Release(t)
+}
+
+// testFcntlReleaseOnClose: subprocess holds a lock then exits (fd closes),
+// test process verifies it can now acquire the lock.
+func testFcntlReleaseOnClose(t *testing.T, fw *FuseTestFramework) {
+	path := filepath.Join(fw.GetMountPoint(), "fcntl_close.txt")
+	require.NoError(t, os.WriteFile(path, []byte("data"), 0644))
+
+	// Subprocess acquires exclusive lock and holds it.
+	holder := startLockHolder(t, path, "F_WRLCK", 0, 0)
+	holder.WaitLocked(t, 5*time.Second)
+
+	// Another subprocess trying to lock should be denied while holder is active.
+	err := tryLockInSubprocess(t, path, "F_WRLCK", 0, 0)
+	assert.ErrorIs(t, err, syscall.EAGAIN, "lock should be held by subprocess")
+
+	// Release subprocess → fd closes → lock released.
+	// The FUSE server may not process the Release immediately, so retry.
+	holder.Release(t)
+
+	f2, err := os.OpenFile(path, os.O_RDWR, 0)
+	require.NoError(t, err)
+	defer f2.Close()
+	var lockErr error
+	for attempt := 0; attempt < 50; attempt++ {
+		lockErr = fcntlSetLk(f2, syscall.F_WRLCK, 0, 0)
+		if lockErr == nil {
+			break
+		}
+		time.Sleep(50 * time.Millisecond)
+	}
+	require.NoError(t, lockErr, "lock should succeed after subprocess exits")
+	require.NoError(t, fcntlSetLk(f2, syscall.F_UNLCK, 0, 0))
+}
+
+// --- Concurrency test (uses flock which has per-fd owners) ---
+
+// testConcurrentLockContention has multiple goroutines competing for an
+// exclusive flock, each appending to the file while holding it.
+// Uses non-blocking flock with retry to avoid exhausting the go-fuse server's
+// reader goroutines (blocking FUSE SETLKW can starve unlock processing).
+func testConcurrentLockContention(t *testing.T, fw *FuseTestFramework) {
+	path := filepath.Join(fw.GetMountPoint(), "lock_contention.txt")
+	// Hold a master fd open for the whole subtest. Workers take flock on their
+	// own fds; master stays passive. Keeping this reference pinned avoids the
+	// re-open-after-concurrent-close code path, which has shown an unresolved
+	// mount/metaCache coherence bug where a freshly-opened fd sees ENOENT for
+	// tens of seconds after a burst of concurrent Release calls.
+	master, err := os.OpenFile(path, os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0644)
+	require.NoError(t, err)
+	defer master.Close()
+
+	numWorkers := 8
+	writesPerWorker := 5
+	var wg sync.WaitGroup
+	var mu sync.Mutex
+	var errs []error
+	var activeHolders atomic.Int32
+	var nextSlot atomic.Int64
+	const recordFormat = "worker %02d write %02d\n"
+	recordSize := len(fmt.Sprintf(recordFormat, 0, 0))
+
+	addError := func(err error) {
+		mu.Lock()
+		defer mu.Unlock()
+		errs = append(errs, err)
+	}
+
+	openWithRetry := func(flags int) (*os.File, error) {
+		var openErr error
+		for attempt := 0; attempt < 400; attempt++ {
+			file, err := os.OpenFile(path, flags, 0)
+			if err == nil {
+				return file, nil
+			}
+			openErr = err
+			if !errors.Is(err, os.ErrNotExist) && !errors.Is(err, syscall.ENOENT) {
+				return nil, err
+			}
+			// Refresh parent directory entries in case the mount cache is stale.
+			_, _ = os.ReadDir(filepath.Dir(path))
+			time.Sleep(50 * time.Millisecond)
+		}
+		return nil, openErr
+	}
+
+	for i := 0; i < numWorkers; i++ {
+		wg.Add(1)
+		go func(id int) {
+			defer wg.Done()
+			f, err := openWithRetry(os.O_RDWR)
+			if err != nil {
+				addError(fmt.Errorf("worker %d open: %v", id, err))
+				return
+			}
+			defer f.Close()
+
+			for j := 0; j < writesPerWorker; j++ {
+				// Non-blocking flock with retry to avoid FUSE server thread starvation.
+				locked := false
+				for attempt := 0; attempt < 3000; attempt++ {
+					if err := syscall.Flock(int(f.Fd()), syscall.LOCK_EX|syscall.LOCK_NB); err == nil {
+						locked = true
+						break
+					}
+					time.Sleep(10 * time.Millisecond)
+				}
+				if !locked {
+					addError(fmt.Errorf("worker %d: failed to acquire flock after retries", id))
+					return
+				}
+
+				if holders := activeHolders.Add(1); holders != 1 {
+					activeHolders.Add(-1)
+					syscall.Flock(int(f.Fd()), syscall.LOCK_UN)
+					addError(fmt.Errorf("worker %d: flock overlap detected with %d holders", id, holders))
+					return
+				}
+				time.Sleep(5 * time.Millisecond)
+
+				msg := fmt.Sprintf(recordFormat, id, j)
+				if len(msg) != recordSize {
+					activeHolders.Add(-1)
+					syscall.Flock(int(f.Fd()), syscall.LOCK_UN)
+					addError(fmt.Errorf("worker %d: unexpected record size %d", id, len(msg)))
+					return
+				}
+
+				slot := nextSlot.Add(1) - 1
+				offset := slot * int64(recordSize)
+				n, err := f.WriteAt([]byte(msg), offset)
+				if err != nil {
+					activeHolders.Add(-1)
+					syscall.Flock(int(f.Fd()), syscall.LOCK_UN)
+					addError(fmt.Errorf("worker %d write: %v", id, err))
+					return
+				}
+				if n != len(msg) {
+					activeHolders.Add(-1)
+					syscall.Flock(int(f.Fd()), syscall.LOCK_UN)
+					addError(fmt.Errorf("worker %d short write: wrote %d of %d bytes", id, n, len(msg)))
+					return
+				}
+
+				activeHolders.Add(-1)
+				syscall.Flock(int(f.Fd()), syscall.LOCK_UN)
+			}
+		}(i)
+	}
+
+	wg.Wait()
+	require.Empty(t, errs, "concurrent lock contention errors: %v", errs)
+
+	// Flush via the master fd. fsync on any fd to the file forces the
+	// kernel to send FUSE_FSYNC, which drains workers' writes to the filer.
+	require.NoError(t, master.Sync())
+
+	expectedLines := numWorkers * writesPerWorker
+	expectedBytes := expectedLines * recordSize
+	var data []byte
+	var lastReadErr error
+	ok := assert.Eventually(t, func() bool {
+		if _, err := master.Seek(0, io.SeekStart); err != nil {
+			lastReadErr = fmt.Errorf("seek: %w", err)
+			return false
+		}
+		d, err := io.ReadAll(master)
+		if err != nil {
+			lastReadErr = fmt.Errorf("read: %w", err)
+			return false
+		}
+		data = d
+		lastReadErr = nil
+		return len(data) == expectedBytes
+	}, 15*time.Second, 100*time.Millisecond, "file should eventually contain exactly %d records from all workers", expectedLines)
+	if !ok {
+		// Diagnostics: dump what the mount currently shows for the parent dir
+		// and the target path, so post-mortem analysis has something to go on.
+		parent := filepath.Dir(path)
+		if entries, readErr := os.ReadDir(parent); readErr != nil {
+			t.Logf("diag: ReadDir(%q) failed: %v", parent, readErr)
+		} else {
+			names := make([]string, 0, len(entries))
+			for _, e := range entries {
+				names = append(names, e.Name())
+			}
+			t.Logf("diag: ReadDir(%q) returned %d entries: %v", parent, len(names), names)
+		}
+		if fi, statErr := os.Stat(path); statErr != nil {
+			t.Logf("diag: Stat(%q) failed: %v", path, statErr)
+		} else {
+			t.Logf("diag: Stat(%q) ok: size=%d mode=%s", path, fi.Size(), fi.Mode())
+		}
+		if freshFd, openErr := os.OpenFile(path, os.O_RDONLY, 0); openErr != nil {
+			t.Logf("diag: fresh open(%q, O_RDONLY) failed: %v", path, openErr)
+		} else {
+			fresh, readErr := io.ReadAll(freshFd)
+			freshFd.Close()
+			t.Logf("diag: fresh open(%q) read %d bytes (err=%v)", path, len(fresh), readErr)
+		}
+		t.Fatalf("read back incomplete: got %d bytes, want %d (lastReadErr=%v)", len(data), expectedBytes, lastReadErr)
+	}
+	actualLines := bytes.Count(data, []byte("\n"))
+	assert.Equal(t, expectedLines, actualLines,
+		"file should contain exactly %d lines from all workers", expectedLines)
+}

test/fuse_integration/sendfile_darwin.go

@@ -0,0 +1,45 @@
+//go:build darwin
+
+package fuse
+
+import (
+	"syscall"
+	"unsafe"
+)
+
+// Sendfile support for macOS
+
+func sendfileTransfer(outFd int, inFd int, offset *int64, count int) (int, error) {
+	// macOS sendfile has different signature: sendfile(in_fd, out_fd, offset, len, hdtr, flags)
+	var off int64
+	if offset != nil {
+		off = *offset
+	}
+
+	length := int64(count)
+
+	_, _, errno := syscall.Syscall6(syscall.SYS_SENDFILE,
+		uintptr(inFd),                    // input fd
+		uintptr(outFd),                   // output fd
+		uintptr(off),                     // offset
+		uintptr(unsafe.Pointer(&length)), // length (in/out parameter)
+		0,                                // hdtr (headers/trailers)
+		0)                                // flags
+
+	if errno != 0 {
+		return 0, errno
+	}
+
+	// Update offset if provided
+	if offset != nil {
+		*offset += length
+	}
+
+	return int(length), nil
+}
+
+func isSendfileSupported() bool {
+	// macOS sendfile(2) requires a socket destination; file-to-file copies
+	// fail with ENOTSOCK.
+	return false
+}

test/fuse_integration/sendfile_linux.go

@@ -0,0 +1,33 @@
+//go:build linux
+
+package fuse
+
+import (
+	"syscall"
+	"unsafe"
+)
+
+// Sendfile support for Linux
+
+func sendfileTransfer(outFd int, inFd int, offset *int64, count int) (int, error) {
+	var offsetPtr uintptr
+	if offset != nil {
+		offsetPtr = uintptr(unsafe.Pointer(offset))
+	}
+
+	n, _, errno := syscall.Syscall6(syscall.SYS_SENDFILE,
+		uintptr(outFd),
+		uintptr(inFd),
+		offsetPtr,
+		uintptr(count),
+		0, 0)
+
+	if errno != 0 {
+		return 0, errno
+	}
+	return int(n), nil
+}
+
+func isSendfileSupported() bool {
+	return true
+}

test/fuse_integration/sendfile_unsupported.go

@@ -0,0 +1,19 @@
+//go:build !linux && !darwin
+
+package fuse
+
+import (
+	"errors"
+)
+
+// Sendfile support for unsupported platforms
+
+var ErrSendfileNotSupported = errors.New("sendfile not supported on this platform")
+
+func sendfileTransfer(outFd int, inFd int, offset *int64, count int) (int, error) {
+	return 0, ErrSendfileNotSupported
+}
+
+func isSendfileSupported() bool {
+	return false
+}

test/fuse_integration/unlinked_open_test.go

@@ -0,0 +1,94 @@
+//go:build linux
+
+package fuse_test
+
+import (
+	"os"
+	"path/filepath"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+	"golang.org/x/sys/unix"
+)
+
+// TestMetadataOnUnlinkedOpen covers the POSIX rule that an inode outlives its
+// last name while a descriptor holds it open: metadata operations through the
+// descriptor keep working between the removal and the final close, for a file
+// after unlink and for a directory after rmdir.
+func TestMetadataOnUnlinkedOpen(t *testing.T) {
+	framework := NewFuseTestFramework(t, DefaultTestConfig())
+	defer framework.Cleanup()
+
+	require.NoError(t, framework.Setup(DefaultTestConfig()))
+
+	t.Run("UnlinkedOpenFile", func(t *testing.T) {
+		testUnlinkedOpenFile(t, framework)
+	})
+
+	t.Run("RemovedOpenDirectory", func(t *testing.T) {
+		testRemovedOpenDirectory(t, framework)
+	})
+}
+
+func testUnlinkedOpenFile(t *testing.T, framework *FuseTestFramework) {
+	path := filepath.Join(framework.GetMountPoint(), "unlinked_open_file")
+	fd, err := unix.Open(path, unix.O_CREAT|unix.O_RDWR, 0644)
+	require.NoError(t, err)
+	defer unix.Close(fd)
+	_, err = unix.Write(fd, []byte("hello"))
+	require.NoError(t, err)
+
+	require.NoError(t, os.Remove(path))
+
+	require.NoError(t, unix.Ftruncate(fd, 2), "ftruncate on an unlinked open file")
+	require.NoError(t, unix.Fchmod(fd, 0600), "fchmod on an unlinked open file")
+	require.NoError(t, unix.Futimes(fd, []unix.Timeval{{Sec: 1234567890}, {Sec: 987654321}}),
+		"futimes on an unlinked open file")
+
+	var st unix.Stat_t
+	require.NoError(t, unix.Fstat(fd, &st), "fstat on an unlinked open file")
+	assert.EqualValues(t, 0, st.Nlink, "nlink after unlink")
+	assert.EqualValues(t, 0600, st.Mode&0777, "mode after fchmod")
+	assert.EqualValues(t, 2, st.Size, "size after ftruncate")
+	assert.EqualValues(t, 1234567890, st.Atim.Sec, "atime after futimes")
+	assert.EqualValues(t, 987654321, st.Mtim.Sec, "mtime after futimes")
+
+	testFdXattrRoundTrip(t, fd)
+}
+
+func testRemovedOpenDirectory(t *testing.T, framework *FuseTestFramework) {
+	path := filepath.Join(framework.GetMountPoint(), "removed_open_dir")
+	require.NoError(t, os.Mkdir(path, 0755))
+	fd, err := unix.Open(path, unix.O_RDONLY|unix.O_DIRECTORY, 0)
+	require.NoError(t, err)
+	defer unix.Close(fd)
+
+	require.NoError(t, os.Remove(path))
+
+	require.NoError(t, unix.Fchmod(fd, 0770), "fchmod on a removed open directory")
+	require.NoError(t, unix.Futimes(fd, []unix.Timeval{{Sec: 1234567890}, {Sec: 987654321}}),
+		"futimes on a removed open directory")
+
+	var st unix.Stat_t
+	require.NoError(t, unix.Fstat(fd, &st), "fstat on a removed open directory")
+	assert.EqualValues(t, unix.S_IFDIR, st.Mode&unix.S_IFMT, "type after rmdir")
+	assert.EqualValues(t, 0770, st.Mode&0777, "mode after fchmod")
+	assert.EqualValues(t, 0, st.Nlink, "nlink after rmdir")
+	assert.EqualValues(t, 987654321, st.Mtim.Sec, "mtime after futimes")
+
+	testFdXattrRoundTrip(t, fd)
+}
+
+func testFdXattrRoundTrip(t *testing.T, fd int) {
+	require.NoError(t, unix.Fsetxattr(fd, "user.k", []byte("v"), 0), "fsetxattr on a removed open inode")
+
+	buf := make([]byte, 8)
+	n, err := unix.Fgetxattr(fd, "user.k", buf)
+	require.NoError(t, err, "fgetxattr on a removed open inode")
+	assert.Equal(t, "v", string(buf[:n]))
+
+	require.NoError(t, unix.Fremovexattr(fd, "user.k"), "fremovexattr on a removed open inode")
+	_, err = unix.Fgetxattr(fd, "user.k", buf)
+	assert.ErrorIs(t, err, unix.ENODATA, "fgetxattr after fremovexattr")
+}

test/fuse_integration/vectored_io_unix.go

@@ -0,0 +1,126 @@
+//go:build unix
+
+package fuse
+
+import (
+	"syscall"
+	"unsafe"
+)
+
+// Vectored I/O support for Unix-like systems
+
+// IOVec represents an I/O vector for readv/writev operations
+type IOVec struct {
+	Base *byte
+	Len  uint64
+}
+
+func readvFile(fd int, iovs []IOVec) (int, error) {
+	if len(iovs) == 0 {
+		return 0, nil
+	}
+
+	n, _, errno := syscall.Syscall(syscall.SYS_READV,
+		uintptr(fd),
+		uintptr(unsafe.Pointer(&iovs[0])),
+		uintptr(len(iovs)))
+
+	if errno != 0 {
+		return 0, errno
+	}
+	return int(n), nil
+}
+
+func writevFile(fd int, iovs []IOVec) (int, error) {
+	if len(iovs) == 0 {
+		return 0, nil
+	}
+
+	n, _, errno := syscall.Syscall(syscall.SYS_WRITEV,
+		uintptr(fd),
+		uintptr(unsafe.Pointer(&iovs[0])),
+		uintptr(len(iovs)))
+
+	if errno != 0 {
+		return 0, errno
+	}
+	return int(n), nil
+}
+
+func preadvFile(fd int, iovs []IOVec, offset int64) (int, error) {
+	if len(iovs) == 0 {
+		return 0, nil
+	}
+
+	// preadv/pwritev may not be available on all Unix systems
+	// Fall back to individual pread calls
+	totalRead := 0
+	currentOffset := offset
+
+	for _, iov := range iovs {
+		if iov.Len == 0 {
+			continue
+		}
+
+		buf := (*[1 << 30]byte)(unsafe.Pointer(iov.Base))[:iov.Len:iov.Len]
+		n, err := syscall.Pread(fd, buf, currentOffset)
+		if err != nil {
+			return totalRead, err
+		}
+		totalRead += n
+		currentOffset += int64(n)
+
+		if n < int(iov.Len) {
+			break // EOF or partial read
+		}
+	}
+
+	return totalRead, nil
+}
+
+func pwritevFile(fd int, iovs []IOVec, offset int64) (int, error) {
+	if len(iovs) == 0 {
+		return 0, nil
+	}
+
+	// preadv/pwritev may not be available on all Unix systems
+	// Fall back to individual pwrite calls
+	totalWritten := 0
+	currentOffset := offset
+
+	for _, iov := range iovs {
+		if iov.Len == 0 {
+			continue
+		}
+
+		buf := (*[1 << 30]byte)(unsafe.Pointer(iov.Base))[:iov.Len:iov.Len]
+		n, err := syscall.Pwrite(fd, buf, currentOffset)
+		if err != nil {
+			return totalWritten, err
+		}
+		totalWritten += n
+		currentOffset += int64(n)
+
+		if n < int(iov.Len) {
+			break // Partial write
+		}
+	}
+
+	return totalWritten, nil
+}
+
+// Helper function to create IOVec from byte slices
+func makeIOVecs(buffers [][]byte) []IOVec {
+	iovs := make([]IOVec, len(buffers))
+	for i, buf := range buffers {
+		if len(buf) > 0 {
+			iovs[i].Base = &buf[0]
+			iovs[i].Len = uint64(len(buf))
+		}
+	}
+	return iovs
+}
+
+func isVectoredIOSupported() bool {
+	return true
+}

test/fuse_integration/vectored_io_unsupported.go

@@ -0,0 +1,41 @@
+//go:build !unix
+
+package fuse
+
+import (
+	"errors"
+)
+
+// Vectored I/O support for unsupported platforms
+
+var ErrVectoredIONotSupported = errors.New("vectored I/O not supported on this platform")
+
+// IOVec represents an I/O vector for readv/writev operations
+type IOVec struct {
+	Base *byte
+	Len  uint64
+}
+
+func readvFile(fd int, iovs []IOVec) (int, error) {
+	return 0, ErrVectoredIONotSupported
+}
+
+func writevFile(fd int, iovs []IOVec) (int, error) {
+	return 0, ErrVectoredIONotSupported
+}
+
+func preadvFile(fd int, iovs []IOVec, offset int64) (int, error) {
+	return 0, ErrVectoredIONotSupported
+}
+
+func pwritevFile(fd int, iovs []IOVec, offset int64) (int, error) {
+	return 0, ErrVectoredIONotSupported
+}
+
+func makeIOVecs(buffers [][]byte) []IOVec {
+	return nil
+}
+
+func isVectoredIOSupported() bool {
+	return false
+}

test/fuse_integration/write_buffer_cap_test.go

@@ -0,0 +1,316 @@
+package fuse_test
+
+import (
+	"bytes"
+	"crypto/rand"
+	"fmt"
+	"io"
+	"net/http"
+	"os"
+	"path/filepath"
+	"runtime"
+	"sync"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/require"
+)
+
+// mountDebugPort holds the debug/pprof port the test passed to the
+// mount process via -debug.port. It is set once at TestWriteBufferCap
+// entry and consulted from the write-timeout paths to fetch the mount
+// process's goroutine dump (the test's own dumpAllGoroutines only
+// covers the test process).
+var mountDebugPort int
+
+// fetchMountGoroutines pulls a full goroutine dump from the mount
+// process's pprof endpoint. If the mount debug port isn't configured
+// or the HTTP call fails, a short explanation is returned instead of
+// an error — this is diagnostic best-effort, not a test assertion.
+func fetchMountGoroutines() string {
+	if mountDebugPort == 0 {
+		return "(mount debug port not configured)"
+	}
+	url := fmt.Sprintf("http://127.0.0.1:%d/debug/pprof/goroutine?debug=2", mountDebugPort)
+	client := &http.Client{Timeout: 10 * time.Second}
+	resp, err := client.Get(url)
+	if err != nil {
+		return fmt.Sprintf("(failed to reach mount pprof at %s: %v)", url, err)
+	}
+	defer resp.Body.Close()
+	body, err := io.ReadAll(resp.Body)
+	if err != nil {
+		return fmt.Sprintf("(failed to read mount pprof body: %v)", err)
+	}
+	return string(body)
+}
+
+// dumpAllGoroutines returns a full stack trace of every live goroutine.
+// Used on write-timeout to give CI actionable diagnosis if the write
+// buffer cap ever re-regresses into a hang.
+func dumpAllGoroutines() string {
+	buf := make([]byte, 1<<20)
+	for {
+		n := runtime.Stack(buf, true)
+		if n < len(buf) {
+			return string(buf[:n])
+		}
+		buf = make([]byte, 2*len(buf))
+	}
+}
+
+// writeBufferCapConfig returns a TestConfig that exercises the new
+// -writeBufferSizeMB flag. The cap is set below the aggregate in-flight
+// write demand of the subtests below, so every new chunk has to pass
+// through the Reserve/Release backpressure path at least some of the
+// time. The cap is intentionally NOT minimal — over-tight settings
+// interact with the per-file writable-chunk limit and the FUSE MaxWrite
+// batching in ways that starve single-handle writers on slow CI.
+//
+// Also enables the mount's pprof debug endpoint so the test can fetch
+// mount-process goroutine dumps on write-timeout, which is the only
+// way to actually diagnose a backpressure deadlock (the test process
+// itself is just blocked in syscall.Write waiting on FUSE).
+func writeBufferCapConfig(debugPort int) *TestConfig {
+	return &TestConfig{
+		Collection:  "",
+		Replication: "000",
+		ChunkSizeMB: 2,   // 2 MiB chunks
+		CacheSizeMB: 100, // read cache (unrelated)
+		NumVolumes:  3,
+		EnableDebug: false,
+		// Route glog to stderr so the framework's process log capture
+		// actually contains something — by default weed sends glog to
+		// /tmp/weed.* files which the CI artifact upload step never
+		// sees. Critical for diagnosing upload/saveToStorage errors
+		// on Linux runs.
+		MountGlobalOptions: []string{
+			"-logtostderr=true",
+			"-v=2",
+		},
+		MountOptions: []string{
+			// 16 MiB total write buffer ⇒ up to 8 chunks in flight
+			// across every open file handle on this mount. Large
+			// enough to avoid starving a single handle on a slow
+			// CI runner, small enough that the concurrent test
+			// below still has to drain through it.
+			"-writeBufferSizeMB=16",
+			"-debug=true",
+			fmt.Sprintf("-debug.port=%d", debugPort),
+		},
+		SkipCleanup: false,
+	}
+}
+
+// writeWithTimeout wraps os.WriteFile with a hard deadline so a stuck
+// write fails the test fast instead of consuming the full job budget.
+// This is belt-and-braces around the 45-minute workflow timeout and
+// makes write-buffer regressions surface as an actionable failure.
+func writeWithTimeout(t *testing.T, path string, data []byte, timeout time.Duration) {
+	t.Helper()
+	done := make(chan error, 1)
+	go func() { done <- os.WriteFile(path, data, 0644) }()
+	select {
+	case err := <-done:
+		if err != nil {
+			// Dump mount goroutines on any write error, not just
+			// timeout — upload failures that surface via close()
+			// as EIO leave the mount process running but in an
+			// informative state (pending sealed chunks, error
+			// counters, etc).
+			t.Logf("write %s failed (%v) — dumping MOUNT goroutines:\n%s", path, err, fetchMountGoroutines())
+		}
+		require.NoError(t, err, "write %s", path)
+	case <-time.After(timeout):
+		t.Logf("write %s did not finish within %v — dumping TEST goroutines:\n%s", path, timeout, dumpAllGoroutines())
+		t.Logf("dumping MOUNT goroutines:\n%s", fetchMountGoroutines())
+		t.Fatalf("write %s timed out — write buffer cap is likely leaking or deadlocking", path)
+	}
+}
+
+// runSubtestWithWatchdog runs body on the current (subtest main)
+// goroutine and starts a watcher goroutine that logs diagnostics and
+// fails the test if body doesn't return within timeout.
+//
+// body must run on the main goroutine because test helpers inside it
+// (require.NoError, writeWithTimeout's own t.Fatalf on its internal
+// timeout) need t.Fatal / t.FailNow, which Go's testing docs restrict
+// to the goroutine running the test function. The watcher goroutine
+// only calls goroutine-safe t methods (t.Log, t.Logf, t.Errorf) so it
+// can mark the test failed and dump diagnostics without violating
+// that contract. If body is stuck past timeout the watcher still
+// surfaces the wedge (test + mount goroutine dumps + a FAIL mark);
+// body itself gets unblocked either by its own inner writeWithTimeout
+// firing t.Fatalf or by Go test's global -timeout.
+func runSubtestWithWatchdog(t *testing.T, timeout time.Duration, body func(t *testing.T)) {
+	t.Helper()
+	stop := make(chan struct{})
+	defer close(stop)
+	go func() {
+		select {
+		case <-stop:
+			return
+		case <-time.After(timeout):
+			t.Logf("subtest exceeded %v watchdog — dumping TEST goroutines:\n%s", timeout, dumpAllGoroutines())
+			t.Logf("dumping MOUNT goroutines:\n%s", fetchMountGoroutines())
+			t.Errorf("subtest exceeded %v watchdog — see goroutine dumps above", timeout)
+		}
+	}()
+	body(t)
+}
+
+// TestWriteBufferCap exercises the end-to-end write-buffer cap on a
+// real FUSE mount. Without the cap, a volume-server stall would let
+// the swap file grow without bound (issue #8777). With the cap, writers
+// must serialize through a bounded budget while still producing correct
+// output — that correctness (and the absence of deadlocks) is what
+// this test verifies.
+//
+// Note: this test deliberately does not assert that Reserve *blocked*
+// at some observed used-byte peak. The mount runs as a subprocess so
+// its in-process WriteBufferAccountant state is not reachable from the
+// test without adding a metrics/RPC surface to the mount binary. The
+// deterministic peak-vs-cap assertion instead lives in the in-package
+// unit test TestWriteBufferCap_SharedAcrossPipelines, which drives a
+// controlled gated uploader and samples Used() throughout the run.
+func TestWriteBufferCap(t *testing.T) {
+	mountDebugPort = freePort(t)
+	config := writeBufferCapConfig(mountDebugPort)
+	framework := NewFuseTestFramework(t, config)
+	defer framework.Cleanup()
+
+	require.NoError(t, framework.Setup(config))
+
+	const subtestTimeout = 3 * time.Minute
+
+	t.Run("ConcurrentWritesUnderCap", func(t *testing.T) {
+		runSubtestWithWatchdog(t, subtestTimeout, func(t *testing.T) {
+			testConcurrentWritesUnderCap(t, framework)
+		})
+	})
+
+	t.Run("LargeFileUnderCap", func(t *testing.T) {
+		runSubtestWithWatchdog(t, subtestTimeout, func(t *testing.T) {
+			testLargeFileUnderCap(t, framework)
+		})
+	})
+
+	t.Run("DoesNotDeadlockAfterPressure", func(t *testing.T) {
+		runSubtestWithWatchdog(t, subtestTimeout, func(t *testing.T) {
+			testWriteBufferNoDeadlockAfterPressure(t, framework)
+		})
+	})
+}
+
+// testConcurrentWritesUnderCap opens several files in parallel with
+// aggregate demand that exceeds the 16 MiB write buffer cap, then
+// verifies every byte survived the round trip.
+func testConcurrentWritesUnderCap(t *testing.T, framework *FuseTestFramework) {
+	const (
+		numFiles = 4
+		fileSize = 8 * 1024 * 1024 // 8 MiB per file ⇒ 32 MiB total vs 16 MiB cap
+	)
+
+	dir := "write_buffer_cap_concurrent"
+	framework.CreateTestDir(dir)
+
+	payloads := make([][]byte, numFiles)
+	for i := range payloads {
+		buf := make([]byte, fileSize)
+		_, err := rand.Read(buf)
+		require.NoError(t, err)
+		payloads[i] = buf
+	}
+
+	start := time.Now()
+	var wg sync.WaitGroup
+	errs := make(chan error, numFiles)
+	timedOut := make(chan struct{}, numFiles)
+	for i := 0; i < numFiles; i++ {
+		i := i
+		wg.Add(1)
+		go func() {
+			defer wg.Done()
+			name := fmt.Sprintf("file_%02d.bin", i)
+			path := filepath.Join(framework.GetMountPoint(), dir, name)
+			done := make(chan error, 1)
+			go func() { done <- os.WriteFile(path, payloads[i], 0644) }()
+			select {
+			case err := <-done:
+				if err != nil {
+					errs <- fmt.Errorf("writer %d: %w", i, err)
+				}
+			case <-time.After(90 * time.Second):
+				timedOut <- struct{}{}
+				errs <- fmt.Errorf("writer %d: timed out after 90s", i)
+			}
+		}()
+	}
+	wg.Wait()
+	close(errs)
+	// If any writer timed out, dump every live goroutine so CI shows the
+	// wedge instead of just a walltime.
+	select {
+	case <-timedOut:
+		t.Logf("at least one concurrent writer timed out — dumping goroutines:\n%s", dumpAllGoroutines())
+	default:
+	}
+	for err := range errs {
+		t.Fatal(err)
+	}
+	t.Logf("wrote %d × %d MiB under 16 MiB cap in %v", numFiles, fileSize/(1024*1024), time.Since(start))
+
+	for i := 0; i < numFiles; i++ {
+		name := fmt.Sprintf("file_%02d.bin", i)
+		path := filepath.Join(framework.GetMountPoint(), dir, name)
+		got, err := os.ReadFile(path)
+		require.NoError(t, err, "read %s", name)
+		require.Equal(t, len(payloads[i]), len(got), "size mismatch for %s", name)
+		if !bytes.Equal(payloads[i], got) {
+			t.Fatalf("content mismatch for %s", name)
+		}
+	}
+}
+
+// testLargeFileUnderCap writes a single file whose size exceeds the
+// 16 MiB cap through a single handle, verifying that the pipeline
+// drains its own earlier chunks and makes forward progress rather than
+// self-deadlocking when the global budget is already full of its own
+// earlier sealed chunks.
+func testLargeFileUnderCap(t *testing.T, framework *FuseTestFramework) {
+	const fileSize = 20 * 1024 * 1024 // 20 MiB ⇒ 10 chunks vs 8-slot budget
+
+	payload := make([]byte, fileSize)
+	_, err := rand.Read(payload)
+	require.NoError(t, err)
+
+	name := "write_buffer_cap_large.bin"
+	path := filepath.Join(framework.GetMountPoint(), name)
+
+	start := time.Now()
+	writeWithTimeout(t, path, payload, 90*time.Second)
+	t.Logf("wrote %d MiB through one handle under 16 MiB cap in %v", fileSize/(1024*1024), time.Since(start))
+
+	got, err := os.ReadFile(path)
+	require.NoError(t, err)
+	require.Equal(t, len(payload), len(got))
+	if !bytes.Equal(payload, got) {
+		t.Fatal("content mismatch on large single-handle write")
+	}
+}
+
+// testWriteBufferNoDeadlockAfterPressure verifies the mount is still
+// healthy after being driven against the cap. A budget-slot leak would
+// eventually cause every new chunk allocation to hang; a quick canary
+// write catches that as a hard failure.
+func testWriteBufferNoDeadlockAfterPressure(t *testing.T, framework *FuseTestFramework) {
+	name := "write_buffer_cap_canary.txt"
+	path := filepath.Join(framework.GetMountPoint(), name)
+	content := []byte("write buffer cap canary — mount still healthy")
+
+	writeWithTimeout(t, path, content, 30*time.Second)
+
+	got, err := os.ReadFile(path)
+	require.NoError(t, err)
+	require.Equal(t, content, got)
+}

test/fuse_integration/writeback_cache_test.go

@@ -0,0 +1,861 @@
+package fuse_test
+
+import (
+	"bytes"
+	"crypto/rand"
+	"fmt"
+	"os"
+	"path/filepath"
+	"runtime"
+	"sync"
+	"sync/atomic"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+// writebackConfig returns a TestConfig with writebackCache enabled
+func writebackConfig() *TestConfig {
+	return &TestConfig{
+		Collection:  "",
+		Replication: "000",
+		ChunkSizeMB: 2,
+		CacheSizeMB: 100,
+		NumVolumes:  3,
+		EnableDebug: false,
+		MountOptions: []string{
+			"-writebackCache",
+		},
+		SkipCleanup: false,
+	}
+}
+
+// waitForFileContent polls until a file has the expected content or timeout expires.
+// This is needed because writebackCache defers data upload to background goroutines,
+// so there is a brief window after close() where the file may not yet be readable.
+func waitForFileContent(t *testing.T, path string, expected []byte, timeout time.Duration) {
+	t.Helper()
+	deadline := time.Now().Add(timeout)
+	var lastErr error
+	for time.Now().Before(deadline) {
+		actual, err := os.ReadFile(path)
+		if err == nil && bytes.Equal(expected, actual) {
+			return
+		}
+		if err != nil {
+			lastErr = err
+		} else {
+			lastErr = fmt.Errorf("content mismatch: got %d bytes, want %d bytes", len(actual), len(expected))
+		}
+		time.Sleep(200 * time.Millisecond)
+	}
+	t.Fatalf("file %s did not have expected content within %v: %v", path, timeout, lastErr)
+}
+
+// waitForFileSize polls until a file reports the expected size or timeout expires.
+func waitForFileSize(t *testing.T, path string, expectedSize int64, timeout time.Duration) {
+	t.Helper()
+	deadline := time.Now().Add(timeout)
+	for time.Now().Before(deadline) {
+		info, err := os.Stat(path)
+		if err == nil && info.Size() == expectedSize {
+			return
+		}
+		time.Sleep(200 * time.Millisecond)
+	}
+	t.Fatalf("file %s did not reach expected size %d within %v", path, expectedSize, timeout)
+}
+
+// TestWritebackCacheBasicOperations tests fundamental file I/O with writebackCache enabled
+func TestWritebackCacheBasicOperations(t *testing.T) {
+	config := writebackConfig()
+	framework := NewFuseTestFramework(t, config)
+	defer framework.Cleanup()
+
+	require.NoError(t, framework.Setup(config))
+
+	t.Run("WriteAndReadBack", func(t *testing.T) {
+		testWritebackWriteAndReadBack(t, framework)
+	})
+
+	t.Run("MultipleFilesSequential", func(t *testing.T) {
+		testWritebackMultipleFilesSequential(t, framework)
+	})
+
+	t.Run("LargeFile", func(t *testing.T) {
+		testWritebackLargeFile(t, framework)
+	})
+
+	t.Run("EmptyFile", func(t *testing.T) {
+		testWritebackEmptyFile(t, framework)
+	})
+
+	t.Run("OverwriteExistingFile", func(t *testing.T) {
+		testWritebackOverwriteFile(t, framework)
+	})
+}
+
+// testWritebackWriteAndReadBack writes a file and verifies it can be read back
+// after the async flush completes.
+func testWritebackWriteAndReadBack(t *testing.T, framework *FuseTestFramework) {
+	filename := "writeback_basic.txt"
+	content := []byte("Hello from writebackCache test!")
+	mountPath := filepath.Join(framework.GetMountPoint(), filename)
+
+	// Write file — close() returns immediately with async flush
+	require.NoError(t, os.WriteFile(mountPath, content, 0644))
+
+	// Wait for async flush to complete and verify content
+	waitForFileContent(t, mountPath, content, 30*time.Second)
+}
+
+// testWritebackMultipleFilesSequential writes multiple files sequentially
+// and verifies all are readable after async flushes complete.
+func testWritebackMultipleFilesSequential(t *testing.T, framework *FuseTestFramework) {
+	dir := "writeback_sequential"
+	framework.CreateTestDir(dir)
+
+	numFiles := 50
+	files := make(map[string][]byte, numFiles)
+
+	// Write files sequentially — each close() returns immediately
+	for i := 0; i < numFiles; i++ {
+		filename := fmt.Sprintf("file_%03d.txt", i)
+		content := []byte(fmt.Sprintf("Sequential file %d content: %s", i, time.Now().Format(time.RFC3339Nano)))
+		path := filepath.Join(framework.GetMountPoint(), dir, filename)
+		require.NoError(t, os.WriteFile(path, content, 0644))
+		files[filename] = content
+	}
+
+	// Verify all files after a brief wait for async flushes
+	for filename, expectedContent := range files {
+		path := filepath.Join(framework.GetMountPoint(), dir, filename)
+		waitForFileContent(t, path, expectedContent, 30*time.Second)
+	}
+}
+
+// testWritebackLargeFile writes a large file (multi-chunk) with writebackCache
+func testWritebackLargeFile(t *testing.T, framework *FuseTestFramework) {
+	filename := "writeback_large.bin"
+	mountPath := filepath.Join(framework.GetMountPoint(), filename)
+
+	// 8MB file (spans multiple 2MB chunks)
+	content := make([]byte, 8*1024*1024)
+	_, err := rand.Read(content)
+	require.NoError(t, err)
+
+	require.NoError(t, os.WriteFile(mountPath, content, 0644))
+
+	// Wait for file to be fully flushed
+	waitForFileContent(t, mountPath, content, 60*time.Second)
+}
+
+// testWritebackEmptyFile creates an empty file with writebackCache
+func testWritebackEmptyFile(t *testing.T, framework *FuseTestFramework) {
+	filename := "writeback_empty.txt"
+	mountPath := filepath.Join(framework.GetMountPoint(), filename)
+
+	// Create empty file
+	f, err := os.Create(mountPath)
+	require.NoError(t, err)
+	require.NoError(t, f.Close())
+
+	// Should exist and be empty
+	info, err := os.Stat(mountPath)
+	require.NoError(t, err)
+	assert.Equal(t, int64(0), info.Size())
+}
+
+// testWritebackOverwriteFile tests overwriting an existing file
+func testWritebackOverwriteFile(t *testing.T, framework *FuseTestFramework) {
+	filename := "writeback_overwrite.txt"
+	mountPath := filepath.Join(framework.GetMountPoint(), filename)
+
+	// First write
+	content1 := []byte("First version of the file")
+	require.NoError(t, os.WriteFile(mountPath, content1, 0644))
+	waitForFileContent(t, mountPath, content1, 30*time.Second)
+
+	// Overwrite with different content
+	content2 := []byte("Second version — overwritten content that is longer than the first")
+	require.NoError(t, os.WriteFile(mountPath, content2, 0644))
+	waitForFileContent(t, mountPath, content2, 30*time.Second)
+}
+
+// TestWritebackCacheFsync tests that fsync still forces synchronous flush
+// even when writebackCache is enabled
+func TestWritebackCacheFsync(t *testing.T) {
+	config := writebackConfig()
+	framework := NewFuseTestFramework(t, config)
+	defer framework.Cleanup()
+
+	require.NoError(t, framework.Setup(config))
+
+	t.Run("FsyncForcesFlush", func(t *testing.T) {
+		testFsyncForcesFlush(t, framework)
+	})
+
+	t.Run("FsyncThenRead", func(t *testing.T) {
+		testFsyncThenRead(t, framework)
+	})
+}
+
+// testFsyncForcesFlush verifies that calling fsync before close ensures
+// data is immediately available for reading, bypassing the async path.
+func testFsyncForcesFlush(t *testing.T, framework *FuseTestFramework) {
+	filename := "writeback_fsync.txt"
+	mountPath := filepath.Join(framework.GetMountPoint(), filename)
+
+	content := []byte("Data that must be flushed synchronously via fsync")
+
+	// Open, write, fsync, close
+	f, err := os.OpenFile(mountPath, os.O_CREATE|os.O_WRONLY, 0644)
+	require.NoError(t, err)
+
+	_, err = f.Write(content)
+	require.NoError(t, err)
+
+	// fsync forces synchronous data+metadata flush
+	require.NoError(t, f.Sync())
+	require.NoError(t, f.Close())
+
+	// Data should be immediately available — no wait needed
+	actual, err := os.ReadFile(mountPath)
+	require.NoError(t, err)
+	assert.Equal(t, content, actual)
+}
+
+// testFsyncThenRead verifies that after fsync, a freshly opened read
+// returns the correct data without any delay.
+func testFsyncThenRead(t *testing.T, framework *FuseTestFramework) {
+	filename := "writeback_fsync_read.txt"
+	mountPath := filepath.Join(framework.GetMountPoint(), filename)
+
+	content := make([]byte, 64*1024) // 64KB
+	_, err := rand.Read(content)
+	require.NoError(t, err)
+
+	// Write with explicit fsync
+	f, err := os.OpenFile(mountPath, os.O_CREATE|os.O_WRONLY, 0644)
+	require.NoError(t, err)
+	_, err = f.Write(content)
+	require.NoError(t, err)
+	require.NoError(t, f.Sync())
+	require.NoError(t, f.Close())
+
+	// Immediate read should succeed
+	actual, err := os.ReadFile(mountPath)
+	require.NoError(t, err)
+	assert.Equal(t, content, actual)
+}
+
+// TestWritebackCacheConcurrentSmallFiles is the primary test for issue #8718:
+// many small files written concurrently should all be eventually readable.
+func TestWritebackCacheConcurrentSmallFiles(t *testing.T) {
+	config := writebackConfig()
+	framework := NewFuseTestFramework(t, config)
+	defer framework.Cleanup()
+
+	require.NoError(t, framework.Setup(config))
+
+	t.Run("ConcurrentSmallFiles", func(t *testing.T) {
+		testWritebackConcurrentSmallFiles(t, framework)
+	})
+
+	t.Run("ConcurrentSmallFilesMultiDir", func(t *testing.T) {
+		testWritebackConcurrentSmallFilesMultiDir(t, framework)
+	})
+
+	t.Run("RapidCreateCloseSequence", func(t *testing.T) {
+		testWritebackRapidCreateClose(t, framework)
+	})
+}
+
+// testWritebackConcurrentSmallFiles simulates the rsync workload from #8718:
+// multiple workers creating many small files in parallel.
+func testWritebackConcurrentSmallFiles(t *testing.T, framework *FuseTestFramework) {
+	dir := "writeback_concurrent_small"
+	framework.CreateTestDir(dir)
+
+	numWorkers := 8
+	filesPerWorker := 20
+	totalFiles := numWorkers * filesPerWorker
+
+	type fileRecord struct {
+		path    string
+		content []byte
+	}
+
+	var mu sync.Mutex
+	var writeErrors []error
+	records := make([]fileRecord, 0, totalFiles)
+
+	// Phase 1: Write files concurrently (simulating rsync workers)
+	var wg sync.WaitGroup
+	for w := 0; w < numWorkers; w++ {
+		wg.Add(1)
+		go func(workerID int) {
+			defer wg.Done()
+
+			for f := 0; f < filesPerWorker; f++ {
+				filename := fmt.Sprintf("w%02d_f%03d.dat", workerID, f)
+				path := filepath.Join(framework.GetMountPoint(), dir, filename)
+
+				// Vary sizes: 100B to 100KB
+				size := 100 + (workerID*filesPerWorker+f)*500
+				if size > 100*1024 {
+					size = 100*1024
+				}
+				content := make([]byte, size)
+				if _, err := rand.Read(content); err != nil {
+					mu.Lock()
+					writeErrors = append(writeErrors, fmt.Errorf("worker %d file %d rand: %v", workerID, f, err))
+					mu.Unlock()
+					return
+				}
+
+				if err := os.WriteFile(path, content, 0644); err != nil {
+					mu.Lock()
+					writeErrors = append(writeErrors, fmt.Errorf("worker %d file %d: %v", workerID, f, err))
+					mu.Unlock()
+					return
+				}
+
+				mu.Lock()
+				records = append(records, fileRecord{path: path, content: content})
+				mu.Unlock()
+			}
+		}(w)
+	}
+	wg.Wait()
+
+	require.Empty(t, writeErrors, "write errors: %v", writeErrors)
+	assert.Equal(t, totalFiles, len(records))
+
+	// Phase 2: Wait for async flushes and verify all files
+	for _, rec := range records {
+		waitForFileContent(t, rec.path, rec.content, 60*time.Second)
+	}
+
+	// Phase 3: Verify directory listing has correct count
+	entries, err := os.ReadDir(filepath.Join(framework.GetMountPoint(), dir))
+	require.NoError(t, err)
+	assert.Equal(t, totalFiles, len(entries))
+}
+
+// testWritebackConcurrentSmallFilesMultiDir tests concurrent writes across
+// multiple directories — a common pattern for parallel copy tools.
+func testWritebackConcurrentSmallFilesMultiDir(t *testing.T, framework *FuseTestFramework) {
+	baseDir := "writeback_multidir"
+	framework.CreateTestDir(baseDir)
+
+	numDirs := 4
+	filesPerDir := 25
+
+	type fileRecord struct {
+		path    string
+		content []byte
+	}
+	var mu sync.Mutex
+	var records []fileRecord
+	var writeErrors []error
+
+	var wg sync.WaitGroup
+	for d := 0; d < numDirs; d++ {
+		subDir := filepath.Join(baseDir, fmt.Sprintf("dir_%02d", d))
+		framework.CreateTestDir(subDir)
+
+		wg.Add(1)
+		go func(dirID int, dirPath string) {
+			defer wg.Done()
+
+			for f := 0; f < filesPerDir; f++ {
+				filename := fmt.Sprintf("file_%03d.txt", f)
+				path := filepath.Join(framework.GetMountPoint(), dirPath, filename)
+				content := []byte(fmt.Sprintf("dir=%d file=%d data=%s", dirID, f, time.Now().Format(time.RFC3339Nano)))
+
+				if err := os.WriteFile(path, content, 0644); err != nil {
+					mu.Lock()
+					writeErrors = append(writeErrors, fmt.Errorf("dir %d file %d: %v", dirID, f, err))
+					mu.Unlock()
+					return
+				}
+
+				mu.Lock()
+				records = append(records, fileRecord{path: path, content: content})
+				mu.Unlock()
+			}
+		}(d, subDir)
+	}
+	wg.Wait()
+
+	require.Empty(t, writeErrors, "write errors: %v", writeErrors)
+
+	// Verify all files
+	for _, rec := range records {
+		waitForFileContent(t, rec.path, rec.content, 60*time.Second)
+	}
+}
+
+// testWritebackRapidCreateClose rapidly creates and closes files to stress
+// the async flush goroutine pool.
+func testWritebackRapidCreateClose(t *testing.T, framework *FuseTestFramework) {
+	dir := "writeback_rapid"
+	framework.CreateTestDir(dir)
+
+	numFiles := 200
+	type fileRecord struct {
+		path    string
+		content []byte
+	}
+	records := make([]fileRecord, numFiles)
+
+	// Rapidly create files without pausing
+	for i := 0; i < numFiles; i++ {
+		path := filepath.Join(framework.GetMountPoint(), dir, fmt.Sprintf("rapid_%04d.bin", i))
+		content := []byte(fmt.Sprintf("rapid-file-%d", i))
+		require.NoError(t, os.WriteFile(path, content, 0644))
+		records[i] = fileRecord{path: path, content: content}
+	}
+
+	// Verify all files eventually appear with correct content
+	for _, rec := range records {
+		waitForFileContent(t, rec.path, rec.content, 60*time.Second)
+	}
+}
+
+// TestWritebackCacheDataIntegrity tests that data integrity is preserved
+// across various write patterns with writebackCache enabled.
+func TestWritebackCacheDataIntegrity(t *testing.T) {
+	config := writebackConfig()
+	framework := NewFuseTestFramework(t, config)
+	defer framework.Cleanup()
+
+	require.NoError(t, framework.Setup(config))
+
+	t.Run("AppendAfterClose", func(t *testing.T) {
+		testWritebackAppendAfterClose(t, framework)
+	})
+
+	t.Run("PartialWrites", func(t *testing.T) {
+		testWritebackPartialWrites(t, framework)
+	})
+
+	t.Run("FileSizeCorrectness", func(t *testing.T) {
+		testWritebackFileSizeCorrectness(t, framework)
+	})
+
+	t.Run("BinaryData", func(t *testing.T) {
+		testWritebackBinaryData(t, framework)
+	})
+}
+
+// testWritebackAppendAfterClose writes a file, closes it (triggering async flush),
+// waits for flush, then reopens and appends more data.
+func testWritebackAppendAfterClose(t *testing.T, framework *FuseTestFramework) {
+	filename := "writeback_append.txt"
+	mountPath := filepath.Join(framework.GetMountPoint(), filename)
+
+	// First write
+	part1 := []byte("First part of the data.\n")
+	require.NoError(t, os.WriteFile(mountPath, part1, 0644))
+
+	// Wait for first async flush
+	waitForFileContent(t, mountPath, part1, 30*time.Second)
+
+	// Append more data
+	part2 := []byte("Second part appended.\n")
+	f, err := os.OpenFile(mountPath, os.O_APPEND|os.O_WRONLY, 0644)
+	require.NoError(t, err)
+	_, err = f.Write(part2)
+	require.NoError(t, err)
+	require.NoError(t, f.Close())
+
+	// Verify combined content
+	expected := append(part1, part2...)
+	waitForFileContent(t, mountPath, expected, 30*time.Second)
+}
+
+// testWritebackPartialWrites tests writing to specific offsets within a file
+func testWritebackPartialWrites(t *testing.T, framework *FuseTestFramework) {
+	filename := "writeback_partial.bin"
+	mountPath := filepath.Join(framework.GetMountPoint(), filename)
+
+	// Create file with initial content
+	initial := bytes.Repeat([]byte("A"), 4096)
+	require.NoError(t, os.WriteFile(mountPath, initial, 0644))
+	waitForFileContent(t, mountPath, initial, 30*time.Second)
+
+	// Open and write at specific offset
+	f, err := os.OpenFile(mountPath, os.O_WRONLY, 0644)
+	require.NoError(t, err)
+	patch := []byte("PATCHED")
+	_, err = f.WriteAt(patch, 100)
+	require.NoError(t, err)
+	require.NoError(t, f.Close())
+
+	// Build expected content
+	expected := make([]byte, 4096)
+	copy(expected, initial)
+	copy(expected[100:], patch)
+
+	waitForFileContent(t, mountPath, expected, 30*time.Second)
+}
+
+// testWritebackFileSizeCorrectness verifies that file sizes are correct
+// after async flush completes.
+func testWritebackFileSizeCorrectness(t *testing.T, framework *FuseTestFramework) {
+	sizes := []int{0, 1, 100, 4096, 65536, 1024 * 1024}
+
+	for _, size := range sizes {
+		filename := fmt.Sprintf("writeback_size_%d.bin", size)
+		mountPath := filepath.Join(framework.GetMountPoint(), filename)
+
+		content := make([]byte, size)
+		if size > 0 {
+			_, err := rand.Read(content)
+			require.NoError(t, err, "rand.Read failed for size %d", size)
+		}
+
+		require.NoError(t, os.WriteFile(mountPath, content, 0644), "failed to write file of size %d", size)
+
+		if size > 0 {
+			waitForFileSize(t, mountPath, int64(size), 30*time.Second)
+			waitForFileContent(t, mountPath, content, 30*time.Second)
+		}
+	}
+}
+
+// testWritebackBinaryData verifies that arbitrary binary data (including null bytes)
+// is preserved correctly through the async flush path.
+func testWritebackBinaryData(t *testing.T, framework *FuseTestFramework) {
+	filename := "writeback_binary.bin"
+	mountPath := filepath.Join(framework.GetMountPoint(), filename)
+
+	// Generate data with all byte values including nulls
+	content := make([]byte, 256*100)
+	for i := range content {
+		content[i] = byte(i % 256)
+	}
+
+	require.NoError(t, os.WriteFile(mountPath, content, 0644))
+	waitForFileContent(t, mountPath, content, 30*time.Second)
+}
+
+// TestWritebackCachePerformance measures whether writebackCache actually
+// improves throughput for small file workloads compared to synchronous flush.
+func TestWritebackCachePerformance(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skipping performance test in short mode")
+	}
+
+	numFiles := 200
+	fileSize := 4096 // 4KB files
+
+	// Generate test data upfront
+	testData := make([][]byte, numFiles)
+	for i := range testData {
+		testData[i] = make([]byte, fileSize)
+		rand.Read(testData[i])
+	}
+
+	// Benchmark with writebackCache enabled
+	t.Run("WithWritebackCache", func(t *testing.T) {
+		config := writebackConfig()
+		framework := NewFuseTestFramework(t, config)
+		defer framework.Cleanup()
+		require.NoError(t, framework.Setup(config))
+
+		dir := "perf_writeback"
+		framework.CreateTestDir(dir)
+
+		start := time.Now()
+		for i := 0; i < numFiles; i++ {
+			path := filepath.Join(framework.GetMountPoint(), dir, fmt.Sprintf("file_%04d.bin", i))
+			require.NoError(t, os.WriteFile(path, testData[i], 0644))
+		}
+		writebackDuration := time.Since(start)
+
+		// Wait for all files to be flushed
+		for i := 0; i < numFiles; i++ {
+			path := filepath.Join(framework.GetMountPoint(), dir, fmt.Sprintf("file_%04d.bin", i))
+			waitForFileContent(t, path, testData[i], 60*time.Second)
+		}
+
+		t.Logf("writebackCache: wrote %d files in %v (%.0f files/sec)",
+			numFiles, writebackDuration, float64(numFiles)/writebackDuration.Seconds())
+	})
+
+	// Benchmark without writebackCache (synchronous flush)
+	t.Run("WithoutWritebackCache", func(t *testing.T) {
+		config := DefaultTestConfig()
+		framework := NewFuseTestFramework(t, config)
+		defer framework.Cleanup()
+		require.NoError(t, framework.Setup(config))
+
+		dir := "perf_sync"
+		framework.CreateTestDir(dir)
+
+		start := time.Now()
+		for i := 0; i < numFiles; i++ {
+			path := filepath.Join(framework.GetMountPoint(), dir, fmt.Sprintf("file_%04d.bin", i))
+			require.NoError(t, os.WriteFile(path, testData[i], 0644))
+		}
+		syncDuration := time.Since(start)
+
+		t.Logf("synchronous: wrote %d files in %v (%.0f files/sec)",
+			numFiles, syncDuration, float64(numFiles)/syncDuration.Seconds())
+	})
+}
+
+// TestWritebackCacheConcurrentMixedOps tests a mix of operations happening
+// concurrently with writebackCache: creates, reads, overwrites, and deletes.
+func TestWritebackCacheConcurrentMixedOps(t *testing.T) {
+	config := writebackConfig()
+	framework := NewFuseTestFramework(t, config)
+	defer framework.Cleanup()
+
+	require.NoError(t, framework.Setup(config))
+
+	dir := "writeback_mixed"
+	framework.CreateTestDir(dir)
+
+	numFiles := 50
+	var mu sync.Mutex
+	var errors []error
+	var completedWrites int64
+
+	addError := func(err error) {
+		mu.Lock()
+		defer mu.Unlock()
+		errors = append(errors, err)
+	}
+
+	// Phase 1: Create initial files and wait for async flushes
+	initialContents := make(map[int][]byte, numFiles)
+	for i := 0; i < numFiles; i++ {
+		path := filepath.Join(framework.GetMountPoint(), dir, fmt.Sprintf("file_%03d.txt", i))
+		content := []byte(fmt.Sprintf("initial content %d", i))
+		require.NoError(t, os.WriteFile(path, content, 0644))
+		initialContents[i] = content
+	}
+
+	// Poll until initial files are flushed (instead of fixed sleep)
+	for i := 0; i < numFiles; i++ {
+		path := filepath.Join(framework.GetMountPoint(), dir, fmt.Sprintf("file_%03d.txt", i))
+		waitForFileContent(t, path, initialContents[i], 30*time.Second)
+	}
+
+	// Phase 2: Concurrent mixed operations
+	var wg sync.WaitGroup
+
+	// Writers: overwrite existing files
+	for i := 0; i < 4; i++ {
+		wg.Add(1)
+		go func(workerID int) {
+			defer wg.Done()
+			for j := 0; j < numFiles; j++ {
+				if j%4 != workerID {
+					continue // each worker handles a subset
+				}
+				path := filepath.Join(framework.GetMountPoint(), dir, fmt.Sprintf("file_%03d.txt", j))
+				content := []byte(fmt.Sprintf("overwritten by worker %d at %s", workerID, time.Now().Format(time.RFC3339Nano)))
+				if err := os.WriteFile(path, content, 0644); err != nil {
+					addError(fmt.Errorf("writer %d file %d: %v", workerID, j, err))
+					return
+				}
+				atomic.AddInt64(&completedWrites, 1)
+			}
+		}(i)
+	}
+
+	// Readers: read files (may see old or new content, but should not error)
+	for i := 0; i < 4; i++ {
+		wg.Add(1)
+		go func(readerID int) {
+			defer wg.Done()
+			for j := 0; j < numFiles; j++ {
+				path := filepath.Join(framework.GetMountPoint(), dir, fmt.Sprintf("file_%03d.txt", j))
+				_, err := os.ReadFile(path)
+				if err != nil && !os.IsNotExist(err) {
+					addError(fmt.Errorf("reader %d file %d: %v", readerID, j, err))
+					return
+				}
+			}
+		}(i)
+	}
+
+	// New file creators
+	wg.Add(1)
+	go func() {
+		defer wg.Done()
+		for i := 0; i < 20; i++ {
+			path := filepath.Join(framework.GetMountPoint(), dir, fmt.Sprintf("new_file_%03d.txt", i))
+			content := []byte(fmt.Sprintf("new file %d", i))
+			if err := os.WriteFile(path, content, 0644); err != nil {
+				addError(fmt.Errorf("creator file %d: %v", i, err))
+				return
+			}
+		}
+	}()
+
+	wg.Wait()
+
+	require.Empty(t, errors, "mixed operation errors: %v", errors)
+	assert.True(t, atomic.LoadInt64(&completedWrites) > 0, "should have completed some writes")
+
+	// Verify new files exist after async flushes complete (poll instead of fixed sleep)
+	for i := 0; i < 20; i++ {
+		path := filepath.Join(framework.GetMountPoint(), dir, fmt.Sprintf("new_file_%03d.txt", i))
+		expected := []byte(fmt.Sprintf("new file %d", i))
+		waitForFileContent(t, path, expected, 30*time.Second)
+	}
+}
+
+// TestWritebackCacheStressSmallFiles is a focused stress test for the
+// async flush path with many small files — the core scenario from #8718.
+func TestWritebackCacheStressSmallFiles(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skipping stress test in short mode")
+	}
+
+	config := writebackConfig()
+	framework := NewFuseTestFramework(t, config)
+	defer framework.Cleanup()
+
+	require.NoError(t, framework.Setup(config))
+
+	dir := "writeback_stress"
+	framework.CreateTestDir(dir)
+
+	numWorkers := 16
+	filesPerWorker := 100
+	totalFiles := numWorkers * filesPerWorker
+
+	type fileRecord struct {
+		path    string
+		content []byte
+	}
+
+	var mu sync.Mutex
+	var writeErrors []error
+	records := make([]fileRecord, 0, totalFiles)
+
+	start := time.Now()
+
+	// Simulate rsync-like workload: many workers each writing small files
+	var wg sync.WaitGroup
+	for w := 0; w < numWorkers; w++ {
+		wg.Add(1)
+		go func(workerID int) {
+			defer wg.Done()
+			for f := 0; f < filesPerWorker; f++ {
+				filename := fmt.Sprintf("w%02d/f%04d.dat", workerID, f)
+				path := filepath.Join(framework.GetMountPoint(), dir, filename)
+
+				// Ensure subdirectory exists
+				if f == 0 {
+					subDir := filepath.Join(framework.GetMountPoint(), dir, fmt.Sprintf("w%02d", workerID))
+					if err := os.MkdirAll(subDir, 0755); err != nil {
+						mu.Lock()
+						writeErrors = append(writeErrors, fmt.Errorf("worker %d mkdir: %v", workerID, err))
+						mu.Unlock()
+						return
+					}
+				}
+
+				// Small file: 1KB-10KB (typical for rsync of config/source files)
+				size := 1024 + (f%10)*1024
+				content := make([]byte, size)
+				if _, err := rand.Read(content); err != nil {
+					mu.Lock()
+					writeErrors = append(writeErrors, fmt.Errorf("worker %d file %d rand: %v", workerID, f, err))
+					mu.Unlock()
+					return
+				}
+
+				if err := os.WriteFile(path, content, 0644); err != nil {
+					mu.Lock()
+					writeErrors = append(writeErrors, fmt.Errorf("worker %d file %d: %v", workerID, f, err))
+					mu.Unlock()
+					return
+				}
+
+				mu.Lock()
+				records = append(records, fileRecord{path: path, content: content})
+				mu.Unlock()
+			}
+		}(w)
+	}
+	wg.Wait()
+
+	writeDuration := time.Since(start)
+	// Stream to stderr so the line is visible even if the test hangs and the
+	// 45m suite alarm panics before t.Logf flushes its buffer.
+	fmt.Fprintf(os.Stderr, "TestWritebackCacheStressSmallFiles: wrote %d files in %v (%.0f files/sec)\n",
+		totalFiles, writeDuration, float64(totalFiles)/writeDuration.Seconds())
+	t.Logf("wrote %d files in %v (%.0f files/sec)",
+		totalFiles, writeDuration, float64(totalFiles)/writeDuration.Seconds())
+
+	require.Empty(t, writeErrors, "write errors: %v", writeErrors)
+	assert.Equal(t, totalFiles, len(records))
+
+	// Verify all files are eventually readable with correct content.
+	// A whole-phase budget bounds total verify time so a stuck mount fails
+	// with diagnostics instead of being swallowed by the 45m suite alarm.
+	const verifyPhaseBudget = 10 * time.Minute
+	const perFileDeadline = 120 * time.Second
+	verifyStart := time.Now()
+	verifyPhaseDeadline := verifyStart.Add(verifyPhaseBudget)
+	var verifyErrors []error
+	for i, rec := range records {
+		if time.Now().After(verifyPhaseDeadline) {
+			buf := make([]byte, 1<<20)
+			n := runtime.Stack(buf, true)
+			fmt.Fprintf(os.Stderr, "TestWritebackCacheStressSmallFiles: verify-phase budget %v exceeded at file %d/%d (%s); goroutine dump:\n%s\n",
+				verifyPhaseBudget, i, len(records), rec.path, buf[:n])
+			t.Fatalf("verify phase exceeded %v at file %d/%d (%s); accumulated errors: %v",
+				verifyPhaseBudget, i, len(records), rec.path, verifyErrors)
+		}
+		fileDeadline := time.Now().Add(perFileDeadline)
+		var lastErr error
+		retries := 0
+		for time.Now().Before(fileDeadline) {
+			actual, err := os.ReadFile(rec.path)
+			if err == nil && bytes.Equal(rec.content, actual) {
+				lastErr = nil
+				break
+			}
+			if err != nil {
+				lastErr = err
+			} else {
+				lastErr = fmt.Errorf("content mismatch for %s: got %d bytes, want %d", rec.path, len(actual), len(rec.content))
+			}
+			retries++
+			// Surface the in-flight error every ~5s of polling so a stuck file
+			// is visible in real time, not only after the test exits.
+			if retries%10 == 0 {
+				fmt.Fprintf(os.Stderr, "TestWritebackCacheStressSmallFiles: file %d/%d %s still failing after %d retries: %v\n",
+					i, len(records), rec.path, retries, lastErr)
+			}
+			time.Sleep(500 * time.Millisecond)
+		}
+		if lastErr != nil {
+			fmt.Fprintf(os.Stderr, "TestWritebackCacheStressSmallFiles: file %d/%d %s gave up after %v: %v\n",
+				i, len(records), rec.path, perFileDeadline, lastErr)
+			verifyErrors = append(verifyErrors, lastErr)
+		}
+		// Periodic progress so the test isn't silent during the verify phase.
+		if (i+1)%100 == 0 {
+			fmt.Fprintf(os.Stderr, "TestWritebackCacheStressSmallFiles: verified %d/%d (elapsed %v, %d errors)\n",
+				i+1, len(records), time.Since(verifyStart), len(verifyErrors))
+		}
+	}
+	require.Empty(t, verifyErrors, "verification errors after stress test: %v", verifyErrors)
+
+	fmt.Fprintf(os.Stderr, "TestWritebackCacheStressSmallFiles: all %d files verified in %v\n",
+		totalFiles, time.Since(verifyStart))
+	t.Logf("all %d files verified successfully", totalFiles)
+}

test/fuse_integration/xattr_darwin.go

@@ -0,0 +1,125 @@
+//go:build darwin
+
+package fuse
+
+import (
+	"syscall"
+	"unsafe"
+)
+
+// Extended attributes support for macOS
+
+const (
+	// macOS-specific flags
+	XATTR_NOFOLLOW = 0x0001
+	XATTR_CREATE   = 0x0002
+	XATTR_REPLACE  = 0x0004
+)
+
+func setXattr(path, name string, value []byte, flags int) error {
+	pathBytes, err := syscall.BytePtrFromString(path)
+	if err != nil {
+		return err
+	}
+	nameBytes, err := syscall.BytePtrFromString(name)
+	if err != nil {
+		return err
+	}
+
+	var valuePtr unsafe.Pointer
+	if len(value) > 0 {
+		valuePtr = unsafe.Pointer(&value[0])
+	}
+
+	_, _, errno := syscall.Syscall6(syscall.SYS_SETXATTR,
+		uintptr(unsafe.Pointer(pathBytes)),
+		uintptr(unsafe.Pointer(nameBytes)),
+		uintptr(valuePtr),
+		uintptr(len(value)),
+		uintptr(0), // position (not used for regular files)
+		uintptr(flags))
+
+	if errno != 0 {
+		return errno
+	}
+	return nil
+}
+
+func getXattr(path, name string, value []byte) (int, error) {
+	pathBytes, err := syscall.BytePtrFromString(path)
+	if err != nil {
+		return 0, err
+	}
+	nameBytes, err := syscall.BytePtrFromString(name)
+	if err != nil {
+		return 0, err
+	}
+
+	var valuePtr unsafe.Pointer
+	if len(value) > 0 {
+		valuePtr = unsafe.Pointer(&value[0])
+	}
+
+	size, _, errno := syscall.Syscall6(syscall.SYS_GETXATTR,
+		uintptr(unsafe.Pointer(pathBytes)),
+		uintptr(unsafe.Pointer(nameBytes)),
+		uintptr(valuePtr),
+		uintptr(len(value)),
+		uintptr(0), // position
+		uintptr(0)) // options
+
+	if errno != 0 {
+		return 0, errno
+	}
+	return int(size), nil
+}
+
+func listXattr(path string, list []byte) (int, error) {
+	pathBytes, err := syscall.BytePtrFromString(path)
+	if err != nil {
+		return 0, err
+	}
+
+	var listPtr unsafe.Pointer
+	if len(list) > 0 {
+		listPtr = unsafe.Pointer(&list[0])
+	}
+
+	size, _, errno := syscall.Syscall6(syscall.SYS_LISTXATTR,
+		uintptr(unsafe.Pointer(pathBytes)),
+		uintptr(listPtr),
+		uintptr(len(list)),
+		uintptr(0), // options
+		0, 0)
+
+	if errno != 0 {
+		return 0, errno
+	}
+	return int(size), nil
+}
+
+func removeXattr(path, name string) error {
+	pathBytes, err := syscall.BytePtrFromString(path)
+	if err != nil {
+		return err
+	}
+	nameBytes, err := syscall.BytePtrFromString(name)
+	if err != nil {
+		return err
+	}
+
+	_, _, errno := syscall.Syscall6(syscall.SYS_REMOVEXATTR,
+		uintptr(unsafe.Pointer(pathBytes)),
+		uintptr(unsafe.Pointer(nameBytes)),
+		uintptr(0), // options
+		0, 0, 0)
+
+	if errno != 0 {
+		return errno
+	}
+	return nil
+}
+
+func isXattrSupported() bool {
+	return true
+}

test/fuse_integration/xattr_linux.go

@@ -0,0 +1,115 @@
+//go:build linux
+
+package fuse
+
+import (
+	"syscall"
+	"unsafe"
+)
+
+// Extended attributes support for Linux
+
+func setXattr(path, name string, value []byte, flags int) error {
+	pathBytes, err := syscall.BytePtrFromString(path)
+	if err != nil {
+		return err
+	}
+	nameBytes, err := syscall.BytePtrFromString(name)
+	if err != nil {
+		return err
+	}
+
+	var valuePtr unsafe.Pointer
+	if len(value) > 0 {
+		valuePtr = unsafe.Pointer(&value[0])
+	}
+
+	_, _, errno := syscall.Syscall6(syscall.SYS_SETXATTR,
+		uintptr(unsafe.Pointer(pathBytes)),
+		uintptr(unsafe.Pointer(nameBytes)),
+		uintptr(valuePtr),
+		uintptr(len(value)),
+		uintptr(flags),
+		0)
+
+	if errno != 0 {
+		return errno
+	}
+	return nil
+}
+
+func getXattr(path, name string, value []byte) (int, error) {
+	pathBytes, err := syscall.BytePtrFromString(path)
+	if err != nil {
+		return 0, err
+	}
+	nameBytes, err := syscall.BytePtrFromString(name)
+	if err != nil {
+		return 0, err
+	}
+
+	var valuePtr unsafe.Pointer
+	if len(value) > 0 {
+		valuePtr = unsafe.Pointer(&value[0])
+	}
+
+	size, _, errno := syscall.Syscall6(syscall.SYS_GETXATTR,
+		uintptr(unsafe.Pointer(pathBytes)),
+		uintptr(unsafe.Pointer(nameBytes)),
+		uintptr(valuePtr),
+		uintptr(len(value)),
+		0,
+		0)
+
+	if errno != 0 {
+		return 0, errno
+	}
+	return int(size), nil
+}
+
+func listXattr(path string, list []byte) (int, error) {
+	pathBytes, err := syscall.BytePtrFromString(path)
+	if err != nil {
+		return 0, err
+	}
+
+	var listPtr unsafe.Pointer
+	if len(list) > 0 {
+		listPtr = unsafe.Pointer(&list[0])
+	}
+
+	size, _, errno := syscall.Syscall(syscall.SYS_LISTXATTR,
+		uintptr(unsafe.Pointer(pathBytes)),
+		uintptr(listPtr),
+		uintptr(len(list)))
+
+	if errno != 0 {
+		return 0, errno
+	}
+	return int(size), nil
+}
+
+func removeXattr(path, name string) error {
+	pathBytes, err := syscall.BytePtrFromString(path)
+	if err != nil {
+		return err
+	}
+	nameBytes, err := syscall.BytePtrFromString(name)
+	if err != nil {
+		return err
+	}
+
+	_, _, errno := syscall.Syscall(syscall.SYS_REMOVEXATTR,
+		uintptr(unsafe.Pointer(pathBytes)),
+		uintptr(unsafe.Pointer(nameBytes)),
+		0)
+
+	if errno != 0 {
+		return errno
+	}
+	return nil
+}
+
+func isXattrSupported() bool {
+	return true
+}

test/fuse_integration/xattr_unsupported.go

@@ -0,0 +1,31 @@
+//go:build !linux && !darwin
+
+package fuse
+
+import (
+	"errors"
+)
+
+// Extended attributes support for unsupported platforms
+
+var ErrXattrNotSupported = errors.New("extended attributes not supported on this platform")
+
+func setXattr(path, name string, value []byte, flags int) error {
+	return ErrXattrNotSupported
+}
+
+func getXattr(path, name string, value []byte) (int, error) {
+	return 0, ErrXattrNotSupported
+}
+
+func listXattr(path string, list []byte) (int, error) {
+	return 0, ErrXattrNotSupported
+}
+
+func removeXattr(path, name string) error {
+	return ErrXattrNotSupported
+}
+
+func isXattrSupported() bool {
+	return false
+}

test/fuse_p2p/framework_test.go

@@ -0,0 +1,436 @@
+//go:build linux || darwin
+
+package fuse_p2p
+
+import (
+	"context"
+	"fmt"
+	"net"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"strconv"
+	"strings"
+	"sync"
+	"syscall"
+	"testing"
+	"time"
+
+	"github.com/seaweedfs/seaweedfs/test/testutil"
+	"github.com/seaweedfs/seaweedfs/weed/pb"
+	"github.com/seaweedfs/seaweedfs/weed/pb/master_pb"
+	"github.com/stretchr/testify/require"
+	"google.golang.org/grpc"
+	"google.golang.org/grpc/credentials/insecure"
+)
+
+// p2pTestCluster manages a minimal SeaweedFS cluster exercising the peer
+// chunk-sharing (p2p) read path: 1 master, 1 volume, 1 filer, and N FUSE
+// mounts that all have -peer.enable set. Three mounts is the sweet spot
+// for integration testing — with 2 mounts the HRW owner of a chunk is
+// ~50% the reader itself (which short-circuits the peer path); with 3+
+// mounts it's ≤ 1/3, so a multi-chunk file almost certainly exercises
+// the remote-owner fan-out.
+type p2pTestCluster struct {
+	t          testing.TB
+	baseDir    string
+	weedBinary string
+
+	masterPort     int
+	masterGrpcPort int
+	volumePort     int
+	volumeGrpcPort int
+	filerPort      int
+	filerGrpcPort  int
+	mountPeerPorts []int
+	mountPoints    []string
+
+	masterCmd *exec.Cmd
+	volumeCmd *exec.Cmd
+	filerCmd  *exec.Cmd
+	mountCmds []*exec.Cmd
+	logFiles  []*os.File
+
+	cleanupOnce sync.Once
+}
+
+// startP2PTestCluster brings up a cluster with numMounts FUSE mounts,
+// every one advertising on its own -peer.listen port. All mounts register
+// with the same filer. Verbose logging (-v=4) is enabled on each mount so
+// peer-read success/failure messages land in the log file and the test
+// can grep them to verify the p2p path fired.
+func startP2PTestCluster(t testing.TB, numMounts int) *p2pTestCluster {
+	require.GreaterOrEqual(t, numMounts, 2, "need at least 2 mounts to exercise p2p")
+	binary := findWeedBinary()
+	if binary == "" {
+		t.Skip("weed binary not found; set WEED_BINARY or ensure it is on PATH")
+	}
+	baseDir, err := os.MkdirTemp("", "seaweedfs_fuse_p2p_test_")
+	require.NoError(t, err)
+
+	c := &p2pTestCluster{
+		t:              t,
+		baseDir:        baseDir,
+		weedBinary:     binary,
+		mountPeerPorts: make([]int, numMounts),
+		mountPoints:    make([]string, numMounts),
+		mountCmds:      make([]*exec.Cmd, numMounts),
+	}
+	t.Cleanup(c.Stop)
+
+	// master(2) + volume(2) + filer(2) + one peer port per mount.
+	// testutil.AllocatePorts holds all listeners open until every port
+	// is reserved, avoiding the brief close→bind race that would
+	// happen with a per-listener close loop.
+	ports, err := testutil.AllocatePorts(6 + numMounts)
+	require.NoError(t, err)
+	c.masterPort = ports[0]
+	c.masterGrpcPort = ports[1]
+	c.volumePort = ports[2]
+	c.volumeGrpcPort = ports[3]
+	c.filerPort = ports[4]
+	c.filerGrpcPort = ports[5]
+	for i := 0; i < numMounts; i++ {
+		c.mountPeerPorts[i] = ports[6+i]
+	}
+
+	configDir := filepath.Join(baseDir, "config")
+	require.NoError(t, os.MkdirAll(configDir, 0755))
+	require.NoError(t, os.WriteFile(filepath.Join(configDir, "security.toml"), []byte(""), 0644))
+
+	require.NoError(t, c.startMaster(configDir))
+	require.NoError(t, c.waitForTCP(c.masterCmd, "master",
+		fmt.Sprintf("127.0.0.1:%d", c.masterPort), 30*time.Second))
+
+	require.NoError(t, c.startVolume(configDir))
+	require.NoError(t, c.waitForTCP(c.volumeCmd, "volume",
+		fmt.Sprintf("127.0.0.1:%d", c.volumePort), 30*time.Second))
+	require.NoError(t, c.waitForVolumeRegistered(30*time.Second),
+		"volume server registration\n%s", c.tailLog("master"))
+
+	require.NoError(t, c.startFiler(configDir))
+	require.NoError(t, c.waitForTCP(c.filerCmd, "filer",
+		fmt.Sprintf("127.0.0.1:%d", c.filerGrpcPort), 30*time.Second))
+
+	for i := 0; i < numMounts; i++ {
+		mp := filepath.Join(baseDir, fmt.Sprintf("mount%d", i))
+		require.NoError(t, os.MkdirAll(mp, 0755))
+		c.mountPoints[i] = mp
+		require.NoError(t, c.startMount(i, configDir))
+		require.NoError(t, c.waitForMount(mp, 30*time.Second),
+			"mount %d not ready\n%s", i, c.tailLog(fmt.Sprintf("mount%d", i)))
+	}
+	return c
+}
+
+func (c *p2pTestCluster) Stop() {
+	if c == nil {
+		return
+	}
+	c.cleanupOnce.Do(func() {
+		for i := len(c.mountCmds) - 1; i >= 0; i-- {
+			stopCmd(c.mountCmds[i])
+			// Backup unmount in case the FUSE teardown didn't clean up.
+			exec.Command("fusermount3", "-u", c.mountPoints[i]).Run()
+			exec.Command("fusermount", "-u", c.mountPoints[i]).Run()
+		}
+		stopCmd(c.filerCmd)
+		stopCmd(c.volumeCmd)
+		stopCmd(c.masterCmd)
+
+		for _, f := range c.logFiles {
+			f.Close()
+		}
+		c.copyLogsForCI()
+		if !c.t.Failed() {
+			os.RemoveAll(c.baseDir)
+		}
+		time.Sleep(2 * time.Second) // let ports drain
+	})
+}
+
+// MountDir returns the filesystem path of the i-th mount.
+func (c *p2pTestCluster) MountDir(i int) string { return c.mountPoints[i] }
+
+// masterAddress / filerAddress return SeaweedFS-style addresses encoding
+// both ports as "host:httpPort.grpcPort". Without this, downstream
+// components fall back to the grpcPort = httpPort + 10000 default,
+// which doesn't match the random port we allocate.
+func (c *p2pTestCluster) masterAddress() string {
+	return string(pb.NewServerAddress("127.0.0.1", c.masterPort, c.masterGrpcPort))
+}
+
+func (c *p2pTestCluster) filerAddress() string {
+	return string(pb.NewServerAddress("127.0.0.1", c.filerPort, c.filerGrpcPort))
+}
+
+// MountLog returns the contents of mount i's log file.
+func (c *p2pTestCluster) MountLog(i int) string {
+	return c.tailLogFull(fmt.Sprintf("mount%d", i))
+}
+
+func (c *p2pTestCluster) startMaster(configDir string) error {
+	c.masterCmd = exec.Command(c.weedBinary,
+		"-logdir="+filepath.Join(c.baseDir, "logs"),
+		"master",
+		"-ip=127.0.0.1",
+		"-ip.bind=127.0.0.1",
+		"-port="+strconv.Itoa(c.masterPort),
+		"-port.grpc="+strconv.Itoa(c.masterGrpcPort),
+		"-mdir="+filepath.Join(c.baseDir, "master"),
+	)
+	return c.startCmd(c.masterCmd, "master")
+}
+
+func (c *p2pTestCluster) startVolume(configDir string) error {
+	volDir := filepath.Join(c.baseDir, "volume")
+	if err := os.MkdirAll(volDir, 0755); err != nil {
+		return fmt.Errorf("create volume dir: %w", err)
+	}
+	c.volumeCmd = exec.Command(c.weedBinary,
+		"-logdir="+filepath.Join(c.baseDir, "logs"),
+		"volume",
+		"-ip=127.0.0.1",
+		"-ip.bind=127.0.0.1",
+		"-port="+strconv.Itoa(c.volumePort),
+		"-port.grpc="+strconv.Itoa(c.volumeGrpcPort),
+		"-master="+c.masterAddress(),
+		"-dir="+volDir,
+		"-max=10",
+	)
+	return c.startCmd(c.volumeCmd, "volume")
+}
+
+func (c *p2pTestCluster) startFiler(configDir string) error {
+	filerDir := filepath.Join(c.baseDir, "filer")
+	if err := os.MkdirAll(filerDir, 0755); err != nil {
+		return fmt.Errorf("create filer dir: %w", err)
+	}
+	c.filerCmd = exec.Command(c.weedBinary,
+		"-logdir="+filepath.Join(c.baseDir, "logs"),
+		"filer",
+		"-ip=127.0.0.1",
+		"-ip.bind=127.0.0.1",
+		"-port="+strconv.Itoa(c.filerPort),
+		"-port.grpc="+strconv.Itoa(c.filerGrpcPort),
+		"-master="+c.masterAddress(),
+		"-defaultStoreDir="+filerDir,
+	)
+	return c.startCmd(c.filerCmd, "filer")
+}
+
+func (c *p2pTestCluster) startMount(idx int, configDir string) error {
+	cacheDir := filepath.Join(c.baseDir, fmt.Sprintf("cache%d", idx))
+	if err := os.MkdirAll(cacheDir, 0755); err != nil {
+		return fmt.Errorf("create cache dir: %w", err)
+	}
+	c.mountCmds[idx] = exec.Command(c.weedBinary,
+		"-logdir="+filepath.Join(c.baseDir, "logs"),
+		"-v=4",
+		"mount",
+		"-filer="+c.filerAddress(),
+		"-dir="+c.mountPoints[idx],
+		"-filer.path=/",
+		"-dirAutoCreate",
+		"-allowOthers=false",
+		"-cacheDir="+cacheDir,
+		"-peer.enable=true",
+		fmt.Sprintf("-peer.listen=127.0.0.1:%d", c.mountPeerPorts[idx]),
+		fmt.Sprintf("-peer.advertise=127.0.0.1:%d", c.mountPeerPorts[idx]),
+		"-peer.dataCenter=dc1",
+		fmt.Sprintf("-peer.rack=rack%d", idx),
+	)
+	return c.startCmd(c.mountCmds[idx], fmt.Sprintf("mount%d", idx))
+}
+
+func (c *p2pTestCluster) startCmd(cmd *exec.Cmd, name string) error {
+	logPath := filepath.Join(c.baseDir, "logs")
+	if err := os.MkdirAll(logPath, 0755); err != nil {
+		return fmt.Errorf("create log dir: %w", err)
+	}
+	logFile, err := os.Create(filepath.Join(logPath, name+".log"))
+	if err != nil {
+		return err
+	}
+	c.logFiles = append(c.logFiles, logFile)
+	cmd.Stdout = logFile
+	cmd.Stderr = logFile
+	return cmd.Start()
+}
+
+func (c *p2pTestCluster) tailLog(name string) string {
+	data, err := os.ReadFile(filepath.Join(c.baseDir, "logs", name+".log"))
+	if err != nil {
+		return fmt.Sprintf("(log %s not available: %v)", name, err)
+	}
+	const maxTail = 8192
+	if len(data) > maxTail {
+		data = data[len(data)-maxTail:]
+	}
+	return string(data)
+}
+
+func (c *p2pTestCluster) tailLogFull(name string) string {
+	data, err := os.ReadFile(filepath.Join(c.baseDir, "logs", name+".log"))
+	if err != nil {
+		return ""
+	}
+	return string(data)
+}
+
+func (c *p2pTestCluster) copyLogsForCI() {
+	// One subdirectory per test: a flat layout lets every teardown overwrite
+	// the previous test's logs, so the CI artifact only ever shows the last
+	// cluster, never the failing one.
+	ciLogDir := filepath.Join("/tmp/seaweedfs-fuse-p2p-logs", strings.ReplaceAll(c.t.Name(), "/", "_"))
+	os.MkdirAll(ciLogDir, 0755)
+	logsDir := filepath.Join(c.baseDir, "logs")
+	entries, err := os.ReadDir(logsDir)
+	if err != nil {
+		return
+	}
+	for _, e := range entries {
+		data, err := os.ReadFile(filepath.Join(logsDir, e.Name()))
+		if err != nil {
+			continue
+		}
+		os.WriteFile(filepath.Join(ciLogDir, e.Name()), data, 0644)
+	}
+}
+
+// waitForVolumeRegistered waits until the volume server shows up in the master
+// topology with its slots reported. An open volume port only means the process
+// is listening: until the master has elected itself and accepted a heartbeat,
+// an assign fails and the first write on a fresh mount surfaces it as ENOSPC.
+func (c *p2pTestCluster) waitForVolumeRegistered(timeout time.Duration) error {
+	addr := fmt.Sprintf("127.0.0.1:%d", c.masterGrpcPort)
+	conn, err := grpc.NewClient(addr, grpc.WithTransportCredentials(insecure.NewCredentials()))
+	if err != nil {
+		return err
+	}
+	defer conn.Close()
+
+	client := master_pb.NewSeaweedClient(conn)
+	deadline := time.Now().Add(timeout)
+	for time.Now().Before(deadline) {
+		ctx, cancel := context.WithTimeout(context.Background(), time.Second)
+		resp, err := client.VolumeList(ctx, &master_pb.VolumeListRequest{})
+		cancel()
+		if err == nil && resp.TopologyInfo != nil {
+			var slots int64
+			for _, dc := range resp.TopologyInfo.DataCenterInfos {
+				for _, rack := range dc.RackInfos {
+					for _, dn := range rack.DataNodeInfos {
+						for _, disk := range dn.DiskInfos {
+							slots += disk.MaxVolumeCount
+						}
+					}
+				}
+			}
+			if slots > 0 {
+				return nil
+			}
+		}
+		time.Sleep(200 * time.Millisecond)
+	}
+	return fmt.Errorf("volume server not registered with the master within %v", timeout)
+}
+
+// waitForTCP polls addr until it accepts a connection, OR the supplied
+// subprocess exits — whichever comes first. Short-circuiting on child
+// exit turns a 30 s-spin-on-dead-process into an immediate failure with
+// the tail of its log. cmd may be nil for callers that don't track a
+// process; in that case we fall back to pure polling.
+//
+// Liveness is checked with signal 0 (POSIX "is this process alive").
+// We deliberately do NOT call cmd.Wait() in a goroutine here because
+// stopCmd() later calls Wait() at shutdown, and only one Wait per
+// process is allowed.
+func (c *p2pTestCluster) waitForTCP(cmd *exec.Cmd, name, addr string, timeout time.Duration) error {
+	deadline := time.Now().Add(timeout)
+	for time.Now().Before(deadline) {
+		conn, err := net.DialTimeout("tcp", addr, time.Second)
+		if err == nil {
+			conn.Close()
+			return nil
+		}
+		if cmd != nil && cmd.Process != nil {
+			// signal 0 doesn't actually send anything; it just asks
+			// the kernel whether the process exists. An error here
+			// (typically "process already finished" or ESRCH) means
+			// the child died before coming up.
+			if sigErr := cmd.Process.Signal(syscall.Signal(0)); sigErr != nil {
+				return fmt.Errorf("%s exited before listening on %s: %v\n%s",
+					name, addr, sigErr, c.tailLog(name))
+			}
+		}
+		time.Sleep(200 * time.Millisecond)
+	}
+	return fmt.Errorf("service at %s not ready within timeout\n%s", addr, c.tailLog(name))
+}
+
+// waitForMount waits for a FUSE filesystem to actually be mounted by
+// watching the device id flip (FUSE mounts have a different Dev than
+// their parent dir).
+func (c *p2pTestCluster) waitForMount(mountPoint string, timeout time.Duration) error {
+	parentDir := filepath.Dir(mountPoint)
+	deadline := time.Now().Add(timeout)
+	for time.Now().Before(deadline) {
+		parentStat, err := os.Stat(parentDir)
+		if err != nil {
+			time.Sleep(200 * time.Millisecond)
+			continue
+		}
+		mountStat, err := os.Stat(mountPoint)
+		if err != nil {
+			time.Sleep(200 * time.Millisecond)
+			continue
+		}
+		parentSys := parentStat.Sys().(*syscall.Stat_t)
+		mountSys := mountStat.Sys().(*syscall.Stat_t)
+		if parentSys.Dev != mountSys.Dev {
+			return nil
+		}
+		time.Sleep(200 * time.Millisecond)
+	}
+	return fmt.Errorf("mount point %s not ready within timeout (FUSE not detected)", mountPoint)
+}
+
+// --- utilities (binary discovery, process shutdown) ---
+//
+// Port allocation is delegated to testutil.MustAllocatePorts, which
+// holds all listeners open until every port is reserved before
+// returning them in a batch — safer than the per-listener
+// close-then-reserve pattern fuse_dlm originally used.
+
+func findWeedBinary() string {
+	if env := os.Getenv("WEED_BINARY"); env != "" {
+		if _, err := os.Stat(env); err == nil {
+			return env
+		}
+	}
+	if p, err := exec.LookPath("weed"); err == nil {
+		return p
+	}
+	return ""
+}
+
+// stopCmd kills a child process with SIGTERM and waits up to 5 s, then
+// escalates to SIGKILL. Tolerates nil and already-exited processes.
+func stopCmd(cmd *exec.Cmd) {
+	if cmd == nil || cmd.Process == nil {
+		return
+	}
+	cmd.Process.Signal(syscall.SIGTERM)
+	done := make(chan struct{})
+	go func() {
+		cmd.Wait()
+		close(done)
+	}()
+	select {
+	case <-done:
+	case <-time.After(5 * time.Second):
+		cmd.Process.Signal(syscall.SIGKILL)
+		<-done
+	}
+}

test/fuse_p2p/peer_chunk_sharing_test.go

@@ -0,0 +1,149 @@
+//go:build linux || darwin
+
+package fuse_p2p
+
+import (
+	"bytes"
+	"crypto/md5"
+	"fmt"
+	"math/rand/v2"
+	"os"
+	"path/filepath"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/require"
+)
+
+// peerReadSuccessMarker is the log line tryPeerRead emits when a peer
+// fetch succeeded. The test greps non-writer mount logs for it to
+// prove the p2p path fired. At glog verbosity 4 (framework sets -v=4).
+const peerReadSuccessMarker = "peer read successful"
+
+// seedConvergenceTimeout bounds how long the test waits for:
+//   - the filer registry to list all mounts,
+//   - every mount's seed view (via MountList poll) to include all peers,
+//   - the first announcer flush cycle to publish chunk holders.
+//
+// With defaults the mount polls MountList every 30 s and flushes
+// ChunkAnnounce every 15 s. Allowing 45 s absorbs one MountList refresh
+// plus one announce cycle plus some slop for CI variance.
+const seedConvergenceTimeout = 90 * time.Second
+
+// TestPeerChunkSharing_ReadersPullFromPeerCache is the headline p2p
+// integration test. It proves at least one non-writer mount can satisfy
+// a read from the writer's chunk cache instead of the volume tier.
+//
+//  1. Bring up 1 master/volume/filer + 3 mounts, all with -peer.enable.
+//  2. Mount 0 writes a ~8 MiB file and reads it back so chunks land in
+//     its local cache and the announcer publishes them.
+//  3. Wait for seed convergence + at least one announcer flush cycle.
+//  4. BOTH mount 1 and mount 2 read the file.
+//
+// Why both readers: with 3 mounts, HRW picks one owner for the chunk.
+// If that owner is mount 1, only mount 2's read will hit the peer
+// path (mount 1's tryPeerRead bails on owner==self). If that owner is
+// mount 2, only mount 1's read will. If that owner is mount 0 (the
+// writer), both can. So by reading from both, we deterministically
+// guarantee at least one non-writer mount exercises the peer path.
+//
+// Once the peer fetch populates its local cache, subsequent reads
+// short-circuit on IsInCache — so we only get one real shot per mount
+// per chunk. That's fine: one success is all the test needs.
+func TestPeerChunkSharing_ReadersPullFromPeerCache(t *testing.T) {
+	c := startP2PTestCluster(t, 3)
+
+	// ~8 MiB, pseudo-random so compression doesn't collapse it to one block.
+	payload := make([]byte, 8*1024*1024)
+	rng := rand.New(rand.NewPCG(1, 2))
+	for i := range payload {
+		payload[i] = byte(rng.Uint32())
+	}
+
+	const relPath = "p2p-test.bin"
+	writer := c.MountDir(0)
+	require.NoError(t, os.WriteFile(filepath.Join(writer, relPath), payload, 0644))
+
+	// Warm mount 0's chunk cache by reading back through its own FUSE.
+	// Without this the chunks are on the volume server but not yet
+	// in anyone's peer-servable cache.
+	readBack, err := os.ReadFile(filepath.Join(writer, relPath))
+	require.NoError(t, err)
+	require.True(t, bytes.Equal(readBack, payload), "write-then-read on writer mount should match")
+
+	waitForSeedConvergence(t, c, seedConvergenceTimeout)
+
+	// Give the announcer several flush windows to push chunk-holder
+	// entries to the HRW owners. First flush may see the writer's seed
+	// view incomplete (only self) and defer all fids; subsequent
+	// flushes re-check against a refreshed seed view. announce interval
+	// is 15 s, so 45 s covers three attempts.
+	time.Sleep(45 * time.Second)
+
+	// Read from both non-writer mounts. For any HRW outcome on any
+	// chunk, at least one of these reads will NOT have the reader as
+	// the HRW owner, so its tryPeerRead will proceed to ChunkLookup +
+	// FetchChunk.
+	for _, idx := range []int{1, 2} {
+		got, err := os.ReadFile(filepath.Join(c.MountDir(idx), relPath))
+		require.NoError(t, err, "read from mount %d must succeed\n--- mount%d ---\n%s",
+			idx, idx, tailLines(c.MountLog(idx), 80))
+		require.Equal(t, md5.Sum(payload), md5.Sum(got),
+			"mount %d returned mismatched bytes (len got=%d want=%d)", idx, len(got), len(payload))
+	}
+
+	// Content matches alone doesn't prove p2p — the volume fallback
+	// would also satisfy the reads. Require at least one non-writer
+	// mount's log to contain the peer-read success marker.
+	var sawPeerRead bool
+	for _, idx := range []int{1, 2} {
+		if strings.Contains(c.MountLog(idx), peerReadSuccessMarker) {
+			sawPeerRead = true
+			break
+		}
+	}
+	if !sawPeerRead {
+		t.Fatalf("no non-writer mount logged %q — peer read path never fired.\n"+
+			"--- mount0 (writer) tail ---\n%s\n--- mount1 tail ---\n%s\n--- mount2 tail ---\n%s",
+			peerReadSuccessMarker,
+			tailLines(c.MountLog(0), 60), tailLines(c.MountLog(1), 60), tailLines(c.MountLog(2), 60))
+	}
+}
+
+// waitForSeedConvergence polls each mount's log looking for any sign
+// that MountList returned a peer list containing the other mounts.
+func waitForSeedConvergence(t *testing.T, c *p2pTestCluster, timeout time.Duration) {
+	t.Helper()
+	// The first MountRegister happens synchronously during mount startup,
+	// and the first MountList is pulled right after. 30 s is the refresh
+	// interval; waiting one full cycle here guarantees every mount has
+	// at minimum observed the others in its seed view.
+	deadline := time.Now().Add(timeout)
+	for time.Now().Before(deadline) {
+		ready := true
+		for i := range c.mountCmds {
+			if !strings.Contains(c.MountLog(i), "peer-grpc listening on") {
+				ready = false
+				break
+			}
+		}
+		if ready {
+			time.Sleep(30 * time.Second)
+			return
+		}
+		time.Sleep(500 * time.Millisecond)
+	}
+	t.Fatalf("peer-grpc servers did not come up within %s", timeout)
+}
+
+// tailLines returns the last n newline-delimited lines of s, or the
+// whole thing if shorter. Keeps test failures readable.
+func tailLines(s string, n int) string {
+	lines := strings.Split(s, "\n")
+	if len(lines) <= n {
+		return s
+	}
+	return fmt.Sprintf("... (%d earlier lines omitted) ...\n%s",
+		len(lines)-n, strings.Join(lines[len(lines)-n:], "\n"))
+}

test/java/spark/docker-compose.yml

@@ -19,9 +19,6 @@ services:
       start_period: 10s
 
   seaweedfs-volume:
-    build:
-      context: ../../../docker
-      dockerfile: Dockerfile.local
     image: seaweedfs:local
     container_name: seaweedfs-spark-volume
     ports:
@@ -43,9 +40,6 @@ services:
       start_period: 10s
 
   seaweedfs-filer:
-    build:
-      context: ../../../docker
-      dockerfile: Dockerfile.local
     image: seaweedfs:local
     container_name: seaweedfs-spark-filer
     ports:

test/java/spark/pom.xml

@@ -16,13 +16,15 @@
         <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
         <maven.compiler.source>11</maven.compiler.source>
         <maven.compiler.target>11</maven.compiler.target>
-        <spark.version>3.5.0</spark.version>
+        <spark.version>3.5.8</spark.version>
         <hadoop.version>3.3.6</hadoop.version>
         <scala.binary.version>2.12</scala.binary.version>
         <junit.version>4.13.2</junit.version>
         <seaweedfs.hadoop3.client.version>4.00</seaweedfs.hadoop3.client.version>
-        <jackson.version>2.18.2</jackson.version> <!-- Upgraded from 2.15.3 -->
-        <netty.version>4.1.129.Final</netty.version> <!-- Upgraded to 4.1.125.Final for security fixes (CVE in netty-codec < 4.1.125.Final, netty-codec-http2 <= 4.1.123.Final) -->
+        <jackson.version>2.22.3</jackson.version> <!-- Upgraded from 2.15.3 -->
+        <!-- jackson-annotations dropped the patch digit in 2.20; matches jackson.version's major.minor -->
+        <jackson.annotations.version>2.22</jackson.annotations.version>
+        <netty.version>4.2.16.Final</netty.version> <!-- Upgraded to 4.1.125.Final for security fixes (CVE in netty-codec < 4.1.125.Final, netty-codec-http2 <= 4.1.123.Final) -->
         <parquet.version>1.15.2</parquet.version> <!-- Upgraded to 1.15.2 for security fix -->
         <parquet.format.version>2.12.0</parquet.format.version>
         <surefire.jvm.args>
@@ -62,7 +64,7 @@
             <dependency>
                 <groupId>com.fasterxml.jackson.core</groupId>
                 <artifactId>jackson-annotations</artifactId>
-                <version>${jackson.version}</version>
+                <version>${jackson.annotations.version}</version>
             </dependency>
             <dependency>
                 <groupId>com.fasterxml.jackson.dataformat</groupId>
@@ -116,14 +118,14 @@
             <dependency>
                 <groupId>org.apache.avro</groupId>
                 <artifactId>avro</artifactId>
-                <version>1.11.4</version>
+                <version>1.11.5</version>
             </dependency>
 
             <!-- Apache ZooKeeper - Fix CVEs -->
             <dependency>
                 <groupId>org.apache.zookeeper</groupId>
                 <artifactId>zookeeper</artifactId>
-                <version>3.9.4</version>
+                <version>3.9.5</version>
             </dependency>
 
             <!-- Apache Commons - Fix CVEs -->

test/kafka/Dockerfile.kafka-gateway

@@ -1,5 +1,5 @@
 # Dockerfile for Kafka Gateway Integration Testing
-FROM golang:1.24-alpine AS builder
+FROM golang:1.26-alpine AS builder
 
 # Install build dependencies
 RUN apk add --no-cache git make gcc musl-dev sqlite-dev
@@ -16,8 +16,9 @@ RUN go mod download
 # Copy source code
 COPY . .
 
-# Build the weed binary with Kafka gateway support
+# Build the weed binaries with Kafka gateway support
 RUN CGO_ENABLED=1 GOOS=linux go build -a -installsuffix cgo -ldflags '-extldflags "-static"' -o weed ./weed
+RUN CGO_ENABLED=1 GOOS=linux go build -a -installsuffix cgo -ldflags '-extldflags "-static"' -o weed-sql ./cmd/weed-sql
 
 # Final stage
 FROM alpine:latest
@@ -32,8 +33,9 @@ RUN addgroup -g 1000 seaweedfs && \
 # Set working directory
 WORKDIR /usr/bin
 
-# Copy binary from builder
+# Copy binaries from builder
 COPY --from=builder /app/weed .
+COPY --from=builder /app/weed-sql .
 
 # Create data directory
 RUN mkdir -p /data && chown seaweedfs:seaweedfs /data

test/kafka/Dockerfile.seaweedfs

@@ -1,5 +1,5 @@
 # Dockerfile for building SeaweedFS components from the current workspace
-FROM golang:1.24-alpine AS builder
+FROM golang:1.26-alpine AS builder
 
 RUN apk add --no-cache git make gcc musl-dev sqlite-dev
 

test/kafka/Dockerfile.test-setup

@@ -1,5 +1,5 @@
 # Dockerfile for Kafka Integration Test Setup
-FROM golang:1.24-alpine AS builder
+FROM golang:1.26-alpine AS builder
 
 # Install build dependencies
 RUN apk add --no-cache git make gcc musl-dev

test/kafka/e2e/offset_management_test.go

@@ -27,17 +27,19 @@ func TestOffsetManagement(t *testing.T) {
 		t.Logf("Running offset management tests with mock backend - offsets are in-memory only")
 	}
 
-	topic := testutil.GenerateUniqueTopicName("offset-management")
+	topic1 := testutil.GenerateUniqueTopicName("offset-management")
+	topic2 := testutil.GenerateUniqueTopicName("offset-resumption")
 	groupID := testutil.GenerateUniqueGroupID("offset-test-group")
 
-	gateway.AddTestTopic(topic)
+	gateway.AddTestTopic(topic1)
+	gateway.AddTestTopic(topic2)
 
 	t.Run("BasicOffsetCommitFetch", func(t *testing.T) {
-		testBasicOffsetCommitFetch(t, addr, topic, groupID)
+		testBasicOffsetCommitFetch(t, addr, topic1, groupID)
 	})
 
 	t.Run("ConsumerGroupResumption", func(t *testing.T) {
-		testConsumerGroupResumption(t, addr, topic, groupID+"2")
+		testConsumerGroupResumption(t, gateway, addr, topic2, groupID+"2")
 	})
 }
 
@@ -76,7 +78,7 @@ func testBasicOffsetCommitFetch(t *testing.T, addr, topic, groupID string) {
 	t.Logf("SUCCESS: Offset management test completed - consumed %d + %d messages", len(consumed1), len(consumed2))
 }
 
-func testConsumerGroupResumption(t *testing.T, addr, topic, groupID string) {
+func testConsumerGroupResumption(t *testing.T, gateway *testutil.GatewayTestServer, addr, topic, groupID string) {
 	client := testutil.NewKafkaGoClient(t, addr)
 	msgGen := testutil.NewMessageGenerator()
 
@@ -95,10 +97,28 @@ func testConsumerGroupResumption(t *testing.T, addr, topic, groupID string) {
 	for i, msg := range consumed1 {
 		t.Logf("  Message %d: offset=%d, partition=%d, value=%s", i, msg.Offset, msg.Partition, string(msg.Value))
 	}
+	gateway.LogConsumerGroupSnapshot(groupID)
 
 	// Simulate consumer restart by consuming remaining messages with same group ID
 	t.Logf("=== Phase 3: Second consumer (simulated restart) - consuming remaining messages with same group %s ===", groupID)
-	consumed2, err := client.ConsumeWithGroup(topic, groupID, 2)
+	consumed2, err := client.ConsumeWithGroupDebug(topic, groupID, 2,
+		func(info testutil.ConsumeGroupRetryDebug) {
+			t.Logf("Consumer restart attempt %d/%d for group %s failed before receiving any messages: %v",
+				info.Attempt, info.MaxAttempts, info.GroupID, info.Err)
+			gateway.LogConsumerGroupSnapshot(groupID)
+		},
+		func() {
+			// Live snapshot taken while the reader is still joined — this is
+			// the only place we get to see the group's transient state
+			// (PreparingRebalance, leader, members) during a stuck attempt.
+			// The onRetry snapshot above runs after reader.Close() and only
+			// ever shows the post-LeaveGroup Empty state.
+			gateway.LogConsumerGroupSnapshot(groupID)
+		},
+	)
+	if err != nil {
+		gateway.LogConsumerGroupSnapshot(groupID)
+	}
 	testutil.AssertNoError(t, err, "Failed to consume after restart")
 	t.Logf("Second consumer consumed %d messages:", len(consumed2))
 	for i, msg := range consumed2 {

test/kafka/go.mod

@@ -1,259 +1,102 @@
 module github.com/seaweedfs/seaweedfs/test/kafka
 
-go 1.24.9
+go 1.26.6
 
 require (
 	github.com/IBM/sarama v1.46.0
-	github.com/linkedin/goavro/v2 v2.14.1
+	github.com/linkedin/goavro/v2 v2.15.0
 	github.com/seaweedfs/seaweedfs v0.0.0-00010101000000-000000000000
 	github.com/segmentio/kafka-go v0.4.49
-	github.com/stretchr/testify v1.11.1
-	google.golang.org/grpc v1.77.0
+	github.com/stretchr/testify v1.12.1
+	google.golang.org/grpc v1.85.0-dev.0.20260915183914-4e49413dcab7
 )
 
 replace github.com/seaweedfs/seaweedfs => ../../
 
 require (
-	cloud.google.com/go/auth v0.17.0 // indirect
-	cloud.google.com/go/auth/oauth2adapt v0.2.8 // indirect
-	cloud.google.com/go/compute/metadata v0.9.0 // indirect
-	github.com/Azure/azure-sdk-for-go/sdk/azcore v1.20.0 // indirect
-	github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.13.1 // indirect
-	github.com/Azure/azure-sdk-for-go/sdk/internal v1.11.2 // indirect
-	github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.6.3 // indirect
-	github.com/Azure/azure-sdk-for-go/sdk/storage/azfile v1.5.2 // indirect
-	github.com/Azure/go-ntlmssp v0.0.0-20221128193559-754e69321358 // indirect
-	github.com/AzureAD/microsoft-authentication-library-for-go v1.6.0 // indirect
-	github.com/Files-com/files-sdk-go/v3 v3.2.218 // indirect
-	github.com/IBM/go-sdk-core/v5 v5.21.0 // indirect
-	github.com/Max-Sum/base32768 v0.0.0-20230304063302-18e6ce5945fd // indirect
-	github.com/Microsoft/go-winio v0.6.2 // indirect
-	github.com/ProtonMail/bcrypt v0.0.0-20211005172633-e235017c1baf // indirect
-	github.com/ProtonMail/gluon v0.17.1-0.20230724134000-308be39be96e // indirect
-	github.com/ProtonMail/go-crypto v1.3.0 // indirect
-	github.com/ProtonMail/go-mime v0.0.0-20230322103455-7d82a3887f2f // indirect
-	github.com/ProtonMail/go-srp v0.0.7 // indirect
-	github.com/ProtonMail/gopenpgp/v2 v2.9.0 // indirect
-	github.com/PuerkitoBio/goquery v1.10.3 // indirect
-	github.com/abbot/go-http-auth v0.4.0 // indirect
-	github.com/andybalholm/brotli v1.2.0 // indirect
-	github.com/andybalholm/cascadia v1.3.3 // indirect
-	github.com/appscode/go-querystring v0.0.0-20170504095604-0126cfb3f1dc // indirect
-	github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 // indirect
+	github.com/andybalholm/brotli v1.2.3 // indirect
 	github.com/aws/aws-sdk-go v1.55.8 // indirect
-	github.com/aws/aws-sdk-go-v2 v1.41.0 // indirect
-	github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.4 // indirect
-	github.com/aws/aws-sdk-go-v2/config v1.32.6 // indirect
-	github.com/aws/aws-sdk-go-v2/credentials v1.19.6 // indirect
-	github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.16 // indirect
-	github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.20.3 // indirect
-	github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.16 // indirect
-	github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.16 // indirect
-	github.com/aws/aws-sdk-go-v2/internal/ini v1.8.4 // indirect
-	github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.15 // indirect
-	github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.4 // indirect
-	github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.6 // indirect
-	github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.16 // indirect
-	github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.15 // indirect
-	github.com/aws/aws-sdk-go-v2/service/s3 v1.93.0 // indirect
-	github.com/aws/aws-sdk-go-v2/service/signin v1.0.4 // indirect
-	github.com/aws/aws-sdk-go-v2/service/sso v1.30.8 // indirect
-	github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.12 // indirect
-	github.com/aws/aws-sdk-go-v2/service/sts v1.41.5 // indirect
-	github.com/aws/smithy-go v1.24.0 // indirect
 	github.com/beorn7/perks v1.0.1 // indirect
-	github.com/bradenaw/juniper v0.15.3 // indirect
-	github.com/bradfitz/iter v0.0.0-20191230175014-e8f45d346db8 // indirect
-	github.com/buengese/sgzip v0.1.1 // indirect
-	github.com/bufbuild/protocompile v0.14.1 // indirect
-	github.com/calebcase/tmpfile v1.0.3 // indirect
 	github.com/cespare/xxhash/v2 v2.3.0 // indirect
-	github.com/chilts/sid v0.0.0-20190607042430-660e94789ec9 // indirect
-	github.com/cloudflare/circl v1.6.1 // indirect
-	github.com/cloudinary/cloudinary-go/v2 v2.12.0 // indirect
-	github.com/cloudsoda/go-smb2 v0.0.0-20250228001242-d4c70e6251cc // indirect
-	github.com/cloudsoda/sddl v0.0.0-20250224235906-926454e91efc // indirect
-	github.com/cognusion/imaging v1.0.2 // indirect
-	github.com/colinmarc/hdfs/v2 v2.4.0 // indirect
-	github.com/coreos/go-semver v0.3.1 // indirect
-	github.com/coreos/go-systemd/v22 v22.5.0 // indirect
-	github.com/creasty/defaults v1.8.0 // indirect
-	github.com/cronokirby/saferith v0.33.0 // indirect
+	github.com/cognusion/imaging v1.0.4 // indirect
 	github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect
-	github.com/dropbox/dropbox-sdk-go-unofficial/v6 v6.0.5 // indirect
-	github.com/dustin/go-humanize v1.0.1 // indirect
+	github.com/dustin/go-humanize v1.1.0 // indirect
 	github.com/eapache/go-resiliency v1.7.0 // indirect
 	github.com/eapache/go-xerial-snappy v0.0.0-20230731223053-c322873962e3 // indirect
 	github.com/eapache/queue v1.1.0 // indirect
-	github.com/ebitengine/purego v0.9.1 // indirect
-	github.com/emersion/go-message v0.18.2 // indirect
-	github.com/emersion/go-vcard v0.0.0-20241024213814-c9703dde27ff // indirect
-	github.com/felixge/httpsnoop v1.0.4 // indirect
-	github.com/flynn/noise v1.1.0 // indirect
+	github.com/ebitengine/purego v0.10.2 // indirect
 	github.com/fsnotify/fsnotify v1.9.0 // indirect
-	github.com/gabriel-vasile/mimetype v1.4.9 // indirect
-	github.com/geoffgarside/ber v1.2.0 // indirect
-	github.com/go-chi/chi/v5 v5.2.2 // indirect
-	github.com/go-darwin/apfs v0.0.0-20211011131704-f84b94dbf348 // indirect
-	github.com/go-jose/go-jose/v4 v4.1.3 // indirect
-	github.com/go-logr/logr v1.4.3 // indirect
-	github.com/go-logr/stdr v1.2.2 // indirect
+	github.com/go-jose/go-jose/v4 v4.1.4 // indirect
 	github.com/go-ole/go-ole v1.3.0 // indirect
-	github.com/go-openapi/errors v0.22.2 // indirect
-	github.com/go-openapi/strfmt v0.23.0 // indirect
-	github.com/go-playground/locales v0.14.1 // indirect
-	github.com/go-playground/universal-translator v0.18.1 // indirect
-	github.com/go-playground/validator/v10 v10.27.0 // indirect
-	github.com/go-resty/resty/v2 v2.16.5 // indirect
-	github.com/go-viper/mapstructure/v2 v2.4.0 // indirect
-	github.com/gofrs/flock v0.12.1 // indirect
-	github.com/gogo/protobuf v1.3.2 // indirect
-	github.com/golang-jwt/jwt/v4 v4.5.2 // indirect
-	github.com/golang-jwt/jwt/v5 v5.3.0 // indirect
+	github.com/go-viper/mapstructure/v2 v2.5.0 // indirect
+	github.com/golang-jwt/jwt/v5 v5.3.1 // indirect
 	github.com/golang/protobuf v1.5.4 // indirect
 	github.com/golang/snappy v1.0.0 // indirect
 	github.com/google/btree v1.1.3 // indirect
-	github.com/google/s2a-go v0.1.9 // indirect
 	github.com/google/uuid v1.6.0 // indirect
-	github.com/googleapis/enterprise-certificate-proxy v0.3.7 // indirect
-	github.com/googleapis/gax-go/v2 v2.15.0 // indirect
 	github.com/gorilla/mux v1.8.1 // indirect
-	github.com/gorilla/schema v1.4.1 // indirect
-	github.com/hashicorp/errwrap v1.1.0 // indirect
-	github.com/hashicorp/go-cleanhttp v0.5.2 // indirect
-	github.com/hashicorp/go-multierror v1.1.1 // indirect
-	github.com/hashicorp/go-retryablehttp v0.7.8 // indirect
 	github.com/hashicorp/go-uuid v1.0.3 // indirect
-	github.com/henrybear327/Proton-API-Bridge v1.0.0 // indirect
-	github.com/henrybear327/go-proton-api v1.0.0 // indirect
+	github.com/hashicorp/golang-lru/v2 v2.0.7 // indirect
 	github.com/jcmturner/aescts/v2 v2.0.0 // indirect
 	github.com/jcmturner/dnsutils/v2 v2.0.0 // indirect
 	github.com/jcmturner/gofork v1.7.6 // indirect
-	github.com/jcmturner/goidentity/v6 v6.0.1 // indirect
 	github.com/jcmturner/gokrb5/v8 v8.4.4 // indirect
 	github.com/jcmturner/rpc/v2 v2.0.3 // indirect
-	github.com/jhump/protoreflect v1.17.0 // indirect
-	github.com/jlaffaye/ftp v0.2.1-0.20240918233326-1b970516f5d3 // indirect
+	github.com/jhump/protoreflect v1.18.0 // indirect
+	github.com/jhump/protoreflect/v2 v2.0.0-beta.1 // indirect
 	github.com/jmespath/go-jmespath v0.4.0 // indirect
-	github.com/jtolds/gls v4.20.0+incompatible // indirect
-	github.com/jtolio/noiseconn v0.0.0-20231127013910-f6d9ecbf1de7 // indirect
-	github.com/jzelinskie/whirlpool v0.0.0-20201016144138-0675e54bb004 // indirect
 	github.com/karlseguin/ccache/v2 v2.0.8 // indirect
-	github.com/klauspost/compress v1.18.2 // indirect
-	github.com/klauspost/cpuid/v2 v2.3.0 // indirect
-	github.com/klauspost/reedsolomon v1.12.6 // indirect
-	github.com/koofr/go-httpclient v0.0.0-20240520111329-e20f8f203988 // indirect
-	github.com/koofr/go-koofrclient v0.0.0-20221207135200-cbd7fc9ad6a6 // indirect
-	github.com/kr/fs v0.1.0 // indirect
-	github.com/kylelemons/godebug v1.1.0 // indirect
-	github.com/lanrat/extsort v1.4.0 // indirect
-	github.com/leodido/go-urn v1.4.0 // indirect
-	github.com/lpar/date v1.0.0 // indirect
-	github.com/lufia/plan9stats v0.0.0-20250317134145-8bc96cf8fc35 // indirect
-	github.com/mattn/go-colorable v0.1.14 // indirect
-	github.com/mattn/go-isatty v0.0.20 // indirect
-	github.com/mattn/go-runewidth v0.0.16 // indirect
-	github.com/mitchellh/go-homedir v1.1.0 // indirect
-	github.com/mitchellh/mapstructure v1.5.1-0.20220423185008-bf980b35cac4 // indirect
+	github.com/klauspost/compress v1.19.2 // indirect
+	github.com/klauspost/cpuid/v2 v2.4.0 // indirect
+	github.com/klauspost/reedsolomon v1.14.2 // indirect
+	github.com/lufia/plan9stats v0.0.0-20260627054121-477a66015f15 // indirect
 	github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
-	github.com/ncw/swift/v2 v2.0.4 // indirect
-	github.com/oklog/ulid v1.3.1 // indirect
-	github.com/oracle/oci-go-sdk/v65 v65.98.0 // indirect
+	github.com/onsi/ginkgo v1.16.5 // indirect
+	github.com/onsi/gomega v1.37.0 // indirect
 	github.com/orcaman/concurrent-map/v2 v2.0.1 // indirect
-	github.com/panjf2000/ants/v2 v2.11.3 // indirect
-	github.com/parquet-go/parquet-go v0.26.3 // indirect
-	github.com/patrickmn/go-cache v2.1.0+incompatible // indirect
-	github.com/pelletier/go-toml/v2 v2.2.4 // indirect
-	github.com/pengsrc/go-shared v0.2.1-0.20190131101655-1999055a4a14 // indirect
-	github.com/peterh/liner v1.2.2 // indirect
-	github.com/pierrec/lz4/v4 v4.1.22 // indirect
-	github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c // indirect
-	github.com/pkg/errors v0.9.1 // indirect
-	github.com/pkg/sftp v1.13.10 // indirect
-	github.com/pkg/xattr v0.4.12 // indirect
-	github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect
+	github.com/parquet-go/bitpack v1.0.0 // indirect
+	github.com/parquet-go/jsonlite v1.0.0 // indirect
+	github.com/parquet-go/parquet-go v0.32.0 // indirect
+	github.com/pelletier/go-toml/v2 v2.4.3 // indirect
+	github.com/petermattis/goid v0.0.0-20260113132338-7c7de50cc741 // indirect
+	github.com/pierrec/lz4/v4 v4.1.29 // indirect
+	github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10 // indirect
 	github.com/power-devops/perfstat v0.0.0-20240221224432-82ca36839d55 // indirect
-	github.com/prometheus/client_golang v1.23.2 // indirect
-	github.com/prometheus/client_model v0.6.2 // indirect
-	github.com/prometheus/common v0.66.1 // indirect
-	github.com/prometheus/procfs v0.19.2 // indirect
-	github.com/putdotio/go-putio/putio v0.0.0-20200123120452-16d982cac2b8 // indirect
-	github.com/rclone/rclone v1.71.2 // indirect
+	github.com/prometheus/client_golang v1.24.1 // indirect
+	github.com/prometheus/client_model v0.6.3 // indirect
+	github.com/prometheus/common v0.71.0 // indirect
+	github.com/prometheus/procfs v0.22.0 // indirect
 	github.com/rcrowley/go-metrics v0.0.0-20250401214520-65e299d6c5c9 // indirect
 	github.com/rdleal/intervalst v1.5.0 // indirect
-	github.com/relvacode/iso8601 v1.6.0 // indirect
-	github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect
-	github.com/rfjakob/eme v1.1.2 // indirect
-	github.com/rivo/uniseg v0.4.7 // indirect
-	github.com/sabhiram/go-gitignore v0.0.0-20210923224102-525f6e181f06 // indirect
 	github.com/sagikazarmark/locafero v0.11.0 // indirect
-	github.com/samber/lo v1.51.0 // indirect
 	github.com/seaweedfs/goexif v1.0.3 // indirect
-	github.com/shirou/gopsutil/v4 v4.25.11 // indirect
-	github.com/sirupsen/logrus v1.9.3 // indirect
-	github.com/skratchdot/open-golang v0.0.0-20200116055534-eef842397966 // indirect
-	github.com/smarty/assertions v1.16.0 // indirect
-	github.com/sony/gobreaker v1.0.0 // indirect
+	github.com/shirou/gopsutil/v4 v4.26.7 // indirect
 	github.com/sourcegraph/conc v0.3.1-0.20240121214520-5f936abd7ae8 // indirect
-	github.com/spacemonkeygo/monkit/v3 v3.0.24 // indirect
 	github.com/spf13/afero v1.15.0 // indirect
 	github.com/spf13/cast v1.10.0 // indirect
 	github.com/spf13/pflag v1.0.10 // indirect
 	github.com/spf13/viper v1.21.0 // indirect
-	github.com/spiffe/go-spiffe/v2 v2.6.0 // indirect
+	github.com/spiffe/go-spiffe/v2 v2.8.1 // indirect
 	github.com/subosito/gotenv v1.6.0 // indirect
 	github.com/syndtr/goleveldb v1.0.1-0.20190318030020-c3a204f8e965 // indirect
-	github.com/t3rm1n4l/go-mega v0.0.0-20250926104142-ccb8d3498e6c // indirect
-	github.com/tklauser/go-sysconf v0.3.16 // indirect
-	github.com/tklauser/numcpus v0.11.0 // indirect
+	github.com/tklauser/go-sysconf v0.4.0 // indirect
+	github.com/tklauser/numcpus v0.12.0 // indirect
+	github.com/twpayne/go-geom v1.6.1 // indirect
 	github.com/tylertreat/BoomFilters v0.0.0-20210315201527-1a82519a3e43 // indirect
-	github.com/unknwon/goconfig v1.0.0 // indirect
 	github.com/valyala/bytebufferpool v1.0.0 // indirect
 	github.com/viant/ptrie v1.0.1 // indirect
-	github.com/xanzy/ssh-agent v0.3.3 // indirect
-	github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f // indirect
+	github.com/xeipuuv/gojsonpointer v0.0.0-20190905194746-02993c407bfb // indirect
 	github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 // indirect
 	github.com/xeipuuv/gojsonschema v1.2.0 // indirect
-	github.com/youmark/pkcs8 v0.0.0-20240726163527-a2c0da244d78 // indirect
-	github.com/yunify/qingstor-sdk-go/v3 v3.2.0 // indirect
 	github.com/yusufpapurcu/wmi v1.2.4 // indirect
-	github.com/zeebo/blake3 v0.2.4 // indirect
-	github.com/zeebo/errs v1.4.0 // indirect
-	github.com/zeebo/xxh3 v1.0.2 // indirect
-	go.etcd.io/bbolt v1.4.2 // indirect
-	go.mongodb.org/mongo-driver v1.17.6 // indirect
-	go.opentelemetry.io/auto/sdk v1.2.1 // indirect
-	go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.62.0 // indirect
-	go.opentelemetry.io/otel v1.38.0 // indirect
-	go.opentelemetry.io/otel/metric v1.38.0 // indirect
-	go.opentelemetry.io/otel/trace v1.38.0 // indirect
-	go.yaml.in/yaml/v2 v2.4.2 // indirect
-	go.yaml.in/yaml/v3 v3.0.4 // indirect
-	golang.org/x/crypto v0.46.0 // indirect
-	golang.org/x/exp v0.0.0-20250811191247-51f88131bc50 // indirect
-	golang.org/x/image v0.34.0 // indirect
-	golang.org/x/net v0.48.0 // indirect
-	golang.org/x/oauth2 v0.34.0 // indirect
-	golang.org/x/sync v0.19.0 // indirect
-	golang.org/x/sys v0.39.0 // indirect
-	golang.org/x/term v0.38.0 // indirect
-	golang.org/x/text v0.32.0 // indirect
-	golang.org/x/time v0.14.0 // indirect
-	google.golang.org/api v0.258.0 // indirect
-	google.golang.org/genproto/googleapis/rpc v0.0.0-20251213004720-97cd9d5aeac2 // indirect
+	go.yaml.in/yaml/v3 v3.0.5 // indirect
+	golang.org/x/crypto v0.57.0 // indirect
+	golang.org/x/image v0.46.0 // indirect
+	golang.org/x/net v0.58.0 // indirect
+	golang.org/x/sync v0.23.0 // indirect
+	golang.org/x/sys v0.48.0 // indirect
+	golang.org/x/text v0.42.0 // indirect
+	google.golang.org/genproto/googleapis/rpc v0.0.0-20260819154853-08b0e4226688 // indirect
 	google.golang.org/grpc/security/advancedtls v1.0.0 // indirect
-	google.golang.org/protobuf v1.36.11 // indirect
-	gopkg.in/natefinch/lumberjack.v2 v2.2.1 // indirect
-	gopkg.in/validator.v2 v2.0.1 // indirect
-	gopkg.in/yaml.v2 v2.4.0 // indirect
-	gopkg.in/yaml.v3 v3.0.1 // indirect
-	modernc.org/mathutil v1.7.1 // indirect
-	moul.io/http2curl/v2 v2.3.0 // indirect
-	sigs.k8s.io/yaml v1.6.0 // indirect
-	storj.io/common v0.0.0-20250808122759-804533d519c1 // indirect
-	storj.io/drpc v0.0.35-0.20250513201419-f7819ea69b55 // indirect
-	storj.io/eventkit v0.0.0-20250410172343-61f26d3de156 // indirect
-	storj.io/infectious v0.0.2 // indirect
-	storj.io/picobuf v0.0.4 // indirect
-	storj.io/uplink v1.13.1 // indirect
+	google.golang.org/protobuf v1.36.12 // indirect
 )

test/kafka/integration/consumer_groups_test.go

@@ -49,12 +49,20 @@ func testConsumerGroupBasicFunctionality(t *testing.T, addr string) {
 	err = client.ProduceMessages(topicName, messages)
 	testutil.AssertNoError(t, err, "Failed to produce messages")
 
-	// Test with multiple consumers in the same group
+	// Test with multiple consumers in the same group. The messages channel is
+	// shared so the assertion can verify total-consumed/no-duplicates across the
+	// group. Each consumer gets its own handler (and its own ready channel) so
+	// we can wait for each distinct consumer to reach Setup, rather than
+	// relying on a single close() that only signals once.
 	numConsumers := 3
-	handler := &ConsumerGroupHandler{
-		messages: make(chan *sarama.ConsumerMessage, len(messages)),
-		ready:    make(chan bool),
-		t:        t,
+	handlers := make([]*ConsumerGroupHandler, numConsumers)
+	sharedMessages := make(chan *sarama.ConsumerMessage, len(messages))
+	for i := 0; i < numConsumers; i++ {
+		handlers[i] = &ConsumerGroupHandler{
+			messages: sharedMessages,
+			ready:    make(chan bool),
+			t:        t,
+		}
 	}
 
 	var wg sync.WaitGroup
@@ -72,35 +80,32 @@ func testConsumerGroupBasicFunctionality(t *testing.T, addr string) {
 			}
 			defer consumerGroup.Close()
 
-			ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+			ctx, cancel := context.WithTimeout(context.Background(), 20*time.Second)
 			defer cancel()
 
-			err = consumerGroup.Consume(ctx, []string{topicName}, handler)
-			if err != nil && err != context.DeadlineExceeded {
-				consumerErrors <- fmt.Errorf("consumer %d: consumption error: %v", consumerID, err)
-				return
-			}
+			runConsumeLoop(t, ctx, fmt.Sprintf("Consumer%d", consumerID),
+				consumerGroup, []string{topicName}, handlers[consumerID])
 		}(i)
 	}
 
-	// Wait for consumers to be ready
-	readyCount := 0
-	for readyCount < numConsumers {
+	// Wait for each consumer to be ready. Multi-consumer rebalance can take
+	// a few heartbeat intervals (default 3s) as the initial leader receives
+	// REBALANCE_IN_PROGRESS from its heartbeat and re-joins, so allow headroom.
+	for i := 0; i < numConsumers; i++ {
 		select {
-		case <-handler.ready:
-			readyCount++
-		case <-time.After(5 * time.Second):
-			t.Fatalf("Timeout waiting for consumers to be ready")
+		case <-handlers[i].ready:
+		case <-time.After(20 * time.Second):
+			t.Fatalf("Timeout waiting for consumer %d to be ready", i)
 		}
 	}
 
 	// Collect consumed messages
 	consumedMessages := make([]*sarama.ConsumerMessage, 0, len(messages))
-	messageTimeout := time.After(10 * time.Second)
+	messageTimeout := time.After(15 * time.Second)
 
 	for len(consumedMessages) < len(messages) {
 		select {
-		case msg := <-handler.messages:
+		case msg := <-sharedMessages:
 			consumedMessages = append(consumedMessages, msg)
 		case err := <-consumerErrors:
 			t.Fatalf("Consumer error: %v", err)

test/kafka/integration/rebalancing_test.go

@@ -2,6 +2,7 @@ package integration
 
 import (
 	"context"
+	"errors"
 	"fmt"
 	"sync"
 	"testing"
@@ -11,6 +12,25 @@ import (
 	"github.com/seaweedfs/seaweedfs/test/kafka/internal/testutil"
 )
 
+// runConsumeLoop drives consumerGroup.Consume in a for-loop until ctx is cancelled
+// or the group is closed. This is the idiomatic Sarama pattern: Consume returns
+// when a rebalance cancels the session, and the caller is expected to re-enter.
+// Tests that observe rebalance behaviour MUST loop — otherwise the consumer goroutine
+// exits on the first rebalance and never sees the post-rebalance assignment.
+func runConsumeLoop(t *testing.T, ctx context.Context, name string, cg sarama.ConsumerGroup, topics []string, handler sarama.ConsumerGroupHandler) {
+	for {
+		if err := cg.Consume(ctx, topics, handler); err != nil {
+			if ctx.Err() != nil || errors.Is(err, sarama.ErrClosedConsumerGroup) {
+				return
+			}
+			t.Logf("%s Consume returned: %v (will retry)", name, err)
+		}
+		if ctx.Err() != nil {
+			return
+		}
+	}
+}
+
 func testSingleConsumerAllPartitions(t *testing.T, addr, topicName, groupID string) {
 	config := sarama.NewConfig()
 	config.Consumer.Group.Rebalance.Strategy = sarama.BalanceStrategyRange
@@ -28,7 +48,7 @@ func testSingleConsumerAllPartitions(t *testing.T, addr, topicName, groupID stri
 	handler := &RebalanceTestHandler{
 		messages:    make(chan *sarama.ConsumerMessage, 20),
 		ready:       make(chan bool),
-		assignments: make(chan []int32, 5),
+		assignments: make(chan []int32, 1),
 		t:           t,
 	}
 
@@ -93,7 +113,7 @@ func testTwoConsumersRebalance(t *testing.T, addr, topicName, groupID string) {
 	handler1 := &RebalanceTestHandler{
 		messages:    make(chan *sarama.ConsumerMessage, 20),
 		ready:       make(chan bool),
-		assignments: make(chan []int32, 5),
+		assignments: make(chan []int32, 1),
 		t:           t,
 		name:        "Consumer1",
 	}
@@ -101,12 +121,7 @@ func testTwoConsumersRebalance(t *testing.T, addr, topicName, groupID string) {
 	ctx1, cancel1 := context.WithTimeout(context.Background(), 45*time.Second)
 	defer cancel1()
 
-	go func() {
-		err := consumerGroup1.Consume(ctx1, []string{topicName}, handler1)
-		if err != nil && err != context.DeadlineExceeded {
-			t.Logf("Consumer1 error: %v", err)
-		}
-	}()
+	go runConsumeLoop(t, ctx1, "Consumer1", consumerGroup1, []string{topicName}, handler1)
 
 	// Wait for first consumer to be ready and get initial assignment
 	<-handler1.ready
@@ -132,7 +147,7 @@ func testTwoConsumersRebalance(t *testing.T, addr, topicName, groupID string) {
 	handler2 := &RebalanceTestHandler{
 		messages:    make(chan *sarama.ConsumerMessage, 20),
 		ready:       make(chan bool),
-		assignments: make(chan []int32, 5),
+		assignments: make(chan []int32, 1),
 		t:           t,
 		name:        "Consumer2",
 	}
@@ -140,12 +155,7 @@ func testTwoConsumersRebalance(t *testing.T, addr, topicName, groupID string) {
 	ctx2, cancel2 := context.WithTimeout(context.Background(), 30*time.Second)
 	defer cancel2()
 
-	go func() {
-		err := consumerGroup2.Consume(ctx2, []string{topicName}, handler2)
-		if err != nil && err != context.DeadlineExceeded {
-			t.Logf("Consumer2 error: %v", err)
-		}
-	}()
+	go runConsumeLoop(t, ctx2, "Consumer2", consumerGroup2, []string{topicName}, handler2)
 
 	// Wait for second consumer to be ready
 	<-handler2.ready
@@ -228,7 +238,7 @@ func testConsumerLeaveRebalance(t *testing.T, addr, topicName, groupID string) {
 	handler1 := &RebalanceTestHandler{
 		messages:    make(chan *sarama.ConsumerMessage, 20),
 		ready:       make(chan bool),
-		assignments: make(chan []int32, 5),
+		assignments: make(chan []int32, 1),
 		t:           t,
 		name:        "Consumer1",
 	}
@@ -236,7 +246,7 @@ func testConsumerLeaveRebalance(t *testing.T, addr, topicName, groupID string) {
 	handler2 := &RebalanceTestHandler{
 		messages:    make(chan *sarama.ConsumerMessage, 20),
 		ready:       make(chan bool),
-		assignments: make(chan []int32, 5),
+		assignments: make(chan []int32, 1),
 		t:           t,
 		name:        "Consumer2",
 	}
@@ -246,20 +256,9 @@ func testConsumerLeaveRebalance(t *testing.T, addr, topicName, groupID string) {
 
 	ctx2, cancel2 := context.WithTimeout(context.Background(), 30*time.Second)
 
-	// Start both consumers
-	go func() {
-		err := consumerGroup1.Consume(ctx1, []string{topicName}, handler1)
-		if err != nil && err != context.DeadlineExceeded {
-			t.Logf("Consumer1 error: %v", err)
-		}
-	}()
-
-	go func() {
-		err := consumerGroup2.Consume(ctx2, []string{topicName}, handler2)
-		if err != nil && err != context.DeadlineExceeded {
-			t.Logf("Consumer2 error: %v", err)
-		}
-	}()
+	// Start both consumers with loops so they survive rebalance session exits.
+	go runConsumeLoop(t, ctx1, "Consumer1", consumerGroup1, []string{topicName}, handler1)
+	go runConsumeLoop(t, ctx2, "Consumer2", consumerGroup2, []string{topicName}, handler2)
 
 	// Wait for both consumers to be ready
 	<-handler1.ready
@@ -315,7 +314,7 @@ func testMultipleConsumersJoin(t *testing.T, addr, topicName, groupID string) {
 		handlers[i] = &RebalanceTestHandler{
 			messages:    make(chan *sarama.ConsumerMessage, 20),
 			ready:       make(chan bool),
-			assignments: make(chan []int32, 5),
+			assignments: make(chan []int32, 1),
 			t:           t,
 			name:        fmt.Sprintf("Consumer%d", i),
 		}
@@ -323,10 +322,8 @@ func testMultipleConsumersJoin(t *testing.T, addr, topicName, groupID string) {
 		contexts[i], cancels[i] = context.WithTimeout(context.Background(), 45*time.Second)
 
 		go func(idx int) {
-			err := consumers[idx].Consume(contexts[idx], []string{topicName}, handlers[idx])
-			if err != nil && err != context.DeadlineExceeded {
-				t.Logf("Consumer%d error: %v", idx, err)
-			}
+			runConsumeLoop(t, contexts[idx], fmt.Sprintf("Consumer%d", idx),
+				consumers[idx], []string{topicName}, handlers[idx])
 		}(i)
 	}
 
@@ -339,26 +336,43 @@ func testMultipleConsumersJoin(t *testing.T, addr, topicName, groupID string) {
 		}
 	}()
 
-	// Wait for all consumers to be ready
+	// Wait for all consumers to be ready. Four consumers racing in produces
+	// several rebalance rounds; each round costs ~HeartbeatInterval (3s) before
+	// the prior leader sees REBALANCE_IN_PROGRESS and rejoins. Give enough
+	// headroom for the group to converge.
 	for i := 0; i < numConsumers; i++ {
 		select {
 		case <-handlers[i].ready:
 			t.Logf("Consumer%d ready", i)
-		case <-time.After(15 * time.Second):
+		case <-time.After(30 * time.Second):
 			t.Fatalf("Timeout waiting for Consumer%d to be ready", i)
 		}
 	}
 
-	// Collect final assignments from all consumers
+	// Collect the settled assignment for each consumer. Because each consumer's
+	// Consume() returns and re-enters on every rebalance, Setup fires multiple
+	// times while the group converges and the handler's assignments channel
+	// accumulates several snapshots. Drain it and keep the most recent value.
 	assignments := make([][]int32, numConsumers)
+	deadline := time.Now().Add(20 * time.Second)
 	for i := 0; i < numConsumers; i++ {
 		select {
 		case partitions := <-handlers[i].assignments:
 			assignments[i] = partitions
-			t.Logf("Consumer%d final assignment: %v", i, partitions)
-		case <-time.After(20 * time.Second):
+		case <-time.After(time.Until(deadline)):
 			t.Errorf("Timeout waiting for Consumer%d assignment", i)
+			continue
 		}
+	drain:
+		for {
+			select {
+			case partitions := <-handlers[i].assignments:
+				assignments[i] = partitions
+			case <-time.After(3 * time.Second):
+				break drain
+			}
+		}
+		t.Logf("Consumer%d final assignment: %v", i, assignments[i])
 	}
 
 	// Verify all partitions are assigned exactly once
@@ -418,13 +432,20 @@ func (h *RebalanceTestHandler) Setup(session sarama.ConsumerGroupSession) error
 		}
 	}
 
-	select {
-	case h.assignments <- partitions:
-	default:
-		// Channel might be full, that's ok
+	// Always publish the latest assignment. If the channel is full, drain one
+	// stale value first so readers see the most recent rebalance outcome (not
+	// whatever arrived first while the group was still converging).
+	for {
+		select {
+		case h.assignments <- partitions:
+			return nil
+		default:
+			select {
+			case <-h.assignments:
+			default:
+			}
+		}
 	}
-
-	return nil
 }
 
 func (h *RebalanceTestHandler) Cleanup(sarama.ConsumerGroupSession) error {

test/kafka/internal/testutil/clients.go

@@ -10,6 +10,14 @@ import (
 	"github.com/segmentio/kafka-go"
 )
 
+const (
+	consumerGroupHeartbeatInterval = 2 * time.Second
+	consumerGroupSessionTimeout    = 6 * time.Second
+	consumerGroupRebalanceTimeout  = 6 * time.Second
+	consumerGroupJoinBackoff       = 250 * time.Millisecond
+	consumerGroupAttemptTimeout    = 15 * time.Second
+)
+
 // KafkaGoClient wraps kafka-go client with test utilities
 type KafkaGoClient struct {
 	brokerAddr string
@@ -23,6 +31,15 @@ type SaramaClient struct {
 	t          *testing.T
 }
 
+type ConsumeGroupRetryDebug struct {
+	Attempt       int
+	MaxAttempts   int
+	Topic         string
+	GroupID       string
+	ExpectedCount int
+	Err           error
+}
+
 // NewKafkaGoClient creates a new kafka-go test client
 func NewKafkaGoClient(t *testing.T, brokerAddr string) *KafkaGoClient {
 	return &KafkaGoClient{
@@ -128,40 +145,131 @@ func (k *KafkaGoClient) ConsumeMessages(topicName string, expectedCount int) ([]
 	return messages, nil
 }
 
-// ConsumeWithGroup consumes messages using consumer group
+// ConsumeWithGroup consumes messages using consumer group.
+// Retries the initial join+fetch with a fresh reader if it fails before any
+// message is received — re-joining an existing group races with the previous
+// member's LeaveGroup / session cleanup and can surface as an i/o timeout on
+// the first FetchMessage.
 func (k *KafkaGoClient) ConsumeWithGroup(topicName, groupID string, expectedCount int) ([]kafka.Message, error) {
+	return k.consumeWithGroup(topicName, groupID, expectedCount, nil, nil)
+}
+
+// ConsumeWithGroupDebug runs ConsumeWithGroup with diagnostic hooks.
+//   - onRetry fires after each failed attempt; the snapshot is post-LeaveGroup
+//     because reader.Close() has already run by then (group will look Empty).
+//   - onTick fires every ~1.5s while an attempt is in flight, so the caller
+//     can observe live group state during the join/sync/fetch cycle — the
+//     only place we get to see PreparingRebalance / CompletingRebalance churn.
+func (k *KafkaGoClient) ConsumeWithGroupDebug(topicName, groupID string, expectedCount int, onRetry func(ConsumeGroupRetryDebug), onTick func()) ([]kafka.Message, error) {
+	return k.consumeWithGroup(topicName, groupID, expectedCount, onRetry, onTick)
+}
+
+func (k *KafkaGoClient) consumeWithGroup(topicName, groupID string, expectedCount int, onRetry func(ConsumeGroupRetryDebug), onTick func()) ([]kafka.Message, error) {
 	k.t.Helper()
 
+	const maxJoinAttempts = 5
+	var lastErr error
+	for attempt := 1; attempt <= maxJoinAttempts; attempt++ {
+		messages, err, progressed := k.consumeWithGroupOnce(topicName, groupID, expectedCount, onTick)
+		if err == nil {
+			return messages, nil
+		}
+		lastErr = err
+		// Only retry if we failed before any message was received. Once we've
+		// fetched at least one message, a partial result is more useful than a
+		// full retry (which would start over from the last committed offset).
+		if progressed {
+			return messages, err
+		}
+		if onRetry != nil {
+			onRetry(ConsumeGroupRetryDebug{
+				Attempt:       attempt,
+				MaxAttempts:   maxJoinAttempts,
+				Topic:         topicName,
+				GroupID:       groupID,
+				ExpectedCount: expectedCount,
+				Err:           err,
+			})
+		}
+		if attempt == maxJoinAttempts {
+			break
+		}
+		backoff := time.Duration(500*(1<<(attempt-1))) * time.Millisecond
+		k.t.Logf("ConsumeWithGroup join attempt %d/%d failed (%v) — retrying after %v", attempt, maxJoinAttempts, err, backoff)
+		time.Sleep(backoff)
+	}
+	return nil, lastErr
+}
+
+// consumeWithGroupOnce runs a single consume attempt. Returns the messages
+// fetched, any error, and whether any message was received (used to decide
+// whether a retry is safe).
+func (k *KafkaGoClient) consumeWithGroupOnce(topicName, groupID string, expectedCount int, onTick func()) ([]kafka.Message, error, bool) {
+	// Give each reader its own ClientID so restarts do not get mistaken for the
+	// still-shutting-down reader they are replacing.
+	dialer := &kafka.Dialer{
+		ClientID: fmt.Sprintf("seaweedfs-e2e-%s-%d", groupID, time.Now().UnixNano()),
+		Timeout:  10 * time.Second,
+	}
+
 	reader := kafka.NewReader(kafka.ReaderConfig{
-		Brokers:        []string{k.brokerAddr},
-		Topic:          topicName,
-		GroupID:        groupID,
-		MinBytes:       1,
-		MaxBytes:       10e6,
-		CommitInterval: 500 * time.Millisecond,
+		Brokers:           []string{k.brokerAddr},
+		Dialer:            dialer,
+		Topic:             topicName,
+		GroupID:           groupID,
+		MinBytes:          1,
+		MaxBytes:          10e6,
+		CommitInterval:    500 * time.Millisecond,
+		HeartbeatInterval: consumerGroupHeartbeatInterval,
+		SessionTimeout:    consumerGroupSessionTimeout,
+		RebalanceTimeout:  consumerGroupRebalanceTimeout,
+		JoinGroupBackoff:  consumerGroupJoinBackoff,
 	})
 	defer reader.Close()
 
-	// Log the initial offset position
 	offset := reader.Offset()
 	k.t.Logf("Consumer group reader created for group %s, initial offset: %d", groupID, offset)
 
-	// Increased timeout for consumer groups - they require coordinator discovery,
-	// offset fetching, and offset commits which can be slow in CI environments
-	ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second)
+	// Keep each attempt short enough that a retry can outlive a stale group
+	// member instead of burning most of the overall test timeout on one try.
+	ctx, cancel := context.WithTimeout(context.Background(), consumerGroupAttemptTimeout)
 	defer cancel()
 
+	// While the reader is still alive (i.e. before defer reader.Close() runs),
+	// periodically invoke onTick so the caller can snapshot the gateway's view
+	// of the group during the join/sync/fetch cycle. Stop it before reader.Close
+	// fires — defers run LIFO, so this defer comes after the reader.Close defer.
+	if onTick != nil {
+		tickStop := make(chan struct{})
+		tickDone := make(chan struct{})
+		go func() {
+			defer close(tickDone)
+			ticker := time.NewTicker(1500 * time.Millisecond)
+			defer ticker.Stop()
+			for {
+				select {
+				case <-tickStop:
+					return
+				case <-ticker.C:
+					onTick()
+				}
+			}
+		}()
+		defer func() {
+			close(tickStop)
+			<-tickDone
+		}()
+	}
+
 	var messages []kafka.Message
 	for i := 0; i < expectedCount; i++ {
-		// Fetch then explicitly commit to better control commit timing
 		msg, err := reader.FetchMessage(ctx)
 		if err != nil {
-			return messages, fmt.Errorf("read message %d: %w", i, err)
+			return messages, fmt.Errorf("read message %d: %w", i, err), len(messages) > 0
 		}
 		messages = append(messages, msg)
 		k.t.Logf("  Fetched message %d: offset=%d, partition=%d", i, msg.Offset, msg.Partition)
 
-		// Commit with simple retry to handle transient connection churn
 		var commitErr error
 		for attempt := 0; attempt < 3; attempt++ {
 			commitErr = reader.CommitMessages(ctx, msg)
@@ -170,16 +278,15 @@ func (k *KafkaGoClient) ConsumeWithGroup(topicName, groupID string, expectedCoun
 				break
 			}
 			k.t.Logf("  Commit attempt %d failed for offset %d: %v", attempt+1, msg.Offset, commitErr)
-			// brief backoff
 			time.Sleep(time.Duration(50*(1<<attempt)) * time.Millisecond)
 		}
 		if commitErr != nil {
-			return messages, fmt.Errorf("committing message %d: %w", i, commitErr)
+			return messages, fmt.Errorf("committing message %d: %w", i, commitErr), true
 		}
 	}
 
 	k.t.Logf("Consumed %d messages from topic %s with group %s", len(messages), topicName, groupID)
-	return messages, nil
+	return messages, nil, true
 }
 
 // CreateTopic creates a topic using Sarama

test/kafka/internal/testutil/gateway.go

@@ -2,6 +2,7 @@ package testutil
 
 import (
 	"context"
+	"encoding/json"
 	"fmt"
 	"net"
 	"os"
@@ -111,6 +112,31 @@ func (g *GatewayTestServer) CleanupAndClose() {
 	}
 }
 
+// LogConsumerGroupSnapshot dumps the gateway's current view of a consumer group.
+func (g *GatewayTestServer) LogConsumerGroupSnapshot(groupID string) {
+	g.t.Helper()
+
+	handler := g.GetHandler()
+	if handler == nil {
+		g.t.Logf("Consumer group snapshot for %s unavailable: handler is nil", groupID)
+		return
+	}
+
+	snapshot := handler.DebugGroupSnapshot(groupID)
+	if snapshot == nil {
+		g.t.Logf("Consumer group snapshot for %s unavailable", groupID)
+		return
+	}
+
+	payload, err := json.MarshalIndent(snapshot, "", "  ")
+	if err != nil {
+		g.t.Logf("Consumer group snapshot for %s could not be marshaled: %v", groupID, err)
+		return
+	}
+
+	g.t.Logf("Consumer group snapshot for %s:\n%s", groupID, string(payload))
+}
+
 // SMQAvailabilityMode indicates whether SeaweedMQ is available for testing
 type SMQAvailabilityMode int
 

test/kafka/kafka-client-loadtest/Dockerfile.loadtest

@@ -2,7 +2,7 @@
 # Multi-stage build for cross-platform support
 
 # Stage 1: Builder
-FROM golang:1.24-alpine AS builder
+FROM golang:1.26-alpine AS builder
 
 WORKDIR /app
 
@@ -17,16 +17,18 @@ COPY test/kafka/kafka-client-loadtest/ ./
 RUN CGO_ENABLED=0 GOOS=linux go build -o /kafka-loadtest ./cmd/loadtest
 
 # Stage 2: Runtime
-FROM ubuntu:22.04
+# Use alpine so we don't depend on Ubuntu apt mirrors, which intermittently
+# refuse connections from GitHub Actions runners and fail the CI build.
+# All runtime dependencies we need (ca-certificates, curl, jq, bash, nc)
+# are in the Alpine main repo.
+FROM alpine:3.20
 
-# Install runtime dependencies
-RUN apt-get update && apt-get install -y \
+RUN apk add --no-cache \
     ca-certificates \
     curl \
     jq \
     bash \
-    netcat \
-    && rm -rf /var/lib/apt/lists/*
+    netcat-openbsd
 
 # Copy built binary from builder stage
 COPY --from=builder /kafka-loadtest /usr/local/bin/kafka-loadtest

test/kafka/kafka-client-loadtest/Dockerfile.seektest

@@ -1,7 +1,8 @@
-FROM openjdk:11-jdk-slim
-
-# Install Maven
-RUN apt-get update && apt-get install -y maven && rm -rf /var/lib/apt/lists/*
+# Use the official Eclipse Temurin + Maven image so we don't depend on
+# Debian apt mirrors, which intermittently refuse connections from
+# GitHub Actions runners and fail the CI build. This image already has
+# JDK 11 and Maven installed.
+FROM maven:3.9-eclipse-temurin-11
 
 WORKDIR /app
 

test/kafka/kafka-client-loadtest/README.md

@@ -314,7 +314,7 @@ make dev-env
 # Build load test binary
 make build
 
-# Run tests locally (requires Go 1.21+)
+# Run tests locally (requires Go 1.26+)
 cd cmd/loadtest && go run main.go -config ../../config/loadtest.yaml
 ```
 

test/kafka/kafka-client-loadtest/go.mod

@@ -1,8 +1,6 @@
 module github.com/seaweedfs/seaweedfs/test/kafka/kafka-client-loadtest
 
-go 1.24.0
-
-toolchain go1.24.7
+go 1.26.6
 
 require (
 	github.com/IBM/sarama v1.46.1
@@ -26,7 +24,7 @@ require (
 	github.com/jcmturner/gofork v1.7.6 // indirect
 	github.com/jcmturner/gokrb5/v8 v8.4.4 // indirect
 	github.com/jcmturner/rpc/v2 v2.0.3 // indirect
-	github.com/klauspost/compress v1.18.0 // indirect
+	github.com/klauspost/compress v1.18.7 // indirect
 	github.com/kr/text v0.2.0 // indirect
 	github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
 	github.com/pierrec/lz4/v4 v4.1.22 // indirect
@@ -35,7 +33,7 @@ require (
 	github.com/prometheus/procfs v0.16.1 // indirect
 	github.com/rcrowley/go-metrics v0.0.0-20250401214520-65e299d6c5c9 // indirect
 	go.yaml.in/yaml/v2 v2.4.2 // indirect
-	golang.org/x/crypto v0.45.0 // indirect
-	golang.org/x/net v0.47.0 // indirect
-	golang.org/x/sys v0.38.0 // indirect
+	golang.org/x/crypto v0.56.0 // indirect
+	golang.org/x/net v0.57.0 // indirect
+	golang.org/x/sys v0.47.0 // indirect
 )

test/kafka/kafka-client-loadtest/go.sum

@@ -38,8 +38,8 @@ github.com/jcmturner/gokrb5/v8 v8.4.4 h1:x1Sv4HaTpepFkXbt2IkL29DXRf8sOfZXo8eRKh6
 github.com/jcmturner/gokrb5/v8 v8.4.4/go.mod h1:1btQEpgT6k+unzCwX1KdWMEwPPkkgBtP+F6aCACiMrs=
 github.com/jcmturner/rpc/v2 v2.0.3 h1:7FXXj8Ti1IaVFpSAziCZWNzbNuZmnvw/i6CqLNdWfZY=
 github.com/jcmturner/rpc/v2 v2.0.3/go.mod h1:VUJYCIDm3PVOEHw8sgt091/20OJjskO/YJki3ELg/Hc=
-github.com/klauspost/compress v1.18.0 h1:c/Cqfb0r+Yi+JtIEq73FWXVkRonBlf0CRNYc8Zttxdo=
-github.com/klauspost/compress v1.18.0/go.mod h1:2Pp+KzxcywXVXMr50+X0Q/Lsb43OQHYWRCY2AiWywWQ=
+github.com/klauspost/compress v1.18.7 h1:aUyZsS4kH3QTKurYhAOwAHxllVPnOthb3vPfnF1Ehjw=
+github.com/klauspost/compress v1.18.7/go.mod h1:cwPg85FWrGar70rWktvGQj8/hthj3wpl0PGDogxkrSQ=
 github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE=
 github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk=
 github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
@@ -84,8 +84,8 @@ go.yaml.in/yaml/v2 v2.4.2/go.mod h1:081UH+NErpNdqlCXm3TtEran0rJZGxAYx9hb/ELlsPU=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
 golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
 golang.org/x/crypto v0.6.0/go.mod h1:OFC/31mSvZgRz0V1QTNCzfAI1aIRzbiufJtkMIlEp58=
-golang.org/x/crypto v0.45.0 h1:jMBrvKuj23MTlT0bQEOBcAE0mjg8mK9RXFhRH6nyF3Q=
-golang.org/x/crypto v0.45.0/go.mod h1:XTGrrkGJve7CYK7J8PEww4aY7gM3qMCElcJQ8n8JdX4=
+golang.org/x/crypto v0.56.0 h1:GUh5Ii4J5jtcseSMiRqr1jXCNHoxjeV9Fmekc2oLy6Y=
+golang.org/x/crypto v0.56.0/go.mod h1:OMW5y6CY9l38uPLmxU6l6pwcXp1obtLo3e6gT7gQR2I=
 golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4=
 golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
@@ -93,8 +93,8 @@ golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v
 golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
 golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
 golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
-golang.org/x/net v0.47.0 h1:Mx+4dIFzqraBXUugkia1OOvlD6LemFo1ALMHjrXDOhY=
-golang.org/x/net v0.47.0/go.mod h1:/jNxtkgq5yWUGYkaZGqo27cfGZ1c5Nen03aYrrKpVRU=
+golang.org/x/net v0.57.0 h1:K5+3DljvIuDG9/Jv9rvyMywYNFCQ9RSUY6OOTTkT+tE=
+golang.org/x/net v0.57.0/go.mod h1:KpXc8iv+r3XplLAG/f7Jsf9RPszJzdR0f58q9vGOuEU=
 golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.17.0 h1:l60nONMj9l5drqw6jlhIELNv9I0A4OFgRsG9k2oT9Ug=
@@ -105,8 +105,8 @@ golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBc
 golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.38.0 h1:3yZWxaJjBmCWXqhN1qh02AkOnCQ1poK6oF+a7xWL6Gc=
-golang.org/x/sys v0.38.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks=
+golang.org/x/sys v0.47.0 h1:o7XGOvZQCADBQQ4Y7VNq2dRWQR7JmOUW8Kxx4ZsNgWs=
+golang.org/x/sys v0.47.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
 golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
 golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k=

test/kafka/kafka-client-loadtest/pom.xml

@@ -11,7 +11,7 @@
     <properties>
         <maven.compiler.source>11</maven.compiler.source>
         <maven.compiler.target>11</maven.compiler.target>
-        <kafka.version>3.9.1</kafka.version>
+        <kafka.version>3.9.2</kafka.version>
     </properties>
 
     <dependencies>

test/kafka/kafka-client-loadtest/tools/go.mod

@@ -5,6 +5,6 @@ go 1.24.7
 require github.com/segmentio/kafka-go v0.4.49
 
 require (
-	github.com/klauspost/compress v1.15.9 // indirect
+	github.com/klauspost/compress v1.18.7 // indirect
 	github.com/pierrec/lz4/v4 v4.1.15 // indirect
 )

test/kafka/kafka-client-loadtest/tools/go.sum

@@ -1,7 +1,7 @@
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
-github.com/klauspost/compress v1.15.9 h1:wKRjX6JRtDdrE9qwa4b/Cip7ACOshUI4smpCQanqjSY=
-github.com/klauspost/compress v1.15.9/go.mod h1:PhcZ0MbTNciWF3rruxRgKxI5NkcHHrHUDtV4Yw2GlzU=
+github.com/klauspost/compress v1.18.7 h1:aUyZsS4kH3QTKurYhAOwAHxllVPnOthb3vPfnF1Ehjw=
+github.com/klauspost/compress v1.18.7/go.mod h1:cwPg85FWrGar70rWktvGQj8/hthj3wpl0PGDogxkrSQ=
 github.com/pierrec/lz4/v4 v4.1.15 h1:MO0/ucJhngq7299dKLwIMtgTfbkoSPF6AoMYDd8Q4q0=
 github.com/pierrec/lz4/v4 v4.1.15/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=

test/kafka/kafka-client-loadtest/tools/pom.xml

@@ -11,7 +11,7 @@
     <properties>
         <maven.compiler.source>11</maven.compiler.source>
         <maven.compiler.target>11</maven.compiler.target>
-        <kafka.version>3.9.1</kafka.version>
+        <kafka.version>3.9.2</kafka.version>
         <confluent.version>7.6.0</confluent.version>
     </properties>
 

test/kafka/simple-consumer/go.mod

@@ -1,10 +1,10 @@
 module simple-consumer
 
-go 1.21
+go 1.24
 
 require github.com/segmentio/kafka-go v0.4.47
 
 require (
-	github.com/klauspost/compress v1.17.0 // indirect
+	github.com/klauspost/compress v1.18.7 // indirect
 	github.com/pierrec/lz4/v4 v4.1.15 // indirect
 )

test/kafka/simple-consumer/go.sum

@@ -2,8 +2,8 @@ github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSs
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/klauspost/compress v1.15.9/go.mod h1:PhcZ0MbTNciWF3rruxRgKxI5NkcHHrHUDtV4Yw2GlzU=
-github.com/klauspost/compress v1.17.0 h1:Rnbp4K9EjcDuVuHtd0dgA4qNuv9yKDYKK1ulpJwgrqM=
-github.com/klauspost/compress v1.17.0/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE=
+github.com/klauspost/compress v1.18.7 h1:aUyZsS4kH3QTKurYhAOwAHxllVPnOthb3vPfnF1Ehjw=
+github.com/klauspost/compress v1.18.7/go.mod h1:cwPg85FWrGar70rWktvGQj8/hthj3wpl0PGDogxkrSQ=
 github.com/pierrec/lz4/v4 v4.1.15 h1:MO0/ucJhngq7299dKLwIMtgTfbkoSPF6AoMYDd8Q4q0=
 github.com/pierrec/lz4/v4 v4.1.15/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=

test/kafka/simple-publisher/go.mod

@@ -1,10 +1,10 @@
 module simple-publisher
 
-go 1.21
+go 1.25
 
 require github.com/segmentio/kafka-go v0.4.47
 
 require (
-	github.com/klauspost/compress v1.17.0 // indirect
+	github.com/klauspost/compress v1.20.3 // indirect
 	github.com/pierrec/lz4/v4 v4.1.15 // indirect
 )

test/kafka/simple-publisher/go.sum

@@ -2,8 +2,8 @@ github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSs
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/klauspost/compress v1.15.9/go.mod h1:PhcZ0MbTNciWF3rruxRgKxI5NkcHHrHUDtV4Yw2GlzU=
-github.com/klauspost/compress v1.17.0 h1:Rnbp4K9EjcDuVuHtd0dgA4qNuv9yKDYKK1ulpJwgrqM=
-github.com/klauspost/compress v1.17.0/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE=
+github.com/klauspost/compress v1.20.3 h1:zWNoFqq/lSuYQRim3dUXlXeA0IqwuZGicdG5ZTfY5j4=
+github.com/klauspost/compress v1.20.3/go.mod h1:LUdAzn7YLVvxLpc7y3V1m40wESHTgc1422pwwBSKYuI=
 github.com/pierrec/lz4/v4 v4.1.15 h1:MO0/ucJhngq7299dKLwIMtgTfbkoSPF6AoMYDd8Q4q0=
 github.com/pierrec/lz4/v4 v4.1.15/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=

test/kafka/test_json_timestamp.sh

@@ -14,8 +14,7 @@ sleep 2
 
 echo "Querying messages..."
 cd /Users/chrislu/go/src/github.com/seaweedfs/seaweedfs/test/kafka/kafka-client-loadtest
-docker compose exec kafka-gateway /usr/local/bin/weed sql \
+docker compose exec kafka-gateway /usr/bin/weed-sql \
   -master=seaweedfs-master:9333 \
   -database=kafka \
   -query="SELECT id, timestamp, producer_id, counter, user_id, event_type FROM \"test-json-topic\" LIMIT 5;"
-

test/kms/Makefile

@@ -7,6 +7,14 @@ SEAWEEDFS_S3_ENDPOINT ?= http://127.0.0.1:8333
 TEST_TIMEOUT ?= 5m
 DOCKER_COMPOSE ?= docker-compose
 
+# CI configuration (build weed from source, run natively)
+SEAWEEDFS_BINARY ?= weed
+S3_PORT ?= 8333
+ACCESS_KEY ?= some_access_key1
+SECRET_KEY ?= some_secret_key1
+TEST_DIR := $(shell pwd)
+SEAWEEDFS_ROOT := $(shell cd ../.. && pwd)
+
 # Colors for output
 BLUE := \033[36m
 GREEN := \033[32m
@@ -14,7 +22,10 @@ YELLOW := \033[33m
 RED := \033[31m
 NC := \033[0m # No Color
 
-.PHONY: help setup test test-unit test-integration test-e2e clean logs status
+.PHONY: help setup test test-unit test-integration test-e2e clean logs status \
+       build-weed check-binary start-openbao-ci stop-openbao-ci \
+       start-seaweedfs-ci stop-seaweedfs-ci clean-ci \
+       test-provider-ci test-s3-kms-ci
 
 help: ## Show this help message
 	@echo "$(BLUE)SeaweedFS KMS Integration Testing$(NC)"
@@ -124,16 +135,146 @@ check-env: ## Check test environment setup
 	@echo "TEST_TIMEOUT: $(TEST_TIMEOUT)"
 	@make install-deps
 
-# CI targets
-ci-test: ## Run tests in CI environment
+# CI targets (docker-compose based)
+ci-test: ## Run tests in CI environment (docker-compose)
 	@echo "$(YELLOW)Running CI tests...$(NC)"
 	@make setup
 	@make test-unit
 	@make test-integration
 	@make clean
 
-ci-e2e: ## Run end-to-end tests in CI
+ci-e2e: ## Run end-to-end tests in CI (docker-compose)
 	@echo "$(YELLOW)Running CI end-to-end tests...$(NC)"
 	@make setup-seaweedfs
 	@make test-e2e
 	@make clean
+
+####################################################
+# GitHub Actions CI targets (build weed from source)
+####################################################
+
+build-weed: ## Build SeaweedFS binary from source
+	@echo "Building SeaweedFS binary..."
+	@cd $(SEAWEEDFS_ROOT)/weed && go install -buildvcs=false
+	@echo "$(GREEN)SeaweedFS binary built successfully$(NC)"
+
+check-binary:
+	@if ! command -v $(SEAWEEDFS_BINARY) > /dev/null 2>&1; then \
+		echo "$(RED)Error: SeaweedFS binary '$(SEAWEEDFS_BINARY)' not found in PATH$(NC)"; \
+		exit 1; \
+	fi
+
+start-openbao-ci: ## Start OpenBao via Docker for CI
+	@echo "Starting OpenBao for CI..."
+	@docker rm -f openbao-ci 2>/dev/null || true
+	@docker run -d --name openbao-ci \
+		-p 8200:8200 \
+		-e BAO_DEV_ROOT_TOKEN_ID=$(OPENBAO_TOKEN) \
+		-e BAO_DEV_LISTEN_ADDRESS=0.0.0.0:8200 \
+		ghcr.io/openbao/openbao:latest \
+		bao server -dev -dev-root-token-id=$(OPENBAO_TOKEN) -dev-listen-address=0.0.0.0:8200
+	@echo "Waiting for OpenBao to be ready..."
+	@for i in $$(seq 1 90); do \
+		if curl -s $(OPENBAO_ADDR)/v1/sys/health > /dev/null 2>&1; then \
+			echo "$(GREEN)OpenBao is ready$(NC)"; \
+			break; \
+		fi; \
+		if [ $$i -eq 90 ]; then \
+			echo "$(RED)Timeout waiting for OpenBao$(NC)"; \
+			docker logs openbao-ci 2>&1 | tail -20; \
+			exit 1; \
+		fi; \
+		sleep 2; \
+	done
+	@chmod +x setup_openbao.sh
+	@OPENBAO_ADDR=$(OPENBAO_ADDR) OPENBAO_TOKEN=$(OPENBAO_TOKEN) ./setup_openbao.sh
+
+stop-openbao-ci: ## Stop OpenBao CI container
+	@echo "Stopping OpenBao..."
+	@docker stop openbao-ci 2>/dev/null || true
+	@docker rm openbao-ci 2>/dev/null || true
+
+start-seaweedfs-ci: check-binary ## Start weed mini with OpenBao KMS config
+	@echo "Starting SeaweedFS with OpenBao KMS..."
+	@mkdir -p /tmp/seaweedfs-test-kms
+	@rm -f /tmp/seaweedfs-kms-*.log || true
+	@sed -e 's/ACCESS_KEY_PLACEHOLDER/$(ACCESS_KEY)/g' \
+	     -e 's/SECRET_KEY_PLACEHOLDER/$(SECRET_KEY)/g' \
+	     -e 's|OPENBAO_ADDR_PLACEHOLDER|$(OPENBAO_ADDR)|g' \
+	     -e 's/OPENBAO_TOKEN_PLACEHOLDER/$(OPENBAO_TOKEN)/g' \
+	     s3-config-openbao-template.json > /tmp/seaweedfs-kms-s3.json
+	@# Start weed mini from the data dir to avoid picking up test/kms/filer.toml
+	@# (filer.toml is read from "." first, and the one here has Docker-only paths)
+	@cd /tmp/seaweedfs-test-kms && \
+		AWS_ACCESS_KEY_ID=$(ACCESS_KEY) AWS_SECRET_ACCESS_KEY=$(SECRET_KEY) GLOG_v=4 $(SEAWEEDFS_BINARY) mini \
+		-dir=/tmp/seaweedfs-test-kms \
+		-s3.port=$(S3_PORT) \
+		-s3.config=/tmp/seaweedfs-kms-s3.json \
+		-ip=127.0.0.1 \
+		> /tmp/seaweedfs-kms-mini.log 2>&1 & echo $$! > /tmp/weed-kms-mini.pid
+	@echo "Checking S3 service is ready..."
+	@for i in $$(seq 1 90); do \
+		if curl -s http://127.0.0.1:$(S3_PORT) > /dev/null 2>&1; then \
+			echo "$(GREEN)S3 service is ready$(NC)"; \
+			break; \
+		fi; \
+		if [ $$i -eq 90 ]; then \
+			echo "$(RED)Timeout waiting for S3 service$(NC)"; \
+			echo "=== Last 50 lines of server log ==="; \
+			tail -50 /tmp/seaweedfs-kms-mini.log 2>/dev/null || echo "No server log found"; \
+			exit 1; \
+		fi; \
+		sleep 1; \
+	done
+
+stop-seaweedfs-ci: ## Stop weed mini (CI-safe)
+	@echo "Stopping SeaweedFS..."
+	@if [ -f /tmp/weed-kms-mini.pid ]; then \
+		kill $$(cat /tmp/weed-kms-mini.pid) 2>/dev/null || true; \
+		rm -f /tmp/weed-kms-mini.pid; \
+	fi
+	@if command -v lsof >/dev/null 2>&1; then \
+		lsof -ti :$(S3_PORT) 2>/dev/null | head -5 | while read pid; do kill -TERM $$pid 2>/dev/null || true; done; \
+	fi
+	@sleep 2
+
+clean-ci: ## Clean up CI test artifacts
+	@rm -rf /tmp/seaweedfs-test-kms
+	@rm -f /tmp/seaweedfs-kms-*.log /tmp/seaweedfs-kms-s3.json /tmp/weed-kms-mini.pid
+
+# Run KMS provider Go integration tests (GitHub Actions compatible)
+test-provider-ci: build-weed ## Run KMS provider tests with OpenBao in CI
+	@echo "$(YELLOW)Starting KMS provider integration tests...$(NC)"
+	@if $(MAKE) start-openbao-ci > /tmp/openbao-ci-setup.log 2>&1; then \
+		echo "$(GREEN)OpenBao started and configured$(NC)"; \
+		trap '$(MAKE) -C $(TEST_DIR) stop-openbao-ci || true' EXIT; \
+		cd $(SEAWEEDFS_ROOT) && go test -v -timeout=$(TEST_TIMEOUT) ./test/kms/... || exit 1; \
+		echo "$(GREEN)KMS provider tests completed successfully$(NC)"; \
+	else \
+		echo "$(RED)Failed to start OpenBao$(NC)"; \
+		cat /tmp/openbao-ci-setup.log 2>/dev/null || true; \
+		exit 1; \
+	fi
+
+# Run S3 KMS end-to-end tests (GitHub Actions compatible)
+test-s3-kms-ci: build-weed ## Run S3 KMS e2e tests with weed mini + OpenBao in CI
+	@echo "$(YELLOW)Starting S3 KMS end-to-end tests...$(NC)"
+	@if $(MAKE) start-openbao-ci > /tmp/openbao-ci-setup.log 2>&1; then \
+		echo "$(GREEN)OpenBao started and configured$(NC)"; \
+		trap 'docker logs openbao-ci > /tmp/openbao-ci-container.log 2>&1 || true; $(MAKE) -C $(TEST_DIR) stop-seaweedfs-ci || true; $(MAKE) -C $(TEST_DIR) stop-openbao-ci || true; $(MAKE) -C $(TEST_DIR) clean-ci || true' EXIT; \
+		if $(MAKE) start-seaweedfs-ci > /tmp/weed-kms-ci-setup.log 2>&1; then \
+			echo "$(GREEN)SeaweedFS started with OpenBao KMS$(NC)"; \
+			sleep 3; \
+			chmod +x test_s3_kms.sh; \
+			SEAWEEDFS_S3_ENDPOINT=http://127.0.0.1:$(S3_PORT) ACCESS_KEY=$(ACCESS_KEY) SECRET_KEY=$(SECRET_KEY) ./test_s3_kms.sh || exit 1; \
+			echo "$(GREEN)S3 KMS e2e tests completed successfully$(NC)"; \
+		else \
+			echo "$(RED)Failed to start SeaweedFS$(NC)"; \
+			cat /tmp/weed-kms-ci-setup.log 2>/dev/null || true; \
+			exit 1; \
+		fi; \
+	else \
+		echo "$(RED)Failed to start OpenBao$(NC)"; \
+		cat /tmp/openbao-ci-setup.log 2>/dev/null || true; \
+		exit 1; \
+	fi