Merge remote-tracking branch 'origin/master' into pr10277-work

# Conflicts:
#	weed/pb/filer_pb/filer.pb.go

.github/actions/cleanup-stale-fuse-mounts/action.yml

@@ -0,0 +1,11 @@
+name: Clean up stale SeaweedFS test mounts
+description: Detach disconnected FUSE test mounts left by earlier jobs on a shared runner.
+runs:
+  using: composite
+  steps:
+    - name: Unmount disconnected SeaweedFS test mounts
+      if: runner.os == 'Linux' && runner.environment == 'self-hosted'
+      shell: bash
+      env:
+        CLEANUP_SCRIPT: ${{ github.action_path }}/cleanup.py
+      run: python3 "$CLEANUP_SCRIPT"

.github/actions/cleanup-stale-fuse-mounts/cleanup.py

@@ -0,0 +1,89 @@
+#!/usr/bin/env python3
+"""Detach disconnected SeaweedFS integration-test mounts without touching live ones."""
+
+import errno
+import os
+from pathlib import Path
+import re
+import shutil
+import subprocess
+import sys
+import tempfile
+
+
+def test_mounts(mountinfo):
+    """Read the kernel table directly: traversing a stale mount may block."""
+    roots = {tempfile.gettempdir(), "/tmp", "/var/tmp"}
+    for line in mountinfo.splitlines():
+        fields, filesystem = line.split(" - ", 1)
+        if filesystem.split()[0].split(".")[0] != "fuse":
+            continue
+        fields = fields.split()
+        # mountinfo escapes whitespace and backslashes with octal sequences.
+        target = re.sub(r"\\([0-7]{3})", lambda m: chr(int(m[1], 8)), fields[4])
+        path = Path(target)
+        # Test mounts are a mount* leaf under a seaweedfs_fuse_* scratch dir
+        # (seaweedfs_fuse_test_*/mount, seaweedfs_fuse_failover_*/mount0, ...)
+        # or the posix-compliance workflow's fixed /tmp/seaweedfs/mount.
+        if (not path.name.startswith("mount")
+                or str(path.parent.parent) not in roots
+                or not (path.parent.name.startswith("seaweedfs_fuse")
+                        or path.parent.name == "seaweedfs")):
+            continue
+        yield fields[0], target
+
+
+def mounted(mount_id, target):
+    return (mount_id, target) in test_mounts(Path("/proc/self/mountinfo").read_text())
+
+
+def cleanup():
+    failed = False
+    for mount_id, target in test_mounts(Path("/proc/self/mountinfo").read_text()):
+        try:
+            # A separate, bounded probe prevents a wedged FUSE daemon from
+            # hanging cleanup. Only ENOTCONN proves the daemon disconnected;
+            # a timeout or another error is not enough to detach a live mount.
+            probe = subprocess.run(
+                [sys.executable, __file__, "--probe", target], timeout=5,
+                capture_output=True, check=False,
+            )
+        except subprocess.TimeoutExpired:
+            print(f"::warning::FUSE probe timed out; leaving {target} mounted")
+            continue
+        if probe.returncode != errno.ENOTCONN or not mounted(mount_id, target):
+            continue
+
+        print(f"Unmounting disconnected FUSE test mount: {target}", flush=True)
+        # Lazy unmount detaches a dead mount even when an old process still
+        # holds it open. Prefer the unprivileged FUSE helper; sudo is fallback.
+        for command in (
+            ["fusermount3", "-uz", "--", target],
+            ["fusermount", "-uz", "--", target],
+            ["sudo", "-n", "umount", "-l", "--", target],
+        ):
+            try:
+                subprocess.run(command, timeout=10, check=False)
+            except (OSError, subprocess.TimeoutExpired) as error:
+                print(f"{command[0]}: {error}", file=sys.stderr)
+            if not mounted(mount_id, target):
+                break
+        else:
+            print(f"::error::Could not unmount disconnected FUSE test mount: {target}")
+            failed = True
+            continue
+        # The detached mount's parent still holds the crashed run's data and
+        # logs; test_mounts only yields targets under a SeaweedFS test dir in
+        # a temp root, so the parent is safe to reclaim here.
+        shutil.rmtree(Path(target).parent, ignore_errors=True)
+    return int(failed)
+
+
+if __name__ == "__main__":
+    if len(sys.argv) == 3 and sys.argv[1] == "--probe":
+        try:
+            os.stat(sys.argv[2])
+        except OSError as error:
+            sys.exit(error.errno)
+        sys.exit(0)
+    sys.exit(cleanup())

.github/actions/cleanup-stale-fuse-mounts/test_cleanup.py

@@ -0,0 +1,137 @@
+import errno
+import importlib.util
+from pathlib import Path
+import subprocess
+import unittest
+from unittest.mock import patch
+
+spec = importlib.util.spec_from_file_location("cleanup", Path(__file__).with_name("cleanup.py"))
+cleanup = importlib.util.module_from_spec(spec)
+spec.loader.exec_module(cleanup)
+
+TARGET = "/tmp/seaweedfs_fuse_test_123/mount"
+MOUNTINFO = f"42 1 0:9 / {TARGET} rw - fuse.seaweedfs seaweedfs rw\n"
+
+
+class CleanupTests(unittest.TestCase):
+    def run_cleanup(self, results, mounted=True):
+        with patch.object(cleanup.Path, "read_text", return_value=MOUNTINFO), \
+                patch.object(cleanup, "mounted", side_effect=mounted if isinstance(mounted, list) else None,
+                             return_value=mounted) as check, \
+                patch.object(cleanup.shutil, "rmtree") as rmtree, \
+                patch.object(cleanup.subprocess, "run", side_effect=results) as run:
+            status = cleanup.cleanup()
+        return status, run, check, rmtree
+
+    def result(self, status):
+        return subprocess.CompletedProcess([], status)
+
+    def test_only_fuse_integration_test_mounts_are_candidates(self):
+        rows = MOUNTINFO
+        rows += MOUNTINFO.replace("fuse.seaweedfs", "ext4")
+        rows += MOUNTINFO.replace(TARGET, "/mnt/production")
+        rows += MOUNTINFO.replace(TARGET, "/tmp/seaweedfs_fuse_test_123/data")
+        rows += MOUNTINFO.replace(TARGET, "/home/user/seaweedfs_fuse_test_123/mount")
+        rows += MOUNTINFO.replace(TARGET, "/tmp/seaweedfs/data")
+        self.assertEqual(list(cleanup.test_mounts(rows)), [("42", TARGET)])
+
+    def test_all_test_mount_layouts_are_candidates(self):
+        targets = (
+            "/tmp/seaweedfs_fuse_test_123/mount",
+            "/tmp/seaweedfs_fuse_failover_456/mount0",
+            "/tmp/seaweedfs_fuse_failover_456/mount1",
+            "/var/tmp/seaweedfs_fuse_p2p_test_789/mount2",
+            "/tmp/seaweedfs/mount",
+        )
+        rows = "".join(
+            MOUNTINFO.replace("42", str(40 + i)).replace(TARGET, target)
+            for i, target in enumerate(targets)
+        )
+        self.assertEqual(
+            list(cleanup.test_mounts(rows)),
+            [(str(40 + i), target) for i, target in enumerate(targets)],
+        )
+
+    def test_mountinfo_paths_are_unescaped_once(self):
+        target = "/tmp/seaweedfs_fuse_test_space and\\slash/mount"
+        encoded = target.replace("\\", r"\134").replace(" ", r"\040")
+        rows = MOUNTINFO.replace(TARGET, encoded)
+        self.assertEqual(list(cleanup.test_mounts(rows)), [("42", target)])
+
+    def test_no_mounts_is_a_noop(self):
+        with patch.object(cleanup.Path, "read_text", return_value=""), \
+                patch.object(cleanup.subprocess, "run") as run:
+            self.assertEqual(cleanup.cleanup(), 0)
+        run.assert_not_called()
+
+    def test_healthy_mount_is_untouched(self):
+        status, run, _, _ = self.run_cleanup([self.result(0)])
+        self.assertEqual(status, 0)
+        self.assertEqual(run.call_count, 1)
+
+    def test_other_probe_errors_are_untouched(self):
+        for error in (errno.EACCES, errno.ENOENT, errno.EIO):
+            with self.subTest(error=error):
+                status, run, _, _ = self.run_cleanup([self.result(error)])
+                self.assertEqual(status, 0)
+                self.assertEqual(run.call_count, 1)
+
+    def test_probe_timeout_does_not_detach_a_possibly_live_mount(self):
+        status, run, _, _ = self.run_cleanup([subprocess.TimeoutExpired("probe", 5)])
+        self.assertEqual(status, 0)
+        self.assertEqual(run.call_count, 1)
+
+    def test_disconnected_mount_is_lazily_unmounted(self):
+        status, run, _, _ = self.run_cleanup([self.result(errno.ENOTCONN), self.result(0)], [True, False])
+        self.assertEqual(status, 0)
+        self.assertEqual(run.call_args_list[1].args[0], ["fusermount3", "-uz", "--", TARGET])
+        self.assertEqual(run.call_args_list[0].kwargs["timeout"], 5)
+        self.assertEqual(run.call_args_list[1].kwargs["timeout"], 10)
+
+    def test_replaced_or_removed_mount_is_untouched(self):
+        status, run, _, _ = self.run_cleanup([self.result(errno.ENOTCONN)], False)
+        self.assertEqual(status, 0)
+        self.assertEqual(run.call_count, 1)
+
+    def test_mount_identity_is_checked_without_stat(self):
+        with patch.object(cleanup.Path, "read_text", return_value=MOUNTINFO):
+            self.assertTrue(cleanup.mounted("42", TARGET))
+            self.assertFalse(cleanup.mounted("43", TARGET))
+
+    def test_missing_or_timed_out_helper_uses_fallback(self):
+        for error in (FileNotFoundError(), subprocess.TimeoutExpired("fusermount3", 10)):
+            with self.subTest(error=error):
+                status, run, _, _ = self.run_cleanup(
+                    [self.result(errno.ENOTCONN), error, self.result(0)], [True, True, False])
+                self.assertEqual(status, 0)
+                self.assertEqual(run.call_args.args[0], ["fusermount", "-uz", "--", TARGET])
+
+    def test_failed_helpers_fall_back_to_sudo(self):
+        status, run, _, _ = self.run_cleanup(
+            [self.result(errno.ENOTCONN), self.result(1), self.result(1), self.result(0)],
+            [True, True, True, False])
+        self.assertEqual(status, 0)
+        self.assertEqual(run.call_args.args[0], ["sudo", "-n", "umount", "-l", "--", TARGET])
+
+    def test_cleanup_fails_when_stale_mount_remains(self):
+        status, run, _, rmtree = self.run_cleanup(
+            [self.result(errno.ENOTCONN), self.result(1), self.result(1), self.result(1)])
+        self.assertEqual(status, 1)
+        self.assertEqual(run.call_count, 4)
+        rmtree.assert_not_called()
+
+    def test_unmounted_test_dir_is_reclaimed(self):
+        status, _, _, rmtree = self.run_cleanup(
+            [self.result(errno.ENOTCONN), self.result(0)], [True, False])
+        self.assertEqual(status, 0)
+        rmtree.assert_called_once_with(Path(TARGET).parent, ignore_errors=True)
+
+    def test_probe_process_reports_errno(self):
+        for target, expected in ((str(Path(__file__)), 0), (str(Path(__file__) / "absent"), errno.ENOTDIR)):
+            with self.subTest(target=target):
+                result = subprocess.run([cleanup.sys.executable, cleanup.__file__, "--probe", target], check=False)
+                self.assertEqual(result.returncode, expected)
+
+
+if __name__ == "__main__":
+    unittest.main()

.github/actions/fix-fusermount-setuid/action.yml

@@ -23,8 +23,14 @@ runs:
 
         bin=$(command -v fusermount3 || true)
         if [ -z "$bin" ]; then
-          echo "no fusermount3 in PATH" >&2
-          exit 1
+          # Self-hosted runners are not guaranteed to carry fuse3.
+          sudo apt-get update
+          sudo apt-get install -y fuse3
+          bin=$(command -v fusermount3 || true)
+          if [ -z "$bin" ]; then
+            echo "no fusermount3 in PATH after installing fuse3" >&2
+            exit 1
+          fi
         fi
         if can_mount "$bin"; then
           ls -l "$bin"

.github/workflows/action-tests.yml

@@ -0,0 +1,30 @@
+name: "actions: unit tests"
+
+on:
+  push:
+    branches: [ master ]
+    paths:
+      - '.github/actions/**'
+      - '.github/workflows/action-tests.yml'
+  pull_request:
+    branches: [ master ]
+    paths:
+      - '.github/actions/**'
+      - '.github/workflows/action-tests.yml'
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || (github.event_name == 'push' && github.ref) || github.run_id }}
+  cancel-in-progress: ${{ github.event_name == 'pull_request' || (github.event_name == 'push' && !startsWith(github.ref, 'refs/tags/')) }}
+
+permissions:
+  contents: read
+
+jobs:
+  cleanup-stale-fuse-mounts:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Check out code
+        uses: actions/checkout@v7
+
+      - name: Run cleanup action unit tests
+        run: python3 .github/actions/cleanup-stale-fuse-mounts/test_cleanup.py

.github/workflows/codeql.yml

@@ -9,8 +9,8 @@ on:
       - '.github/workflows/codeql.yml'
 
 concurrency:
-  group: ${{ github.head_ref }}/codeql
-  cancel-in-progress: true
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || (github.event_name == 'push' && github.ref) || github.run_id }}
+  cancel-in-progress: ${{ github.event_name == 'pull_request' || (github.event_name == 'push' && !startsWith(github.ref, 'refs/tags/')) }}
 
 jobs:
   CodeQL-Build:
@@ -25,6 +25,12 @@ jobs:
       - name: Checkout repository
         uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1
 
+      - name: Set up Go
+        uses: actions/setup-go@v7
+        with:
+          go-version-file: 'go.mod'
+          cache: ${{ runner.environment == 'github-hosted' }}
+
       # Initializes the CodeQL tools for scanning.
       - name: Initialize CodeQL
         uses: github/codeql-action/init@v4.39.0

.github/workflows/codespell.yml

@@ -8,6 +8,10 @@ on:
   pull_request:
     branches: [master]
 
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || (github.event_name == 'push' && github.ref) || github.run_id }}
+  cancel-in-progress: ${{ github.event_name == 'pull_request' || (github.event_name == 'push' && !startsWith(github.ref, 'refs/tags/')) }}
+
 permissions:
   contents: read
 

.github/workflows/container_dev.yml

@@ -13,6 +13,10 @@ on:
       - 'go.sum'
       - '.github/workflows/container_dev.yml'
 
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || (github.event_name == 'push' && github.ref) || github.run_id }}
+  cancel-in-progress: ${{ github.event_name == 'pull_request' || (github.event_name == 'push' && !startsWith(github.ref, 'refs/tags/')) }}
+
 permissions:
   contents: read
 
@@ -51,6 +55,7 @@ jobs:
           echo "CFLAGS_aarch64_unknown_linux_musl=-U_FORTIFY_SOURCE" >> "$GITHUB_ENV"
 
       - name: Cache cargo registry and target
+        if: runner.environment == 'github-hosted'
         uses: actions/cache@v6
         with:
           path: |
@@ -193,6 +198,12 @@ jobs:
           platforms: linux/amd64, linux/arm64
           tags: ${{ steps.docker_meta.outputs.tags }}
           labels: ${{ steps.docker_meta.outputs.labels }}
+          cache-from: type=gha
+          cache-to: type=gha,mode=max
+          # Key the checkout layer on the pushed sha so cached layers can't
+          # serve stale code; the Dockerfile fetch fallback picks it up.
+          build-args: |
+            BRANCH=${{ github.sha }}
 
       - name: Sign
         if: github.event_name != 'pull_request'

.github/workflows/container_foundationdb_version.yml

@@ -24,6 +24,10 @@ on:
         required: false
         default: ''
 
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || (github.event_name == 'push' && github.ref) || github.event.inputs.image_tag || format('foundationdb_{0}_seaweedfs_{1}', github.event.inputs.fdb_version || '7.4.5', github.event.inputs.seaweedfs_ref || github.ref_name) }}
+  cancel-in-progress: ${{ github.event_name == 'pull_request' || (github.event_name == 'push' && !startsWith(github.ref, 'refs/tags/')) }}
+
 permissions:
   contents: read
 
@@ -64,6 +68,7 @@ jobs:
         uses: actions/setup-go@v7
         with:
           go-version-file: go.mod
+          cache: ${{ runner.environment == 'github-hosted' }}
 
       - name: Run FoundationDB tagged tests
         env:

.github/workflows/container_latest.yml

@@ -30,6 +30,11 @@ on:
         type: boolean
         default: false
 
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || (github.event_name == 'push' && github.ref) || github.event.inputs.image_tag || github.run_id }}
+  cancel-in-progress: false
+  queue: max
+
 permissions:
   contents: read
   security-events: write
@@ -94,6 +99,7 @@ jobs:
           echo "CFLAGS_aarch64_unknown_linux_musl=-U_FORTIFY_SOURCE" >> "$GITHUB_ENV"
 
       - name: Cache cargo registry and target
+        if: runner.environment == 'github-hosted'
         uses: actions/cache@v6
         with:
           path: |
@@ -164,10 +170,13 @@ jobs:
         uses: actions/checkout@v7
         with:
           ref: ${{ github.event_name == 'workflow_dispatch' && github.event.inputs.source_ref || github.ref }}
+      - name: Clean up stale FUSE mounts
+        uses: ./.github/actions/cleanup-stale-fuse-mounts
+
       - name: Free Disk Space
         run: |
           echo "Available disk space before cleanup:"
-          df -h
+          timeout 15 df -h || true
           # Remove pre-installed tools
           sudo rm -rf /usr/share/dotnet /usr/local/lib/android /opt/ghc /opt/hostedtoolcache/CodeQL
           # Clean package managers
@@ -179,7 +188,7 @@ jobs:
           [ -d ~/.cache/go-build ] && rm -rf ~/.cache/go-build || true
           [ -d /go/pkg ] && rm -rf /go/pkg || true
           echo "Available disk space after cleanup:"
-          df -h
+          timeout 15 df -h || true
       
       - name: Configure variant
         id: config

.github/workflows/container_release_foundationdb.yml

@@ -11,6 +11,11 @@ on:
         required: true
         default: ''
 
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || (github.event_name == 'push' && github.ref) || format('refs/tags/{0}', github.event.inputs.release_tag) || github.run_id }}
+  cancel-in-progress: false
+  queue: max
+
 permissions:
   contents: read
 

.github/workflows/container_release_unified.yml

@@ -25,7 +25,7 @@ on:
       rocksdb_version:
         description: 'RocksDB git tag to use when variant=rocksdb'
         required: false
-        default: 'v10.10.1'
+        default: 'v11.1.2'
 
 permissions:
   contents: read
@@ -78,6 +78,7 @@ jobs:
           echo "CFLAGS_aarch64_unknown_linux_musl=-U_FORTIFY_SOURCE" >> "$GITHUB_ENV"
 
       - name: Cache cargo registry and target
+        if: runner.environment == 'github-hosted'
         uses: actions/cache@v6
         with:
           path: |
@@ -179,6 +180,10 @@ jobs:
         if: github.event_name != 'workflow_dispatch' || github.event.inputs.variant == 'all' || github.event.inputs.variant == matrix.variant
         uses: actions/checkout@v7
 
+      - name: Clean up stale FUSE mounts
+        if: github.event_name != 'workflow_dispatch' || github.event.inputs.variant == 'all' || github.event.inputs.variant == matrix.variant
+        uses: ./.github/actions/cleanup-stale-fuse-mounts
+
       - name: Download pre-built Rust binaries
         if: github.event_name != 'workflow_dispatch' || github.event.inputs.variant == 'all' || github.event.inputs.variant == matrix.variant
         uses: actions/download-artifact@v8
@@ -218,7 +223,7 @@ jobs:
           sudo docker system prune -af --volumes
           [ -d ~/.cache/go-build ] && rm -rf ~/.cache/go-build || true
           [ -d /go/pkg ] && rm -rf /go/pkg || true
-          df -h
+          timeout 15 df -h || true
 
       - name: Docker meta
         if: github.event_name != 'workflow_dispatch' || github.event.inputs.variant == 'all' || github.event.inputs.variant == matrix.variant
@@ -270,8 +275,11 @@ jobs:
           file: ${{ matrix.dockerfile }}
           platforms: ${{ matrix.platform }}
           labels: ${{ steps.docker_meta.outputs.labels }}
-          # Flat single-platform manifest so imagetools create assembles cleanly.
-          provenance: false
+          # mode=max keeps resolvedDependencies complete and records the
+          # builder identity; attestations ride inside each pushed index and
+          # survive the imagetools merge into the multi-arch manifest.
+          provenance: mode=max
+          sbom: true
           outputs: type=image,name=${{ env.IMAGE }},push-by-digest=true,name-canonical=true,push=true
           cache-from: type=gha,scope=${{ matrix.variant }}-${{ matrix.arch }}
           # max only for rocksdb: its RocksDB compile is sha-independent and worth
@@ -281,7 +289,7 @@ jobs:
             ${{ matrix.build_args }}
             BUILDKIT_INLINE_CACHE=1
             BRANCH=${{ github.sha }}
-            ${{ matrix.variant == 'rocksdb' && format('ROCKSDB_VERSION={0}', github.event.inputs.rocksdb_version || 'v10.10.1') || '' }}
+            ${{ matrix.variant == 'rocksdb' && format('ROCKSDB_VERSION={0}', github.event.inputs.rocksdb_version || 'v11.1.2') || '' }}
 
       - name: Export digest
         if: github.event_name != 'workflow_dispatch' || github.event.inputs.variant == 'all' || github.event.inputs.variant == matrix.variant
@@ -307,6 +315,8 @@ jobs:
     permissions:
       contents: read
       id-token: write
+      attestations: write
+      packages: write
     strategy:
       fail-fast: false
       matrix:
@@ -414,6 +424,45 @@ jobs:
             ${{ env.IMAGE }}@${{ steps.manifest.outputs.digest }}
             chrislusf/seaweedfs@${{ steps.manifest.outputs.digest }}
 
+      # Attest the merged multi-arch digest itself, so digest-pinning consumers
+      # can verify provenance against the exact index they pull. Both steps
+      # reuse the GHCR login above and push as OCI referrers.
+      - name: Attest build provenance for ${{ matrix.variant }}
+        if: github.event_name != 'workflow_dispatch' || github.event.inputs.variant == 'all' || github.event.inputs.variant == matrix.variant
+        uses: actions/attest-build-provenance@v4
+        with:
+          subject-name: ${{ env.IMAGE }}
+          subject-digest: ${{ steps.manifest.outputs.digest }}
+          push-to-registry: true
+
+      # SBOMs are generated and attested per platform manifest: an index-wide
+      # scan only describes syft's default platform, while the manifests it
+      # merges differ (some ship the real binary, others placeholders), so the
+      # SBOM has to be bound to the platform digest a consumer actually pulls.
+      - name: Install syft
+        if: github.event_name != 'workflow_dispatch' || github.event.inputs.variant == 'all' || github.event.inputs.variant == matrix.variant
+        uses: anchore/sbom-action/download-syft@v0
+
+      - name: Attest per-platform SBOMs for ${{ matrix.variant }}
+        if: github.event_name != 'workflow_dispatch' || github.event.inputs.variant == 'all' || github.event.inputs.variant == matrix.variant
+        working-directory: /tmp/digests
+        run: |
+          set -euo pipefail
+          for f in *; do
+            # Each saved digest is an OCI index (image + attestation manifests);
+            # scan and attest the image manifest a same-platform pull resolves.
+            ref="${{ env.IMAGE }}@sha256:${f}"
+            image=$(crane manifest "$ref" | jq -r '
+              [.manifests[]? | select((.annotations["vnd.docker.reference.type"] // "") != "attestation-manifest"
+                and (.platform.os // "unknown") != "unknown") | .digest][0] // empty')
+            ref="${{ env.IMAGE }}@${image:-sha256:${f}}"
+            syft "registry:${ref}" -o "spdx-json=/tmp/sbom-${f}.spdx.json"
+            cosign attest --yes \
+              --predicate "/tmp/sbom-${f}.spdx.json" \
+              --type spdxjson \
+              "$ref"
+          done
+
   # Report-only trivy scan: uploads fixable HIGH/CRITICAL findings to GitHub
   # Security for visibility, but never blocks the release. Releases (including
   # `latest`) ship regardless — vulnerabilities are tracked, not gated, since
@@ -452,7 +501,9 @@ jobs:
           scanners: vuln
           vuln-type: os,library
           severity: HIGH,CRITICAL
-          ignore-unfixed: true
+          # Keep unfixed findings in the report so digest-pinning consumers see
+          # the complete picture, not just what a patch could resolve.
+          ignore-unfixed: false
           limit-severities-for-sarif: true
           format: sarif
           output: trivy-results.sarif

.github/workflows/container_rocksdb_version.yml

@@ -4,9 +4,9 @@ on:
   workflow_dispatch:
     inputs:
       rocksdb_version:
-        description: 'RocksDB git tag or branch to build (e.g. v10.10.1)'
+        description: 'RocksDB git tag or branch to build (e.g. v11.1.2)'
         required: true
-        default: 'v10.10.1'
+        default: 'v11.1.2'
       seaweedfs_ref:
         description: 'SeaweedFS git tag, branch, or commit to build'
         required: true
@@ -16,6 +16,11 @@ on:
         required: false
         default: ''
 
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.inputs.image_tag || format('rocksdb_{0}_seaweedfs_{1}', github.event.inputs.rocksdb_version, github.event.inputs.seaweedfs_ref) }}
+  cancel-in-progress: false
+  queue: max
+
 permissions:
   contents: read
 

.github/workflows/deploy_telemetry.yml

@@ -27,6 +27,7 @@ jobs:
         uses: actions/setup-go@v7
         with:
           go-version-file: 'telemetry/server/go.mod'
+          cache: ${{ runner.environment == 'github-hosted' }}
 
       - name: Build Telemetry Server
         if: github.event_name == 'workflow_dispatch' && inputs.deploy

.github/workflows/depsreview.yml

@@ -1,6 +1,10 @@
 name: 'Dependency Review'
 on: [pull_request]
 
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || (github.event_name == 'push' && github.ref) || github.run_id }}
+  cancel-in-progress: ${{ github.event_name == 'pull_request' || (github.event_name == 'push' && !startsWith(github.ref, 'refs/tags/')) }}
+
 permissions:
   contents: read
 

.github/workflows/e2e.yml

@@ -19,8 +19,8 @@ on:
       - '.github/workflows/e2e.yml'
 
 concurrency:
-  group: ${{ github.head_ref }}/e2e
-  cancel-in-progress: true
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || (github.event_name == 'push' && github.ref) || github.run_id }}
+  cancel-in-progress: ${{ github.event_name == 'pull_request' || (github.event_name == 'push' && !startsWith(github.ref, 'refs/tags/')) }}
 
 permissions:
   contents: read
@@ -42,6 +42,7 @@ jobs:
       uses: actions/setup-go@v7
       with:
         go-version-file: 'go.mod'
+        cache: ${{ runner.environment == 'github-hosted' }}
 
     - name: Configure Docker Hub mirror
       run: |
@@ -52,6 +53,7 @@ jobs:
       uses: docker/setup-buildx-action@v4
       
     - name: Cache Docker layers
+      if: runner.environment == 'github-hosted'
       uses: actions/cache@v6
       with:
         path: /tmp/.buildx-cache

.github/workflows/ec-integration-tests.yml

@@ -18,6 +18,10 @@ on:
       - 'go.sum'
       - '.github/workflows/ec-integration-tests.yml'
 
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || (github.event_name == 'push' && github.ref) || github.run_id }}
+  cancel-in-progress: ${{ github.event_name == 'pull_request' || (github.event_name == 'push' && !startsWith(github.ref, 'refs/tags/')) }}
+
 permissions:
   contents: read
 
@@ -30,6 +34,7 @@ jobs:
     - name: Set up Go 1.x
       uses: actions/setup-go@v7
       with:
+        cache: ${{ runner.environment == 'github-hosted' }}
         go-version: ^1.26
       id: go
 

.github/workflows/fuse-dlm-integration.yml

@@ -19,8 +19,8 @@ on:
       - '.github/actions/fix-fusermount-setuid/**'
 
 concurrency:
-  group: ${{ github.head_ref || github.ref }}/fuse-dlm-integration
-  cancel-in-progress: true
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || (github.event_name == 'push' && github.ref) || github.run_id }}
+  cancel-in-progress: ${{ github.event_name == 'pull_request' || (github.event_name == 'push' && !startsWith(github.ref, 'refs/tags/')) }}
 
 permissions:
   contents: read
@@ -39,6 +39,7 @@ jobs:
         uses: actions/setup-go@v7
         with:
           go-version-file: 'go.mod'
+          cache: ${{ runner.environment == 'github-hosted' }}
 
       - name: Configure FUSE
         run: |

.github/workflows/fuse-failover.yml

@@ -24,8 +24,8 @@ on:
       - '.github/actions/fix-fusermount-setuid/**'
 
 concurrency:
-  group: ${{ github.head_ref || github.ref }}/fuse-failover
-  cancel-in-progress: true
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || (github.event_name == 'push' && github.ref) || github.run_id }}
+  cancel-in-progress: ${{ github.event_name == 'pull_request' || (github.event_name == 'push' && !startsWith(github.ref, 'refs/tags/')) }}
 
 permissions:
   contents: read
@@ -42,10 +42,14 @@ jobs:
         with:
           persist-credentials: false
 
+      - name: Clean up stale FUSE mounts
+        uses: ./.github/actions/cleanup-stale-fuse-mounts
+
       - name: Set up Go
         uses: actions/setup-go@v7
         with:
           go-version-file: 'go.mod'
+          cache: ${{ runner.environment == 'github-hosted' }}
 
       - name: Configure FUSE
         run: |

.github/workflows/fuse-integration.yml

@@ -17,8 +17,8 @@ on:
       - '.github/actions/fix-fusermount-setuid/**'
 
 concurrency:
-  group: ${{ github.head_ref }}/fuse-integration
-  cancel-in-progress: true
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || (github.event_name == 'push' && github.ref) || github.run_id }}
+  cancel-in-progress: ${{ github.event_name == 'pull_request' || (github.event_name == 'push' && !startsWith(github.ref, 'refs/tags/')) }}
 
 permissions:
   contents: read
@@ -37,6 +37,7 @@ jobs:
       uses: actions/setup-go@v7
       with:
         go-version-file: 'go.mod'
+        cache: ${{ runner.environment == 'github-hosted' }}
 
     - name: Configure FUSE
       run: |

.github/workflows/fuse-p2p-integration.yml

@@ -25,8 +25,8 @@ on:
       - '.github/actions/fix-fusermount-setuid/**'
 
 concurrency:
-  group: ${{ github.head_ref || github.ref }}/fuse-p2p-integration
-  cancel-in-progress: true
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || (github.event_name == 'push' && github.ref) || github.run_id }}
+  cancel-in-progress: ${{ github.event_name == 'pull_request' || (github.event_name == 'push' && !startsWith(github.ref, 'refs/tags/')) }}
 
 permissions:
   contents: read
@@ -41,10 +41,14 @@ jobs:
       - name: Check out code
         uses: actions/checkout@v7
 
+      - name: Clean up stale FUSE mounts
+        uses: ./.github/actions/cleanup-stale-fuse-mounts
+
       - name: Set up Go
         uses: actions/setup-go@v7
         with:
           go-version-file: 'go.mod'
+          cache: ${{ runner.environment == 'github-hosted' }}
 
       - name: Configure FUSE
         run: |

.github/workflows/go.yml

@@ -17,8 +17,8 @@ on:
       - '.github/workflows/go.yml'
 
 concurrency:
-  group: ${{ github.head_ref }}/go
-  cancel-in-progress: true
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || (github.event_name == 'push' && github.ref) || github.run_id }}
+  cancel-in-progress: ${{ github.event_name == 'pull_request' || (github.event_name == 'push' && !startsWith(github.ref, 'refs/tags/')) }}
 
 permissions:
   contents: read
@@ -35,6 +35,7 @@ jobs:
       uses: actions/setup-go@v7
       with:
         go-version-file: 'go.mod'
+        cache: ${{ runner.environment == 'github-hosted' }}
     - name: Get dependencies
       run: |
         cd weed; go get -v -t -d ./...
@@ -57,6 +58,7 @@ jobs:
       uses: actions/setup-go@v7
       with:
         go-version-file: 'go.mod'
+        cache: ${{ runner.environment == 'github-hosted' }}
     - name: Go Vet linux/386 (type-checks code and tests for 32-bit int overflows)
       run: |
         GOOS=linux GOARCH=386 go vet ./... 2>&1 | grep -v "MessageState contains sync.Mutex" | grep -v "IdentityAccessManagement contains sync.RWMutex" | tee vet-32bit-output.txt
@@ -72,6 +74,7 @@ jobs:
       uses: actions/setup-go@v7
       with:
         go-version-file: 'go.mod'
+        cache: ${{ runner.environment == 'github-hosted' }}
     - name: Build
       run: cd weed; go build -tags "elastic gocdk sqlite ydb tarantool tikv rclone" -v .
 
@@ -94,6 +97,7 @@ jobs:
       uses: actions/setup-go@v7
       with:
         go-version-file: 'go.mod'
+        cache: ${{ runner.environment == 'github-hosted' }}
     # Nothing else on a PR compiles these, so per-OS syscall constants creep
     # back into shared files unnoticed and only a release build catches it.
     - name: Cross-compile
@@ -128,6 +132,7 @@ jobs:
       uses: actions/setup-go@v7
       with:
         go-version-file: 'go.mod'
+        cache: ${{ runner.environment == 'github-hosted' }}
     - name: Test
       env:
         RUN_REDIS_TESTS: "1"
@@ -143,6 +148,7 @@ jobs:
       uses: actions/setup-go@v7
       with:
         go-version-file: 'go.mod'
+        cache: ${{ runner.environment == 'github-hosted' }}
     # 386 test binaries run natively on the amd64 runner. This catches what vet
     # can't: unaligned 64-bit atomics and arithmetic that wraps at runtime.
     # -short skips the e2e suites already covered on amd64.
@@ -159,6 +165,7 @@ jobs:
       uses: actions/setup-go@v7
       with:
         go-version-file: 'go.mod'
+        cache: ${{ runner.environment == 'github-hosted' }}
     # weed/kms/azure is excluded from every default build, so without this
     # nothing compiles it and tag-gated code silently rots.
     - name: Build and test with azurekms

.github/workflows/helm_ci.yml

@@ -8,6 +8,10 @@ on:
     branches: [ master ]
     paths: ['k8s/**', '.github/workflows/helm_ci.yml', '.github/scripts/helm_jwt_keys.py']
 
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || (github.event_name == 'push' && github.ref) || github.run_id }}
+  cancel-in-progress: ${{ github.event_name == 'pull_request' || (github.event_name == 'push' && !startsWith(github.ref, 'refs/tags/')) }}
+
 permissions:
   contents: read
 

.github/workflows/java_integration_tests.yml

@@ -14,6 +14,10 @@ on:
       - 'weed/**'
       - '.github/workflows/java_integration_tests.yml'
 
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || (github.event_name == 'push' && github.ref) || github.run_id }}
+  cancel-in-progress: ${{ github.event_name == 'pull_request' || (github.event_name == 'push' && !startsWith(github.ref, 'refs/tags/')) }}
+
 jobs:
   test:
     name: Java Integration Tests
@@ -31,6 +35,7 @@ jobs:
         uses: actions/setup-go@v7
         with:
           go-version-file: 'go.mod'
+          cache: ${{ runner.environment == 'github-hosted' }}
         id: go
       
       - name: Set up Java
@@ -38,7 +43,7 @@ jobs:
         with:
           java-version: ${{ matrix.java }}
           distribution: 'temurin'
-          cache: 'maven'
+          cache: ${{ runner.environment == 'github-hosted' && 'maven' || '' }}
       
       - name: Build SeaweedFS
         run: |
@@ -55,6 +60,7 @@ jobs:
           # Start SeaweedFS with optimized settings for CI
           # Include S3 API for s3copier integration tests
           weed server -dir="$WEED_DATA_DIR" \
+            -ip=127.0.0.1 -ip.bind=127.0.0.1 \
             -master.raftHashicorp \
             -master.electionTimeout=1s \
             -master.volumeSizeLimitMB=100 \

.github/workflows/java_release.yml

@@ -77,7 +77,7 @@ jobs:
         with:
           java-version: '17'
           distribution: 'temurin'
-          cache: 'maven'
+          cache: ${{ runner.environment == 'github-hosted' && 'maven' || '' }}
           server-id: central
           server-username: MAVEN_CENTRAL_USERNAME
           server-password: MAVEN_CENTRAL_PASSWORD

.github/workflows/java_unit_tests.yml

@@ -12,6 +12,10 @@ on:
       - 'other/java/**'
       - '.github/workflows/java_unit_tests.yml'
 
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || (github.event_name == 'push' && github.ref) || github.run_id }}
+  cancel-in-progress: ${{ github.event_name == 'pull_request' || (github.event_name == 'push' && !startsWith(github.ref, 'refs/tags/')) }}
+
 jobs:
   test:
     name: Java Unit Tests
@@ -30,7 +34,7 @@ jobs:
         with:
           java-version: ${{ matrix.java }}
           distribution: 'temurin'
-          cache: 'maven'
+          cache: ${{ runner.environment == 'github-hosted' && 'maven' || '' }}
       
       - name: Build and Install SeaweedFS Client
         working-directory: other/java/client

.github/workflows/kafka-quicktest.yml

@@ -24,8 +24,8 @@ on:
   workflow_dispatch:  # Allow manual trigger
 
 concurrency:
-  group: ${{ github.head_ref }}/kafka-quicktest
-  cancel-in-progress: true
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || (github.event_name == 'push' && github.ref) || github.run_id }}
+  cancel-in-progress: ${{ github.event_name == 'pull_request' || (github.event_name == 'push' && !startsWith(github.ref, 'refs/tags/')) }}
 
 permissions:
   contents: read
@@ -43,7 +43,7 @@ jobs:
       uses: actions/setup-go@v7
       with:
         go-version: ^1.26
-        cache: true
+        cache: ${{ runner.environment == 'github-hosted' }}
         cache-dependency-path: |
           **/go.sum
       id: go

.github/workflows/kafka-tests.yml

@@ -23,8 +23,8 @@ on:
       - '.github/workflows/kafka-tests.yml'
 
 concurrency:
-  group: ${{ github.head_ref }}/kafka-tests
-  cancel-in-progress: true
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || (github.event_name == 'push' && github.ref) || github.run_id }}
+  cancel-in-progress: ${{ github.event_name == 'pull_request' || (github.event_name == 'push' && !startsWith(github.ref, 'refs/tags/')) }}
 
 # Force different runners for better isolation
 env:
@@ -53,6 +53,7 @@ jobs:
     - name: Set up Go 1.x
       uses: actions/setup-go@v7
       with:
+        cache: ${{ runner.environment == 'github-hosted' }}
         go-version: ^1.26
       id: go
 
@@ -98,6 +99,7 @@ jobs:
     - name: Set up Go 1.x
       uses: actions/setup-go@v7
       with:
+        cache: ${{ runner.environment == 'github-hosted' }}
         go-version: ^1.26
       id: go
 
@@ -149,7 +151,7 @@ jobs:
       uses: actions/setup-go@v7
       with:
         go-version: ^1.26
-        cache: true
+        cache: ${{ runner.environment == 'github-hosted' }}
         cache-dependency-path: |
           **/go.sum
       id: go
@@ -328,7 +330,7 @@ jobs:
       uses: actions/setup-go@v7
       with:
         go-version: ^1.26
-        cache: true
+        cache: ${{ runner.environment == 'github-hosted' }}
         cache-dependency-path: |
           **/go.sum
       id: go
@@ -490,7 +492,7 @@ jobs:
       uses: actions/setup-go@v7
       with:
         go-version: ^1.26
-        cache: true
+        cache: ${{ runner.environment == 'github-hosted' }}
         cache-dependency-path: |
           **/go.sum
       id: go
@@ -648,7 +650,7 @@ jobs:
       uses: actions/setup-go@v7
       with:
         go-version: ^1.26
-        cache: true
+        cache: ${{ runner.environment == 'github-hosted' }}
         cache-dependency-path: |
           **/go.sum
       id: go
@@ -805,6 +807,7 @@ jobs:
     - name: Set up Go 1.x
       uses: actions/setup-go@v7
       with:
+        cache: ${{ runner.environment == 'github-hosted' }}
         go-version: ^1.26
       id: go
 

.github/workflows/kms-tests.yml

@@ -19,8 +19,8 @@ on:
       - 'test/kms/**'
 
 concurrency:
-  group: ${{ github.head_ref || github.ref }}-kms-tests
-  cancel-in-progress: true
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || (github.event_name == 'push' && github.ref) || github.run_id }}
+  cancel-in-progress: ${{ github.event_name == 'pull_request' || (github.event_name == 'push' && !startsWith(github.ref, 'refs/tags/')) }}
 
 permissions:
   contents: read
@@ -43,6 +43,7 @@ jobs:
         uses: actions/setup-go@v7
         with:
           go-version-file: 'go.mod'
+          cache: ${{ runner.environment == 'github-hosted' }}
         id: go
 
       - name: Install SeaweedFS
@@ -88,6 +89,7 @@ jobs:
         uses: actions/setup-go@v7
         with:
           go-version-file: 'go.mod'
+          cache: ${{ runner.environment == 'github-hosted' }}
         id: go
 
       - name: Install SeaweedFS

.github/workflows/master-cold-start-tests.yml

@@ -21,8 +21,8 @@ on:
       - '.github/workflows/master-cold-start-tests.yml'
 
 concurrency:
-  group: ${{ github.head_ref || github.ref }}/master-cold-start-tests
-  cancel-in-progress: true
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || (github.event_name == 'push' && github.ref) || github.run_id }}
+  cancel-in-progress: ${{ github.event_name == 'pull_request' || (github.event_name == 'push' && !startsWith(github.ref, 'refs/tags/')) }}
 
 permissions:
   contents: read
@@ -42,6 +42,7 @@ jobs:
         uses: actions/setup-go@v7
         with:
           go-version-file: 'go.mod'
+          cache: ${{ runner.environment == 'github-hosted' }}
 
       - name: Build weed binary
         run: |

.github/workflows/metadata-subscribe-tests.yml

@@ -25,8 +25,8 @@ on:
       - '.github/workflows/metadata-subscribe-tests.yml'
 
 concurrency:
-  group: ${{ github.head_ref }}/metadata-subscribe-tests
-  cancel-in-progress: true
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || (github.event_name == 'push' && github.ref) || github.run_id }}
+  cancel-in-progress: ${{ github.event_name == 'pull_request' || (github.event_name == 'push' && !startsWith(github.ref, 'refs/tags/')) }}
 
 permissions:
   contents: read
@@ -48,6 +48,7 @@ jobs:
       uses: actions/setup-go@v7
       with:
         go-version-file: 'go.mod'
+        cache: ${{ runner.environment == 'github-hosted' }}
         
     - name: Build SeaweedFS
       run: |

.github/workflows/mount-benchmark.yml

@@ -10,8 +10,8 @@ on:
     branches: [ 'winfsp-bench**' ]
 
 concurrency:
-  group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }}
-  cancel-in-progress: true
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || (github.event_name == 'push' && github.ref) || github.run_id }}
+  cancel-in-progress: ${{ github.event_name == 'pull_request' || (github.event_name == 'push' && !startsWith(github.ref, 'refs/tags/')) }}
 
 permissions:
   contents: read
@@ -32,6 +32,7 @@ jobs:
     - uses: actions/setup-go@v7
       with:
         go-version-file: 'go.mod'
+        cache: ${{ runner.environment == 'github-hosted' }}
 
     - name: Install WinFsp and rclone
       run: choco install winfsp rclone -y --no-progress
@@ -150,6 +151,7 @@ jobs:
     - uses: actions/setup-go@v7
       with:
         go-version-file: 'go.mod'
+        cache: ${{ runner.environment == 'github-hosted' }}
 
     - name: Repair the fusermount3 setuid bit
       uses: ./.github/actions/fix-fusermount-setuid

.github/workflows/mount-windows-conformance.yml

@@ -24,8 +24,8 @@ on:
       - '.github/workflows/mount-windows-conformance.yml'
 
 concurrency:
-  group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }}
-  cancel-in-progress: true
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || (github.event_name == 'push' && github.ref) || github.run_id }}
+  cancel-in-progress: ${{ github.event_name == 'pull_request' || (github.event_name == 'push' && !startsWith(github.ref, 'refs/tags/')) }}
 
 permissions:
   contents: read
@@ -49,6 +49,7 @@ jobs:
     - uses: actions/setup-go@v7
       with:
         go-version-file: 'go.mod'
+        cache: ${{ runner.environment == 'github-hosted' }}
 
     # cgofuse loads winfsp-x64.dll at run time, so WinFsp is needed here but
     # not to build.

.github/workflows/mount-windows.yml

@@ -24,8 +24,8 @@ on:
       - '.github/workflows/mount-windows.yml'
 
 concurrency:
-  group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }}
-  cancel-in-progress: true
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || (github.event_name == 'push' && github.ref) || github.run_id }}
+  cancel-in-progress: ${{ github.event_name == 'pull_request' || (github.event_name == 'push' && !startsWith(github.ref, 'refs/tags/')) }}
 
 permissions:
   contents: read
@@ -49,6 +49,7 @@ jobs:
     - uses: actions/setup-go@v7
       with:
         go-version-file: 'go.mod'
+        cache: ${{ runner.environment == 'github-hosted' }}
 
     # cgofuse loads winfsp-x64.dll at run time, so WinFsp is needed here but
     # not to build.

.github/workflows/multi-master-tests.yml

@@ -21,8 +21,8 @@ on:
       - '.github/workflows/multi-master-tests.yml'
 
 concurrency:
-  group: ${{ github.head_ref || github.ref }}/multi-master-tests
-  cancel-in-progress: true
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || (github.event_name == 'push' && github.ref) || github.run_id }}
+  cancel-in-progress: ${{ github.event_name == 'pull_request' || (github.event_name == 'push' && !startsWith(github.ref, 'refs/tags/')) }}
 
 permissions:
   contents: read
@@ -40,6 +40,7 @@ jobs:
         uses: actions/setup-go@v7
         with:
           go-version-file: 'go.mod'
+          cache: ${{ runner.environment == 'github-hosted' }}
 
       - name: Install SeaweedFS
         run: |

.github/workflows/performance.yml

@@ -45,8 +45,8 @@ on:
         type: string
 
 concurrency:
-  group: ${{ github.head_ref || github.ref }}/performance
-  cancel-in-progress: true
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || (github.event_name == 'push' && github.ref) || github.run_id }}
+  cancel-in-progress: ${{ github.event_name == 'pull_request' || (github.event_name == 'push' && !startsWith(github.ref, 'refs/tags/')) }}
 
 permissions:
   contents: read
@@ -67,6 +67,7 @@ jobs:
         uses: actions/setup-go@v7
         with:
           go-version-file: 'go.mod'
+          cache: ${{ runner.environment == 'github-hosted' }}
 
       - name: Build weed
         run: go build -o weed_bin ./weed
@@ -175,13 +176,14 @@ jobs:
         uses: actions/setup-go@v7
         with:
           go-version-file: 'go.mod'
+          cache: ${{ runner.environment == 'github-hosted' }}
 
       - name: Install Rust toolchain
         if: matrix.impl == 'rust'
         uses: dtolnay/rust-toolchain@stable
 
       - name: Cache cargo registry and target
-        if: matrix.impl == 'rust'
+        if: matrix.impl == 'rust' && runner.environment == 'github-hosted'
         uses: actions/cache@v6
         with:
           path: |
@@ -327,13 +329,14 @@ jobs:
         uses: actions/setup-go@v7
         with:
           go-version-file: 'go.mod'
+          cache: ${{ runner.environment == 'github-hosted' }}
 
       - name: Install Rust toolchain
         if: matrix.impl == 'rust'
         uses: dtolnay/rust-toolchain@stable
 
       - name: Cache cargo registry and target
-        if: matrix.impl == 'rust'
+        if: matrix.impl == 'rust' && runner.environment == 'github-hosted'
         uses: actions/cache@v6
         with:
           path: |

.github/workflows/pjdfstest.yml

@@ -38,15 +38,23 @@ jobs:
       uses: actions/setup-go@v7
       with:
         go-version-file: 'go.mod'
+        cache: ${{ runner.environment == 'github-hosted' }}
 
     - name: Start local Docker registry
-      run: docker run -d --restart=always -p 5000:5000 --name registry registry:2
+      run: |
+        if ! docker start registry 2>/dev/null; then
+          docker rm -f registry >/dev/null 2>&1 || true
+          docker run -d --restart=always -p 5000:5000 --name registry registry:2
+        fi
 
     - name: Set up Docker Buildx
       uses: docker/setup-buildx-action@v4
       with:
         driver-opts: network=host
 
+    - name: Set up Docker Compose
+      uses: docker/setup-compose-action@v2
+
     - name: Build weed race binary
       run: |
         cd docker

.github/workflows/plugin-workers.yml

@@ -18,6 +18,10 @@ on:
       - 'go.sum'
       - '.github/workflows/plugin-workers.yml'
 
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || (github.event_name == 'push' && github.ref) || github.run_id }}
+  cancel-in-progress: ${{ github.event_name == 'pull_request' || (github.event_name == 'push' && !startsWith(github.ref, 'refs/tags/')) }}
+
 permissions:
   contents: read
 
@@ -41,6 +45,7 @@ jobs:
       - name: Set up Go 1.x
         uses: actions/setup-go@v7
         with:
+          cache: ${{ runner.environment == 'github-hosted' }}
           go-version: ^1.26
         id: go
 

.github/workflows/posix-compliance.yml

@@ -58,11 +58,14 @@ jobs:
       with:
         fetch-depth: 1
     
+    - name: Clean up stale FUSE mounts
+      uses: ./.github/actions/cleanup-stale-fuse-mounts
+
     - name: Set up Go ${{ env.GO_VERSION }}
       uses: actions/setup-go@v7
       with:
         go-version: ${{ env.GO_VERSION }}
-        cache: true
+        cache: ${{ runner.environment == 'github-hosted' }}
         cache-dependency-path: |
           go.sum
           test/fuse_integration/go.sum
@@ -98,7 +101,11 @@ jobs:
         
         # Set permissions on FUSE device
         sudo chmod 666 /dev/fuse
-        
+
+        # Allow non-root mounts to pass -o allow_other (weed -allowOthers)
+        echo 'user_allow_other' | sudo tee -a /etc/fuse.conf
+        sudo chmod 644 /etc/fuse.conf
+
         # Ensure fuse module is loaded
         sudo modprobe fuse || true
         
@@ -389,7 +396,7 @@ jobs:
         echo "" >> reports/debug/system_info.txt
         
         echo "Disk Space" >> reports/debug/system_info.txt
-        df -h >> reports/debug/system_info.txt
+        timeout 15 df -h >> reports/debug/system_info.txt || true
         echo "" >> reports/debug/system_info.txt
         
         echo "Memory Information" >> reports/debug/system_info.txt
@@ -564,7 +571,7 @@ jobs:
       uses: actions/setup-go@v7
       with:
         go-version: ${{ env.GO_VERSION }}
-        cache: true
+        cache: ${{ runner.environment == 'github-hosted' }}
     
     - name: Install macFUSE
       run: |
@@ -617,6 +624,7 @@ jobs:
       uses: actions/setup-go@v7
       with:
         go-version: ${{ env.GO_VERSION }}
+        cache: ${{ runner.environment == 'github-hosted' }}
     
     - name: Install security analysis tools
       run: |
@@ -673,6 +681,7 @@ jobs:
       uses: actions/setup-go@v7
       with:
         go-version: ${{ env.GO_VERSION }}
+        cache: ${{ runner.environment == 'github-hosted' }}
     
     - name: Install dependencies
       run: |
@@ -680,6 +689,8 @@ jobs:
         sudo apt-get install -y fuse libfuse-dev fio
         sudo usermod -a -G fuse $USER
         sudo chmod 666 /dev/fuse
+        # The posix suites mount with -allowOthers; allow_other needs this
+        echo 'user_allow_other' | sudo tee -a /etc/fuse.conf
     
     - name: Build SeaweedFS
       run: make
@@ -723,12 +734,15 @@ jobs:
       uses: actions/setup-go@v7
       with:
         go-version: ${{ env.GO_VERSION }}
+        cache: ${{ runner.environment == 'github-hosted' }}
     
     - name: Install dependencies
       run: |
         sudo apt-get update
         sudo apt-get install -y fuse libfuse-dev
         pip3 install --user nfstest
+        # The posix suites mount with -allowOthers; allow_other needs this
+        echo 'user_allow_other' | sudo tee -a /etc/fuse.conf
     
     - name: Build SeaweedFS
       run: make

.github/workflows/postgres-tests.yml

@@ -23,8 +23,8 @@ on:
       - '.github/workflows/postgres-tests.yml'
 
 concurrency:
-  group: ${{ github.head_ref }}/postgres-tests
-  cancel-in-progress: true
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || (github.event_name == 'push' && github.ref) || github.run_id }}
+  cancel-in-progress: ${{ github.event_name == 'pull_request' || (github.event_name == 'push' && !startsWith(github.ref, 'refs/tags/')) }}
 
 permissions:
   contents: read
@@ -41,6 +41,7 @@ jobs:
     - name: Set up Go 1.x
       uses: actions/setup-go@v7
       with:
+        cache: ${{ runner.environment == 'github-hosted' }}
         go-version: ^1.26
       id: go
 
@@ -56,6 +57,7 @@ jobs:
       uses: docker/setup-buildx-action@v4
 
     - name: Cache Docker layers
+      if: runner.environment == 'github-hosted'
       uses: actions/cache@v6
       with:
         path: /tmp/.buildx-cache

.github/workflows/rust-volume-server-tests.yml

@@ -21,8 +21,8 @@ on:
       - '.github/workflows/rust-volume-server-tests.yml'
 
 concurrency:
-  group: ${{ github.workflow }}-${{ github.head_ref || github.ref }}
-  cancel-in-progress: true
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || (github.event_name == 'push' && github.ref) || github.run_id }}
+  cancel-in-progress: ${{ github.event_name == 'pull_request' || (github.event_name == 'push' && !startsWith(github.ref, 'refs/tags/')) }}
 
 permissions:
   contents: read
@@ -52,6 +52,65 @@ jobs:
               - 'seaweed-volume/**'
               - '.github/workflows/rust-volume-server-tests.yml'
 
+  build-test-binaries:
+    name: Build Test Binaries
+    runs-on: ubuntu-22.04
+    timeout-minutes: 30
+
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v7
+
+      - name: Set up Go
+        uses: actions/setup-go@v7
+        with:
+          go-version-file: 'go.mod'
+          cache: ${{ runner.environment == 'github-hosted' }}
+
+      - name: Install Rust toolchain
+        uses: dtolnay/rust-toolchain@stable
+
+      # cargo tracks its own inputs but not the runner's C toolchain, so a cached
+      # target/ can carry C objects built against a different glibc than we link against.
+      - name: Fingerprint build toolchain
+        id: toolchain
+        run: echo "fingerprint=$(getconf GNU_LIBC_VERSION | tr ' ' '-')-rustc-$(rustc -V | awk '{print $2}')" >> "$GITHUB_OUTPUT"
+
+      - name: Cache cargo registry and target
+        if: runner.environment == 'github-hosted'
+        uses: actions/cache@v6
+        with:
+          path: |
+            ~/.cargo/registry
+            ~/.cargo/git
+            seaweed-volume/target
+          key: rust-${{ steps.toolchain.outputs.fingerprint }}-${{ hashFiles('seaweed-volume/Cargo.lock', 'seaweed-common/Cargo.lock') }}
+          restore-keys: |
+            rust-${{ steps.toolchain.outputs.fingerprint }}-
+
+      - name: Build Go weed binary
+        run: |
+          cd weed
+          go build -tags 5BytesOffset -o weed .
+          chmod +x weed
+          ./weed version
+
+      - name: Build Rust volume binary
+        run: cd seaweed-volume && cargo build --release
+
+      - name: Package binaries
+        run: tar cf rust-test-binaries.tar weed/weed seaweed-volume/target/release/weed-volume
+
+      - name: Upload test binaries
+        uses: actions/upload-artifact@v7
+        with:
+          # archive layout mirrors the workspace: weed/weed and
+          # seaweed-volume/target/release/weed-volume
+          name: rust-test-binaries
+          if-no-files-found: error
+          retention-days: 1
+          path: rust-test-binaries.tar
+
   rust-unit-tests:
     name: Rust Unit Tests
     runs-on: ubuntu-22.04
@@ -71,6 +130,7 @@ jobs:
         run: echo "fingerprint=$(getconf GNU_LIBC_VERSION | tr ' ' '-')-rustc-$(rustc -V | awk '{print $2}')" >> "$GITHUB_OUTPUT"
 
       - name: Cache cargo registry and target
+        if: runner.environment == 'github-hosted'
         uses: actions/cache@v6
         with:
           path: |
@@ -134,6 +194,7 @@ jobs:
         run: echo "fingerprint=windows-rustc-$(rustc -V | awk '{print $2}')" >> "$GITHUB_OUTPUT"
 
       - name: Cache cargo registry and target
+        if: runner.environment == 'github-hosted'
         uses: actions/cache@v6
         with:
           path: |
@@ -154,6 +215,7 @@ jobs:
     name: Rust Integration Tests
     runs-on: ubuntu-22.04
     timeout-minutes: 30
+    needs: [build-test-binaries]
 
     steps:
       - name: Checkout code
@@ -163,36 +225,18 @@ jobs:
         uses: actions/setup-go@v7
         with:
           go-version-file: 'go.mod'
+          cache: ${{ runner.environment == 'github-hosted' }}
 
-      - name: Install Rust toolchain
-        uses: dtolnay/rust-toolchain@stable
-
-      # cargo tracks its own inputs but not the runner's C toolchain, so a cached
-      # target/ can carry C objects built against a different glibc than we link against.
-      - name: Fingerprint build toolchain
-        id: toolchain
-        run: echo "fingerprint=$(getconf GNU_LIBC_VERSION | tr ' ' '-')-rustc-$(rustc -V | awk '{print $2}')" >> "$GITHUB_OUTPUT"
-
-      - name: Cache cargo registry and target
-        uses: actions/cache@v6
+      - name: Download test binaries
+        uses: actions/download-artifact@v7
         with:
-          path: |
-            ~/.cargo/registry
-            ~/.cargo/git
-            seaweed-volume/target
-          key: rust-${{ steps.toolchain.outputs.fingerprint }}-${{ hashFiles('seaweed-volume/Cargo.lock', 'seaweed-common/Cargo.lock') }}
-          restore-keys: |
-            rust-${{ steps.toolchain.outputs.fingerprint }}-
+          name: rust-test-binaries
 
-      - name: Build Go weed binary
+      - name: Restore binaries
         run: |
-          cd weed
-          go build -tags 5BytesOffset -o weed .
-          chmod +x weed
-          ./weed version
-
-      - name: Build Rust volume binary
-        run: cd seaweed-volume && cargo build --release
+          tar xf rust-test-binaries.tar
+          chmod +x weed/weed seaweed-volume/target/release/weed-volume
+          ./weed/weed version
 
       - name: Run integration tests
         env:
@@ -228,6 +272,7 @@ jobs:
     name: Go Tests with Rust Volume (${{ matrix.test-type }} - Shard ${{ matrix.shard }})
     runs-on: ubuntu-22.04
     timeout-minutes: 45
+    needs: [build-test-binaries]
     env:
       # Keep in step with the length of matrix.shard below.
       SHARD_COUNT: 3
@@ -245,36 +290,19 @@ jobs:
         uses: actions/setup-go@v7
         with:
           go-version-file: 'go.mod'
+          cache: ${{ runner.environment == 'github-hosted' }}
 
-      - name: Install Rust toolchain
-        uses: dtolnay/rust-toolchain@stable
-
-      # cargo tracks its own inputs but not the runner's C toolchain, so a cached
-      # target/ can carry C objects built against a different glibc than we link against.
-      - name: Fingerprint build toolchain
-        id: toolchain
-        run: echo "fingerprint=$(getconf GNU_LIBC_VERSION | tr ' ' '-')-rustc-$(rustc -V | awk '{print $2}')" >> "$GITHUB_OUTPUT"
-
-      - name: Cache cargo registry and target
-        uses: actions/cache@v6
+      - name: Download test binaries
+        uses: actions/download-artifact@v7
         with:
-          path: |
-            ~/.cargo/registry
-            ~/.cargo/git
-            seaweed-volume/target
-          key: rust-${{ steps.toolchain.outputs.fingerprint }}-${{ hashFiles('seaweed-volume/Cargo.lock', 'seaweed-common/Cargo.lock') }}
-          restore-keys: |
-            rust-${{ steps.toolchain.outputs.fingerprint }}-
+          name: rust-test-binaries
 
-      - name: Build Go weed binary
+      - name: Restore binaries
         run: |
-          cd weed
-          go build -tags 5BytesOffset -o weed .
-          chmod +x weed
-          ./weed version
-
-      - name: Build Rust volume binary
-        run: cd seaweed-volume && cargo build --release
+          tar xf rust-test-binaries.tar
+          chmod +x weed/weed seaweed-volume/target/release/weed-volume
+          ./weed/weed version
+          ./seaweed-volume/target/release/weed-volume --help >/dev/null || true
 
       # Dealing the listed tests out one by one keeps the shards even. Bucketing
       # them by first letter did not: names cluster, so ^Test[I-S] drew 50 of

.github/workflows/rust-worker-tests.yml

@@ -17,8 +17,8 @@ on:
       - '.github/workflows/rust-worker-tests.yml'
 
 concurrency:
-  group: ${{ github.workflow }}-${{ github.head_ref || github.ref }}
-  cancel-in-progress: true
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || (github.event_name == 'push' && github.ref) || github.run_id }}
+  cancel-in-progress: ${{ github.event_name == 'pull_request' || (github.event_name == 'push' && !startsWith(github.ref, 'refs/tags/')) }}
 
 permissions:
   contents: read
@@ -45,6 +45,7 @@ jobs:
         run: echo "fingerprint=$(getconf GNU_LIBC_VERSION | tr ' ' '-')-rustc-$(rustc -V | awk '{print $2}')" >> "$GITHUB_OUTPUT"
 
       - name: Cache cargo registry and target
+        if: runner.environment == 'github-hosted'
         uses: actions/cache@v6
         with:
           path: |

.github/workflows/rust_binaries_dev.yml

@@ -46,6 +46,7 @@ jobs:
         uses: dtolnay/rust-toolchain@stable
 
       - name: Cache cargo registry and target
+        if: runner.environment == 'github-hosted'
         uses: actions/cache@v6
         with:
           path: |
@@ -114,6 +115,7 @@ jobs:
           targets: ${{ matrix.target }}
 
       - name: Cache cargo registry and target
+        if: runner.environment == 'github-hosted'
         uses: actions/cache@v6
         with:
           path: |

.github/workflows/rust_binaries_release.yml

@@ -48,6 +48,7 @@ jobs:
           echo "OPENSSL_LIB_DIR=/usr/lib/aarch64-linux-gnu" >> "$GITHUB_ENV"
 
       - name: Cache cargo registry and target
+        if: runner.environment == 'github-hosted'
         uses: actions/cache@v6
         with:
           path: |
@@ -153,6 +154,7 @@ jobs:
           echo "CARGO_TARGET_AARCH64_UNKNOWN_LINUX_GNU_LINKER=aarch64-linux-gnu-gcc" >> "$GITHUB_ENV"
 
       - name: Cache cargo registry and target
+        if: runner.environment == 'github-hosted'
         uses: actions/cache@v6
         with:
           path: |
@@ -230,6 +232,7 @@ jobs:
           targets: ${{ matrix.target }}
 
       - name: Cache cargo registry and target
+        if: runner.environment == 'github-hosted'
         uses: actions/cache@v6
         with:
           path: |
@@ -305,6 +308,7 @@ jobs:
         uses: dtolnay/rust-toolchain@stable
 
       - name: Cache cargo registry and target
+        if: runner.environment == 'github-hosted'
         uses: actions/cache@v6
         with:
           path: |

.github/workflows/s3-etag-acl-tests.yml

@@ -21,8 +21,8 @@ on:
       - '.github/workflows/s3-etag-acl-tests.yml'
 
 concurrency:
-  group: ${{ github.head_ref || github.ref }}/s3-etag-acl-tests
-  cancel-in-progress: true
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || (github.event_name == 'push' && github.ref) || github.run_id }}
+  cancel-in-progress: ${{ github.event_name == 'pull_request' || (github.event_name == 'push' && !startsWith(github.ref, 'refs/tags/')) }}
 
 permissions:
   contents: read
@@ -40,6 +40,7 @@ jobs:
         uses: actions/setup-go@v7
         with:
           go-version-file: 'go.mod'
+          cache: ${{ runner.environment == 'github-hosted' }}
 
       - name: Install SeaweedFS
         run: |

.github/workflows/s3-example-integration-tests.yml

@@ -13,8 +13,8 @@ on:
       - '.github/workflows/s3-example-integration-tests.yml'
 
 concurrency:
-  group: ${{ github.head_ref }}/s3-integration-tests
-  cancel-in-progress: true
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || (github.event_name == 'push' && github.ref) || github.run_id }}
+  cancel-in-progress: ${{ github.event_name == 'pull_request' || (github.event_name == 'push' && !startsWith(github.ref, 'refs/tags/')) }}
 
 permissions:
   contents: read
@@ -33,6 +33,7 @@ jobs:
         uses: actions/setup-go@v7
         with:
           go-version-file: 'go.mod'
+          cache: ${{ runner.environment == 'github-hosted' }}
         id: go
 
       - name: Build SeaweedFS

.github/workflows/s3-filer-group-tests.yml

@@ -12,8 +12,8 @@ on:
       - '.github/workflows/s3-filer-group-tests.yml'
 
 concurrency:
-  group: ${{ github.head_ref }}/s3-filer-group-tests
-  cancel-in-progress: true
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || (github.event_name == 'push' && github.ref) || github.run_id }}
+  cancel-in-progress: ${{ github.event_name == 'pull_request' || (github.event_name == 'push' && !startsWith(github.ref, 'refs/tags/')) }}
 
 permissions:
   contents: read
@@ -32,10 +32,14 @@ jobs:
       - name: Check out code
         uses: actions/checkout@v7
 
+      - name: Clean up stale FUSE mounts
+        uses: ./.github/actions/cleanup-stale-fuse-mounts
+
       - name: Set up Go
         uses: actions/setup-go@v7
         with:
           go-version-file: 'go.mod'
+          cache: ${{ runner.environment == 'github-hosted' }}
         id: go
 
       - name: Install SeaweedFS
@@ -50,7 +54,7 @@ jobs:
           echo "=== System Information ==="
           uname -a
           free -h
-          df -h
+          timeout 15 df -h || true
           echo "=== Starting Filer Group Tests ==="
           
           # Run tests with automatic server management

.github/workflows/s3-go-tests.yml

@@ -12,8 +12,8 @@ on:
       - '.github/workflows/s3-go-tests.yml'
 
 concurrency:
-  group: ${{ github.head_ref }}/s3-go-tests
-  cancel-in-progress: true
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || (github.event_name == 'push' && github.ref) || github.run_id }}
+  cancel-in-progress: ${{ github.event_name == 'pull_request' || (github.event_name == 'push' && !startsWith(github.ref, 'refs/tags/')) }}
 
 permissions:
   contents: read
@@ -35,10 +35,14 @@ jobs:
       - name: Check out code
         uses: actions/checkout@v7
 
+      - name: Clean up stale FUSE mounts
+        uses: ./.github/actions/cleanup-stale-fuse-mounts
+
       - name: Set up Go
         uses: actions/setup-go@v7
         with:
           go-version-file: 'go.mod'
+          cache: ${{ runner.environment == 'github-hosted' }}
         id: go
 
       - name: Install SeaweedFS
@@ -53,7 +57,7 @@ jobs:
           echo "=== System Information ==="
           uname -a
           free -h
-          df -h
+          timeout 15 df -h || true
           echo "=== Starting Tests ==="
           
           # Run tests with automatic server management
@@ -103,6 +107,7 @@ jobs:
         uses: actions/setup-go@v7
         with:
           go-version-file: 'go.mod'
+          cache: ${{ runner.environment == 'github-hosted' }}
         id: go
 
       - name: Install SeaweedFS
@@ -151,6 +156,7 @@ jobs:
         uses: actions/setup-go@v7
         with:
           go-version-file: 'go.mod'
+          cache: ${{ runner.environment == 'github-hosted' }}
         id: go
 
       - name: Install SeaweedFS
@@ -198,10 +204,14 @@ jobs:
       - name: Check out code
         uses: actions/checkout@v7
 
+      - name: Clean up stale FUSE mounts
+        uses: ./.github/actions/cleanup-stale-fuse-mounts
+
       - name: Set up Go
         uses: actions/setup-go@v7
         with:
           go-version-file: 'go.mod'
+          cache: ${{ runner.environment == 'github-hosted' }}
         id: go
 
       - name: Install SeaweedFS
@@ -216,7 +226,7 @@ jobs:
           echo "=== System Information ==="
           uname -a
           free -h
-          df -h
+          timeout 15 df -h || true
           echo "=== Starting Tests ==="
           
           # Run tests with automatic server management
@@ -283,6 +293,7 @@ jobs:
           - TestLifecycleSuspendedVersioningExpiration
           - TestLifecycleTagFilter
           - TestLifecycleVersionedBucketCreatesDeleteMarker
+          - TestLifecycleVersionedBucketExpiresOnSecondRun
 
     steps:
       - name: Check out code
@@ -292,6 +303,7 @@ jobs:
         uses: actions/setup-go@v7
         with:
           go-version-file: 'go.mod'
+          cache: ${{ runner.environment == 'github-hosted' }}
         id: go
 
       - name: Install SeaweedFS
@@ -333,10 +345,14 @@ jobs:
       - name: Check out code
         uses: actions/checkout@v7
 
+      - name: Clean up stale FUSE mounts
+        uses: ./.github/actions/cleanup-stale-fuse-mounts
+
       - name: Set up Go
         uses: actions/setup-go@v7
         with:
           go-version-file: 'go.mod'
+          cache: ${{ runner.environment == 'github-hosted' }}
         id: go
 
       - name: Install SeaweedFS
@@ -351,7 +367,7 @@ jobs:
           echo "=== System Information ==="
           uname -a
           free -h
-          df -h
+          timeout 15 df -h || true
           echo "=== Starting Tests ==="
           make test-with-server
 
@@ -391,10 +407,14 @@ jobs:
       - name: Check out code
         uses: actions/checkout@v7
 
+      - name: Clean up stale FUSE mounts
+        uses: ./.github/actions/cleanup-stale-fuse-mounts
+
       - name: Set up Go
         uses: actions/setup-go@v7
         with:
           go-version-file: 'go.mod'
+          cache: ${{ runner.environment == 'github-hosted' }}
         id: go
 
       - name: Install SeaweedFS
@@ -409,7 +429,7 @@ jobs:
           echo "=== System Information ==="
           uname -a
           free -h
-          df -h
+          timeout 15 df -h || true
           echo "=== Starting Tests ==="
           
           # Run tests with automatic server management
@@ -459,6 +479,7 @@ jobs:
         uses: actions/setup-go@v7
         with:
           go-version-file: 'go.mod'
+          cache: ${{ runner.environment == 'github-hosted' }}
         id: go
 
       - name: Install SeaweedFS
@@ -508,6 +529,7 @@ jobs:
         uses: actions/setup-go@v7
         with:
           go-version-file: 'go.mod'
+          cache: ${{ runner.environment == 'github-hosted' }}
         id: go
 
       - name: Install SeaweedFS
@@ -576,6 +598,7 @@ jobs:
         uses: actions/setup-go@v7
         with:
           go-version-file: 'go.mod'
+          cache: ${{ runner.environment == 'github-hosted' }}
         id: go
 
       - name: Install SeaweedFS
@@ -631,6 +654,7 @@ jobs:
         uses: actions/setup-go@v7
         with:
           go-version-file: 'go.mod'
+          cache: ${{ runner.environment == 'github-hosted' }}
         id: go
 
       - name: Install SeaweedFS

.github/workflows/s3-iam-tests.yml

@@ -20,8 +20,8 @@ on:
       - '.github/workflows/s3-iam-tests.yml'
   
 concurrency:
-  group: ${{ github.head_ref }}/s3-iam-tests
-  cancel-in-progress: true
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || (github.event_name == 'push' && github.ref) || github.run_id }}
+  cancel-in-progress: ${{ github.event_name == 'pull_request' || (github.event_name == 'push' && !startsWith(github.ref, 'refs/tags/')) }}
 
 permissions:
   contents: read
@@ -45,6 +45,7 @@ jobs:
         uses: actions/setup-go@v7
         with:
           go-version-file: 'go.mod'
+          cache: ${{ runner.environment == 'github-hosted' }}
         id: go
 
       - name: Get dependencies
@@ -90,10 +91,14 @@ jobs:
       - name: Check out code
         uses: actions/checkout@v7
 
+      - name: Clean up stale FUSE mounts
+        uses: ./.github/actions/cleanup-stale-fuse-mounts
+
       - name: Set up Go
         uses: actions/setup-go@v7
         with:
           go-version-file: 'go.mod'
+          cache: ${{ runner.environment == 'github-hosted' }}
         id: go
 
       - name: Install SeaweedFS
@@ -109,7 +114,7 @@ jobs:
           echo "=== System Information ==="
           uname -a
           free -h
-          df -h
+          timeout 15 df -h || true
           echo "=== Starting S3 IAM Integration Tests (${{ matrix.test-type }}) ==="
           
           # Set WEED_BINARY to use the installed version
@@ -208,6 +213,7 @@ jobs:
         uses: actions/setup-go@v7
         with:
           go-version-file: 'go.mod'
+          cache: ${{ runner.environment == 'github-hosted' }}
         id: go
 
       - name: Install SeaweedFS
@@ -268,6 +274,7 @@ jobs:
         uses: actions/setup-go@v7
         with:
           go-version-file: 'go.mod'
+          cache: ${{ runner.environment == 'github-hosted' }}
         id: go
 
       - name: Install SeaweedFS

.github/workflows/s3-keycloak-tests.yml

@@ -16,8 +16,8 @@ on:
       - '.github/workflows/s3-keycloak-tests.yml'
   
 concurrency:
-  group: ${{ github.head_ref }}/s3-keycloak-tests
-  cancel-in-progress: true
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || (github.event_name == 'push' && github.ref) || github.run_id }}
+  cancel-in-progress: ${{ github.event_name == 'pull_request' || (github.event_name == 'push' && !startsWith(github.ref, 'refs/tags/')) }}
 
 permissions:
   contents: read
@@ -37,10 +37,14 @@ jobs:
       - name: Check out code
         uses: actions/checkout@v7
 
+      - name: Clean up stale FUSE mounts
+        uses: ./.github/actions/cleanup-stale-fuse-mounts
+
       - name: Set up Go
         uses: actions/setup-go@v7
         with:
           go-version-file: 'go.mod'
+          cache: ${{ runner.environment == 'github-hosted' }}
         id: go
 
       - name: Install SeaweedFS
@@ -56,7 +60,7 @@ jobs:
           echo "=== System Information ==="
           uname -a
           free -h
-          df -h
+          timeout 15 df -h || true
           echo "=== Starting S3 Keycloak Integration Tests ==="
           
           # Set WEED_BINARY to use the installed version

.github/workflows/s3-mutation-regression-tests.yml

@@ -12,8 +12,8 @@ on:
       - '.github/workflows/s3-mutation-regression-tests.yml'
 
 concurrency:
-  group: ${{ github.head_ref }}/s3-mutation-regression-tests
-  cancel-in-progress: true
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || (github.event_name == 'push' && github.ref) || github.run_id }}
+  cancel-in-progress: ${{ github.event_name == 'pull_request' || (github.event_name == 'push' && !startsWith(github.ref, 'refs/tags/')) }}
 
 permissions:
   contents: read
@@ -32,6 +32,7 @@ jobs:
         uses: actions/setup-go@v7
         with:
           go-version-file: 'go.mod'
+          cache: ${{ runner.environment == 'github-hosted' }}
 
       - name: Run S3 versioning regression tests
         timeout-minutes: 20
@@ -85,6 +86,7 @@ jobs:
         uses: actions/setup-go@v7
         with:
           go-version-file: 'go.mod'
+          cache: ${{ runner.environment == 'github-hosted' }}
 
       - name: Run S3 delete regression tests
         timeout-minutes: 15
@@ -123,6 +125,7 @@ jobs:
         uses: actions/setup-go@v7
         with:
           go-version-file: 'go.mod'
+          cache: ${{ runner.environment == 'github-hosted' }}
 
       - name: Build SeaweedFS
         run: |

.github/workflows/s3-parquet-tests.yml

@@ -23,6 +23,10 @@ env:
   S3_ENDPOINT_URL: http://localhost:8333
   BUCKET_NAME: test-parquet-bucket
 
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || (github.event_name == 'push' && github.ref) || github.run_id }}
+  cancel-in-progress: ${{ github.event_name == 'pull_request' || (github.event_name == 'push' && !startsWith(github.ref, 'refs/tags/')) }}
+
 jobs:
   parquet-integration-tests:
     name: PyArrow Parquet Tests (Python ${{ matrix.python-version }})
@@ -42,13 +46,13 @@ jobs:
         uses: actions/setup-go@v7
         with:
           go-version: ^1.26
-          cache: true
+          cache: ${{ runner.environment == 'github-hosted' }}
       
       - name: Set up Python ${{ matrix.python-version }}
         uses: actions/setup-python@v7
         with:
           python-version: ${{ matrix.python-version }}
-          cache: 'pip'
+          cache: ${{ runner.environment == 'github-hosted' && 'pip' || '' }}
           cache-dependency-path: 'test/s3/parquet/requirements.txt'
       
       - name: Install system dependencies
@@ -150,7 +154,7 @@ jobs:
         uses: actions/setup-go@v7
         with:
           go-version: ^1.26
-          cache: true
+          cache: ${{ runner.environment == 'github-hosted' }}
       
       - name: Run Go unit tests
         run: |

.github/workflows/s3-policy-tests.yml

@@ -24,8 +24,8 @@ on:
       - '.github/workflows/s3-policy-tests.yml'
 
 concurrency:
-  group: ${{ github.head_ref }}/s3-policy-tests
-  cancel-in-progress: true
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || (github.event_name == 'push' && github.ref) || github.run_id }}
+  cancel-in-progress: ${{ github.event_name == 'pull_request' || (github.event_name == 'push' && !startsWith(github.ref, 'refs/tags/')) }}
 
 permissions:
   contents: read
@@ -49,6 +49,7 @@ jobs:
         uses: actions/setup-go@v7
         with:
           go-version-file: 'go.mod'
+          cache: ${{ runner.environment == 'github-hosted' }}
         id: go
 
       - name: Get dependencies
@@ -88,10 +89,14 @@ jobs:
       - name: Check out code
         uses: actions/checkout@v7
 
+      - name: Clean up stale FUSE mounts
+        uses: ./.github/actions/cleanup-stale-fuse-mounts
+
       - name: Set up Go
         uses: actions/setup-go@v7
         with:
           go-version-file: 'go.mod'
+          cache: ${{ runner.environment == 'github-hosted' }}
         id: go
 
       - name: Install SeaweedFS
@@ -106,7 +111,7 @@ jobs:
           echo "=== System Information ==="
           uname -a
           free -h
-          df -h
+          timeout 15 df -h || true
           
           echo "=== Starting S3 Policy Variables Integration Tests ==="
           
@@ -197,10 +202,14 @@ jobs:
       - name: Check out code
         uses: actions/checkout@v7
 
+      - name: Clean up stale FUSE mounts
+        uses: ./.github/actions/cleanup-stale-fuse-mounts
+
       - name: Set up Go
         uses: actions/setup-go@v7
         with:
           go-version-file: 'go.mod'
+          cache: ${{ runner.environment == 'github-hosted' }}
         id: go
 
       - name: Install SeaweedFS
@@ -215,7 +224,7 @@ jobs:
           echo "=== System Information ==="
           uname -a
           free -h
-          df -h
+          timeout 15 df -h || true
           
           echo "=== Starting S3 Policy Enforcement Tests (${{ matrix.test-case }}) ==="
           
@@ -319,6 +328,7 @@ jobs:
         uses: actions/setup-go@v7
         with:
           go-version-file: 'go.mod'
+          cache: ${{ runner.environment == 'github-hosted' }}
         id: go
 
       - name: Install SeaweedFS
@@ -406,6 +416,7 @@ jobs:
         uses: actions/setup-go@v7
         with:
           go-version-file: 'go.mod'
+          cache: ${{ runner.environment == 'github-hosted' }}
         id: go
 
       - name: Install SeaweedFS

.github/workflows/s3-proxy-signature-tests.yml

@@ -21,8 +21,8 @@ on:
       - '.github/workflows/s3-proxy-signature-tests.yml'
 
 concurrency:
-  group: ${{ github.head_ref || github.ref }}/s3-proxy-signature-tests
-  cancel-in-progress: true
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || (github.event_name == 'push' && github.ref) || github.run_id }}
+  cancel-in-progress: ${{ github.event_name == 'pull_request' || (github.event_name == 'push' && !startsWith(github.ref, 'refs/tags/')) }}
 
 permissions:
   contents: read
@@ -40,6 +40,7 @@ jobs:
         uses: actions/setup-go@v7
         with:
           go-version-file: 'go.mod'
+          cache: ${{ runner.environment == 'github-hosted' }}
         id: go
 
       - name: Configure Docker Hub mirror

.github/workflows/s3-sdk-v2-routing-tests.yml

@@ -15,8 +15,8 @@ on:
       - '.github/workflows/s3-sdk-v2-routing-tests.yml'
 
 concurrency:
-  group: ${{ github.head_ref || github.ref }}/s3-sdk-v2-routing-tests
-  cancel-in-progress: true
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || (github.event_name == 'push' && github.ref) || github.run_id }}
+  cancel-in-progress: ${{ github.event_name == 'pull_request' || (github.event_name == 'push' && !startsWith(github.ref, 'refs/tags/')) }}
 
 permissions:
   contents: read
@@ -34,6 +34,7 @@ jobs:
         uses: actions/setup-go@v7
         with:
           go-version-file: 'go.mod'
+          cache: ${{ runner.environment == 'github-hosted' }}
 
       - name: Install SeaweedFS
         run: |

.github/workflows/s3-snowflake-tests.yml

@@ -60,6 +60,7 @@ jobs:
         uses: actions/setup-go@v7
         with:
           go-version-file: 'go.mod'
+          cache: ${{ runner.environment == 'github-hosted' }}
         id: go
 
       - name: Set up Java
@@ -67,7 +68,7 @@ jobs:
         with:
           java-version: '17'
           distribution: 'temurin'
-          cache: 'maven'
+          cache: ${{ runner.environment == 'github-hosted' && 'maven' || '' }}
 
       - name: Install SeaweedFS
         run: |

.github/workflows/s3-spark-tests.yml

@@ -11,8 +11,8 @@ on:
   workflow_dispatch:
 
 concurrency:
-  group: ${{ github.head_ref }}/s3-spark-tests
-  cancel-in-progress: true
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || (github.event_name == 'push' && github.ref) || github.run_id }}
+  cancel-in-progress: ${{ github.event_name == 'pull_request' || (github.event_name == 'push' && !startsWith(github.ref, 'refs/tags/')) }}
 
 permissions:
   contents: read
@@ -27,10 +27,14 @@ jobs:
       - name: Check out code
         uses: actions/checkout@v7
 
+      - name: Clean up stale FUSE mounts
+        uses: ./.github/actions/cleanup-stale-fuse-mounts
+
       - name: Set up Go
         uses: actions/setup-go@v7
         with:
           go-version-file: 'go.mod'
+          cache: ${{ runner.environment == 'github-hosted' }}
         id: go
 
       - name: Configure Docker Hub mirror
@@ -56,7 +60,7 @@ jobs:
           echo "=== System Information ==="
           uname -a
           free -h
-          df -h
+          timeout 15 df -h || true
           echo "=== Starting S3 Spark Integration Tests ==="
 
           go test -v -timeout 30m . 2>&1 | tee test-output.log || {

.github/workflows/s3-sse-tests.yml

@@ -23,8 +23,8 @@ on:
       - 'test/s3/sse/**'
   
 concurrency:
-  group: ${{ github.head_ref }}/s3-sse-tests
-  cancel-in-progress: true
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || (github.event_name == 'push' && github.ref) || github.run_id }}
+  cancel-in-progress: ${{ github.event_name == 'pull_request' || (github.event_name == 'push' && !startsWith(github.ref, 'refs/tags/')) }}
 
 permissions:
   contents: read
@@ -46,10 +46,14 @@ jobs:
       - name: Check out code
         uses: actions/checkout@v7
 
+      - name: Clean up stale FUSE mounts
+        uses: ./.github/actions/cleanup-stale-fuse-mounts
+
       - name: Set up Go
         uses: actions/setup-go@v7
         with:
           go-version-file: 'go.mod'
+          cache: ${{ runner.environment == 'github-hosted' }}
         id: go
 
       - name: Install SeaweedFS
@@ -64,7 +68,7 @@ jobs:
           echo "=== System Information ==="
           uname -a
           free -h
-          df -h
+          timeout 15 df -h || true
           echo "=== Starting SSE Tests ==="
           
           # Run tests with automatic server management
@@ -118,6 +122,7 @@ jobs:
         uses: actions/setup-go@v7
         with:
           go-version-file: 'go.mod'
+          cache: ${{ runner.environment == 'github-hosted' }}
         id: go
 
       - name: Install SeaweedFS
@@ -166,6 +171,7 @@ jobs:
         uses: actions/setup-go@v7
         with:
           go-version-file: 'go.mod'
+          cache: ${{ runner.environment == 'github-hosted' }}
         id: go
 
       - name: Install SeaweedFS
@@ -215,6 +221,7 @@ jobs:
         uses: actions/setup-go@v7
         with:
           go-version-file: 'go.mod'
+          cache: ${{ runner.environment == 'github-hosted' }}
         id: go
 
       - name: Install SeaweedFS
@@ -264,6 +271,7 @@ jobs:
         uses: actions/setup-go@v7
         with:
           go-version-file: 'go.mod'
+          cache: ${{ runner.environment == 'github-hosted' }}
         id: go
 
       - name: Install SeaweedFS
@@ -315,6 +323,7 @@ jobs:
         uses: actions/setup-go@v7
         with:
           go-version-file: 'go.mod'
+          cache: ${{ runner.environment == 'github-hosted' }}
         id: go
 
       - name: Install SeaweedFS
@@ -363,6 +372,7 @@ jobs:
         uses: actions/setup-go@v7
         with:
           go-version-file: 'go.mod'
+          cache: ${{ runner.environment == 'github-hosted' }}
         id: go
 
       - name: Install SeaweedFS

.github/workflows/s3-tables-tests.yml

@@ -12,6 +12,10 @@ on:
       - 'go.sum'
       - '.github/workflows/s3-tables-tests.yml'
 
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || (github.event_name == 'push' && github.ref) || github.run_id }}
+  cancel-in-progress: ${{ github.event_name == 'pull_request' || (github.event_name == 'push' && !startsWith(github.ref, 'refs/tags/')) }}
+
 permissions:
   contents: read
 
@@ -26,10 +30,14 @@ jobs:
       - name: Check out code
         uses: actions/checkout@v7
 
+      - name: Clean up stale FUSE mounts
+        uses: ./.github/actions/cleanup-stale-fuse-mounts
+
       - name: Set up Go
         uses: actions/setup-go@v7
         with:
           go-version-file: 'go.mod'
+          cache: ${{ runner.environment == 'github-hosted' }}
         id: go
 
       - name: Run go mod tidy
@@ -48,7 +56,8 @@ jobs:
           echo "=== System Information ==="
           uname -a
           free -h
-          df -h
+          # A stale FUSE mount can hang or fail df; keep diagnostics best effort.
+          timeout 15 df -h || true
           echo "=== Starting S3 Tables Tests ==="
           
           # Run S3 Tables integration tests
@@ -86,10 +95,14 @@ jobs:
       - name: Check out code
         uses: actions/checkout@v7
 
+      - name: Clean up stale FUSE mounts
+        uses: ./.github/actions/cleanup-stale-fuse-mounts
+
       - name: Set up Go
         uses: actions/setup-go@v7
         with:
           go-version-file: 'go.mod'
+          cache: ${{ runner.environment == 'github-hosted' }}
         id: go
 
       - name: Configure Docker Hub mirror
@@ -109,7 +122,7 @@ jobs:
           echo "=== System Information ==="
           uname -a
           free -h
-          df -h
+          timeout 15 df -h || true
           echo "=== Starting Iceberg Catalog Tests ==="
           
           # Run Iceberg catalog integration tests using Makefile (handles build)
@@ -147,10 +160,14 @@ jobs:
       - name: Check out code
         uses: actions/checkout@v7
 
+      - name: Clean up stale FUSE mounts
+        uses: ./.github/actions/cleanup-stale-fuse-mounts
+
       - name: Set up Go
         uses: actions/setup-go@v7
         with:
           go-version-file: 'go.mod'
+          cache: ${{ runner.environment == 'github-hosted' }}
         id: go
 
       - name: Configure Docker Hub mirror
@@ -179,7 +196,7 @@ jobs:
           echo "=== System Information ==="
           uname -a
           free -h
-          df -h
+          timeout 15 df -h || true
           echo "=== Starting Trino Iceberg Catalog Tests ==="
           
           # Run Trino + Iceberg catalog integration tests
@@ -217,10 +234,14 @@ jobs:
       - name: Check out code
         uses: actions/checkout@v7
 
+      - name: Clean up stale FUSE mounts
+        uses: ./.github/actions/cleanup-stale-fuse-mounts
+
       - name: Set up Go
         uses: actions/setup-go@v7
         with:
           go-version-file: 'go.mod'
+          cache: ${{ runner.environment == 'github-hosted' }}
         id: go
 
       - name: Configure Docker Hub mirror
@@ -250,7 +271,7 @@ jobs:
           echo "=== System Information ==="
           uname -a
           free -h
-          df -h
+          timeout 15 df -h || true
           docker info
           echo "=== Starting Dremio Iceberg Catalog Tests ==="
 
@@ -290,10 +311,14 @@ jobs:
       - name: Check out code
         uses: actions/checkout@v7
 
+      - name: Clean up stale FUSE mounts
+        uses: ./.github/actions/cleanup-stale-fuse-mounts
+
       - name: Set up Go
         uses: actions/setup-go@v7
         with:
           go-version-file: 'go.mod'
+          cache: ${{ runner.environment == 'github-hosted' }}
         id: go
 
       - name: Configure Docker Hub mirror
@@ -323,7 +348,7 @@ jobs:
           echo "=== System Information ==="
           uname -a
           free -h
-          df -h
+          timeout 15 df -h || true
           docker info
           echo "=== Starting Doris Iceberg Catalog Tests ==="
 
@@ -373,10 +398,14 @@ jobs:
       - name: Check out code
         uses: actions/checkout@v7
 
+      - name: Clean up stale FUSE mounts
+        uses: ./.github/actions/cleanup-stale-fuse-mounts
+
       - name: Set up Go
         uses: actions/setup-go@v7
         with:
           go-version-file: 'go.mod'
+          cache: ${{ runner.environment == 'github-hosted' }}
         id: go
 
       - name: Configure Docker Hub mirror
@@ -408,7 +437,7 @@ jobs:
           echo "=== System Information ==="
           uname -a
           free -h
-          df -h
+          timeout 15 df -h || true
           docker info
           echo "=== Starting ClickHouse Iceberg Catalog Tests ==="
 
@@ -460,10 +489,14 @@ jobs:
       - name: Check out code
         uses: actions/checkout@v7
 
+      - name: Clean up stale FUSE mounts
+        uses: ./.github/actions/cleanup-stale-fuse-mounts
+
       - name: Set up Go
         uses: actions/setup-go@v7
         with:
           go-version-file: 'go.mod'
+          cache: ${{ runner.environment == 'github-hosted' }}
         id: go
 
       - name: Configure Docker Hub mirror
@@ -496,7 +529,7 @@ jobs:
           echo "=== System Information ==="
           uname -a
           free -h
-          df -h
+          timeout 15 df -h || true
           docker info
           echo "=== Starting OLake Iceberg Catalog Tests ==="
 
@@ -559,10 +592,14 @@ jobs:
       - name: Check out code
         uses: actions/checkout@v7
 
+      - name: Clean up stale FUSE mounts
+        uses: ./.github/actions/cleanup-stale-fuse-mounts
+
       - name: Set up Go
         uses: actions/setup-go@v7
         with:
           go-version-file: 'go.mod'
+          cache: ${{ runner.environment == 'github-hosted' }}
         id: go
 
       - name: Run go mod tidy
@@ -590,7 +627,7 @@ jobs:
           echo "=== System Information ==="
           uname -a
           free -h
-          df -h
+          timeout 15 df -h || true
           echo "=== Starting Polaris Tests ==="
 
           go test -v -timeout 20m ./test/s3tables/polaris 2>&1 | tee test/s3tables/polaris/test-output.log || {
@@ -627,10 +664,14 @@ jobs:
       - name: Check out code
         uses: actions/checkout@v7
 
+      - name: Clean up stale FUSE mounts
+        uses: ./.github/actions/cleanup-stale-fuse-mounts
+
       - name: Set up Go
         uses: actions/setup-go@v7
         with:
           go-version-file: 'go.mod'
+          cache: ${{ runner.environment == 'github-hosted' }}
         id: go
 
       - name: Configure Docker Hub mirror
@@ -659,7 +700,7 @@ jobs:
           echo "=== System Information ==="
           uname -a
           free -h
-          df -h
+          timeout 15 df -h || true
           echo "=== Starting Spark Iceberg Catalog Tests ==="
           
           # Run Spark + Iceberg catalog integration tests
@@ -697,10 +738,14 @@ jobs:
       - name: Check out code
         uses: actions/checkout@v7
 
+      - name: Clean up stale FUSE mounts
+        uses: ./.github/actions/cleanup-stale-fuse-mounts
+
       - name: Set up Go
         uses: actions/setup-go@v7
         with:
           go-version-file: 'go.mod'
+          cache: ${{ runner.environment == 'github-hosted' }}
         id: go
 
       - name: Configure Docker Hub mirror
@@ -730,7 +775,7 @@ jobs:
           echo "=== System Information ==="
           uname -a
           free -h
-          df -h
+          timeout 15 df -h || true
           echo "=== Starting RisingWave Catalog Tests ==="
           
           # Run RisingWave catalog integration tests
@@ -768,10 +813,14 @@ jobs:
       - name: Check out code
         uses: actions/checkout@v7
 
+      - name: Clean up stale FUSE mounts
+        uses: ./.github/actions/cleanup-stale-fuse-mounts
+
       - name: Set up Go
         uses: actions/setup-go@v7
         with:
           go-version-file: 'go.mod'
+          cache: ${{ runner.environment == 'github-hosted' }}
         id: go
 
       - name: Configure Docker Hub mirror
@@ -785,6 +834,7 @@ jobs:
 
       - name: Restore python:3 image cache
         id: python-image
+        if: runner.environment == 'github-hosted'
         uses: actions/cache@v6
         with:
           path: /tmp/python3-image.tar
@@ -824,7 +874,7 @@ jobs:
           echo "=== System Information ==="
           uname -a
           free -h
-          df -h
+          timeout 15 df -h || true
           echo "=== Starting STS Integration Tests ==="
           
           # Run STS integration tests
@@ -862,10 +912,14 @@ jobs:
       - name: Check out code
         uses: actions/checkout@v7
 
+      - name: Clean up stale FUSE mounts
+        uses: ./.github/actions/cleanup-stale-fuse-mounts
+
       - name: Set up Go
         uses: actions/setup-go@v7
         with:
           go-version-file: 'go.mod'
+          cache: ${{ runner.environment == 'github-hosted' }}
         id: go
 
       - name: Run go mod tidy
@@ -884,7 +938,7 @@ jobs:
           echo "=== System Information ==="
           uname -a
           free -h
-          df -h
+          timeout 15 df -h || true
           echo "=== Starting Lakekeeper Integration Tests ==="
           
           # Run Lakekeeper integration tests
@@ -922,10 +976,14 @@ jobs:
       - name: Check out code
         uses: actions/checkout@v7
 
+      - name: Clean up stale FUSE mounts
+        uses: ./.github/actions/cleanup-stale-fuse-mounts
+
       - name: Set up Go
         uses: actions/setup-go@v7
         with:
           go-version-file: 'go.mod'
+          cache: ${{ runner.environment == 'github-hosted' }}
         id: go
 
       - name: Configure Docker Hub mirror
@@ -955,7 +1013,7 @@ jobs:
           echo "=== System Information ==="
           uname -a
           free -h
-          df -h
+          timeout 15 df -h || true
           docker info
           echo "=== Starting Unity Catalog Tests ==="
 
@@ -995,10 +1053,14 @@ jobs:
       - name: Check out code
         uses: actions/checkout@v7
 
+      - name: Clean up stale FUSE mounts
+        uses: ./.github/actions/cleanup-stale-fuse-mounts
+
       - name: Set up Go
         uses: actions/setup-go@v7
         with:
           go-version-file: 'go.mod'
+          cache: ${{ runner.environment == 'github-hosted' }}
         id: go
 
       - name: Configure Docker Hub mirror
@@ -1027,7 +1089,7 @@ jobs:
           echo "=== System Information ==="
           uname -a
           free -h
-          df -h
+          timeout 15 df -h || true
           docker info
           echo "=== Starting LanceDB Namespace Tests ==="
 
@@ -1069,6 +1131,7 @@ jobs:
         uses: actions/setup-go@v7
         with:
           go-version-file: 'go.mod'
+          cache: ${{ runner.environment == 'github-hosted' }}
         id: go
 
       - name: Configure Docker Hub mirror
@@ -1137,10 +1200,14 @@ jobs:
           # so do not leave a token in the checkout for it to pick up.
           persist-credentials: false
 
+      - name: Clean up stale FUSE mounts
+        uses: ./.github/actions/cleanup-stale-fuse-mounts
+
       - name: Set up Go
         uses: actions/setup-go@v7
         with:
           go-version-file: 'go.mod'
+          cache: ${{ runner.environment == 'github-hosted' }}
         id: go
 
       - name: Configure Docker Hub mirror
@@ -1170,7 +1237,7 @@ jobs:
           echo "=== System Information ==="
           uname -a
           free -h
-          df -h
+          timeout 15 df -h || true
           docker info
           echo "=== Starting DuckDB Lance Tests ==="
 
@@ -1212,10 +1279,14 @@ jobs:
           # so do not leave a token in the checkout for it to pick up.
           persist-credentials: false
 
+      - name: Clean up stale FUSE mounts
+        uses: ./.github/actions/cleanup-stale-fuse-mounts
+
       - name: Set up Go
         uses: actions/setup-go@v7
         with:
           go-version-file: 'go.mod'
+          cache: ${{ runner.environment == 'github-hosted' }}
         id: go
 
       - name: Configure Docker Hub mirror
@@ -1244,7 +1315,7 @@ jobs:
           echo "=== System Information ==="
           uname -a
           free -h
-          df -h
+          timeout 15 df -h || true
           docker info
           echo "=== Starting Spark Lance Namespace Tests ==="
 
@@ -1286,6 +1357,7 @@ jobs:
         uses: actions/setup-go@v7
         with:
           go-version-file: 'go.mod'
+          cache: ${{ runner.environment == 'github-hosted' }}
         id: go
 
       - name: Verify S3 Tables Package Builds
@@ -1351,6 +1423,7 @@ jobs:
         uses: actions/setup-go@v7
         with:
           go-version-file: 'go.mod'
+          cache: ${{ runner.environment == 'github-hosted' }}
         id: go
 
       - name: Check Go Format
@@ -1402,6 +1475,7 @@ jobs:
         uses: actions/setup-go@v7
         with:
           go-version-file: 'go.mod'
+          cache: ${{ runner.environment == 'github-hosted' }}
         id: go
 
       - name: Run Go Vet

.github/workflows/s3tests.yml

@@ -25,8 +25,8 @@ on:
       - '.github/workflows/s3tests.yml'
 
 concurrency:
-  group: ${{ github.head_ref }}/s3tests
-  cancel-in-progress: true
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || (github.event_name == 'push' && github.ref) || github.run_id }}
+  cancel-in-progress: ${{ github.event_name == 'pull_request' || (github.event_name == 'push' && !startsWith(github.ref, 'refs/tags/')) }}
 
 permissions:
   contents: read
@@ -44,6 +44,7 @@ jobs:
         uses: actions/setup-go@v7
         with:
           go-version-file: 'go.mod'
+          cache: ${{ runner.environment == 'github-hosted' }}
         id: go
 
       - name: Set up Python
@@ -57,9 +58,9 @@ jobs:
           cd s3-tests
           sudo apt-get update -qq
           sudo apt-get install -y -qq libxml2-dev libxslt1-dev zlib1g-dev
-          pip install -r requirements.txt
-          pip install tox
-          pip install -e .
+          python3 -m pip install -r requirements.txt
+          python3 -m pip install tox
+          python3 -m pip install -e .
 
       - name: Fix S3 tests bucket creation conflicts
         run: |
@@ -396,6 +397,7 @@ jobs:
         uses: actions/setup-go@v7
         with:
           go-version-file: 'go.mod'
+          cache: ${{ runner.environment == 'github-hosted' }}
         id: go
 
       - name: Set up Python
@@ -409,9 +411,9 @@ jobs:
           cd s3-tests
           sudo apt-get update -qq
           sudo apt-get install -y -qq libxml2-dev libxslt1-dev zlib1g-dev
-          pip install -r requirements.txt
-          pip install tox
-          pip install -e .
+          python3 -m pip install -r requirements.txt
+          python3 -m pip install tox
+          python3 -m pip install -e .
 
       - name: Fix S3 tests bucket creation conflicts
         run: |
@@ -566,6 +568,7 @@ jobs:
         uses: actions/setup-go@v7
         with:
           go-version-file: 'go.mod'
+          cache: ${{ runner.environment == 'github-hosted' }}
         id: go
 
       - name: Set up Python
@@ -579,9 +582,9 @@ jobs:
           cd s3-tests
           sudo apt-get update -qq
           sudo apt-get install -y -qq libxml2-dev libxslt1-dev zlib1g-dev
-          pip install -r requirements.txt
-          pip install tox
-          pip install -e .
+          python3 -m pip install -r requirements.txt
+          python3 -m pip install tox
+          python3 -m pip install -e .
 
       - name: Run S3 CORS tests
         timeout-minutes: 10
@@ -691,6 +694,7 @@ jobs:
         uses: actions/setup-go@v7
         with:
           go-version-file: 'go.mod'
+          cache: ${{ runner.environment == 'github-hosted' }}
         id: go
 
       - name: Run SeaweedFS Custom S3 Copy tests
@@ -793,6 +797,7 @@ jobs:
         uses: actions/setup-go@v7
         with:
           go-version-file: 'go.mod'
+          cache: ${{ runner.environment == 'github-hosted' }}
         id: go
 
       - name: Set up Python
@@ -806,9 +811,9 @@ jobs:
           cd s3-tests
           sudo apt-get update -qq
           sudo apt-get install -y -qq libxml2-dev libxslt1-dev zlib1g-dev
-          pip install -r requirements.txt
-          pip install tox
-          pip install -e .
+          python3 -m pip install -r requirements.txt
+          python3 -m pip install tox
+          python3 -m pip install -e .
 
       - name: Run Ceph S3 tests with SQL store
         timeout-minutes: 15

.github/workflows/samba-integration.yml

@@ -40,15 +40,23 @@ jobs:
       uses: actions/setup-go@v7
       with:
         go-version-file: 'go.mod'
+        cache: ${{ runner.environment == 'github-hosted' }}
 
     - name: Start local Docker registry
-      run: docker run -d --restart=always -p 5000:5000 --name registry registry:2
+      run: |
+        if ! docker start registry 2>/dev/null; then
+          docker rm -f registry >/dev/null 2>&1 || true
+          docker run -d --restart=always -p 5000:5000 --name registry registry:2
+        fi
 
     - name: Set up Docker Buildx
       uses: docker/setup-buildx-action@v4
       with:
         driver-opts: network=host
 
+    - name: Set up Docker Compose
+      uses: docker/setup-compose-action@v2
+
     - name: Build weed race binary
       run: |
         cd docker

.github/workflows/sftp-tests.yml

@@ -17,8 +17,8 @@ on:
       - '.github/workflows/sftp-tests.yml'
 
 concurrency:
-  group: ${{ github.head_ref }}/sftp-tests
-  cancel-in-progress: true
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || (github.event_name == 'push' && github.ref) || github.run_id }}
+  cancel-in-progress: ${{ github.event_name == 'pull_request' || (github.event_name == 'push' && !startsWith(github.ref, 'refs/tags/')) }}
 
 permissions:
   contents: read
@@ -40,6 +40,7 @@ jobs:
       uses: actions/setup-go@v7
       with:
         go-version-file: 'go.mod'
+        cache: ${{ runner.environment == 'github-hosted' }}
         
     - name: Install dependencies
       run: |

.github/workflows/spark-integration-tests.yml

@@ -15,6 +15,10 @@ on:
       - '.github/workflows/spark-integration-tests.yml'
   workflow_dispatch:
 
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || (github.event_name == 'push' && github.ref) || github.run_id }}
+  cancel-in-progress: ${{ github.event_name == 'pull_request' || (github.event_name == 'push' && !startsWith(github.ref, 'refs/tags/')) }}
+
 permissions:
   contents: read
   checks: write
@@ -38,12 +42,13 @@ jobs:
       with:
         java-version: '11'
         distribution: 'temurin'
-        cache: maven
+        cache: ${{ runner.environment == 'github-hosted' && 'maven' || '' }}
 
     - name: Set up Go
       uses: actions/setup-go@v7
       with:
         go-version-file: 'go.mod'
+        cache: ${{ runner.environment == 'github-hosted' }}
 
     - name: Build SeaweedFS binary
       run: |

.github/workflows/star_history.yml

@@ -10,12 +10,11 @@ permissions:
   contents: write
 
 concurrency:
-  # Only one chart regeneration per branch at a time; a newer run on the same
-  # branch cancels an in-flight one so overlapping runs never conflict on
-  # note/star_history.svg during rebase. Scoped by ref so a manual run on
-  # another branch can't cancel the daily master update.
+  # Only one chart regeneration per branch at a time so overlapping runs never
+  # conflict on note/star_history.svg during rebase. Scoped by ref so a manual
+  # run on another branch can't cancel the daily master update.
   group: star-history-${{ github.ref }}
-  cancel-in-progress: true
+  cancel-in-progress: ${{ github.event_name == 'pull_request' || (github.event_name == 'push' && !startsWith(github.ref, 'refs/tags/')) }}
 
 jobs:
   render:
@@ -33,7 +32,7 @@ jobs:
         uses: actions/setup-python@v7
         with:
           python-version: "3.x"
-          cache: pip
+          cache: ${{ runner.environment == 'github-hosted' && 'pip' || '' }}
           cache-dependency-path: .github/scripts/star_history.py
 
       - name: Install matplotlib

.github/workflows/telemetry-integration.yml

@@ -14,6 +14,10 @@ on:
       - 'weed/telemetry/**'
       - '.github/workflows/telemetry-integration.yml'
 
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || (github.event_name == 'push' && github.ref) || github.run_id }}
+  cancel-in-progress: ${{ github.event_name == 'pull_request' || (github.event_name == 'push' && !startsWith(github.ref, 'refs/tags/')) }}
+
 permissions:
   contents: read
 
@@ -30,6 +34,7 @@ jobs:
       uses: actions/setup-go@v7
       with:
         go-version-file: 'go.mod'
+        cache: ${{ runner.environment == 'github-hosted' }}
 
     - name: Build telemetry server
       run: cd telemetry/server && go build -o telemetry-server .

.github/workflows/terraform_ci.yml

@@ -8,6 +8,10 @@ on:
     branches: [ master ]
     paths: ['terraform/**', '.github/workflows/terraform_ci.yml']
 
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || (github.event_name == 'push' && github.ref) || github.run_id }}
+  cancel-in-progress: ${{ github.event_name == 'pull_request' || (github.event_name == 'push' && !startsWith(github.ref, 'refs/tags/')) }}
+
 permissions:
   contents: read
 
@@ -62,6 +66,7 @@ jobs:
         uses: actions/setup-go@v7
         with:
           go-version-file: go.mod
+          cache: ${{ runner.environment == 'github-hosted' }}
 
       - name: Build weed
         run: go build -o "$RUNNER_TEMP/weed" ./weed

.github/workflows/test-s3-over-https-using-awscli.yml

@@ -27,6 +27,10 @@ defaults:
   run:
     working-directory: weed
 
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || (github.event_name == 'push' && github.ref) || github.run_id }}
+  cancel-in-progress: ${{ github.event_name == 'pull_request' || (github.event_name == 'push' && !startsWith(github.ref, 'refs/tags/')) }}
+
 jobs:
   awscli-tests:
     runs-on: ubuntu-latest
@@ -37,6 +41,7 @@ jobs:
       - uses: actions/setup-go@v7
         with:
           go-version: ^1.26
+          cache: ${{ runner.environment == 'github-hosted' }}
 
       - name: Build SeaweedFS
         run: |

.github/workflows/tls-rotation-tests.yml

@@ -18,6 +18,10 @@ on:
       - 'go.sum'
       - '.github/workflows/tls-rotation-tests.yml'
 
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || (github.event_name == 'push' && github.ref) || github.run_id }}
+  cancel-in-progress: ${{ github.event_name == 'pull_request' || (github.event_name == 'push' && !startsWith(github.ref, 'refs/tags/')) }}
+
 permissions:
   contents: read
 
@@ -30,6 +34,7 @@ jobs:
     - name: Set up Go 1.x
       uses: actions/setup-go@v7
       with:
+        cache: ${{ runner.environment == 'github-hosted' }}
         go-version: ^1.26
       id: go
 

.github/workflows/tus-tests.yml

@@ -11,8 +11,8 @@ on:
       - '.github/workflows/tus-tests.yml'
 
 concurrency:
-  group: ${{ github.head_ref || github.ref }}/tus-tests
-  cancel-in-progress: true
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || (github.event_name == 'push' && github.ref) || github.run_id }}
+  cancel-in-progress: ${{ github.event_name == 'pull_request' || (github.event_name == 'push' && !startsWith(github.ref, 'refs/tags/')) }}
 
 permissions:
   contents: read
@@ -31,10 +31,14 @@ jobs:
       - name: Check out code
         uses: actions/checkout@v7
 
+      - name: Clean up stale FUSE mounts
+        uses: ./.github/actions/cleanup-stale-fuse-mounts
+
       - name: Set up Go
         uses: actions/setup-go@v7
         with:
           go-version-file: 'go.mod'
+          cache: ${{ runner.environment == 'github-hosted' }}
         id: go
 
       - name: Install SeaweedFS
@@ -49,7 +53,7 @@ jobs:
           echo "=== System Information ==="
           uname -a
           free -h
-          df -h
+          timeout 15 df -h || true
           echo "=== Starting TUS Tests ==="
           
           # Run tests with automatic server management

.github/workflows/vacuum-integration-tests.yml

@@ -18,6 +18,10 @@ on:
       - 'go.sum'
       - '.github/workflows/vacuum-integration-tests.yml'
 
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || (github.event_name == 'push' && github.ref) || github.run_id }}
+  cancel-in-progress: ${{ github.event_name == 'pull_request' || (github.event_name == 'push' && !startsWith(github.ref, 'refs/tags/')) }}
+
 permissions:
   contents: read
 
@@ -30,6 +34,7 @@ jobs:
     - name: Set up Go 1.x
       uses: actions/setup-go@v7
       with:
+        cache: ${{ runner.environment == 'github-hosted' }}
         go-version: ^1.26
       id: go
 

.github/workflows/volume-server-integration-tests.yml

@@ -21,8 +21,8 @@ on:
       - '.github/workflows/volume-server-integration-tests.yml'
 
 concurrency:
-  group: ${{ github.head_ref || github.ref }}/volume-server-integration-tests
-  cancel-in-progress: true
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || (github.event_name == 'push' && github.ref) || github.run_id }}
+  cancel-in-progress: ${{ github.event_name == 'pull_request' || (github.event_name == 'push' && !startsWith(github.ref, 'refs/tags/')) }}
 
 permissions:
   contents: read
@@ -33,10 +33,44 @@ env:
   SHARD_COUNT: 3
 
 jobs:
+  build-weed:
+    name: Build SeaweedFS Binary
+    runs-on: ubuntu-22.04
+    timeout-minutes: 15
+
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v7
+
+      - name: Set up Go
+        uses: actions/setup-go@v7
+        with:
+          go-version-file: 'go.mod'
+          cache: ${{ runner.environment == 'github-hosted' }}
+
+      - name: Build SeaweedFS binary
+        run: |
+          cd weed
+          go build -o weed .
+          chmod +x weed
+          ./weed version
+
+      - name: Package binary
+        run: tar cf weed-binary.tar weed/weed
+
+      - name: Upload binary
+        uses: actions/upload-artifact@v7
+        with:
+          name: weed-binary
+          if-no-files-found: error
+          retention-days: 1
+          path: weed-binary.tar
+
   volume-server-integration-tests:
     name: Volume Server Integration Tests (${{ matrix.test-type }} - Shard ${{ matrix.shard }})
     runs-on: ubuntu-22.04
     timeout-minutes: 45
+    needs: [build-weed]
     strategy:
       fail-fast: false
       matrix:
@@ -51,13 +85,18 @@ jobs:
         uses: actions/setup-go@v7
         with:
           go-version-file: 'go.mod'
+          cache: ${{ runner.environment == 'github-hosted' }}
 
-      - name: Build SeaweedFS binary
+      - name: Download binary
+        uses: actions/download-artifact@v7
+        with:
+          name: weed-binary
+
+      - name: Restore binary
         run: |
-          cd weed
-          go build -o weed .
-          chmod +x weed
-          ./weed version
+          tar xf weed-binary.tar
+          chmod +x weed/weed
+          ./weed/weed version
 
       # Dealing the listed tests out one by one keeps the shards even. Bucketing
       # them by first letter did not: names cluster, so ^Test[I-S] drew 50 of

.gitignore

@@ -144,3 +144,4 @@ weed_bin
 telemetry/server/telemetry-server
 .aider*
 /seaweed-volume/docs
+__pycache__/

docker/Dockerfile.rocksdb_dev_env

@@ -3,7 +3,7 @@ FROM golang:1.26 AS builder
 RUN apt-get update
 RUN apt-get install -y build-essential libsnappy-dev zlib1g-dev libbz2-dev libgflags-dev liblz4-dev libzstd-dev
 
-ARG ROCKSDB_VERSION=v10.10.1
+ARG ROCKSDB_VERSION=v11.1.2
 ENV ROCKSDB_VERSION=${ROCKSDB_VERSION}
 
 # build RocksDB

docker/Dockerfile.rocksdb_large

@@ -3,7 +3,7 @@ FROM golang:1.26 AS builder
 RUN apt-get update
 RUN apt-get install -y build-essential libsnappy-dev zlib1g-dev libbz2-dev libgflags-dev liblz4-dev libzstd-dev
 
-ARG ROCKSDB_VERSION=v10.10.1
+ARG ROCKSDB_VERSION=v11.1.2
 ENV ROCKSDB_VERSION=${ROCKSDB_VERSION}
 
 # build RocksDB

go.mod

@@ -362,7 +362,7 @@ require (
 	github.com/flynn/noise v1.1.0 // indirect
 	github.com/gabriel-vasile/mimetype v1.4.13 // indirect
 	github.com/geoffgarside/ber v1.2.0 // indirect
-	github.com/go-chi/chi/v5 v5.3.1 // indirect
+	github.com/go-chi/chi/v5 v5.4.0 // indirect
 	github.com/go-darwin/apfs v0.0.0-20211011131704-f84b94dbf348 // indirect
 	github.com/go-jose/go-jose/v4 v4.1.5 // indirect
 	github.com/go-logr/logr v1.4.4 // indirect

go.sum

@@ -483,8 +483,8 @@ github.com/gin-gonic/gin v1.11.0 h1:OW/6PLjyusp2PPXtyxKHU0RbX6I/l28FTdDlae5ueWk=
 github.com/gin-gonic/gin v1.11.0/go.mod h1:+iq/FyxlGzII0KHiBGjuNn4UNENUlKbGlNmc+W50Dls=
 github.com/go-asn1-ber/asn1-ber v1.5.8 h1:H9AZkK22UOmfX8J84ubyaZxKJZ3FMHVwn8swoMML7iQ=
 github.com/go-asn1-ber/asn1-ber v1.5.8/go.mod h1:hEBeB/ic+5LoWskz+yKT7vGhhPYkProFKoKdwZRWMe0=
-github.com/go-chi/chi/v5 v5.3.1 h1:3j4HZLGZQ3JpMCrPJF/Jl3mYJfWLKBfNJ6quurUGCf8=
-github.com/go-chi/chi/v5 v5.3.1/go.mod h1:R+tYY2hNuVUUjxoPtqUdgBqevM9s9njzkTLutVsOCto=
+github.com/go-chi/chi/v5 v5.4.0 h1:+ufzvFTGKyin26Or5DLrlIDq4ygIDBVcj7oVNeeaYFE=
+github.com/go-chi/chi/v5 v5.4.0/go.mod h1:d1IG7GtKPSjPyBP365PH8sAGVHk3qfB/IhjFNRH1wWM=
 github.com/go-darwin/apfs v0.0.0-20211011131704-f84b94dbf348 h1:JnrjqG5iR07/8k7NqrLNilRsl3s1EPRQEGvbPyOce68=
 github.com/go-darwin/apfs v0.0.0-20211011131704-f84b94dbf348/go.mod h1:Czxo/d1g948LtrALAZdL04TL/HnkopquAjxYUuI02bo=
 github.com/go-errors/errors v1.5.1 h1:ZwEMSLRCapFLflTpT7NKaAc7ukJ8ZPEjzlxt8rPN8bk=

k8s/charts/seaweedfs/templates/shared/post-install-bucket-hook.yaml

@@ -172,7 +172,8 @@ spec:
             {{- $bucketName := .name }}
             {{- $bucketLock := or .lock .objectLock .withLock }}
             bucket_list=$(/bin/echo 's3.bucket.list' | /usr/bin/weed shell) || { echo "Error listing s3 buckets"; exit 1; }
-            if echo "$bucket_list" | awk '{print $1}' | grep -Fxq "{{ $bucketName }}"; then
+            {{- /* One awk: grep -q exits early, awk takes SIGPIPE, pipefail fails the test. */}}
+            if printf '%s\n' "$bucket_list" | awk -v bucket="{{ $bucketName }}" '$1 "" == bucket { found = 1 } END { exit !found }'; then
               echo "Bucket '{{ $bucketName }}' already exists, skipping creation."
             else
               echo "Creating bucket '{{ $bucketName }}'..."

other/java/client/src/main/proto/filer.proto

@@ -771,7 +771,8 @@ message FilerConf {
         string replication = 3;
         string ttl = 4;
         string disk_type = 5;
-        bool fsync = 6;
+        // unset inherits from the enclosing path rule, set overrides it
+        optional bool fsync = 6;
         uint32 volume_growth_count = 7;
         bool read_only = 8;
         string data_center = 9;

seaweed-volume/Cargo.toml

@@ -157,6 +157,8 @@ windows-sys = { version = "0.61", features = ["Win32_Storage_FileSystem"] }
 
 [dev-dependencies]
 tempfile = "3"
+# Paused-time tests drive the write-queue batch window deterministically.
+tokio = { version = "1", features = ["full", "test-util"] }
 # Already a transitive dependency of aws-sdk-s3 at a single locked version;
 # needed directly only for the canned HttpClient in remote_storage::s3 tests.
 aws-smithy-runtime-api = "1"

seaweed-volume/src/config.rs

@@ -233,6 +233,25 @@ pub struct Cli {
     #[arg(long = "volume.allowUntrustedRemoteEndpoints", default_value_t = false)]
     pub allow_untrusted_remote_endpoints: bool,
 
+    /// Minimum bytes an fsync batch gathers before a drained queue flushes
+    /// without waiting.
+    #[arg(long = "fsyncMinBatchBytes", default_value_t = 1024 * 1024)]
+    pub fsync_min_batch_bytes: u64,
+
+    /// Minimum requests an fsync batch gathers before a drained queue flushes
+    /// without waiting.
+    #[arg(long = "fsyncMinBatchRequests", default_value_t = 128)]
+    pub fsync_min_batch_requests: usize,
+
+    /// How long a below-minimum fsync batch waits for more requests once the
+    /// queue drains; 0 flushes immediately.
+    #[arg(
+        long = "fsyncBatchWait",
+        default_value = "500us",
+        value_parser = parse_go_duration
+    )]
+    pub fsync_batch_wait: std::time::Duration,
+
     /// A file of command line options, each line in optionName=optionValue format.
     #[arg(long = "options", default_value = "")]
     pub options: String,
@@ -308,6 +327,14 @@ pub struct VolumeServerConfig {
     /// If true, FetchAndWriteNeedle skips remote S3 endpoint validation
     /// (allows loopback / link-local / metadata hosts).
     pub allow_untrusted_remote_endpoints: bool,
+    /// Minimum bytes an fsync batch gathers before a drained queue flushes
+    /// without waiting.
+    pub fsync_min_batch_bytes: u64,
+    /// Minimum requests an fsync batch gathers before a drained queue flushes
+    /// without waiting.
+    pub fsync_min_batch_requests: usize,
+    /// How long a below-minimum fsync batch waits for stragglers.
+    pub fsync_batch_wait: std::time::Duration,
 }
 
 pub use crate::storage::needle_map::NeedleMapKind;
@@ -938,6 +965,9 @@ fn resolve_config_with_env(cli: Cli, env: EnvLookup<'_>) -> VolumeServerConfig {
         concurrent_download_limit: cli.concurrent_download_limit_mb as i64 * 1024 * 1024,
         inflight_upload_data_timeout: cli.inflight_upload_data_timeout,
         inflight_download_data_timeout: cli.inflight_download_data_timeout,
+        fsync_min_batch_bytes: cli.fsync_min_batch_bytes,
+        fsync_min_batch_requests: cli.fsync_min_batch_requests,
+        fsync_batch_wait: cli.fsync_batch_wait,
         has_slow_read: cli.has_slow_read,
         read_buffer_size_mb: cli.read_buffer_size_mb,
         ldb_timeout: cli.ldb_timeout,

seaweed-volume/src/main.rs

@@ -23,7 +23,7 @@ use seaweed_volume::server::request_id::GrpcRequestIdLayer;
 use seaweed_volume::server::volume_server::{
     RuntimeMetricsConfig, VolumeServerState, build_metrics_router,
 };
-use seaweed_volume::server::write_queue::WriteQueue;
+use seaweed_volume::server::write_queue::{self, WriteQueue};
 use seaweed_volume::storage::store::Store;
 use seaweed_volume::storage::types::DiskType;
 
@@ -436,7 +436,17 @@ async fn run(
     // The write queue is always on: durable (fsync) uploads go through it so
     // a batch shares one .dat/.idx sync, the way Go's per-volume async writer
     // group-commits. Non-fsync writes stay on the synchronous path.
-    let wq = WriteQueue::new(state.clone(), 128);
+    let min_bytes = usize::try_from(config.fsync_min_batch_bytes)
+        .map_err(|_| "fsyncMinBatchBytes exceeds this platform's maximum usize value")?;
+    let wq = WriteQueue::new(
+        state.clone(),
+        128,
+        write_queue::SyncBatchOption {
+            min_bytes,
+            min_requests: config.fsync_min_batch_requests,
+            wait: config.fsync_batch_wait,
+        },
+    );
     let _ = state.write_queue.set(wq);
 
     // Set initial metric gauges for concurrent limits and max volumes

seaweed-volume/src/server/grpc_client.rs

@@ -393,6 +393,9 @@ mod tests {
             fix_jpg_orientation: false,
             read_mode: ReadMode::Local,
             allow_untrusted_remote_endpoints: false,
+            fsync_min_batch_bytes: 1024 * 1024,
+            fsync_min_batch_requests: 128,
+            fsync_batch_wait: std::time::Duration::from_micros(500),
             cpu_profile: String::new(),
             mem_profile: String::new(),
             compaction_byte_per_second: 0,

seaweed-volume/src/server/handlers.rs

@@ -207,9 +207,15 @@ fn read_needle_for_get(
     if request.try_meta_first(plan.size()) {
         plan.read_meta(&mut n)?;
         if n.cookie != cookie {
+            // The meta read answered the request.
+            plan.note_meta_only_success();
             return Ok(None);
         }
         if request.served_from_source(&n) {
+            if request.is_head {
+                // HEAD ends here; no payload read will record the success.
+                plan.note_meta_only_success();
+            }
             let info = plan.into_stream_info(&n);
             return Ok(Some((n, Some(info))));
         }
@@ -237,6 +243,9 @@ struct StreamingBody {
     hold_read_lock_for_stream: bool,
     _held_read_lease: Option<crate::storage::volume::DataFileReadLease>,
     io_unavailable: Arc<crate::storage::volume::IoUnavailable>,
+    /// Chunk read failures feed the volume's io-error streak, and finishing
+    /// the stream clears it — Go readNeedleDataInto parity.
+    io_errors: Arc<crate::storage::io_error::IoErrorTracker>,
     /// Pending chunk read; it hands `buf` back with the result.
     pending: Option<tokio::task::JoinHandle<(bytes::BytesMut, std::io::Result<bytes::Bytes>)>>,
     /// Chunk buffer, reclaimed once the previous frame has been written out.
@@ -298,12 +307,18 @@ impl http_body::Body for StreamingBody {
                                             ),
                                         )));
                                     }
+                                    // Hyper may not poll again once Content-Length is
+                                    // satisfied, so the read's success is recorded here.
+                                    self.io_errors.check_read_write_error(None);
                                 }
                                 return std::task::Poll::Ready(Some(Ok(http_body::Frame::data(
                                     chunk,
                                 ))));
                             }
-                            Ok(Err(e)) => return std::task::Poll::Ready(Some(Err(e))),
+                            Ok(Err(e)) => {
+                                self.io_errors.check_read_write_error(Some(&e));
+                                return std::task::Poll::Ready(Some(Err(e)));
+                            }
                             Err(e) => {
                                 return std::task::Poll::Ready(Some(Err(std::io::Error::other(e))));
                             }
@@ -314,6 +329,7 @@ impl http_body::Body for StreamingBody {
 
             let total = self.data_size as usize;
             if self.pos >= total {
+                self.io_errors.check_read_write_error(None);
                 return std::task::Poll::Ready(None);
             }
 
@@ -1105,6 +1121,10 @@ async fn get_or_head_handler_inner(
 
     let (etag, last_modified_str) = etag_and_last_modified(&n);
     if let Some(resp) = not_modified_response(&n, &headers, &etag, &last_modified_str) {
+        // A request that ends here was answered by the meta read alone.
+        if let Some(io_errors) = strategy.io_errors() {
+            io_errors.check_read_write_error(None);
+        }
         return resp;
     }
 
@@ -1198,6 +1218,19 @@ struct ReadPlan {
     strategy: ReadStrategy,
 }
 
+impl ReadStrategy {
+    /// The tracker a terminal meta read reports success to, when the reply
+    /// is served from the data file.
+    fn io_errors(&self) -> Option<&Arc<crate::storage::io_error::IoErrorTracker>> {
+        match self {
+            ReadStrategy::Stream(info)
+            | ReadStrategy::HeadFromMeta(info)
+            | ReadStrategy::RangeFromSource(info) => Some(&info.io_errors),
+            ReadStrategy::Buffered => None,
+        }
+    }
+}
+
 /// The 401 reply for a bad read JWT; Go checks it before NewVolumeId, so a
 /// bad path with JWT enabled is a 401, not a 400.
 fn reject_read_jwt(
@@ -1486,41 +1519,78 @@ fn not_modified_response(
     etag: &str,
     last_modified_str: &Option<String>,
 ) -> Option<Response> {
-    // If-Modified-Since first (Go order), then If-None-Match.
+    let not_modified = || {
+        let mut resp = StatusCode::NOT_MODIFIED.into_response();
+        resp.headers_mut()
+            .insert(header::ETAG, etag.parse().unwrap());
+        if let Some(lm) = last_modified_str {
+            resp.headers_mut()
+                .insert(header::LAST_MODIFIED, lm.parse().unwrap());
+        }
+        resp
+    };
+
+    // If-Modified-Since applies only when there is no If-None-Match (RFC 9110 13.2.2).
+    if headers.contains_key(header::IF_NONE_MATCH) {
+        let inm = headers
+            .get_all(header::IF_NONE_MATCH)
+            .iter()
+            .map(|v| v.to_str().unwrap_or(""))
+            .collect::<Vec<_>>()
+            .join(",");
+        return etag_list_matches(&inm, etag).then(not_modified);
+    }
+
     if n.last_modified > 0
-        && let Some(ims_header) = headers.get(header::IF_MODIFIED_SINCE)
-        && let Ok(ims_str) = ims_header.to_str()
+        && let Some(ims_str) = headers
+            .get(header::IF_MODIFIED_SINCE)
+            .and_then(|v| v.to_str().ok())
     {
         // Parse HTTP date format: "Mon, 02 Jan 2006 15:04:05 GMT"
         if let Ok(ims_time) =
             chrono::NaiveDateTime::parse_from_str(ims_str, "%a, %d %b %Y %H:%M:%S GMT")
             && (n.last_modified as i64) <= ims_time.and_utc().timestamp()
         {
-            let mut resp = StatusCode::NOT_MODIFIED.into_response();
-            if let Some(lm) = last_modified_str {
-                resp.headers_mut()
-                    .insert(header::LAST_MODIFIED, lm.parse().unwrap());
-            }
-            // Go sets ETag after the 304 paths, so a 304 has no ETag.
-            return Some(resp);
+            return Some(not_modified());
         }
     }
-
-    if let Some(if_none_match) = headers.get(header::IF_NONE_MATCH)
-        && let Ok(inm) = if_none_match.to_str()
-        && inm == etag
-    {
-        let mut resp = StatusCode::NOT_MODIFIED.into_response();
-        if let Some(lm) = last_modified_str {
-            resp.headers_mut()
-                .insert(header::LAST_MODIFIED, lm.parse().unwrap());
-        }
-        // Go sets ETag after the 304 paths, so a 304 has no ETag.
-        return Some(resp);
-    }
     None
 }
 
+/// Weak comparison against "*" or a comma-separated entity-tag list (RFC 9110 13.1.2).
+fn etag_list_matches(header: &str, etag: &str) -> bool {
+    if header.trim() == "*" {
+        return true;
+    }
+    let etag = etag.trim_matches('"');
+    split_etag_list(header).into_iter().any(|tag| {
+        let tag = tag.trim();
+        let tag = tag.strip_prefix("W/").unwrap_or(tag);
+        // Empty list elements are ignored, so padding like a trailing
+        // comma cannot match an empty opaque tag.
+        !tag.is_empty() && tag.trim_matches('"') == etag
+    })
+}
+
+/// Splits an entity-tag list on the commas between tags; a comma inside a
+/// quoted opaque tag does not split.
+fn split_etag_list(header: &str) -> Vec<&str> {
+    let mut tags = Vec::new();
+    let (mut start, mut quoted) = (0, false);
+    for (i, b) in header.bytes().enumerate() {
+        match b {
+            b'"' => quoted = !quoted,
+            b',' if !quoted => {
+                tags.push(&header[start..i]);
+                start = i + 1;
+            }
+            _ => {}
+        }
+    }
+    tags.push(&header[start..]);
+    tags
+}
+
 /// The 200 reply headers, and the extension, which falls back to the stored name's.
 fn read_response_headers(
     n: &Needle,
@@ -1722,6 +1792,7 @@ fn stream_response(
         data_file_access_control: info.data_file_access_control,
         hold_read_lock_for_stream: !state.has_slow_read,
         io_unavailable: info.io_unavailable,
+        io_errors: info.io_errors,
         pending: None,
         buf: bytes::BytesMut::new(),
         state: tracking_state,
@@ -2121,9 +2192,21 @@ fn handle_range_request_from_source(
             return Ok(Vec::new());
         }
         let mut buf = vec![0u8; length as usize];
-        info.source
-            .read_exact_at(&mut buf, info.data_file_offset + start as u64)?;
-        Ok(buf)
+        match info
+            .source
+            .read_exact_at(&mut buf, info.data_file_offset + start as u64)
+        {
+            // A finished range read clears the streak a failed meta read
+            // counted, like readNeedleDataInto's terminal report.
+            Ok(()) => {
+                info.io_errors.check_read_write_error(None);
+                Ok(buf)
+            }
+            Err(e) => {
+                info.io_errors.check_read_write_error(Some(&e));
+                Err(e)
+            }
+        }
     };
 
     if ranges.len() == 1 {
@@ -5225,6 +5308,7 @@ mod tests {
             hold_read_lock_for_stream: true,
             _held_read_lease: None,
             io_unavailable: Arc::default(),
+            io_errors: Arc::default(),
             pending: None,
             buf: bytes::BytesMut::new(),
             state: None,
@@ -5597,6 +5681,80 @@ mod tests {
         delete_handler(State(state.clone()), request).await.status()
     }
 
+    #[test]
+    fn test_not_modified_checks_if_none_match_first() {
+        let n = Needle {
+            last_modified: 1700000000,
+            ..Needle::default()
+        };
+        let (etag, last_modified_str) = etag_and_last_modified(&n);
+        let bare = n.etag();
+        let before = "Tue, 14 Nov 2023 22:12:20 GMT".to_string();
+        let after = "Tue, 14 Nov 2023 22:14:20 GMT".to_string();
+
+        let cases: Vec<(Vec<(axum::http::HeaderName, String)>, bool)> = vec![
+            (vec![(header::IF_NONE_MATCH, format!("\"{bare}\""))], true),
+            (vec![(header::IF_NONE_MATCH, format!("W/\"{bare}\""))], true),
+            (
+                vec![(header::IF_NONE_MATCH, format!("\"x\", \"{bare}\""))],
+                true,
+            ),
+            (vec![(header::IF_NONE_MATCH, "*".into())], true),
+            (vec![(header::IF_NONE_MATCH, "\"x\"".into())], false),
+            (
+                vec![
+                    (header::IF_NONE_MATCH, "\"x\"".into()),
+                    (header::IF_NONE_MATCH, format!("\"{bare}\"")),
+                ],
+                true,
+            ),
+            (
+                vec![(header::IF_NONE_MATCH, format!("\"{bare},old\""))],
+                false,
+            ),
+            (
+                vec![(header::IF_NONE_MATCH, format!("\"x\",,\"{bare}\""))],
+                true,
+            ),
+            (
+                vec![
+                    (header::IF_NONE_MATCH, "\"x\"".into()),
+                    (header::IF_MODIFIED_SINCE, after.clone()),
+                ],
+                false,
+            ),
+            (
+                vec![
+                    (header::IF_NONE_MATCH, "".into()),
+                    (header::IF_MODIFIED_SINCE, after.clone()),
+                ],
+                false,
+            ),
+            (vec![(header::IF_MODIFIED_SINCE, after)], true),
+            (vec![(header::IF_MODIFIED_SINCE, before)], false),
+        ];
+        for (pairs, want) in cases {
+            let mut headers = HeaderMap::new();
+            for (name, value) in &pairs {
+                headers.append(name, value.parse().unwrap());
+            }
+            assert_eq!(
+                not_modified_response(&n, &headers, &etag, &last_modified_str).is_some(),
+                want,
+                "{pairs:?}"
+            );
+        }
+
+        let mut headers = HeaderMap::new();
+        headers.insert(
+            header::IF_NONE_MATCH,
+            format!("\"{bare}\"").parse().unwrap(),
+        );
+        let resp = not_modified_response(&n, &headers, &etag, &last_modified_str).unwrap();
+        assert!(resp.headers().contains_key(header::ETAG));
+        assert!(resp.headers().contains_key(header::LAST_MODIFIED));
+    }
+
     /// A GET whose needle read is parked must not hold the store lock: a
     /// writer, here a real append to the same volume, must get through.
     #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
@@ -5771,6 +5929,44 @@ mod tests {
         assert_eq!(status, StatusCode::NOT_FOUND);
     }
 
+    /// A ranged read served from the data file reports its outcome to the
+    /// io-error streak, like readNeedleDataInto: a finished range clears the
+    /// count a failed meta read left.
+    #[test]
+    fn test_range_from_source_reports_to_io_error_streak() {
+        use std::io::Write;
+
+        let tmp = tempfile::TempDir::new().unwrap();
+        let file = std::fs::OpenOptions::new()
+            .read(true)
+            .write(true)
+            .create(true)
+            .open(tmp.path().join("1.dat"))
+            .unwrap();
+        let data: Vec<u8> = (0..256u32).map(|i| i as u8).collect();
+        (&file).write_all(&data).unwrap();
+
+        let io_errors = Arc::new(crate::storage::io_error::IoErrorTracker::default());
+        let eio = crate::storage::volume::media_error_for_test();
+        io_errors.check_read_write_error(Some(&eio));
+        io_errors.check_read_write_error(Some(&eio));
+        assert_eq!(io_errors.get_io_error_state().1, 2);
+
+        let info = crate::storage::volume::NeedleStreamInfo {
+            source: crate::storage::volume::NeedleStreamSource::Local(file),
+            data_file_offset: 0,
+            data_size: data.len() as u32,
+            data_file_access_control: Arc::default(),
+            io_unavailable: Arc::default(),
+            io_errors: io_errors.clone(),
+            needle_id: NeedleId(1),
+            checksum: 0,
+        };
+        let resp = handle_range_request_from_source("bytes=10-19", info, HeaderMap::new(), None);
+        assert_eq!(resp.status(), StatusCode::PARTIAL_CONTENT);
+        assert_eq!(io_errors.get_io_error_state().1, 0);
+    }
+
     /// Range is read as Go reads it: a byte its parseRange cannot parse is a
     /// 416, Unicode whitespace around a range is trimmed, an empty value is no
     /// range, and HEAD ignores it.
@@ -7223,6 +7419,7 @@ mod tests {
                 .set(crate::server::write_queue::WriteQueue::new(
                     state.clone(),
                     128,
+                    crate::server::write_queue::SyncBatchOption::default(),
                 ))
                 .is_ok()
         );

seaweed-volume/src/server/profiling.rs

@@ -132,6 +132,9 @@ mod tests {
             fix_jpg_orientation: false,
             read_mode: ReadMode::Local,
             allow_untrusted_remote_endpoints: false,
+            fsync_min_batch_bytes: 1024 * 1024,
+            fsync_min_batch_requests: 128,
+            fsync_batch_wait: std::time::Duration::from_micros(500),
             cpu_profile: String::new(),
             mem_profile: String::new(),
             compaction_byte_per_second: 0,

seaweed-volume/src/server/write_queue.rs

@@ -56,6 +56,27 @@ const MAX_CONCURRENT_VOLUME_WRITES: usize = 8;
 /// retirement is the volume leaving the store.
 const IDLE_WORKER_POLL: std::time::Duration = std::time::Duration::from_secs(60);
 
+/// Tunes the per-volume durable-write worker. Once the queue drains with the
+/// batch still below min_bytes and min_requests, the batch stays open up to
+/// wait for stragglers instead of paying an fsync per request. Mirrors Go's
+/// storage.SyncBatchOption.
+#[derive(Clone, Copy)]
+pub struct SyncBatchOption {
+    pub min_bytes: usize,
+    pub min_requests: usize,
+    pub wait: std::time::Duration,
+}
+
+impl Default for SyncBatchOption {
+    fn default() -> Self {
+        SyncBatchOption {
+            min_bytes: 1024 * 1024,
+            min_requests: MAX_BATCH_SIZE,
+            wait: std::time::Duration::from_micros(500),
+        }
+    }
+}
+
 /// Handle for submitting write requests to the per-volume workers.
 #[derive(Clone)]
 pub struct WriteQueue {
@@ -67,23 +88,35 @@ pub struct WriteQueue {
     /// alone would multiply the old single queue's capacity by volume count.
     queue_permits: Arc<Semaphore>,
     workers: Arc<Mutex<HashMap<VolumeId, mpsc::Sender<WriteRequest>>>>,
+    sync_batch: SyncBatchOption,
     /// Submissions accepted, for tests asserting which writes route here.
     #[cfg(test)]
     submits: Arc<std::sync::atomic::AtomicU64>,
+    /// Times a worker armed the straggler window, for tests that must land a
+    /// write inside an already-open batch.
+    #[cfg(test)]
+    windows_armed: Arc<std::sync::atomic::AtomicU64>,
 }
 
 impl WriteQueue {
     /// Create a new write queue. Per-volume workers are spawned lazily on the
     /// first submission for that volume.
-    pub fn new(state: Arc<VolumeServerState>, capacity: usize) -> Self {
+    pub fn new(
+        state: Arc<VolumeServerState>,
+        capacity: usize,
+        sync_batch: SyncBatchOption,
+    ) -> Self {
         WriteQueue {
             state,
             capacity,
             io_permits: Arc::new(Semaphore::new(MAX_CONCURRENT_VOLUME_WRITES)),
             queue_permits: Arc::new(Semaphore::new(capacity)),
             workers: Arc::new(Mutex::new(HashMap::new())),
+            sync_batch,
             #[cfg(test)]
             submits: Arc::new(std::sync::atomic::AtomicU64::new(0)),
+            #[cfg(test)]
+            windows_armed: Arc::new(std::sync::atomic::AtomicU64::new(0)),
         }
     }
 
@@ -93,6 +126,12 @@ impl WriteQueue {
         self.submits.load(std::sync::atomic::Ordering::Relaxed)
     }
 
+    /// How many batch windows workers have armed so far.
+    #[cfg(test)]
+    fn windows_armed_for_test(&self) -> u64 {
+        self.windows_armed.load(std::sync::atomic::Ordering::Relaxed)
+    }
+
     /// Sender for the volume's worker, spawning one on first use.
     fn worker_for(&self, volume_id: VolumeId) -> mpsc::Sender<WriteRequest> {
         let mut workers = self.workers.lock().unwrap();
@@ -107,6 +146,9 @@ impl WriteQueue {
                         state: self.state.clone(),
                         io_permits: self.io_permits.clone(),
                         workers: self.workers.clone(),
+                        sync_batch: self.sync_batch,
+                        #[cfg(test)]
+                        windows_armed: self.windows_armed.clone(),
                     }
                     .run(),
                 );
@@ -191,6 +233,9 @@ struct VolumeWriteWorker {
     state: Arc<VolumeServerState>,
     io_permits: Arc<Semaphore>,
     workers: Arc<Mutex<HashMap<VolumeId, mpsc::Sender<WriteRequest>>>>,
+    sync_batch: SyncBatchOption,
+    #[cfg(test)]
+    windows_armed: Arc<std::sync::atomic::AtomicU64>,
 }
 
 impl VolumeWriteWorker {
@@ -238,14 +283,40 @@ impl VolumeWriteWorker {
         let mut batch = Vec::with_capacity(MAX_BATCH_SIZE);
         let mut batch_bytes: usize = first.needle.data.len();
         batch.push(first);
+        // Armed once the queue first drains below the minimum batch; holds the
+        // batch open for stragglers so the wait bound is per batch, not per
+        // lull (Go's batchWait).
+        let mut window_deadline = None;
         while batch.len() < MAX_BATCH_SIZE && batch_bytes < MAX_BATCH_BYTES {
-            match self.rx.try_recv() {
-                Ok(req) => {
-                    batch_bytes += req.needle.data.len();
-                    batch.push(req);
-                }
-                Err(_) => break,
+            if window_deadline.is_some_and(|d| tokio::time::Instant::now() >= d) {
+                break;
             }
+            let req = match self.rx.try_recv() {
+                Ok(req) => req,
+                Err(mpsc::error::TryRecvError::Empty) => {
+                    let sync = self.sync_batch;
+                    if sync.wait.is_zero()
+                        || batch_bytes >= sync.min_bytes
+                        || batch.len() >= sync.min_requests
+                    {
+                        // a drained queue flushes right away once it holds enough
+                        break;
+                    }
+                    let deadline = *window_deadline.get_or_insert_with(|| {
+                        #[cfg(test)]
+                        self.windows_armed
+                            .fetch_add(1, std::sync::atomic::Ordering::Relaxed);
+                        tokio::time::Instant::now() + sync.wait
+                    });
+                    match tokio::time::timeout_at(deadline, self.rx.recv()).await {
+                        Ok(Some(req)) => req,
+                        Ok(None) | Err(_) => break,
+                    }
+                }
+                Err(mpsc::error::TryRecvError::Disconnected) => break,
+            };
+            batch_bytes += req.needle.data.len();
+            batch.push(req);
         }
         debug!("processing write batch of {} requests", batch.len());
 
@@ -380,7 +451,7 @@ mod tests {
     async fn test_write_queue_submit_no_volume() {
         // Submit a write to a non-existent volume -- should return VolumeError::NotFound.
         let state = make_test_state();
-        let queue = WriteQueue::new(state, MAX_BATCH_SIZE);
+        let queue = WriteQueue::new(state, MAX_BATCH_SIZE, SyncBatchOption::default());
 
         let needle = Needle {
             id: 1.into(),
@@ -402,7 +473,7 @@ mod tests {
     async fn test_write_queue_concurrent_submissions() {
         // Submit multiple concurrent writes -- all should complete (with errors since no volume).
         let state = make_test_state();
-        let queue = WriteQueue::new(state, MAX_BATCH_SIZE);
+        let queue = WriteQueue::new(state, MAX_BATCH_SIZE, SyncBatchOption::default());
 
         let mut handles = Vec::new();
         for i in 0..10u64 {
@@ -430,7 +501,7 @@ mod tests {
     async fn test_write_queue_batching() {
         // Verify that many concurrent writes get processed (testing the batching path).
         let state = make_test_state();
-        let queue = WriteQueue::new(state, MAX_BATCH_SIZE);
+        let queue = WriteQueue::new(state, MAX_BATCH_SIZE, SyncBatchOption::default());
 
         // Submit MAX_BATCH_SIZE requests concurrently
         let mut handles = Vec::new();
@@ -522,7 +593,7 @@ mod tests {
     async fn test_write_queue_dropped_sender() {
         // When the queue is dropped, subsequent submits should fail gracefully.
         let state = make_test_state();
-        let queue = WriteQueue::new(state, 1);
+        let queue = WriteQueue::new(state, 1, SyncBatchOption::default());
 
         // Clone then drop the original -- the worker keeps running via its rx handle.
         let queue2 = queue.clone();
@@ -665,7 +736,7 @@ mod tests {
                     .unwrap();
             }
         }
-        let queue = WriteQueue::new(state, MAX_BATCH_SIZE);
+        let queue = WriteQueue::new(state, MAX_BATCH_SIZE, SyncBatchOption::default());
 
         let in_append = Arc::new(Barrier::new(2));
         let release = Arc::new(Barrier::new(2));
@@ -734,4 +805,85 @@ mod tests {
         assert!(a1.await.unwrap().is_ok());
         assert!(a2.await.unwrap().is_ok());
     }
+
+    /// A durable write below the minimum batch rides out its window so a
+    /// straggler joins the same sync instead of each paying an fsync alone —
+    /// Go's batchWait. Paused time makes the hold observable: a lone write can
+    /// only complete once the runtime auto-advances to the window's deadline.
+    #[tokio::test(start_paused = true)]
+    async fn test_batch_window_defers_flush_for_stragglers() {
+        use crate::config::MinFreeSpace;
+        use crate::storage::types::{DiskType, NeedleId};
+        use crate::storage::volume::VolumeSpec;
+        use std::time::Duration;
+
+        let tmp = tempfile::TempDir::new().unwrap();
+        let dir = tmp.path().to_str().unwrap();
+        let state = make_test_state();
+        {
+            let mut store = state.store.write().unwrap();
+            store
+                .add_location(
+                    dir,
+                    dir,
+                    10,
+                    DiskType::HardDrive,
+                    MinFreeSpace::Percent(1.0),
+                    Vec::new(),
+                )
+                .unwrap();
+            store
+                .add_volume(VolumeId(1), DiskType::HardDrive, &VolumeSpec::default())
+                .unwrap();
+        }
+
+        let queue = WriteQueue::new(
+            state,
+            MAX_BATCH_SIZE,
+            SyncBatchOption {
+                min_bytes: usize::MAX,
+                min_requests: MAX_BATCH_SIZE,
+                wait: Duration::from_secs(30),
+            },
+        );
+
+        let needle = |id: u64| Needle {
+            id: NeedleId(id),
+            cookie: 0x1111.into(),
+            data: vec![id as u8; 8],
+            data_size: 8,
+            ..Needle::default()
+        };
+
+        let start = tokio::time::Instant::now();
+        queue.submit(VolumeId(1), needle(1), true).await.unwrap();
+        assert!(start.elapsed() >= Duration::from_secs(30));
+
+        // Wait for the worker to take the second write and open its batch
+        // window, so the straggler provably lands inside an open batch
+        // instead of arriving during the initial drain.
+        let armed = queue.windows_armed_for_test();
+        let second = tokio::spawn({
+            let queue = queue.clone();
+            async move { queue.submit(VolumeId(1), needle(2), true).await }
+        });
+        for _ in 0..10_000 {
+            if queue.windows_armed_for_test() > armed {
+                break;
+            }
+            tokio::task::yield_now().await;
+        }
+        assert!(queue.windows_armed_for_test() > armed);
+        let third = tokio::spawn({
+            let queue = queue.clone();
+            async move { queue.submit(VolumeId(1), needle(3), true).await }
+        });
+        let (second, third) = tokio::join!(second, third);
+        second.unwrap().unwrap();
+        third.unwrap().unwrap();
+        // One shared window ends both writes ~30s after their batch armed;
+        // back-to-back windows would keep the third out to ~90s.
+        let elapsed = start.elapsed();
+        assert!(elapsed >= Duration::from_secs(60) && elapsed < Duration::from_secs(90));
+    }
 }

seaweed-volume/src/storage/volume.rs

@@ -836,7 +836,7 @@ impl NeedleReadPlan {
     pub(crate) fn read_meta(&mut self, n: &mut Needle) -> Result<(), VolumeError> {
         let (source, size, version) = (&self.source, self.size, self.version);
         let read_at = |buf: &mut [u8], at: u64| Ok(source.read_exact_at(buf, at)?);
-        with_offset_retry(&mut self.offset, |off| {
+        let result = with_offset_retry(&mut self.offset, |off| {
             if version == VERSION_1 {
                 // A V1 body is all data, so its "meta" is the whole record.
                 let mut buf = vec![0u8; get_actual_size(size, version) as usize];
@@ -847,13 +847,26 @@ impl NeedleReadPlan {
             } else {
                 Volume::read_needle_meta_blob_and_parse(read_at, n, off, size, version)
             }
-        })?;
+        });
+        // Report failures only: a successful meta read precedes the payload
+        // read, so clearing the streak here would reset the count before
+        // every failing download and the volume could never reach quarantine.
+        if let Err(VolumeError::Io(e)) = &result {
+            self.io_errors.check_read_write_error(Some(e));
+        }
+        result?;
         if needle_expired(n) {
             return Err(VolumeError::NotFound);
         }
         Ok(())
     }
 
+    /// A successful meta read ended the request (e.g. HEAD), so it may reset
+    /// the streak — unlike a meta read that precedes a payload read.
+    pub(crate) fn note_meta_only_success(&self) {
+        self.io_errors.check_read_write_error(None);
+    }
+
     /// For a reply served from the data file, after `read_meta` filled `n`.
     pub(crate) fn into_stream_info(self, n: &Needle) -> NeedleStreamInfo {
         // V1 data starts right after the header, V2/V3 after the DataSize field.
@@ -868,6 +881,7 @@ impl NeedleReadPlan {
             data_size: n.data_size,
             data_file_access_control: self.data_file_access_control,
             io_unavailable: self.io_unavailable,
+            io_errors: self.io_errors,
             needle_id: self.needle_id,
             checksum: n.checksum.0,
         }
@@ -1144,6 +1158,9 @@ pub struct NeedleStreamInfo {
     pub data_file_access_control: Arc<DataFileAccessControl>,
     /// Checked before each chunk: the volume can become unavailable mid-stream.
     pub(crate) io_unavailable: Arc<IoUnavailable>,
+    /// Streamed chunks report read failures here and clear the streak when
+    /// the whole body has been emitted, mirroring readNeedleDataInto.
+    pub(crate) io_errors: Arc<IoErrorTracker>,
     pub needle_id: NeedleId,
     /// Checksum stored in the needle tail, verified once the last chunk has
     /// been read — before that frame is emitted.
@@ -2918,18 +2935,52 @@ impl Volume {
         }
 
         // Cookie validation for existing needle (matches Go: check whenever nm.Get returns ok)
-        if let Some(nm) = &io.nm
-            && let Some(nv) = nm.get(n.id)?
+        let indexed = match io.nm.as_ref() {
+            Some(nm) => nm.get(n.id)?,
+            None => None,
+        };
+        if let Some(nv) = indexed
+            && !nv.offset.is_zero()
         {
             let mut existing = Needle::default();
             // Read only the header to check cookie
             self.read_needle_header_unlocked(io, &mut existing, nv.offset.to_actual_offset())?;
 
-            if n.cookie.0 == 0 && !check_cookie {
-                n.cookie = existing.cookie;
+            if existing.id != n.id && existing.size != nv.size {
+                // a 4-byte index entry can index past the 32 GiB wrap: the
+                // low offset may hold another needle while the real record
+                // sits at the wrapped offset, where reads already retry
+                #[cfg(not(feature = "5bytes"))]
+                if nv.offset.to_actual_offset() < MAX_POSSIBLE_VOLUME_SIZE as i64 {
+                    let mut wrapped = Needle::default();
+                    match self.read_needle_header_unlocked(
+                        io,
+                        &mut wrapped,
+                        nv.offset.to_actual_offset() + MAX_POSSIBLE_VOLUME_SIZE as i64,
+                    ) {
+                        Ok(()) => existing = wrapped,
+                        Err(VolumeError::Io(e)) if e.kind() == io::ErrorKind::UnexpectedEof => {}
+                        Err(e) => return Err(e),
+                    }
+                }
             }
-            if existing.cookie != n.cookie {
-                return Err(VolumeError::CookieMismatch(n.cookie.0));
+
+            if existing.id != n.id && existing.size != nv.size {
+                // the entry resolves to a record that is neither the indexed
+                // key nor the indexed size, so it is stale rather than a
+                // duplicate — skip the cookie check and let publish_write
+                // overwrite it once the append commits
+                warn!(
+                    "volume {} index entry for needle {} resolves to needle {}; overwriting stale index entry",
+                    self.id.0, n.id.0, existing.id.0
+                );
+            } else {
+                if n.cookie.0 == 0 && !check_cookie {
+                    n.cookie = existing.cookie;
+                }
+                if existing.cookie != n.cookie {
+                    return Err(VolumeError::CookieMismatch(n.cookie.0));
+                }
             }
         }
         Ok(false)
@@ -6067,7 +6118,7 @@ fn preallocate_file(file: &File, size: u64) {
 /// An OS error the platform reports for failing storage media, which is
 /// what counts toward the I/O error streak.
 #[cfg(test)]
-fn media_error_for_test() -> io::Error {
+pub(crate) fn media_error_for_test() -> io::Error {
     #[cfg(unix)]
     {
         io::Error::from_raw_os_error(libc::EIO)
@@ -9044,6 +9095,107 @@ mod tests {
         assert!(matches!(err, VolumeError::CookieMismatch(_)));
     }
 
+    /// A damaged index entry resolving to a different needle must miss on
+    /// reads and be dropped by the next write for the key, not rejected as a
+    /// cookie conflict.
+    #[test]
+    fn test_stale_index_entry_heals_on_write() {
+        let tmp = TempDir::new().unwrap();
+        let dir = tmp.path().to_str().unwrap();
+        let v = make_test_volume(dir);
+
+        let mut first = Needle {
+            id: NeedleId(2),
+            cookie: Cookie(0x2222),
+            data: b"second".to_vec(),
+            data_size: 6,
+            ..Needle::default()
+        };
+        v.write_needle(&mut first, true, false).unwrap();
+
+        // point key 9 at needle 2's record with a size that does not match
+        // the record, as a corrupt index entry would look
+        {
+            let mut io = v.io.write().unwrap();
+            let nm = io.nm.as_mut().unwrap();
+            let nv = nm.get(NeedleId(2)).unwrap().unwrap();
+            nm.put(NeedleId(9), nv.offset, Size(nv.size.0 + 1)).unwrap();
+        }
+
+        let mut probe = Needle {
+            id: NeedleId(9),
+            ..Needle::default()
+        };
+        assert!(v.read_needle(&mut probe).is_err());
+
+        let mut n = Needle {
+            id: NeedleId(9),
+            cookie: Cookie(0xabcd),
+            data: b"ninth".to_vec(),
+            data_size: 5,
+            ..Needle::default()
+        };
+        v.write_needle(&mut n, true, false).unwrap();
+
+        let mut back = Needle {
+            id: NeedleId(9),
+            ..Needle::default()
+        };
+        v.read_needle(&mut back).unwrap();
+        assert_eq!(back.data, b"ninth");
+        assert_eq!(back.cookie, Cookie(0xabcd));
+    }
+
+    /// A stale index entry is only replaced once the new record commits: a
+    /// durable write rolled back by a failed fsync leaves the index exactly
+    /// as it found it.
+    #[test]
+    fn test_stale_index_entry_survives_failed_write() {
+        let tmp = TempDir::new().unwrap();
+        let dir = tmp.path().to_str().unwrap();
+        let v = make_test_volume(dir);
+
+        let mut existing = Needle {
+            id: NeedleId(2),
+            cookie: Cookie(0x2222),
+            data: b"second".to_vec(),
+            data_size: 6,
+            ..Needle::default()
+        };
+        v.write_needle(&mut existing, true, false).unwrap();
+
+        let stale_nv = {
+            let mut io = v.io.write().unwrap();
+            let nm = io.nm.as_mut().unwrap();
+            let nv = nm.get(NeedleId(2)).unwrap().unwrap();
+            nm.put(NeedleId(9), nv.offset, Size(nv.size.0 + 1)).unwrap();
+            nm.get(NeedleId(9)).unwrap().unwrap()
+        };
+
+        v.fail_next_fsync_for_test(true);
+        let mut replacement = Needle {
+            id: NeedleId(9),
+            cookie: Cookie(0xabcd),
+            data: b"ninth".to_vec(),
+            data_size: 5,
+            ..Needle::default()
+        };
+        assert!(v.write_needle(&mut replacement, true, true).is_err());
+        v.fail_next_fsync_for_test(false);
+
+        let now = {
+            let io = v.io.read().unwrap();
+            io.nm
+                .as_ref()
+                .unwrap()
+                .get(NeedleId(9))
+                .unwrap()
+                .expect("a failed write must not drop the index entry it meant to replace")
+        };
+        assert_eq!(now.offset, stale_nv.offset);
+        assert_eq!(now.size, stale_nv.size);
+    }
+
     // A size disagreeing with the blob's own header indexes the needle at the wrong
     // length and, on v3, stamps the append timestamp into the middle of the needle.
     #[test]

seaweed-volume/tests/http_integration.rs

@@ -1340,11 +1340,11 @@ async fn non_ascii_fid_and_ttl_are_rejected_not_panicked() {
 
 #[tokio::test]
 async fn write_queue_upload_returns_same_etag_as_direct_write() {
-    use seaweed_volume::server::write_queue::WriteQueue;
+    use seaweed_volume::server::write_queue::{SyncBatchOption, WriteQueue};
 
     let (direct_state, _direct_tmp) = test_state();
     let (queued_state, _queued_tmp) = test_state();
-    let wq = WriteQueue::new(queued_state.clone(), 128);
+    let wq = WriteQueue::new(queued_state.clone(), 128, SyncBatchOption::default());
     let _ = queued_state.write_queue.set(wq);
 
     let compressible = "seaweedfs ".repeat(200).into_bytes();

test/erasure_coding/ec_integration_test.go

@@ -3,9 +3,12 @@ package erasure_coding
 import (
 	"bytes"
 	"context"
+	"encoding/json"
 	"fmt"
 	"io"
 	"math"
+	"net"
+	"net/http"
 	"os"
 	"os/exec"
 	"path/filepath"
@@ -13,6 +16,7 @@ import (
 	"testing"
 	"time"
 
+	"github.com/seaweedfs/seaweedfs/test/testutil"
 	"github.com/seaweedfs/seaweedfs/weed/operation"
 	"github.com/seaweedfs/seaweedfs/weed/pb"
 	"github.com/seaweedfs/seaweedfs/weed/pb/volume_server_pb"
@@ -47,17 +51,15 @@ func TestECEncodingVolumeLocationTimingBug(t *testing.T) {
 	defer cluster.Stop()
 
 	// Wait for servers to be ready
-	require.NoError(t, waitForServer("127.0.0.1:9333", 30*time.Second))
-	require.NoError(t, waitForServer("127.0.0.1:8080", 30*time.Second))
-	require.NoError(t, waitForServer("127.0.0.1:8081", 30*time.Second))
-	require.NoError(t, waitForServer("127.0.0.1:8082", 30*time.Second))
-	require.NoError(t, waitForServer("127.0.0.1:8083", 30*time.Second))
-	require.NoError(t, waitForServer("127.0.0.1:8084", 30*time.Second))
-	require.NoError(t, waitForServer("127.0.0.1:8085", 30*time.Second))
+	require.NoError(t, waitForServer(cluster.masterAddr, 30*time.Second))
+	for _, addr := range cluster.volumeAddrs {
+		require.NoError(t, waitForServer(addr, 30*time.Second))
+	}
+	require.NoError(t, waitForMasterLeader(cluster.masterAddr, 30*time.Second))
 
 	// Create command environment
 	options := &shell.ShellOptions{
-		Masters:        stringPtr("127.0.0.1:9333"),
+		Masters:        stringPtr(cluster.masterAddr),
 		GrpcDialOption: grpc.WithInsecure(),
 		FilerGroup:     stringPtr("default"),
 	}
@@ -71,7 +73,7 @@ func TestECEncodingVolumeLocationTimingBug(t *testing.T) {
 
 	// Upload some test data to create volumes
 	testData := []byte("This is test data for EC encoding integration test")
-	volumeId, err := uploadTestData(testData, "127.0.0.1:9333")
+	volumeId, err := uploadTestData(testData, cluster.masterAddr)
 	require.NoError(t, err)
 	t.Logf("Created volume %d with test data", volumeId)
 
@@ -257,12 +259,13 @@ func TestECEncodingMasterTimingRaceCondition(t *testing.T) {
 	defer cluster.Stop()
 
 	// Wait for servers to be ready
-	require.NoError(t, waitForServer("127.0.0.1:9333", 30*time.Second))
-	require.NoError(t, waitForServer("127.0.0.1:8080", 30*time.Second))
+	require.NoError(t, waitForServer(cluster.masterAddr, 30*time.Second))
+	require.NoError(t, waitForServer(cluster.volumeAddrs[0], 30*time.Second))
+	require.NoError(t, waitForMasterLeader(cluster.masterAddr, 30*time.Second))
 
 	// Create command environment
 	options := &shell.ShellOptions{
-		Masters:        stringPtr("127.0.0.1:9333"),
+		Masters:        stringPtr(cluster.masterAddr),
 		GrpcDialOption: grpc.WithInsecure(),
 		FilerGroup:     stringPtr("default"),
 	}
@@ -276,7 +279,7 @@ func TestECEncodingMasterTimingRaceCondition(t *testing.T) {
 
 	// Upload test data
 	testData := []byte("Race condition test data")
-	volumeId, err := uploadTestData(testData, "127.0.0.1:9333")
+	volumeId, err := uploadTestData(testData, cluster.masterAddr)
 	require.NoError(t, err)
 	t.Logf("Created volume %d for race condition test", volumeId)
 
@@ -366,6 +369,8 @@ func TestECEncodingMasterTimingRaceCondition(t *testing.T) {
 type TestCluster struct {
 	masterCmd     *exec.Cmd
 	volumeServers []*exec.Cmd
+	masterAddr    string
+	volumeAddrs   []string
 }
 
 func (c *TestCluster) Stop() {
@@ -402,9 +407,23 @@ func startSeaweedFSCluster(ctx context.Context, dataDir string) (*TestCluster, e
 	securityToml := filepath.Join(dataDir, "security.toml")
 	os.WriteFile(securityToml, []byte("# Empty security config for testing\n"), 0644)
 
+	// Allocate ports instead of using the defaults: on a shared or reused
+	// runner a stale weed process can still hold 9333/808x, which kills the
+	// freshly started master at bind time and leaves the test talking to a
+	// raft follower that answers every assign with "not current leader".
+	// Mini-safe ports guarantee each HTTP port's derived gRPC port
+	// (port + 10000) is free as well.
+	ports, err := testutil.AllocateMiniPorts(7)
+	if err != nil {
+		return nil, fmt.Errorf("failed to allocate ports: %v", err)
+	}
+	masterPort := ports[0]
+	masterAddr := fmt.Sprintf("127.0.0.1:%d", masterPort)
+	cluster.masterAddr = masterAddr
+
 	// Start master server
 	masterCmd := exec.CommandContext(ctx, weedBinary, "master",
-		"-port", "9333",
+		"-port", fmt.Sprintf("%d", masterPort),
 		"-mdir", masterDir,
 		"-volumeSizeLimitMB", "10", // Small volumes for testing
 		"-ip", "127.0.0.1",
@@ -432,13 +451,14 @@ func startSeaweedFSCluster(ctx context.Context, dataDir string) (*TestCluster, e
 		volumeDir := filepath.Join(dataDir, fmt.Sprintf("volume%d", i))
 		os.MkdirAll(volumeDir, 0755)
 
-		port := fmt.Sprintf("808%d", i)
+		port := fmt.Sprintf("%d", ports[1+i])
+		cluster.volumeAddrs = append(cluster.volumeAddrs, fmt.Sprintf("127.0.0.1:%d", ports[1+i]))
 		rack := fmt.Sprintf("rack%d", i)
 		volumeCmd := exec.CommandContext(ctx, weedBinary, "volume",
 			"-port", port,
 			"-dir", volumeDir,
 			"-max", "10",
-			"-master", "127.0.0.1:9333",
+			"-master", masterAddr,
 			"-volume.allowUntrustedRemoteEndpoints",
 			"-ip", "127.0.0.1",
 			"-dataCenter", "dc1",
@@ -498,7 +518,8 @@ func findWeedBinary() string {
 func waitForServer(address string, timeout time.Duration) error {
 	start := time.Now()
 	for time.Since(start) < timeout {
-		if conn, err := grpc.NewClient(address, grpc.WithInsecure()); err == nil {
+		conn, err := net.DialTimeout("tcp", address, time.Second)
+		if err == nil {
 			conn.Close()
 			return nil
 		}
@@ -507,6 +528,29 @@ func waitForServer(address string, timeout time.Duration) error {
 	return fmt.Errorf("timeout waiting for server %s", address)
 }
 
+// waitForMasterLeader polls the master's /cluster/status endpoint until it
+// reports itself as the raft leader. A goraft master can take up to the
+// configured election timeout (10s by default) before it is ready to serve
+// raft-gated requests like volume assign, so a bare port check is not enough.
+func waitForMasterLeader(masterAddr string, timeout time.Duration) error {
+	start := time.Now()
+	for time.Since(start) < timeout {
+		resp, err := http.Get(fmt.Sprintf("http://%s/cluster/status", masterAddr))
+		if err == nil {
+			var status struct {
+				IsLeader bool `json:"IsLeader"`
+			}
+			err = json.NewDecoder(resp.Body).Decode(&status)
+			resp.Body.Close()
+			if err == nil && status.IsLeader {
+				return nil
+			}
+		}
+		time.Sleep(500 * time.Millisecond)
+	}
+	return fmt.Errorf("timeout waiting for master %s to become raft leader", masterAddr)
+}
+
 func uploadTestData(data []byte, masterAddress string) (needle.VolumeId, error) {
 	// Upload data to get a file ID
 	assignResult, err := operation.Assign(context.Background(), func(ctx context.Context) pb.ServerAddress {
@@ -2209,14 +2253,15 @@ func TestECEncodeReplicatedVolumeSync(t *testing.T) {
 	defer cluster.Stop()
 
 	// Wait for servers to be ready
-	require.NoError(t, waitForServer("127.0.0.1:9333", 30*time.Second))
-	for i := 0; i < 6; i++ {
-		require.NoError(t, waitForServer(fmt.Sprintf("127.0.0.1:%d", 8080+i), 30*time.Second))
+	require.NoError(t, waitForServer(cluster.masterAddr, 30*time.Second))
+	for _, addr := range cluster.volumeAddrs {
+		require.NoError(t, waitForServer(addr, 30*time.Second))
 	}
+	require.NoError(t, waitForMasterLeader(cluster.masterAddr, 30*time.Second))
 
 	// Create command environment
 	options := &shell.ShellOptions{
-		Masters:        stringPtr("127.0.0.1:9333"),
+		Masters:        stringPtr(cluster.masterAddr),
 		GrpcDialOption: grpc.WithInsecure(),
 		FilerGroup:     stringPtr("default"),
 	}
@@ -2240,7 +2285,7 @@ func TestECEncodeReplicatedVolumeSync(t *testing.T) {
 		var volumeId needle.VolumeId
 		var uploadErr error
 		for retry := 0; retry < 5; retry++ {
-			volumeId, uploadErr = uploadTestDataWithReplication(t, "127.0.0.1:9333", testCollection, testReplication)
+			volumeId, uploadErr = uploadTestDataWithReplication(t, cluster.masterAddr, testCollection, testReplication)
 			if uploadErr == nil {
 				break
 			}
@@ -2290,7 +2335,7 @@ func TestECEncodeReplicatedVolumeSync(t *testing.T) {
 		var volumeId needle.VolumeId
 		var uploadErr error
 		for retry := 0; retry < 5; retry++ {
-			volumeId, uploadErr = uploadTestDataWithReplication(t, "127.0.0.1:9333", testCollection, testReplication)
+			volumeId, uploadErr = uploadTestDataWithReplication(t, cluster.masterAddr, testCollection, testReplication)
 			if uploadErr == nil {
 				break
 			}

test/fuse_integration/framework.go

@@ -4,9 +4,12 @@ import (
 	"fmt"
 	"io/fs"
 	"net"
+	"net/http"
 	"os"
 	"os/exec"
 	"path/filepath"
+	"strconv"
+	"strings"
 	"syscall"
 	"testing"
 	"time"
@@ -14,19 +17,19 @@ import (
 	"github.com/stretchr/testify/require"
 )
 
-// FuseTestFramework provides utilities for FUSE integration testing
+// FuseTestFramework provides utilities for FUSE integration testing.
+// It starts a single "weed mini" process (master+volume+filer in one)
+// and a separate "weed mount" process for the FUSE filesystem.
 type FuseTestFramework struct {
 	t             *testing.T
 	tempDir       string
 	mountPoint    string
 	dataDir       string
-	masterProcess *os.Process
-	volumeProcess *os.Process
-	filerProcess  *os.Process
-	mountProcess  *os.Process
-	masterAddr    string
-	volumeAddr    string
+	logDir        string
+	miniProcess   *managedProcess
+	mountProcess  *managedProcess
 	filerAddr     string
+	filerPort     int
 	weedBinary    string
 	isSetup       bool
 	config        *TestConfig
@@ -35,14 +38,18 @@ type FuseTestFramework struct {
 
 // TestConfig holds configuration for FUSE tests
 type TestConfig struct {
-	Collection   string
-	Replication  string
-	ChunkSizeMB  int
-	CacheSizeMB  int
-	NumVolumes   int
-	EnableDebug  bool
-	MountOptions []string
-	SkipCleanup  bool // for debugging failed tests
+	Collection  string
+	Replication string
+	ChunkSizeMB int
+	CacheSizeMB int
+	NumVolumes  int
+	EnableDebug bool
+	// MountGlobalOptions are glog flags (-v, -logtostderr, ...) registered on
+	// weed's global flagset; they must precede the subcommand name or the
+	// mount process dies at flag parsing.
+	MountGlobalOptions []string
+	MountOptions       []string
+	SkipCleanup        bool // for debugging failed tests
 }
 
 // DefaultTestConfig returns a default configuration for FUSE tests
@@ -59,7 +66,7 @@ func DefaultTestConfig() *TestConfig {
 	}
 }
 
-// NewFuseTestFramework creates a new FUSE testing framework
+// NewFuseTestFramework creates a new FUSE testing framework.
 func NewFuseTestFramework(t *testing.T, config *TestConfig) *FuseTestFramework {
 	if config == nil {
 		config = DefaultTestConfig()
@@ -68,20 +75,63 @@ func NewFuseTestFramework(t *testing.T, config *TestConfig) *FuseTestFramework {
 	tempDir, err := os.MkdirTemp("", "seaweedfs_fuse_test_")
 	require.NoError(t, err)
 
+	filerPort := freePort(t)
+
 	return &FuseTestFramework{
 		t:          t,
 		tempDir:    tempDir,
 		mountPoint: filepath.Join(tempDir, "mount"),
 		dataDir:    filepath.Join(tempDir, "data"),
-		masterAddr: "127.0.0.1:19333",
-		volumeAddr: "127.0.0.1:18080",
-		filerAddr:  "127.0.0.1:18888",
+		logDir:     filepath.Join(tempDir, "logs"),
+		filerPort:  filerPort,
+		filerAddr:  fmt.Sprintf("127.0.0.1:%d", filerPort),
 		weedBinary: findWeedBinary(),
 		isSetup:    false,
 		config:     config,
 	}
 }
 
+// grpcPortOffset mirrors weed's HTTP->gRPC port convention (gRPC = HTTP + offset);
+// "weed mount" derives the filer gRPC port from the filer HTTP address the same way.
+const grpcPortOffset = 10000
+
+// freePort returns a free filer HTTP port whose gRPC sibling (port+grpcPortOffset)
+// is also free. Both must stay below the Linux ephemeral floor (32768): a gRPC port
+// above it can be transiently grabbed by an outbound connection, forcing mini to
+// relocate its filer gRPC port while "weed mount" keeps dialing HTTP+10000 — the
+// mount then never connects and the test times out. Capping HTTP at 22000 keeps the
+// gRPC port at or below 32000.
+func freePort(t *testing.T) int {
+	t.Helper()
+	const (
+		minServicePort = 20000
+		maxServicePort = 22000
+	)
+
+	portCount := maxServicePort - minServicePort + 1
+	start := minServicePort + int(time.Now().UnixNano()%int64(portCount))
+
+	for attempt := 0; attempt < portCount; attempt++ {
+		port := minServicePort + (start-minServicePort+attempt)%portCount
+		if portIsFree(port) && portIsFree(port+grpcPortOffset) {
+			return port
+		}
+	}
+
+	t.Fatalf("failed to allocate a free HTTP/gRPC port pair in [%d,%d]", minServicePort, maxServicePort)
+	return 0
+}
+
+// portIsFree reports whether a TCP port can currently be bound on 127.0.0.1.
+func portIsFree(port int) bool {
+	l, err := net.Listen("tcp", net.JoinHostPort("127.0.0.1", strconv.Itoa(port)))
+	if err != nil {
+		return false
+	}
+	l.Close()
+	return true
+}
+
 // Setup starts SeaweedFS cluster and mounts FUSE filesystem
 func (f *FuseTestFramework) Setup(config *TestConfig) error {
 	if f.isSetup {
@@ -159,41 +209,22 @@ func (f *FuseTestFramework) Setup(config *TestConfig) error {
 	}
 
 	// Create directories for full cluster setup
-	dirs := []string{f.mountPoint, f.dataDir}
+	dirs := []string{f.mountPoint, f.logDir, f.dataDir}
 	for _, dir := range dirs {
 		if err := os.MkdirAll(dir, 0755); err != nil {
 			return fmt.Errorf("failed to create directory %s: %v", dir, err)
 		}
 	}
 
-	// Start master
-	if err := f.startMaster(config); err != nil {
-		return fmt.Errorf("failed to start master: %v", err)
+	// Start weed mini (master + volume + filer in one process)
+	if err := f.startMini(config); err != nil {
+		return fmt.Errorf("failed to start weed mini: %v", err)
 	}
 
-	// Wait for master to be ready
-	if err := f.waitForService(f.masterAddr, 30*time.Second); err != nil {
-		return fmt.Errorf("master not ready: %v", err)
-	}
-
-	// Start volume servers
-	if err := f.startVolumeServers(config); err != nil {
-		return fmt.Errorf("failed to start volume servers: %v", err)
-	}
-
-	// Wait for volume server to be ready
-	if err := f.waitForService(f.volumeAddr, 30*time.Second); err != nil {
-		return fmt.Errorf("volume server not ready: %v", err)
-	}
-
-	// Start filer
-	if err := f.startFiler(config); err != nil {
-		return fmt.Errorf("failed to start filer: %v", err)
-	}
-
-	// Wait for filer to be ready
-	if err := f.waitForService(f.filerAddr, 30*time.Second); err != nil {
-		return fmt.Errorf("filer not ready: %v", err)
+	// Wait for filer to be ready (mini starts all services on filerPort)
+	if err := f.waitForService(f.miniProcess, f.filerAddr, 30*time.Second); err != nil {
+		f.dumpLog("mini")
+		return fmt.Errorf("weed mini not ready: %v", err)
 	}
 
 	// Skip rather than fail on a host with no FUSE device to mount through
@@ -208,14 +239,24 @@ func (f *FuseTestFramework) Setup(config *TestConfig) error {
 
 	// Wait for mount to be ready
 	if err := f.waitForMount(30 * time.Second); err != nil {
+		f.dumpLog("mount")
 		return fmt.Errorf("FUSE mount not ready: %v", err)
 	}
 
+	// A mounted filesystem still cannot write until a volume registers and a
+	// grow round finishes, so gate on a real write instead of letting the
+	// first test hits ride out the startup window as EIO on close.
+	if err := f.waitForWritable(90 * time.Second); err != nil {
+		f.dumpLog("mount")
+		return fmt.Errorf("FUSE mount not writable: %v", err)
+	}
+
 	f.isSetup = true
 	return nil
 }
 
-// Cleanup stops all processes and removes temporary files
+// Cleanup stops all processes and removes temporary files.
+// If the test failed, it dumps logs automatically.
 func (f *FuseTestFramework) Cleanup() {
 	// An external mount needs no teardown; only this run's work dir is ours.
 	// A TEST_SKIP_CLUSTER_SETUP that fell through to a self-started cluster
@@ -228,25 +269,32 @@ func (f *FuseTestFramework) Cleanup() {
 		return
 	}
 
-	if f.mountProcess != nil {
-		f.unmountFuse()
+	if f.t.Failed() {
+		f.DumpLogs()
 	}
 
 	// Stop processes in reverse order
-	processes := []*os.Process{f.mountProcess, f.filerProcess, f.volumeProcess, f.masterProcess}
-	for _, proc := range processes {
-		if proc != nil {
-			proc.Signal(syscall.SIGTERM)
-			proc.Wait()
-		}
+	f.unmountFuse()
+	if f.miniProcess != nil {
+		f.miniProcess.stop()
+		f.miniProcess = nil
 	}
 
+	f.copyLogsForCI()
+
 	// Remove temp directory
 	if !f.config.SkipCleanup {
 		os.RemoveAll(f.tempDir)
 	}
 }
 
+// DumpLogs prints the tail of all SeaweedFS process logs to test output.
+func (f *FuseTestFramework) DumpLogs() {
+	for _, name := range []string{"mini", "mount"} {
+		f.dumpLog(name)
+	}
+}
+
 // GetMountPoint returns the FUSE mount point path
 func (f *FuseTestFramework) GetMountPoint() string {
 	return f.mountPoint
@@ -257,88 +305,164 @@ func (f *FuseTestFramework) GetFilerAddr() string {
 	return f.filerAddr
 }
 
-// startMaster starts the SeaweedFS master server
-func (f *FuseTestFramework) startMaster(config *TestConfig) error {
+// managedProcess is a started weed sub-command whose exit is watched, so a wait
+// for it to come up ends the moment it dies instead of burning its full timeout.
+type managedProcess struct {
+	cmd     *exec.Cmd
+	done    chan struct{}
+	err     error // exit error, read only after done is closed
+	logPath string
+}
+
+// exited returns the exit error once the process is gone, nil while it runs.
+func (p *managedProcess) exited() error {
+	select {
+	case <-p.done:
+		if p.err != nil {
+			return p.err
+		}
+		return fmt.Errorf("exit status 0")
+	default:
+		return nil
+	}
+}
+
+// stop asks the process to terminate and waits for it to go away.
+func (p *managedProcess) stop() {
+	// Signal fails with os.ErrProcessDone when the child is already gone, which
+	// is exactly the case the select below handles.
+	_ = p.cmd.Process.Signal(syscall.SIGTERM)
+	select {
+	case <-p.done:
+	case <-time.After(10 * time.Second):
+		p.cmd.Process.Kill()
+		<-p.done
+	}
+}
+
+// startProcess is a helper that starts a weed sub-command with output captured
+// to a log file in f.logDir.
+func (f *FuseTestFramework) startProcess(name string, args []string) (*managedProcess, error) {
+	logPath := filepath.Join(f.logDir, name+".log")
+	logFile, err := os.Create(logPath)
+	if err != nil {
+		return nil, fmt.Errorf("create log file: %v", err)
+	}
+	cmd := exec.Command(f.weedBinary, args...)
+	cmd.Dir = f.tempDir
+	cmd.Stdout = logFile
+	cmd.Stderr = logFile
+	if err := cmd.Start(); err != nil {
+		logFile.Close()
+		return nil, err
+	}
+	// Close the file handle — the child process inherited it.
+	logFile.Close()
+
+	p := &managedProcess{cmd: cmd, done: make(chan struct{}), logPath: logPath}
+	go func() {
+		p.err = cmd.Wait()
+		close(p.done)
+	}()
+	return p, nil
+}
+
+// logTail returns the last bytes of a captured process log so a startup
+// failure message carries the fatal line instead of a bare timeout.
+func logTail(path string) string {
+	data, err := os.ReadFile(path)
+	if err != nil || len(data) == 0 {
+		return ""
+	}
+	const maxTail = 2 * 1024
+	if len(data) > maxTail {
+		data = data[len(data)-maxTail:]
+	}
+	return string(data)
+}
+
+// dumpLog prints the last lines of a process log file to the test output
+// for debugging when a service fails to start or a test fails.
+func (f *FuseTestFramework) dumpLog(name string) {
+	data, err := os.ReadFile(filepath.Join(f.logDir, name+".log"))
+	if err != nil {
+		f.t.Logf("[%s log] (not available: %v)", name, err)
+		return
+	}
+	// Show last 16KB on failure for meaningful context.
+	const maxTail = 16 * 1024
+	if len(data) > maxTail {
+		data = data[len(data)-maxTail:]
+	}
+	f.t.Logf("[%s log tail (%d bytes)]\n%s", name, len(data), string(data))
+}
+
+// copyLogsForCI copies SeaweedFS process logs under a per-test directory in
+// /tmp/seaweedfs-fuse-logs/ so the CI workflow can upload them as artifacts.
+// Each framework instance gets its own directory: every test in the suite
+// builds one, and fixed filenames would let a later passing test overwrite
+// the logs of an earlier failing one.
+func (f *FuseTestFramework) copyLogsForCI() {
+	ciLogDir := filepath.Join("/tmp/seaweedfs-fuse-logs", strings.NewReplacer("/", "_", " ", "_").Replace(f.t.Name()))
+	if err := os.MkdirAll(ciLogDir, 0755); err != nil {
+		return
+	}
+	for _, name := range []string{"mini", "mount"} {
+		src := filepath.Join(f.logDir, name+".log")
+		data, err := os.ReadFile(src)
+		if err != nil {
+			continue
+		}
+		os.WriteFile(filepath.Join(ciLogDir, name+".log"), data, 0644)
+	}
+}
+
+// startMini starts "weed mini" which runs master+volume+filer in one process.
+func (f *FuseTestFramework) startMini(config *TestConfig) error {
 	var args []string
 	if config.EnableDebug {
 		// -v is a global flag; before the subcommand or flag parsing dies
 		args = append(args, "-v=4")
 	}
-	args = append(args,
-		"master",
-		"-ip=127.0.0.1",
-		"-port=19333",
-		"-mdir=" + filepath.Join(f.dataDir, "master"),
-		"-raftBootstrap",
-	)
-
-	cmd := exec.Command(f.weedBinary, args...)
-	cmd.Dir = f.tempDir
-	if err := cmd.Start(); err != nil {
-		return err
-	}
-	f.masterProcess = cmd.Process
-	return nil
-}
-
-// startVolumeServers starts SeaweedFS volume servers
-func (f *FuseTestFramework) startVolumeServers(config *TestConfig) error {
-	var args []string
-	if config.EnableDebug {
-		args = append(args, "-v=4")
+	adminPort := freePort(f.t)
+	for adminPort == f.filerPort {
+		adminPort = freePort(f.t)
 	}
 	args = append(args,
-		"volume",
-		"-mserver=" + f.masterAddr,
+		"mini",
+		"-dir="+f.dataDir,
 		"-ip=127.0.0.1",
-		"-port=18080",
-		"-dir=" + filepath.Join(f.dataDir, "volume"),
-		fmt.Sprintf("-max=%d", config.NumVolumes),
+		"-ip.bind=127.0.0.1",
+		"-filer.port="+strconv.Itoa(f.filerPort),
+		"-s3=false",
+		"-webdav=false",
+		"-admin.ui=false",
+		"-admin.port="+strconv.Itoa(adminPort),
+		"-admin.port.grpc="+strconv.Itoa(adminPort+grpcPortOffset),
+		"-admin.port.legacy=0",
 	)
 
-	cmd := exec.Command(f.weedBinary, args...)
-	cmd.Dir = f.tempDir
-	if err := cmd.Start(); err != nil {
+	proc, err := f.startProcess("mini", args)
+	if err != nil {
 		return err
 	}
-	f.volumeProcess = cmd.Process
-	return nil
-}
-
-// startFiler starts the SeaweedFS filer server
-func (f *FuseTestFramework) startFiler(config *TestConfig) error {
-	var args []string
-	if config.EnableDebug {
-		args = append(args, "-v=4")
-	}
-	args = append(args,
-		"filer",
-		"-master=" + f.masterAddr,
-		"-ip=127.0.0.1",
-		"-port=18888",
-	)
-
-	cmd := exec.Command(f.weedBinary, args...)
-	cmd.Dir = f.tempDir
-	if err := cmd.Start(); err != nil {
-		return err
-	}
-	f.filerProcess = cmd.Process
+	f.miniProcess = proc
 	return nil
 }
 
 // mountFuse mounts the SeaweedFS FUSE filesystem
 func (f *FuseTestFramework) mountFuse(config *TestConfig) error {
-	var args []string
+	args := append([]string{}, config.MountGlobalOptions...)
 	if config.EnableDebug {
 		args = append(args, "-v=4")
 	}
 	args = append(args,
 		"mount",
-		"-filer=" + f.filerAddr,
-		"-dir=" + f.mountPoint,
+		"-filer="+f.filerAddr,
+		"-dir="+f.mountPoint,
 		"-filer.path=/",
 		"-dirAutoCreate",
+		"-allowOthers=false",
 	)
 
 	if config.Collection != "" {
@@ -351,55 +475,63 @@ func (f *FuseTestFramework) mountFuse(config *TestConfig) error {
 		args = append(args, fmt.Sprintf("-chunkSizeLimitMB=%d", config.ChunkSizeMB))
 	}
 	if config.CacheSizeMB > 0 {
-		args = append(args, fmt.Sprintf("-cacheSizeMB=%d", config.CacheSizeMB))
+		args = append(args, fmt.Sprintf("-cacheCapacityMB=%d", config.CacheSizeMB))
 	}
 
 	args = append(args, config.MountOptions...)
 
-	cmd := exec.Command(f.weedBinary, args...)
-	cmd.Dir = f.tempDir
-	if err := cmd.Start(); err != nil {
+	proc, err := f.startProcess("mount", args)
+	if err != nil {
 		return err
 	}
-	f.mountProcess = cmd.Process
+	f.mountProcess = proc
 	return nil
 }
 
 // unmountFuse unmounts the FUSE filesystem
 func (f *FuseTestFramework) unmountFuse() error {
 	if f.mountProcess != nil {
-		f.mountProcess.Signal(syscall.SIGTERM)
-		f.mountProcess.Wait()
+		f.mountProcess.stop()
 		f.mountProcess = nil
 	}
 
 	// Also try system unmount as backup
+	exec.Command("fusermount3", "-u", f.mountPoint).Run()
+	exec.Command("fusermount", "-u", f.mountPoint).Run()
 	exec.Command("umount", f.mountPoint).Run()
 	return nil
 }
 
-// waitForService waits for a service to be available
-func (f *FuseTestFramework) waitForService(addr string, timeout time.Duration) error {
+// waitForService waits until the weed process answers its own /healthz, so a
+// foreign listener on the port cannot pass for readiness.
+func (f *FuseTestFramework) waitForService(proc *managedProcess, addr string, timeout time.Duration) error {
 	deadline := time.Now().Add(timeout)
+	client := &http.Client{Timeout: time.Second}
 	for time.Now().Before(deadline) {
-		conn, err := net.DialTimeout("tcp", addr, 1*time.Second)
+		resp, err := client.Get("http://" + addr + "/healthz")
 		if err == nil {
-			conn.Close()
-			return nil
+			resp.Body.Close()
+			if resp.StatusCode == http.StatusOK && proc.exited() == nil {
+				return nil
+			}
+		}
+		if exitErr := proc.exited(); exitErr != nil {
+			return fmt.Errorf("process exited (%v) before %s accepted connections: %s", exitErr, addr, logTail(proc.logPath))
 		}
 		time.Sleep(100 * time.Millisecond)
 	}
-	return fmt.Errorf("service at %s not ready within timeout", addr)
+	return fmt.Errorf("service at %s not ready within timeout: %s", addr, logTail(proc.logPath))
 }
 
-// waitForMount waits for the FUSE mount to be ready
+// waitForMount waits for the FUSE mount to be ready. A stat/ReadDir probe
+// alone is not enough: the bare mount point directory passes both before the
+// mount process finishes starting, letting tests race ahead and write to the
+// local disk underneath the mount. The mount point's device ID differing from
+// its parent's confirms a filesystem is actually mounted there.
 func (f *FuseTestFramework) waitForMount(timeout time.Duration) error {
-	// The mount point is created before weed mount starts, so Stat/ReadDir
-	// alone succeed on the plain directory. The device id only differs from
-	// its parent's once a filesystem is actually mounted on it.
 	parentDev, err := deviceID(filepath.Dir(f.mountPoint))
 	if err != nil {
-		return fmt.Errorf("stat mount parent: %w", err)
+		return fmt.Errorf("stat mount point parent: %w", err)
 	}
 	deadline := time.Now().Add(timeout)
 	for time.Now().Before(deadline) {
@@ -408,9 +540,76 @@ func (f *FuseTestFramework) waitForMount(timeout time.Duration) error {
 				return nil
 			}
 		}
+		// A mount that cannot mount at all (no /dev/fuse, fusermount not setuid)
+		// dies within a second; reporting that beats waiting out the timeout.
+		if exitErr := f.mountProcess.exited(); exitErr != nil {
+			return fmt.Errorf("mount process exited (%v): %s", exitErr, logTail(f.mountProcess.logPath))
+		}
 		time.Sleep(100 * time.Millisecond)
 	}
-	return fmt.Errorf("mount point not ready within timeout")
+	return fmt.Errorf("mount point not ready within timeout: %s", logTail(f.mountProcess.logPath))
+}
+
+// waitForWritable probes the mount with a real write+read+remove until one
+// succeeds, covering the window where a fresh cluster cannot yet assign a
+// file id (volume not registered, grow pending, leader settle). Each attempt
+// runs off the loop because a wedged mount can hold a file op open, and one
+// attempt stays in flight until it finishes or the overall deadline passes.
+func (f *FuseTestFramework) waitForWritable(timeout time.Duration) error {
+	probe := filepath.Join(f.mountPoint, ".framework_write_probe")
+	payload := []byte("probe")
+	deadline := time.Now().Add(timeout)
+	var lastErr error
+	for time.Now().Before(deadline) {
+		if exitErr := f.mountProcess.exited(); exitErr != nil {
+			return fmt.Errorf("mount process exited (%v): %s", exitErr, logTail(f.mountProcess.logPath))
+		}
+		result := make(chan error, 1)
+		go func() {
+			if err := os.WriteFile(probe, payload, 0644); err != nil {
+				result <- err
+				return
+			}
+			data, err := os.ReadFile(probe)
+			if err != nil {
+				result <- err
+				return
+			}
+			if len(data) != len(payload) {
+				result <- fmt.Errorf("read back %d bytes, want %d", len(data), len(payload))
+				return
+			}
+			result <- os.Remove(probe)
+		}()
+		tick := time.NewTicker(500 * time.Millisecond)
+		attemptOpen := true
+		for attemptOpen {
+			select {
+			case err := <-result:
+				attemptOpen = false
+				if err == nil {
+					tick.Stop()
+					return nil
+				}
+				lastErr = err
+			case <-tick.C:
+				if exitErr := f.mountProcess.exited(); exitErr != nil {
+					tick.Stop()
+					return fmt.Errorf("mount process exited (%v): %s", exitErr, logTail(f.mountProcess.logPath))
+				}
+				if !time.Now().Before(deadline) {
+					tick.Stop()
+					if lastErr == nil {
+						return fmt.Errorf("write probe did not complete within timeout")
+					}
+					return fmt.Errorf("write probe failed within timeout: %v", lastErr)
+				}
+			}
+		}
+		tick.Stop()
+		time.Sleep(500 * time.Millisecond)
+	}
+	return fmt.Errorf("write probe failed within timeout: %v", lastErr)
 }
 
 // fuseAvailable reports whether the host has a FUSE device node to mount

test/fuse_integration/framework_test.go

@@ -8,6 +8,7 @@ import (
 	"os/exec"
 	"path/filepath"
 	"strconv"
+	"strings"
 	"syscall"
 	"testing"
 	"time"
@@ -152,7 +153,7 @@ func (f *FuseTestFramework) Setup(config *TestConfig) error {
 	}
 
 	// Wait for filer to be ready (mini starts all services on filerPort)
-	if err := f.waitForService(f.miniProcess, f.filerAddr, 30*time.Second); err != nil {
+	if err := f.waitForService(f.miniProcess, f.filerAddr, 90*time.Second); err != nil {
 		f.dumpLog("mini")
 		return fmt.Errorf("weed mini not ready: %v", err)
 	}
@@ -163,11 +164,18 @@ func (f *FuseTestFramework) Setup(config *TestConfig) error {
 	}
 
 	// Wait for mount to be ready
-	if err := f.waitForMount(30 * time.Second); err != nil {
+	if err := f.waitForMount(90 * time.Second); err != nil {
 		f.dumpLog("mount")
 		return fmt.Errorf("FUSE mount not ready: %v", err)
 	}
 
+	// Gate on a real write: a fresh mini still cannot assign a file id until
+	// its volume registers and a grow round finishes.
+	if err := f.waitForWritable(90 * time.Second); err != nil {
+		f.dumpLog("mount")
+		return fmt.Errorf("FUSE mount not writable: %v", err)
+	}
+
 	f.isSetup = true
 	return nil
 }
@@ -286,11 +294,16 @@ func (f *FuseTestFramework) dumpLog(name string) {
 	f.t.Logf("[%s log tail (%d bytes)]\n%s", name, len(data), string(data))
 }
 
-// copyLogsForCI copies SeaweedFS process logs to /tmp/seaweedfs-fuse-logs/
-// so the CI workflow can upload them as artifacts.
+// copyLogsForCI copies SeaweedFS process logs under a per-test directory in
+// /tmp/seaweedfs-fuse-logs/ so the CI workflow can upload them as artifacts.
+// Each framework instance gets its own directory: every test in the suite
+// builds one, and fixed filenames would let a later passing test overwrite
+// the logs of an earlier failing one.
 func (f *FuseTestFramework) copyLogsForCI() {
-	ciLogDir := "/tmp/seaweedfs-fuse-logs"
-	os.MkdirAll(ciLogDir, 0755)
+	ciLogDir := filepath.Join("/tmp/seaweedfs-fuse-logs", strings.NewReplacer("/", "_", " ", "_").Replace(f.t.Name()))
+	if err := os.MkdirAll(ciLogDir, 0755); err != nil {
+		return
+	}
 	for _, name := range []string{"mini", "mount"} {
 		src := filepath.Join(f.logDir, name+".log")
 		data, err := os.ReadFile(src)
@@ -307,6 +320,10 @@ func (f *FuseTestFramework) startMini(config *TestConfig) error {
 	if config.EnableDebug {
 		args = append(args, "-v=4")
 	}
+	adminPort := freePort(f.t)
+	for adminPort == f.filerPort {
+		adminPort = freePort(f.t)
+	}
 	args = append(args,
 		"mini",
 		"-dir="+f.dataDir,
@@ -316,6 +333,9 @@ func (f *FuseTestFramework) startMini(config *TestConfig) error {
 		"-s3=false",
 		"-webdav=false",
 		"-admin.ui=false",
+		"-admin.port="+strconv.Itoa(adminPort),
+		"-admin.port.grpc="+strconv.Itoa(adminPort+grpcPortOffset),
+		"-admin.port.legacy=0",
 	)
 
 	proc, err := f.startProcess("mini", args)
@@ -421,6 +441,68 @@ func (f *FuseTestFramework) waitForMount(timeout time.Duration) error {
 	return fmt.Errorf("mount point not ready within timeout")
 }
 
+// waitForWritable probes the mount with a real write+read+remove until one
+// succeeds, covering the window where a fresh cluster cannot yet assign a
+// file id (volume not registered, grow pending, leader settle). Each attempt
+// runs off the loop because a wedged mount can hold a file op open, and one
+// attempt stays in flight until it finishes or the overall deadline passes.
+func (f *FuseTestFramework) waitForWritable(timeout time.Duration) error {
+	probe := filepath.Join(f.mountPoint, ".framework_write_probe")
+	payload := []byte("probe")
+	deadline := time.Now().Add(timeout)
+	var lastErr error
+	for time.Now().Before(deadline) {
+		if exitErr := f.mountProcess.exited(); exitErr != nil {
+			return fmt.Errorf("mount process exited (%v)", exitErr)
+		}
+		result := make(chan error, 1)
+		go func() {
+			if err := os.WriteFile(probe, payload, 0644); err != nil {
+				result <- err
+				return
+			}
+			data, err := os.ReadFile(probe)
+			if err != nil {
+				result <- err
+				return
+			}
+			if len(data) != len(payload) {
+				result <- fmt.Errorf("read back %d bytes, want %d", len(data), len(payload))
+				return
+			}
+			result <- os.Remove(probe)
+		}()
+		tick := time.NewTicker(500 * time.Millisecond)
+		attemptOpen := true
+		for attemptOpen {
+			select {
+			case err := <-result:
+				attemptOpen = false
+				if err == nil {
+					tick.Stop()
+					return nil
+				}
+				lastErr = err
+			case <-tick.C:
+				if exitErr := f.mountProcess.exited(); exitErr != nil {
+					tick.Stop()
+					return fmt.Errorf("mount process exited (%v)", exitErr)
+				}
+				if !time.Now().Before(deadline) {
+					tick.Stop()
+					if lastErr == nil {
+						return fmt.Errorf("write probe did not complete within timeout")
+					}
+					return fmt.Errorf("write probe failed within timeout: %v", lastErr)
+				}
+			}
+		}
+		tick.Stop()
+		time.Sleep(500 * time.Millisecond)
+	}
+	return fmt.Errorf("write probe failed within timeout: %v", lastErr)
+}
+
 // deviceID returns the device ID of the filesystem containing path.
 func deviceID(path string) (uint64, error) {
 	var st syscall.Stat_t

test/fuse_integration/posix_external_test.go

@@ -102,10 +102,13 @@ func (s *ExternalPOSIXTestSuite) TestPjdFsTest(t *testing.T) {
 		}
 	}
 
-	// Run each test category
+	// pjdfstest asserts full POSIX semantics the mount does not provide
+	// yet, so category failures are report-only unless the run asked for
+	// external tools to gate.
+	strict := os.Getenv("POSIX_REQUIRE_EXTERNAL_TOOLS") == "true"
 	for _, test := range pjdTests {
 		t.Run(test.name, func(t *testing.T) {
-			s.runPjdTest(t, pjdDir, test.testPath, testDir, test.critical)
+			s.runPjdTest(t, pjdDir, test.testPath, testDir, strict && test.critical)
 		})
 	}
 }

test/kms/Makefile

@@ -174,12 +174,12 @@ start-openbao-ci: ## Start OpenBao via Docker for CI
 		ghcr.io/openbao/openbao:latest \
 		bao server -dev -dev-root-token-id=$(OPENBAO_TOKEN) -dev-listen-address=0.0.0.0:8200
 	@echo "Waiting for OpenBao to be ready..."
-	@for i in $$(seq 1 30); do \
+	@for i in $$(seq 1 90); do \
 		if curl -s $(OPENBAO_ADDR)/v1/sys/health > /dev/null 2>&1; then \
 			echo "$(GREEN)OpenBao is ready$(NC)"; \
 			break; \
 		fi; \
-		if [ $$i -eq 30 ]; then \
+		if [ $$i -eq 90 ]; then \
 			echo "$(RED)Timeout waiting for OpenBao$(NC)"; \
 			docker logs openbao-ci 2>&1 | tail -20; \
 			exit 1; \
@@ -213,12 +213,12 @@ start-seaweedfs-ci: check-binary ## Start weed mini with OpenBao KMS config
 		-ip=127.0.0.1 \
 		> /tmp/seaweedfs-kms-mini.log 2>&1 & echo $$! > /tmp/weed-kms-mini.pid
 	@echo "Checking S3 service is ready..."
-	@for i in $$(seq 1 30); do \
+	@for i in $$(seq 1 90); do \
 		if curl -s http://127.0.0.1:$(S3_PORT) > /dev/null 2>&1; then \
 			echo "$(GREEN)S3 service is ready$(NC)"; \
 			break; \
 		fi; \
-		if [ $$i -eq 30 ]; then \
+		if [ $$i -eq 90 ]; then \
 			echo "$(RED)Timeout waiting for S3 service$(NC)"; \
 			echo "=== Last 50 lines of server log ==="; \
 			tail -50 /tmp/seaweedfs-kms-mini.log 2>/dev/null || echo "No server log found"; \

test/s3/copying/Makefile

@@ -76,12 +76,12 @@ start-seaweedfs: check-binary
 		-ip=127.0.0.1 \
 		> /tmp/seaweedfs-mini.log 2>&1 & echo $$! > /tmp/weed-mini.pid
 	@echo "$(YELLOW)Waiting for S3 service to be ready...$(NC)"
-	@for i in $$(seq 1 30); do \
+	@for i in $$(seq 1 90); do \
 		if curl -s -f http://127.0.0.1:$(S3_PORT) > /dev/null 2>&1; then \
 			echo "$(GREEN)S3 service is ready$(NC)"; \
 			break; \
 		fi; \
-		echo "Waiting for S3 service... ($$i/30)"; \
+		echo "Waiting for S3 service... ($$i/90)"; \
 		sleep 1; \
 	done
 	@sleep 2

test/s3/cors/Makefile

@@ -92,7 +92,7 @@ start-server: check-deps
 		echo $$! > weed-server.pid
 	
 	@echo "Waiting for S3 server to be ready..."
-	@for i in $$(seq 1 30); do \
+	@for i in $$(seq 1 90); do \
 		if echo | nc -z localhost $(S3_PORT); then \
 			echo "S3 server is ready!"; \
 			exit 0; \

test/s3/filer_group/Makefile

@@ -79,14 +79,14 @@ start-server: check-deps
 		echo $$! > weed-server.pid
 	
 	@echo "Waiting for S3 server to be ready..."
-	@for i in $$(seq 1 30); do \
+	@for i in $$(seq 1 90); do \
 		if echo | nc -z localhost $(S3_PORT); then \
 			echo "S3 server is ready!"; \
 			exit 0; \
 		fi; \
 		sleep 1; \
 	done; \
-	echo "❌ Server failed to start within 30 seconds"; \
+	echo "❌ Server failed to start within 90 seconds"; \
 	if [ -f weed-server.log ]; then \
 		cat weed-server.log; \
 	fi; \

test/s3/iam/Makefile

@@ -81,7 +81,7 @@ start-services: ## Start SeaweedFS services for testing
 wait-for-services: ## Wait for all services to be ready
 	@echo "Waiting for services to be ready..."
 	@echo "Checking S3 API server..."
-	@for i in $$(seq 1 30); do \
+	@for i in $$(seq 1 90); do \
 		if curl -s http://localhost:$(S3_PORT) > /dev/null 2>&1; then \
 			echo "OK S3 API server is ready"; \
 			exit 0; \

test/s3/iam/s3_iam_admin_test.go

@@ -295,8 +295,8 @@ func TestIAMPolicyManagement(t *testing.T) {
 		assert.Equal(t, true, aws.BoolValue(listVersionsResp.Versions[0].IsDefaultVersion))
 
 		getVersionResp, err := iamClient.GetPolicyVersion(&iam.GetPolicyVersionInput{
-			PolicyArn:  createResp.Policy.Arn,
-			VersionId:  aws.String("v1"),
+			PolicyArn: createResp.Policy.Arn,
+			VersionId: aws.String("v1"),
 		})
 		require.NoError(t, err)
 		require.NotNil(t, getVersionResp.PolicyVersion)

test/s3/iam/s3_sts_get_federation_token_test.go

@@ -14,8 +14,8 @@ import (
 	"github.com/aws/aws-sdk-go/aws"
 	"github.com/aws/aws-sdk-go/aws/credentials"
 	"github.com/aws/aws-sdk-go/aws/session"
-	"github.com/aws/aws-sdk-go/service/s3"
 	v4 "github.com/aws/aws-sdk-go/aws/signer/v4"
+	"github.com/aws/aws-sdk-go/service/s3"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 )

test/s3/lifecycle/Makefile

@@ -42,6 +42,8 @@ start-server: build-weed
 	@mkdir -p $(SERVER_DIR)
 	@AWS_ACCESS_KEY_ID=$(ACCESS_KEY) AWS_SECRET_ACCESS_KEY=$(SECRET_KEY) $(WEED_BINARY) mini \
 		-dir=$(SERVER_DIR) \
+		-ip=127.0.0.1 \
+		-ip.bind=127.0.0.1 \
 		-s3.port=$(S3_PORT) \
 		-filer.port.grpc=$(FILER_GRPC_PORT) \
 		> weed-test.log 2>&1 & \