From 34c7bd03add0394c9911d2f1c77b038e70fa9e65 Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Sat, 24 Oct 2026 07:07:55 -0700 Subject: [PATCH] ci: clean up stale FUSE mounts before running tests (#12020) * ci(s3tables): tolerate stale FUSE mounts in disk diagnostics * ci(s3tables): bound df diagnostics with timeout, explain once A stale FUSE mount can also hang statfs, not just fail it, so wrap df in timeout 15; one comment at the first site covers all sixteen. * ci: tolerate stale FUSE mounts in df diagnostics elsewhere The same self-hosted runners serve the other workflow files; a bare df -h there can stall or fail a job the same way. * ci: unmount disconnected FUSE test mounts before jobs start * ci: skip FUSE cleanup when the variant checkout is skipped * ci: reclaim the crashed run's test dir once its stale FUSE mount is detached * ci: stop tracking python bytecode for the FUSE cleanup action The helper's __pycache__ slipped into the tree; ignore it so local test runs cannot re-add it. Generated with [Devin](https://devin.ai) Co-Authored-By: Devin <158243242+devin-ai-integration[bot]@users.noreply.github.com> * ci: run the FUSE cleanup action's unit tests on action changes test_cleanup.py was never wired into CI, so a regression in cleanup.py would ship silently. A path-filtered workflow runs it on every change under .github/actions/. Generated with [Devin](https://devin.ai) Co-Authored-By: Devin <158243242+devin-ai-integration[bot]@users.noreply.github.com> * ci: match every FUSE test mount layout in the cleanup action The candidates list only matched seaweedfs_fuse_test_*/mount, so stale mounts left by fuse_failover and fuse_p2p runs (mountN leaves under seaweedfs_fuse_* dirs) and the posix-compliance workflow's fixed /tmp/seaweedfs/mount were never reclaimed and could still wedge df. * ci: run the FUSE mount cleanup in the fuse_failover and fuse_p2p workflows --------- Co-authored-by: Devin <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../cleanup-stale-fuse-mounts/action.yml | 11 ++ .../cleanup-stale-fuse-mounts/cleanup.py | 89 ++++++++++++ .../cleanup-stale-fuse-mounts/test_cleanup.py | 137 ++++++++++++++++++ .github/workflows/action-tests.yml | 30 ++++ .github/workflows/container_latest.yml | 7 +- .../workflows/container_release_unified.yml | 6 +- .github/workflows/fuse-failover.yml | 3 + .github/workflows/fuse-p2p-integration.yml | 3 + .github/workflows/posix-compliance.yml | 5 +- .github/workflows/s3-filer-group-tests.yml | 5 +- .github/workflows/s3-go-tests.yml | 20 ++- .github/workflows/s3-iam-tests.yml | 5 +- .github/workflows/s3-keycloak-tests.yml | 5 +- .github/workflows/s3-policy-tests.yml | 10 +- .github/workflows/s3-spark-tests.yml | 5 +- .github/workflows/s3-sse-tests.yml | 5 +- .github/workflows/s3-tables-tests.yml | 81 +++++++++-- .github/workflows/tus-tests.yml | 5 +- .gitignore | 1 + 19 files changed, 401 insertions(+), 32 deletions(-) create mode 100644 .github/actions/cleanup-stale-fuse-mounts/action.yml create mode 100644 .github/actions/cleanup-stale-fuse-mounts/cleanup.py create mode 100644 .github/actions/cleanup-stale-fuse-mounts/test_cleanup.py create mode 100644 .github/workflows/action-tests.yml diff --git a/.github/actions/cleanup-stale-fuse-mounts/action.yml b/.github/actions/cleanup-stale-fuse-mounts/action.yml new file mode 100644 index 0000000000..7e6ddab7bd --- /dev/null +++ b/.github/actions/cleanup-stale-fuse-mounts/action.yml @@ -0,0 +1,11 @@ +name: Clean up stale SeaweedFS test mounts +description: Detach disconnected FUSE test mounts left by earlier jobs on a shared runner. +runs: + using: composite + steps: + - name: Unmount disconnected SeaweedFS test mounts + if: runner.os == 'Linux' && runner.environment == 'self-hosted' + shell: bash + env: + CLEANUP_SCRIPT: ${{ github.action_path }}/cleanup.py + run: python3 "$CLEANUP_SCRIPT" diff --git a/.github/actions/cleanup-stale-fuse-mounts/cleanup.py b/.github/actions/cleanup-stale-fuse-mounts/cleanup.py new file mode 100644 index 0000000000..ad17a767fa --- /dev/null +++ b/.github/actions/cleanup-stale-fuse-mounts/cleanup.py @@ -0,0 +1,89 @@ +#!/usr/bin/env python3 +"""Detach disconnected SeaweedFS integration-test mounts without touching live ones.""" + +import errno +import os +from pathlib import Path +import re +import shutil +import subprocess +import sys +import tempfile + + +def test_mounts(mountinfo): + """Read the kernel table directly: traversing a stale mount may block.""" + roots = {tempfile.gettempdir(), "/tmp", "/var/tmp"} + for line in mountinfo.splitlines(): + fields, filesystem = line.split(" - ", 1) + if filesystem.split()[0].split(".")[0] != "fuse": + continue + fields = fields.split() + # mountinfo escapes whitespace and backslashes with octal sequences. + target = re.sub(r"\\([0-7]{3})", lambda m: chr(int(m[1], 8)), fields[4]) + path = Path(target) + # Test mounts are a mount* leaf under a seaweedfs_fuse_* scratch dir + # (seaweedfs_fuse_test_*/mount, seaweedfs_fuse_failover_*/mount0, ...) + # or the posix-compliance workflow's fixed /tmp/seaweedfs/mount. + if (not path.name.startswith("mount") + or str(path.parent.parent) not in roots + or not (path.parent.name.startswith("seaweedfs_fuse") + or path.parent.name == "seaweedfs")): + continue + yield fields[0], target + + +def mounted(mount_id, target): + return (mount_id, target) in test_mounts(Path("/proc/self/mountinfo").read_text()) + + +def cleanup(): + failed = False + for mount_id, target in test_mounts(Path("/proc/self/mountinfo").read_text()): + try: + # A separate, bounded probe prevents a wedged FUSE daemon from + # hanging cleanup. Only ENOTCONN proves the daemon disconnected; + # a timeout or another error is not enough to detach a live mount. + probe = subprocess.run( + [sys.executable, __file__, "--probe", target], timeout=5, + capture_output=True, check=False, + ) + except subprocess.TimeoutExpired: + print(f"::warning::FUSE probe timed out; leaving {target} mounted") + continue + if probe.returncode != errno.ENOTCONN or not mounted(mount_id, target): + continue + + print(f"Unmounting disconnected FUSE test mount: {target}", flush=True) + # Lazy unmount detaches a dead mount even when an old process still + # holds it open. Prefer the unprivileged FUSE helper; sudo is fallback. + for command in ( + ["fusermount3", "-uz", "--", target], + ["fusermount", "-uz", "--", target], + ["sudo", "-n", "umount", "-l", "--", target], + ): + try: + subprocess.run(command, timeout=10, check=False) + except (OSError, subprocess.TimeoutExpired) as error: + print(f"{command[0]}: {error}", file=sys.stderr) + if not mounted(mount_id, target): + break + else: + print(f"::error::Could not unmount disconnected FUSE test mount: {target}") + failed = True + continue + # The detached mount's parent still holds the crashed run's data and + # logs; test_mounts only yields targets under a SeaweedFS test dir in + # a temp root, so the parent is safe to reclaim here. + shutil.rmtree(Path(target).parent, ignore_errors=True) + return int(failed) + + +if __name__ == "__main__": + if len(sys.argv) == 3 and sys.argv[1] == "--probe": + try: + os.stat(sys.argv[2]) + except OSError as error: + sys.exit(error.errno) + sys.exit(0) + sys.exit(cleanup()) diff --git a/.github/actions/cleanup-stale-fuse-mounts/test_cleanup.py b/.github/actions/cleanup-stale-fuse-mounts/test_cleanup.py new file mode 100644 index 0000000000..b8ee3fa028 --- /dev/null +++ b/.github/actions/cleanup-stale-fuse-mounts/test_cleanup.py @@ -0,0 +1,137 @@ +import errno +import importlib.util +from pathlib import Path +import subprocess +import unittest +from unittest.mock import patch + +spec = importlib.util.spec_from_file_location("cleanup", Path(__file__).with_name("cleanup.py")) +cleanup = importlib.util.module_from_spec(spec) +spec.loader.exec_module(cleanup) + +TARGET = "/tmp/seaweedfs_fuse_test_123/mount" +MOUNTINFO = f"42 1 0:9 / {TARGET} rw - fuse.seaweedfs seaweedfs rw\n" + + +class CleanupTests(unittest.TestCase): + def run_cleanup(self, results, mounted=True): + with patch.object(cleanup.Path, "read_text", return_value=MOUNTINFO), \ + patch.object(cleanup, "mounted", side_effect=mounted if isinstance(mounted, list) else None, + return_value=mounted) as check, \ + patch.object(cleanup.shutil, "rmtree") as rmtree, \ + patch.object(cleanup.subprocess, "run", side_effect=results) as run: + status = cleanup.cleanup() + return status, run, check, rmtree + + def result(self, status): + return subprocess.CompletedProcess([], status) + + def test_only_fuse_integration_test_mounts_are_candidates(self): + rows = MOUNTINFO + rows += MOUNTINFO.replace("fuse.seaweedfs", "ext4") + rows += MOUNTINFO.replace(TARGET, "/mnt/production") + rows += MOUNTINFO.replace(TARGET, "/tmp/seaweedfs_fuse_test_123/data") + rows += MOUNTINFO.replace(TARGET, "/home/user/seaweedfs_fuse_test_123/mount") + rows += MOUNTINFO.replace(TARGET, "/tmp/seaweedfs/data") + self.assertEqual(list(cleanup.test_mounts(rows)), [("42", TARGET)]) + + def test_all_test_mount_layouts_are_candidates(self): + targets = ( + "/tmp/seaweedfs_fuse_test_123/mount", + "/tmp/seaweedfs_fuse_failover_456/mount0", + "/tmp/seaweedfs_fuse_failover_456/mount1", + "/var/tmp/seaweedfs_fuse_p2p_test_789/mount2", + "/tmp/seaweedfs/mount", + ) + rows = "".join( + MOUNTINFO.replace("42", str(40 + i)).replace(TARGET, target) + for i, target in enumerate(targets) + ) + self.assertEqual( + list(cleanup.test_mounts(rows)), + [(str(40 + i), target) for i, target in enumerate(targets)], + ) + + def test_mountinfo_paths_are_unescaped_once(self): + target = "/tmp/seaweedfs_fuse_test_space and\\slash/mount" + encoded = target.replace("\\", r"\134").replace(" ", r"\040") + rows = MOUNTINFO.replace(TARGET, encoded) + self.assertEqual(list(cleanup.test_mounts(rows)), [("42", target)]) + + def test_no_mounts_is_a_noop(self): + with patch.object(cleanup.Path, "read_text", return_value=""), \ + patch.object(cleanup.subprocess, "run") as run: + self.assertEqual(cleanup.cleanup(), 0) + run.assert_not_called() + + def test_healthy_mount_is_untouched(self): + status, run, _, _ = self.run_cleanup([self.result(0)]) + self.assertEqual(status, 0) + self.assertEqual(run.call_count, 1) + + def test_other_probe_errors_are_untouched(self): + for error in (errno.EACCES, errno.ENOENT, errno.EIO): + with self.subTest(error=error): + status, run, _, _ = self.run_cleanup([self.result(error)]) + self.assertEqual(status, 0) + self.assertEqual(run.call_count, 1) + + def test_probe_timeout_does_not_detach_a_possibly_live_mount(self): + status, run, _, _ = self.run_cleanup([subprocess.TimeoutExpired("probe", 5)]) + self.assertEqual(status, 0) + self.assertEqual(run.call_count, 1) + + def test_disconnected_mount_is_lazily_unmounted(self): + status, run, _, _ = self.run_cleanup([self.result(errno.ENOTCONN), self.result(0)], [True, False]) + self.assertEqual(status, 0) + self.assertEqual(run.call_args_list[1].args[0], ["fusermount3", "-uz", "--", TARGET]) + self.assertEqual(run.call_args_list[0].kwargs["timeout"], 5) + self.assertEqual(run.call_args_list[1].kwargs["timeout"], 10) + + def test_replaced_or_removed_mount_is_untouched(self): + status, run, _, _ = self.run_cleanup([self.result(errno.ENOTCONN)], False) + self.assertEqual(status, 0) + self.assertEqual(run.call_count, 1) + + def test_mount_identity_is_checked_without_stat(self): + with patch.object(cleanup.Path, "read_text", return_value=MOUNTINFO): + self.assertTrue(cleanup.mounted("42", TARGET)) + self.assertFalse(cleanup.mounted("43", TARGET)) + + def test_missing_or_timed_out_helper_uses_fallback(self): + for error in (FileNotFoundError(), subprocess.TimeoutExpired("fusermount3", 10)): + with self.subTest(error=error): + status, run, _, _ = self.run_cleanup( + [self.result(errno.ENOTCONN), error, self.result(0)], [True, True, False]) + self.assertEqual(status, 0) + self.assertEqual(run.call_args.args[0], ["fusermount", "-uz", "--", TARGET]) + + def test_failed_helpers_fall_back_to_sudo(self): + status, run, _, _ = self.run_cleanup( + [self.result(errno.ENOTCONN), self.result(1), self.result(1), self.result(0)], + [True, True, True, False]) + self.assertEqual(status, 0) + self.assertEqual(run.call_args.args[0], ["sudo", "-n", "umount", "-l", "--", TARGET]) + + def test_cleanup_fails_when_stale_mount_remains(self): + status, run, _, rmtree = self.run_cleanup( + [self.result(errno.ENOTCONN), self.result(1), self.result(1), self.result(1)]) + self.assertEqual(status, 1) + self.assertEqual(run.call_count, 4) + rmtree.assert_not_called() + + def test_unmounted_test_dir_is_reclaimed(self): + status, _, _, rmtree = self.run_cleanup( + [self.result(errno.ENOTCONN), self.result(0)], [True, False]) + self.assertEqual(status, 0) + rmtree.assert_called_once_with(Path(TARGET).parent, ignore_errors=True) + + def test_probe_process_reports_errno(self): + for target, expected in ((str(Path(__file__)), 0), (str(Path(__file__) / "absent"), errno.ENOTDIR)): + with self.subTest(target=target): + result = subprocess.run([cleanup.sys.executable, cleanup.__file__, "--probe", target], check=False) + self.assertEqual(result.returncode, expected) + + +if __name__ == "__main__": + unittest.main() diff --git a/.github/workflows/action-tests.yml b/.github/workflows/action-tests.yml new file mode 100644 index 0000000000..88f33ed93c --- /dev/null +++ b/.github/workflows/action-tests.yml @@ -0,0 +1,30 @@ +name: "actions: unit tests" + +on: + push: + branches: [ master ] + paths: + - '.github/actions/**' + - '.github/workflows/action-tests.yml' + pull_request: + branches: [ master ] + paths: + - '.github/actions/**' + - '.github/workflows/action-tests.yml' + +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || (github.event_name == 'push' && github.ref) || github.run_id }} + cancel-in-progress: ${{ github.event_name == 'pull_request' || (github.event_name == 'push' && !startsWith(github.ref, 'refs/tags/')) }} + +permissions: + contents: read + +jobs: + cleanup-stale-fuse-mounts: + runs-on: ubuntu-latest + steps: + - name: Check out code + uses: actions/checkout@v7 + + - name: Run cleanup action unit tests + run: python3 .github/actions/cleanup-stale-fuse-mounts/test_cleanup.py diff --git a/.github/workflows/container_latest.yml b/.github/workflows/container_latest.yml index 0dc49db29c..8eccfc89d4 100644 --- a/.github/workflows/container_latest.yml +++ b/.github/workflows/container_latest.yml @@ -170,10 +170,13 @@ jobs: uses: actions/checkout@v7 with: ref: ${{ github.event_name == 'workflow_dispatch' && github.event.inputs.source_ref || github.ref }} + - name: Clean up stale FUSE mounts + uses: ./.github/actions/cleanup-stale-fuse-mounts + - name: Free Disk Space run: | echo "Available disk space before cleanup:" - df -h + timeout 15 df -h || true # Remove pre-installed tools sudo rm -rf /usr/share/dotnet /usr/local/lib/android /opt/ghc /opt/hostedtoolcache/CodeQL # Clean package managers @@ -185,7 +188,7 @@ jobs: [ -d ~/.cache/go-build ] && rm -rf ~/.cache/go-build || true [ -d /go/pkg ] && rm -rf /go/pkg || true echo "Available disk space after cleanup:" - df -h + timeout 15 df -h || true - name: Configure variant id: config diff --git a/.github/workflows/container_release_unified.yml b/.github/workflows/container_release_unified.yml index 320436cb2e..17d976509e 100644 --- a/.github/workflows/container_release_unified.yml +++ b/.github/workflows/container_release_unified.yml @@ -180,6 +180,10 @@ jobs: if: github.event_name != 'workflow_dispatch' || github.event.inputs.variant == 'all' || github.event.inputs.variant == matrix.variant uses: actions/checkout@v7 + - name: Clean up stale FUSE mounts + if: github.event_name != 'workflow_dispatch' || github.event.inputs.variant == 'all' || github.event.inputs.variant == matrix.variant + uses: ./.github/actions/cleanup-stale-fuse-mounts + - name: Download pre-built Rust binaries if: github.event_name != 'workflow_dispatch' || github.event.inputs.variant == 'all' || github.event.inputs.variant == matrix.variant uses: actions/download-artifact@v8 @@ -219,7 +223,7 @@ jobs: sudo docker system prune -af --volumes [ -d ~/.cache/go-build ] && rm -rf ~/.cache/go-build || true [ -d /go/pkg ] && rm -rf /go/pkg || true - df -h + timeout 15 df -h || true - name: Docker meta if: github.event_name != 'workflow_dispatch' || github.event.inputs.variant == 'all' || github.event.inputs.variant == matrix.variant diff --git a/.github/workflows/fuse-failover.yml b/.github/workflows/fuse-failover.yml index 21646dbe17..2c03cf9369 100644 --- a/.github/workflows/fuse-failover.yml +++ b/.github/workflows/fuse-failover.yml @@ -42,6 +42,9 @@ jobs: with: persist-credentials: false + - name: Clean up stale FUSE mounts + uses: ./.github/actions/cleanup-stale-fuse-mounts + - name: Set up Go uses: actions/setup-go@v7 with: diff --git a/.github/workflows/fuse-p2p-integration.yml b/.github/workflows/fuse-p2p-integration.yml index 78df490b43..a65f5ddbc2 100644 --- a/.github/workflows/fuse-p2p-integration.yml +++ b/.github/workflows/fuse-p2p-integration.yml @@ -41,6 +41,9 @@ jobs: - name: Check out code uses: actions/checkout@v7 + - name: Clean up stale FUSE mounts + uses: ./.github/actions/cleanup-stale-fuse-mounts + - name: Set up Go uses: actions/setup-go@v7 with: diff --git a/.github/workflows/posix-compliance.yml b/.github/workflows/posix-compliance.yml index 25fbfe5096..66efc0de44 100644 --- a/.github/workflows/posix-compliance.yml +++ b/.github/workflows/posix-compliance.yml @@ -58,6 +58,9 @@ jobs: with: fetch-depth: 1 + - name: Clean up stale FUSE mounts + uses: ./.github/actions/cleanup-stale-fuse-mounts + - name: Set up Go ${{ env.GO_VERSION }} uses: actions/setup-go@v7 with: @@ -393,7 +396,7 @@ jobs: echo "" >> reports/debug/system_info.txt echo "Disk Space" >> reports/debug/system_info.txt - df -h >> reports/debug/system_info.txt + timeout 15 df -h >> reports/debug/system_info.txt || true echo "" >> reports/debug/system_info.txt echo "Memory Information" >> reports/debug/system_info.txt diff --git a/.github/workflows/s3-filer-group-tests.yml b/.github/workflows/s3-filer-group-tests.yml index 18f64fd78c..d714ca98e3 100644 --- a/.github/workflows/s3-filer-group-tests.yml +++ b/.github/workflows/s3-filer-group-tests.yml @@ -32,6 +32,9 @@ jobs: - name: Check out code uses: actions/checkout@v7 + - name: Clean up stale FUSE mounts + uses: ./.github/actions/cleanup-stale-fuse-mounts + - name: Set up Go uses: actions/setup-go@v7 with: @@ -51,7 +54,7 @@ jobs: echo "=== System Information ===" uname -a free -h - df -h + timeout 15 df -h || true echo "=== Starting Filer Group Tests ===" # Run tests with automatic server management diff --git a/.github/workflows/s3-go-tests.yml b/.github/workflows/s3-go-tests.yml index 7660ce1d71..1260e5af41 100644 --- a/.github/workflows/s3-go-tests.yml +++ b/.github/workflows/s3-go-tests.yml @@ -35,6 +35,9 @@ jobs: - name: Check out code uses: actions/checkout@v7 + - name: Clean up stale FUSE mounts + uses: ./.github/actions/cleanup-stale-fuse-mounts + - name: Set up Go uses: actions/setup-go@v7 with: @@ -54,7 +57,7 @@ jobs: echo "=== System Information ===" uname -a free -h - df -h . /tmp + timeout 15 df -h || true echo "=== Starting Tests ===" # Run tests with automatic server management @@ -201,6 +204,9 @@ jobs: - name: Check out code uses: actions/checkout@v7 + - name: Clean up stale FUSE mounts + uses: ./.github/actions/cleanup-stale-fuse-mounts + - name: Set up Go uses: actions/setup-go@v7 with: @@ -220,7 +226,7 @@ jobs: echo "=== System Information ===" uname -a free -h - df -h . /tmp + timeout 15 df -h || true echo "=== Starting Tests ===" # Run tests with automatic server management @@ -339,6 +345,9 @@ jobs: - name: Check out code uses: actions/checkout@v7 + - name: Clean up stale FUSE mounts + uses: ./.github/actions/cleanup-stale-fuse-mounts + - name: Set up Go uses: actions/setup-go@v7 with: @@ -358,7 +367,7 @@ jobs: echo "=== System Information ===" uname -a free -h - df -h . /tmp + timeout 15 df -h || true echo "=== Starting Tests ===" make test-with-server @@ -398,6 +407,9 @@ jobs: - name: Check out code uses: actions/checkout@v7 + - name: Clean up stale FUSE mounts + uses: ./.github/actions/cleanup-stale-fuse-mounts + - name: Set up Go uses: actions/setup-go@v7 with: @@ -417,7 +429,7 @@ jobs: echo "=== System Information ===" uname -a free -h - df -h . /tmp + timeout 15 df -h || true echo "=== Starting Tests ===" # Run tests with automatic server management diff --git a/.github/workflows/s3-iam-tests.yml b/.github/workflows/s3-iam-tests.yml index ee12afd842..9d57baf978 100644 --- a/.github/workflows/s3-iam-tests.yml +++ b/.github/workflows/s3-iam-tests.yml @@ -91,6 +91,9 @@ jobs: - name: Check out code uses: actions/checkout@v7 + - name: Clean up stale FUSE mounts + uses: ./.github/actions/cleanup-stale-fuse-mounts + - name: Set up Go uses: actions/setup-go@v7 with: @@ -111,7 +114,7 @@ jobs: echo "=== System Information ===" uname -a free -h - df -h + timeout 15 df -h || true echo "=== Starting S3 IAM Integration Tests (${{ matrix.test-type }}) ===" # Set WEED_BINARY to use the installed version diff --git a/.github/workflows/s3-keycloak-tests.yml b/.github/workflows/s3-keycloak-tests.yml index d893b1063b..f3955ca3fc 100644 --- a/.github/workflows/s3-keycloak-tests.yml +++ b/.github/workflows/s3-keycloak-tests.yml @@ -37,6 +37,9 @@ jobs: - name: Check out code uses: actions/checkout@v7 + - name: Clean up stale FUSE mounts + uses: ./.github/actions/cleanup-stale-fuse-mounts + - name: Set up Go uses: actions/setup-go@v7 with: @@ -57,7 +60,7 @@ jobs: echo "=== System Information ===" uname -a free -h - df -h + timeout 15 df -h || true echo "=== Starting S3 Keycloak Integration Tests ===" # Set WEED_BINARY to use the installed version diff --git a/.github/workflows/s3-policy-tests.yml b/.github/workflows/s3-policy-tests.yml index caa9f9de6f..2fbeff3474 100644 --- a/.github/workflows/s3-policy-tests.yml +++ b/.github/workflows/s3-policy-tests.yml @@ -89,6 +89,9 @@ jobs: - name: Check out code uses: actions/checkout@v7 + - name: Clean up stale FUSE mounts + uses: ./.github/actions/cleanup-stale-fuse-mounts + - name: Set up Go uses: actions/setup-go@v7 with: @@ -108,7 +111,7 @@ jobs: echo "=== System Information ===" uname -a free -h - df -h + timeout 15 df -h || true echo "=== Starting S3 Policy Variables Integration Tests ===" @@ -199,6 +202,9 @@ jobs: - name: Check out code uses: actions/checkout@v7 + - name: Clean up stale FUSE mounts + uses: ./.github/actions/cleanup-stale-fuse-mounts + - name: Set up Go uses: actions/setup-go@v7 with: @@ -218,7 +224,7 @@ jobs: echo "=== System Information ===" uname -a free -h - df -h + timeout 15 df -h || true echo "=== Starting S3 Policy Enforcement Tests (${{ matrix.test-case }}) ===" diff --git a/.github/workflows/s3-spark-tests.yml b/.github/workflows/s3-spark-tests.yml index 39ce8d9991..59fae691a8 100644 --- a/.github/workflows/s3-spark-tests.yml +++ b/.github/workflows/s3-spark-tests.yml @@ -27,6 +27,9 @@ jobs: - name: Check out code uses: actions/checkout@v7 + - name: Clean up stale FUSE mounts + uses: ./.github/actions/cleanup-stale-fuse-mounts + - name: Set up Go uses: actions/setup-go@v7 with: @@ -57,7 +60,7 @@ jobs: echo "=== System Information ===" uname -a free -h - df -h + timeout 15 df -h || true echo "=== Starting S3 Spark Integration Tests ===" go test -v -timeout 30m . 2>&1 | tee test-output.log || { diff --git a/.github/workflows/s3-sse-tests.yml b/.github/workflows/s3-sse-tests.yml index 4c44e42448..2ea0531f0a 100644 --- a/.github/workflows/s3-sse-tests.yml +++ b/.github/workflows/s3-sse-tests.yml @@ -46,6 +46,9 @@ jobs: - name: Check out code uses: actions/checkout@v7 + - name: Clean up stale FUSE mounts + uses: ./.github/actions/cleanup-stale-fuse-mounts + - name: Set up Go uses: actions/setup-go@v7 with: @@ -65,7 +68,7 @@ jobs: echo "=== System Information ===" uname -a free -h - df -h + timeout 15 df -h || true echo "=== Starting SSE Tests ===" # Run tests with automatic server management diff --git a/.github/workflows/s3-tables-tests.yml b/.github/workflows/s3-tables-tests.yml index baa50a4cb9..19674a3325 100644 --- a/.github/workflows/s3-tables-tests.yml +++ b/.github/workflows/s3-tables-tests.yml @@ -30,6 +30,9 @@ jobs: - name: Check out code uses: actions/checkout@v7 + - name: Clean up stale FUSE mounts + uses: ./.github/actions/cleanup-stale-fuse-mounts + - name: Set up Go uses: actions/setup-go@v7 with: @@ -53,7 +56,8 @@ jobs: echo "=== System Information ===" uname -a free -h - df -h + # A stale FUSE mount can hang or fail df; keep diagnostics best effort. + timeout 15 df -h || true echo "=== Starting S3 Tables Tests ===" # Run S3 Tables integration tests @@ -91,6 +95,9 @@ jobs: - name: Check out code uses: actions/checkout@v7 + - name: Clean up stale FUSE mounts + uses: ./.github/actions/cleanup-stale-fuse-mounts + - name: Set up Go uses: actions/setup-go@v7 with: @@ -115,7 +122,7 @@ jobs: echo "=== System Information ===" uname -a free -h - df -h + timeout 15 df -h || true echo "=== Starting Iceberg Catalog Tests ===" # Run Iceberg catalog integration tests using Makefile (handles build) @@ -153,6 +160,9 @@ jobs: - name: Check out code uses: actions/checkout@v7 + - name: Clean up stale FUSE mounts + uses: ./.github/actions/cleanup-stale-fuse-mounts + - name: Set up Go uses: actions/setup-go@v7 with: @@ -186,7 +196,7 @@ jobs: echo "=== System Information ===" uname -a free -h - df -h + timeout 15 df -h || true echo "=== Starting Trino Iceberg Catalog Tests ===" # Run Trino + Iceberg catalog integration tests @@ -224,6 +234,9 @@ jobs: - name: Check out code uses: actions/checkout@v7 + - name: Clean up stale FUSE mounts + uses: ./.github/actions/cleanup-stale-fuse-mounts + - name: Set up Go uses: actions/setup-go@v7 with: @@ -258,7 +271,7 @@ jobs: echo "=== System Information ===" uname -a free -h - df -h + timeout 15 df -h || true docker info echo "=== Starting Dremio Iceberg Catalog Tests ===" @@ -298,6 +311,9 @@ jobs: - name: Check out code uses: actions/checkout@v7 + - name: Clean up stale FUSE mounts + uses: ./.github/actions/cleanup-stale-fuse-mounts + - name: Set up Go uses: actions/setup-go@v7 with: @@ -332,7 +348,7 @@ jobs: echo "=== System Information ===" uname -a free -h - df -h + timeout 15 df -h || true docker info echo "=== Starting Doris Iceberg Catalog Tests ===" @@ -382,6 +398,9 @@ jobs: - name: Check out code uses: actions/checkout@v7 + - name: Clean up stale FUSE mounts + uses: ./.github/actions/cleanup-stale-fuse-mounts + - name: Set up Go uses: actions/setup-go@v7 with: @@ -418,7 +437,7 @@ jobs: echo "=== System Information ===" uname -a free -h - df -h + timeout 15 df -h || true docker info echo "=== Starting ClickHouse Iceberg Catalog Tests ===" @@ -470,6 +489,9 @@ jobs: - name: Check out code uses: actions/checkout@v7 + - name: Clean up stale FUSE mounts + uses: ./.github/actions/cleanup-stale-fuse-mounts + - name: Set up Go uses: actions/setup-go@v7 with: @@ -507,7 +529,7 @@ jobs: echo "=== System Information ===" uname -a free -h - df -h + timeout 15 df -h || true docker info echo "=== Starting OLake Iceberg Catalog Tests ===" @@ -570,6 +592,9 @@ jobs: - name: Check out code uses: actions/checkout@v7 + - name: Clean up stale FUSE mounts + uses: ./.github/actions/cleanup-stale-fuse-mounts + - name: Set up Go uses: actions/setup-go@v7 with: @@ -602,7 +627,7 @@ jobs: echo "=== System Information ===" uname -a free -h - df -h + timeout 15 df -h || true echo "=== Starting Polaris Tests ===" go test -v -timeout 20m ./test/s3tables/polaris 2>&1 | tee test/s3tables/polaris/test-output.log || { @@ -639,6 +664,9 @@ jobs: - name: Check out code uses: actions/checkout@v7 + - name: Clean up stale FUSE mounts + uses: ./.github/actions/cleanup-stale-fuse-mounts + - name: Set up Go uses: actions/setup-go@v7 with: @@ -672,7 +700,7 @@ jobs: echo "=== System Information ===" uname -a free -h - df -h + timeout 15 df -h || true echo "=== Starting Spark Iceberg Catalog Tests ===" # Run Spark + Iceberg catalog integration tests @@ -710,6 +738,9 @@ jobs: - name: Check out code uses: actions/checkout@v7 + - name: Clean up stale FUSE mounts + uses: ./.github/actions/cleanup-stale-fuse-mounts + - name: Set up Go uses: actions/setup-go@v7 with: @@ -744,7 +775,7 @@ jobs: echo "=== System Information ===" uname -a free -h - df -h + timeout 15 df -h || true echo "=== Starting RisingWave Catalog Tests ===" # Run RisingWave catalog integration tests @@ -782,6 +813,9 @@ jobs: - name: Check out code uses: actions/checkout@v7 + - name: Clean up stale FUSE mounts + uses: ./.github/actions/cleanup-stale-fuse-mounts + - name: Set up Go uses: actions/setup-go@v7 with: @@ -840,7 +874,7 @@ jobs: echo "=== System Information ===" uname -a free -h - df -h + timeout 15 df -h || true echo "=== Starting STS Integration Tests ===" # Run STS integration tests @@ -878,6 +912,9 @@ jobs: - name: Check out code uses: actions/checkout@v7 + - name: Clean up stale FUSE mounts + uses: ./.github/actions/cleanup-stale-fuse-mounts + - name: Set up Go uses: actions/setup-go@v7 with: @@ -901,7 +938,7 @@ jobs: echo "=== System Information ===" uname -a free -h - df -h + timeout 15 df -h || true echo "=== Starting Lakekeeper Integration Tests ===" # Run Lakekeeper integration tests @@ -939,6 +976,9 @@ jobs: - name: Check out code uses: actions/checkout@v7 + - name: Clean up stale FUSE mounts + uses: ./.github/actions/cleanup-stale-fuse-mounts + - name: Set up Go uses: actions/setup-go@v7 with: @@ -973,7 +1013,7 @@ jobs: echo "=== System Information ===" uname -a free -h - df -h + timeout 15 df -h || true docker info echo "=== Starting Unity Catalog Tests ===" @@ -1013,6 +1053,9 @@ jobs: - name: Check out code uses: actions/checkout@v7 + - name: Clean up stale FUSE mounts + uses: ./.github/actions/cleanup-stale-fuse-mounts + - name: Set up Go uses: actions/setup-go@v7 with: @@ -1046,7 +1089,7 @@ jobs: echo "=== System Information ===" uname -a free -h - df -h + timeout 15 df -h || true docker info echo "=== Starting LanceDB Namespace Tests ===" @@ -1157,6 +1200,9 @@ jobs: # so do not leave a token in the checkout for it to pick up. persist-credentials: false + - name: Clean up stale FUSE mounts + uses: ./.github/actions/cleanup-stale-fuse-mounts + - name: Set up Go uses: actions/setup-go@v7 with: @@ -1191,7 +1237,7 @@ jobs: echo "=== System Information ===" uname -a free -h - df -h + timeout 15 df -h || true docker info echo "=== Starting DuckDB Lance Tests ===" @@ -1233,6 +1279,9 @@ jobs: # so do not leave a token in the checkout for it to pick up. persist-credentials: false + - name: Clean up stale FUSE mounts + uses: ./.github/actions/cleanup-stale-fuse-mounts + - name: Set up Go uses: actions/setup-go@v7 with: @@ -1266,7 +1315,7 @@ jobs: echo "=== System Information ===" uname -a free -h - df -h + timeout 15 df -h || true docker info echo "=== Starting Spark Lance Namespace Tests ===" diff --git a/.github/workflows/tus-tests.yml b/.github/workflows/tus-tests.yml index 699c36b0b5..1140c7f505 100644 --- a/.github/workflows/tus-tests.yml +++ b/.github/workflows/tus-tests.yml @@ -31,6 +31,9 @@ jobs: - name: Check out code uses: actions/checkout@v7 + - name: Clean up stale FUSE mounts + uses: ./.github/actions/cleanup-stale-fuse-mounts + - name: Set up Go uses: actions/setup-go@v7 with: @@ -50,7 +53,7 @@ jobs: echo "=== System Information ===" uname -a free -h - df -h + timeout 15 df -h || true echo "=== Starting TUS Tests ===" # Run tests with automatic server management diff --git a/.gitignore b/.gitignore index 9a3bf80cd3..c383623df9 100644 --- a/.gitignore +++ b/.gitignore @@ -144,3 +144,4 @@ weed_bin telemetry/server/telemetry-server .aider* /seaweed-volume/docs +__pycache__/