mirror of
https://github.com/seaweedfs/seaweedfs.git
synced 2026-11-15 08:18:42 +01:00
Chris LuandDevin
34c7bd03ad
* ci(s3tables): tolerate stale FUSE mounts in disk diagnostics * ci(s3tables): bound df diagnostics with timeout, explain once A stale FUSE mount can also hang statfs, not just fail it, so wrap df in timeout 15; one comment at the first site covers all sixteen. * ci: tolerate stale FUSE mounts in df diagnostics elsewhere The same self-hosted runners serve the other workflow files; a bare df -h there can stall or fail a job the same way. * ci: unmount disconnected FUSE test mounts before jobs start * ci: skip FUSE cleanup when the variant checkout is skipped * ci: reclaim the crashed run's test dir once its stale FUSE mount is detached * ci: stop tracking python bytecode for the FUSE cleanup action The helper's __pycache__ slipped into the tree; ignore it so local test runs cannot re-add it. Generated with [Devin](https://devin.ai) Co-Authored-By: Devin <158243242+devin-ai-integration[bot]@users.noreply.github.com> * ci: run the FUSE cleanup action's unit tests on action changes test_cleanup.py was never wired into CI, so a regression in cleanup.py would ship silently. A path-filtered workflow runs it on every change under .github/actions/. Generated with [Devin](https://devin.ai) Co-Authored-By: Devin <158243242+devin-ai-integration[bot]@users.noreply.github.com> * ci: match every FUSE test mount layout in the cleanup action The candidates list only matched seaweedfs_fuse_test_*/mount, so stale mounts left by fuse_failover and fuse_p2p runs (mountN leaves under seaweedfs_fuse_* dirs) and the posix-compliance workflow's fixed /tmp/seaweedfs/mount were never reclaimed and could still wedge df. * ci: run the FUSE mount cleanup in the fuse_failover and fuse_p2p workflows --------- Co-authored-by: Devin <158243242+devin-ai-integration[bot]@users.noreply.github.com>
90 lines
3.5 KiB
Python
90 lines
3.5 KiB
Python
#!/usr/bin/env python3
|
|
"""Detach disconnected SeaweedFS integration-test mounts without touching live ones."""
|
|
|
|
import errno
|
|
import os
|
|
from pathlib import Path
|
|
import re
|
|
import shutil
|
|
import subprocess
|
|
import sys
|
|
import tempfile
|
|
|
|
|
|
def test_mounts(mountinfo):
|
|
"""Read the kernel table directly: traversing a stale mount may block."""
|
|
roots = {tempfile.gettempdir(), "/tmp", "/var/tmp"}
|
|
for line in mountinfo.splitlines():
|
|
fields, filesystem = line.split(" - ", 1)
|
|
if filesystem.split()[0].split(".")[0] != "fuse":
|
|
continue
|
|
fields = fields.split()
|
|
# mountinfo escapes whitespace and backslashes with octal sequences.
|
|
target = re.sub(r"\\([0-7]{3})", lambda m: chr(int(m[1], 8)), fields[4])
|
|
path = Path(target)
|
|
# Test mounts are a mount* leaf under a seaweedfs_fuse_* scratch dir
|
|
# (seaweedfs_fuse_test_*/mount, seaweedfs_fuse_failover_*/mount0, ...)
|
|
# or the posix-compliance workflow's fixed /tmp/seaweedfs/mount.
|
|
if (not path.name.startswith("mount")
|
|
or str(path.parent.parent) not in roots
|
|
or not (path.parent.name.startswith("seaweedfs_fuse")
|
|
or path.parent.name == "seaweedfs")):
|
|
continue
|
|
yield fields[0], target
|
|
|
|
|
|
def mounted(mount_id, target):
|
|
return (mount_id, target) in test_mounts(Path("/proc/self/mountinfo").read_text())
|
|
|
|
|
|
def cleanup():
|
|
failed = False
|
|
for mount_id, target in test_mounts(Path("/proc/self/mountinfo").read_text()):
|
|
try:
|
|
# A separate, bounded probe prevents a wedged FUSE daemon from
|
|
# hanging cleanup. Only ENOTCONN proves the daemon disconnected;
|
|
# a timeout or another error is not enough to detach a live mount.
|
|
probe = subprocess.run(
|
|
[sys.executable, __file__, "--probe", target], timeout=5,
|
|
capture_output=True, check=False,
|
|
)
|
|
except subprocess.TimeoutExpired:
|
|
print(f"::warning::FUSE probe timed out; leaving {target} mounted")
|
|
continue
|
|
if probe.returncode != errno.ENOTCONN or not mounted(mount_id, target):
|
|
continue
|
|
|
|
print(f"Unmounting disconnected FUSE test mount: {target}", flush=True)
|
|
# Lazy unmount detaches a dead mount even when an old process still
|
|
# holds it open. Prefer the unprivileged FUSE helper; sudo is fallback.
|
|
for command in (
|
|
["fusermount3", "-uz", "--", target],
|
|
["fusermount", "-uz", "--", target],
|
|
["sudo", "-n", "umount", "-l", "--", target],
|
|
):
|
|
try:
|
|
subprocess.run(command, timeout=10, check=False)
|
|
except (OSError, subprocess.TimeoutExpired) as error:
|
|
print(f"{command[0]}: {error}", file=sys.stderr)
|
|
if not mounted(mount_id, target):
|
|
break
|
|
else:
|
|
print(f"::error::Could not unmount disconnected FUSE test mount: {target}")
|
|
failed = True
|
|
continue
|
|
# The detached mount's parent still holds the crashed run's data and
|
|
# logs; test_mounts only yields targets under a SeaweedFS test dir in
|
|
# a temp root, so the parent is safe to reclaim here.
|
|
shutil.rmtree(Path(target).parent, ignore_errors=True)
|
|
return int(failed)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
if len(sys.argv) == 3 and sys.argv[1] == "--probe":
|
|
try:
|
|
os.stat(sys.argv[2])
|
|
except OSError as error:
|
|
sys.exit(error.errno)
|
|
sys.exit(0)
|
|
sys.exit(cleanup())
|