mirror of
https://github.com/seaweedfs/seaweedfs.git
synced 2026-11-21 11:55:18 +01:00
Compare commits
5
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
9d84ecca2f | ||
|
|
a1d2ebbb9b | ||
|
|
371d320854 | ||
|
|
84cc179b80 | ||
|
|
c8002d63bb |
No files matched your search
@@ -1,11 +0,0 @@
|
||||
name: Clean up stale SeaweedFS test mounts
|
||||
description: Detach disconnected FUSE test mounts left by earlier jobs on a shared runner.
|
||||
runs:
|
||||
using: composite
|
||||
steps:
|
||||
- name: Unmount disconnected SeaweedFS test mounts
|
||||
if: runner.os == 'Linux' && runner.environment == 'self-hosted'
|
||||
shell: bash
|
||||
env:
|
||||
CLEANUP_SCRIPT: ${{ github.action_path }}/cleanup.py
|
||||
run: python3 "$CLEANUP_SCRIPT"
|
||||
@@ -1,89 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Detach disconnected SeaweedFS integration-test mounts without touching live ones."""
|
||||
|
||||
import errno
|
||||
import os
|
||||
from pathlib import Path
|
||||
import re
|
||||
import shutil
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
|
||||
|
||||
def test_mounts(mountinfo):
|
||||
"""Read the kernel table directly: traversing a stale mount may block."""
|
||||
roots = {tempfile.gettempdir(), "/tmp", "/var/tmp"}
|
||||
for line in mountinfo.splitlines():
|
||||
fields, filesystem = line.split(" - ", 1)
|
||||
if filesystem.split()[0].split(".")[0] != "fuse":
|
||||
continue
|
||||
fields = fields.split()
|
||||
# mountinfo escapes whitespace and backslashes with octal sequences.
|
||||
target = re.sub(r"\\([0-7]{3})", lambda m: chr(int(m[1], 8)), fields[4])
|
||||
path = Path(target)
|
||||
# Test mounts are a mount* leaf under a seaweedfs_fuse_* scratch dir
|
||||
# (seaweedfs_fuse_test_*/mount, seaweedfs_fuse_failover_*/mount0, ...)
|
||||
# or the posix-compliance workflow's fixed /tmp/seaweedfs/mount.
|
||||
if (not path.name.startswith("mount")
|
||||
or str(path.parent.parent) not in roots
|
||||
or not (path.parent.name.startswith("seaweedfs_fuse")
|
||||
or path.parent.name == "seaweedfs")):
|
||||
continue
|
||||
yield fields[0], target
|
||||
|
||||
|
||||
def mounted(mount_id, target):
|
||||
return (mount_id, target) in test_mounts(Path("/proc/self/mountinfo").read_text())
|
||||
|
||||
|
||||
def cleanup():
|
||||
failed = False
|
||||
for mount_id, target in test_mounts(Path("/proc/self/mountinfo").read_text()):
|
||||
try:
|
||||
# A separate, bounded probe prevents a wedged FUSE daemon from
|
||||
# hanging cleanup. Only ENOTCONN proves the daemon disconnected;
|
||||
# a timeout or another error is not enough to detach a live mount.
|
||||
probe = subprocess.run(
|
||||
[sys.executable, __file__, "--probe", target], timeout=5,
|
||||
capture_output=True, check=False,
|
||||
)
|
||||
except subprocess.TimeoutExpired:
|
||||
print(f"::warning::FUSE probe timed out; leaving {target} mounted")
|
||||
continue
|
||||
if probe.returncode != errno.ENOTCONN or not mounted(mount_id, target):
|
||||
continue
|
||||
|
||||
print(f"Unmounting disconnected FUSE test mount: {target}", flush=True)
|
||||
# Lazy unmount detaches a dead mount even when an old process still
|
||||
# holds it open. Prefer the unprivileged FUSE helper; sudo is fallback.
|
||||
for command in (
|
||||
["fusermount3", "-uz", "--", target],
|
||||
["fusermount", "-uz", "--", target],
|
||||
["sudo", "-n", "umount", "-l", "--", target],
|
||||
):
|
||||
try:
|
||||
subprocess.run(command, timeout=10, check=False)
|
||||
except (OSError, subprocess.TimeoutExpired) as error:
|
||||
print(f"{command[0]}: {error}", file=sys.stderr)
|
||||
if not mounted(mount_id, target):
|
||||
break
|
||||
else:
|
||||
print(f"::error::Could not unmount disconnected FUSE test mount: {target}")
|
||||
failed = True
|
||||
continue
|
||||
# The detached mount's parent still holds the crashed run's data and
|
||||
# logs; test_mounts only yields targets under a SeaweedFS test dir in
|
||||
# a temp root, so the parent is safe to reclaim here.
|
||||
shutil.rmtree(Path(target).parent, ignore_errors=True)
|
||||
return int(failed)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
if len(sys.argv) == 3 and sys.argv[1] == "--probe":
|
||||
try:
|
||||
os.stat(sys.argv[2])
|
||||
except OSError as error:
|
||||
sys.exit(error.errno)
|
||||
sys.exit(0)
|
||||
sys.exit(cleanup())
|
||||
@@ -1,137 +0,0 @@
|
||||
import errno
|
||||
import importlib.util
|
||||
from pathlib import Path
|
||||
import subprocess
|
||||
import unittest
|
||||
from unittest.mock import patch
|
||||
|
||||
spec = importlib.util.spec_from_file_location("cleanup", Path(__file__).with_name("cleanup.py"))
|
||||
cleanup = importlib.util.module_from_spec(spec)
|
||||
spec.loader.exec_module(cleanup)
|
||||
|
||||
TARGET = "/tmp/seaweedfs_fuse_test_123/mount"
|
||||
MOUNTINFO = f"42 1 0:9 / {TARGET} rw - fuse.seaweedfs seaweedfs rw\n"
|
||||
|
||||
|
||||
class CleanupTests(unittest.TestCase):
|
||||
def run_cleanup(self, results, mounted=True):
|
||||
with patch.object(cleanup.Path, "read_text", return_value=MOUNTINFO), \
|
||||
patch.object(cleanup, "mounted", side_effect=mounted if isinstance(mounted, list) else None,
|
||||
return_value=mounted) as check, \
|
||||
patch.object(cleanup.shutil, "rmtree") as rmtree, \
|
||||
patch.object(cleanup.subprocess, "run", side_effect=results) as run:
|
||||
status = cleanup.cleanup()
|
||||
return status, run, check, rmtree
|
||||
|
||||
def result(self, status):
|
||||
return subprocess.CompletedProcess([], status)
|
||||
|
||||
def test_only_fuse_integration_test_mounts_are_candidates(self):
|
||||
rows = MOUNTINFO
|
||||
rows += MOUNTINFO.replace("fuse.seaweedfs", "ext4")
|
||||
rows += MOUNTINFO.replace(TARGET, "/mnt/production")
|
||||
rows += MOUNTINFO.replace(TARGET, "/tmp/seaweedfs_fuse_test_123/data")
|
||||
rows += MOUNTINFO.replace(TARGET, "/home/user/seaweedfs_fuse_test_123/mount")
|
||||
rows += MOUNTINFO.replace(TARGET, "/tmp/seaweedfs/data")
|
||||
self.assertEqual(list(cleanup.test_mounts(rows)), [("42", TARGET)])
|
||||
|
||||
def test_all_test_mount_layouts_are_candidates(self):
|
||||
targets = (
|
||||
"/tmp/seaweedfs_fuse_test_123/mount",
|
||||
"/tmp/seaweedfs_fuse_failover_456/mount0",
|
||||
"/tmp/seaweedfs_fuse_failover_456/mount1",
|
||||
"/var/tmp/seaweedfs_fuse_p2p_test_789/mount2",
|
||||
"/tmp/seaweedfs/mount",
|
||||
)
|
||||
rows = "".join(
|
||||
MOUNTINFO.replace("42", str(40 + i)).replace(TARGET, target)
|
||||
for i, target in enumerate(targets)
|
||||
)
|
||||
self.assertEqual(
|
||||
list(cleanup.test_mounts(rows)),
|
||||
[(str(40 + i), target) for i, target in enumerate(targets)],
|
||||
)
|
||||
|
||||
def test_mountinfo_paths_are_unescaped_once(self):
|
||||
target = "/tmp/seaweedfs_fuse_test_space and\\slash/mount"
|
||||
encoded = target.replace("\\", r"\134").replace(" ", r"\040")
|
||||
rows = MOUNTINFO.replace(TARGET, encoded)
|
||||
self.assertEqual(list(cleanup.test_mounts(rows)), [("42", target)])
|
||||
|
||||
def test_no_mounts_is_a_noop(self):
|
||||
with patch.object(cleanup.Path, "read_text", return_value=""), \
|
||||
patch.object(cleanup.subprocess, "run") as run:
|
||||
self.assertEqual(cleanup.cleanup(), 0)
|
||||
run.assert_not_called()
|
||||
|
||||
def test_healthy_mount_is_untouched(self):
|
||||
status, run, _, _ = self.run_cleanup([self.result(0)])
|
||||
self.assertEqual(status, 0)
|
||||
self.assertEqual(run.call_count, 1)
|
||||
|
||||
def test_other_probe_errors_are_untouched(self):
|
||||
for error in (errno.EACCES, errno.ENOENT, errno.EIO):
|
||||
with self.subTest(error=error):
|
||||
status, run, _, _ = self.run_cleanup([self.result(error)])
|
||||
self.assertEqual(status, 0)
|
||||
self.assertEqual(run.call_count, 1)
|
||||
|
||||
def test_probe_timeout_does_not_detach_a_possibly_live_mount(self):
|
||||
status, run, _, _ = self.run_cleanup([subprocess.TimeoutExpired("probe", 5)])
|
||||
self.assertEqual(status, 0)
|
||||
self.assertEqual(run.call_count, 1)
|
||||
|
||||
def test_disconnected_mount_is_lazily_unmounted(self):
|
||||
status, run, _, _ = self.run_cleanup([self.result(errno.ENOTCONN), self.result(0)], [True, False])
|
||||
self.assertEqual(status, 0)
|
||||
self.assertEqual(run.call_args_list[1].args[0], ["fusermount3", "-uz", "--", TARGET])
|
||||
self.assertEqual(run.call_args_list[0].kwargs["timeout"], 5)
|
||||
self.assertEqual(run.call_args_list[1].kwargs["timeout"], 10)
|
||||
|
||||
def test_replaced_or_removed_mount_is_untouched(self):
|
||||
status, run, _, _ = self.run_cleanup([self.result(errno.ENOTCONN)], False)
|
||||
self.assertEqual(status, 0)
|
||||
self.assertEqual(run.call_count, 1)
|
||||
|
||||
def test_mount_identity_is_checked_without_stat(self):
|
||||
with patch.object(cleanup.Path, "read_text", return_value=MOUNTINFO):
|
||||
self.assertTrue(cleanup.mounted("42", TARGET))
|
||||
self.assertFalse(cleanup.mounted("43", TARGET))
|
||||
|
||||
def test_missing_or_timed_out_helper_uses_fallback(self):
|
||||
for error in (FileNotFoundError(), subprocess.TimeoutExpired("fusermount3", 10)):
|
||||
with self.subTest(error=error):
|
||||
status, run, _, _ = self.run_cleanup(
|
||||
[self.result(errno.ENOTCONN), error, self.result(0)], [True, True, False])
|
||||
self.assertEqual(status, 0)
|
||||
self.assertEqual(run.call_args.args[0], ["fusermount", "-uz", "--", TARGET])
|
||||
|
||||
def test_failed_helpers_fall_back_to_sudo(self):
|
||||
status, run, _, _ = self.run_cleanup(
|
||||
[self.result(errno.ENOTCONN), self.result(1), self.result(1), self.result(0)],
|
||||
[True, True, True, False])
|
||||
self.assertEqual(status, 0)
|
||||
self.assertEqual(run.call_args.args[0], ["sudo", "-n", "umount", "-l", "--", TARGET])
|
||||
|
||||
def test_cleanup_fails_when_stale_mount_remains(self):
|
||||
status, run, _, rmtree = self.run_cleanup(
|
||||
[self.result(errno.ENOTCONN), self.result(1), self.result(1), self.result(1)])
|
||||
self.assertEqual(status, 1)
|
||||
self.assertEqual(run.call_count, 4)
|
||||
rmtree.assert_not_called()
|
||||
|
||||
def test_unmounted_test_dir_is_reclaimed(self):
|
||||
status, _, _, rmtree = self.run_cleanup(
|
||||
[self.result(errno.ENOTCONN), self.result(0)], [True, False])
|
||||
self.assertEqual(status, 0)
|
||||
rmtree.assert_called_once_with(Path(TARGET).parent, ignore_errors=True)
|
||||
|
||||
def test_probe_process_reports_errno(self):
|
||||
for target, expected in ((str(Path(__file__)), 0), (str(Path(__file__) / "absent"), errno.ENOTDIR)):
|
||||
with self.subTest(target=target):
|
||||
result = subprocess.run([cleanup.sys.executable, cleanup.__file__, "--probe", target], check=False)
|
||||
self.assertEqual(result.returncode, expected)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -23,14 +23,8 @@ runs:
|
||||
|
||||
bin=$(command -v fusermount3 || true)
|
||||
if [ -z "$bin" ]; then
|
||||
# Self-hosted runners are not guaranteed to carry fuse3.
|
||||
sudo apt-get update
|
||||
sudo apt-get install -y fuse3
|
||||
bin=$(command -v fusermount3 || true)
|
||||
if [ -z "$bin" ]; then
|
||||
echo "no fusermount3 in PATH after installing fuse3" >&2
|
||||
exit 1
|
||||
fi
|
||||
echo "no fusermount3 in PATH" >&2
|
||||
exit 1
|
||||
fi
|
||||
if can_mount "$bin"; then
|
||||
ls -l "$bin"
|
||||
|
||||
@@ -1,30 +0,0 @@
|
||||
name: "actions: unit tests"
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [ master ]
|
||||
paths:
|
||||
- '.github/actions/**'
|
||||
- '.github/workflows/action-tests.yml'
|
||||
pull_request:
|
||||
branches: [ master ]
|
||||
paths:
|
||||
- '.github/actions/**'
|
||||
- '.github/workflows/action-tests.yml'
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.event.pull_request.number || (github.event_name == 'push' && github.ref) || github.run_id }}
|
||||
cancel-in-progress: ${{ github.event_name == 'pull_request' || (github.event_name == 'push' && !startsWith(github.ref, 'refs/tags/')) }}
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
jobs:
|
||||
cleanup-stale-fuse-mounts:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Check out code
|
||||
uses: actions/checkout@v7
|
||||
|
||||
- name: Run cleanup action unit tests
|
||||
run: python3 .github/actions/cleanup-stale-fuse-mounts/test_cleanup.py
|
||||
@@ -9,8 +9,8 @@ on:
|
||||
- '.github/workflows/codeql.yml'
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.event.pull_request.number || (github.event_name == 'push' && github.ref) || github.run_id }}
|
||||
cancel-in-progress: ${{ github.event_name == 'pull_request' || (github.event_name == 'push' && !startsWith(github.ref, 'refs/tags/')) }}
|
||||
group: ${{ github.head_ref }}/codeql
|
||||
cancel-in-progress: true
|
||||
|
||||
jobs:
|
||||
CodeQL-Build:
|
||||
@@ -25,12 +25,6 @@ jobs:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v7
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
cache: ${{ runner.environment == 'github-hosted' }}
|
||||
|
||||
# Initializes the CodeQL tools for scanning.
|
||||
- name: Initialize CodeQL
|
||||
uses: github/codeql-action/init@v4.39.0
|
||||
|
||||
@@ -8,10 +8,6 @@ on:
|
||||
pull_request:
|
||||
branches: [master]
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.event.pull_request.number || (github.event_name == 'push' && github.ref) || github.run_id }}
|
||||
cancel-in-progress: ${{ github.event_name == 'pull_request' || (github.event_name == 'push' && !startsWith(github.ref, 'refs/tags/')) }}
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
|
||||
@@ -13,10 +13,6 @@ on:
|
||||
- 'go.sum'
|
||||
- '.github/workflows/container_dev.yml'
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.event.pull_request.number || (github.event_name == 'push' && github.ref) || github.run_id }}
|
||||
cancel-in-progress: ${{ github.event_name == 'pull_request' || (github.event_name == 'push' && !startsWith(github.ref, 'refs/tags/')) }}
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
@@ -24,7 +20,7 @@ jobs:
|
||||
|
||||
# ── Pre-build the Rust binaries natively ────────────────────────────
|
||||
build-rust-binaries:
|
||||
runs-on: ubuntu-24.04
|
||||
runs-on: ubuntu-22.04
|
||||
strategy:
|
||||
matrix:
|
||||
include:
|
||||
@@ -55,7 +51,6 @@ jobs:
|
||||
echo "CFLAGS_aarch64_unknown_linux_musl=-U_FORTIFY_SOURCE" >> "$GITHUB_ENV"
|
||||
|
||||
- name: Cache cargo registry and target
|
||||
if: runner.environment == 'github-hosted'
|
||||
uses: actions/cache@v6
|
||||
with:
|
||||
path: |
|
||||
@@ -198,12 +193,6 @@ jobs:
|
||||
platforms: linux/amd64, linux/arm64
|
||||
tags: ${{ steps.docker_meta.outputs.tags }}
|
||||
labels: ${{ steps.docker_meta.outputs.labels }}
|
||||
cache-from: type=gha
|
||||
cache-to: type=gha,mode=max
|
||||
# Key the checkout layer on the pushed sha so cached layers can't
|
||||
# serve stale code; the Dockerfile fetch fallback picks it up.
|
||||
build-args: |
|
||||
BRANCH=${{ github.sha }}
|
||||
|
||||
- name: Sign
|
||||
if: github.event_name != 'pull_request'
|
||||
|
||||
@@ -24,10 +24,6 @@ on:
|
||||
required: false
|
||||
default: ''
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.event.pull_request.number || (github.event_name == 'push' && github.ref) || github.event.inputs.image_tag || format('foundationdb_{0}_seaweedfs_{1}', github.event.inputs.fdb_version || '7.4.5', github.event.inputs.seaweedfs_ref || github.ref_name) }}
|
||||
cancel-in-progress: ${{ github.event_name == 'pull_request' || (github.event_name == 'push' && !startsWith(github.ref, 'refs/tags/')) }}
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
@@ -68,7 +64,6 @@ jobs:
|
||||
uses: actions/setup-go@v7
|
||||
with:
|
||||
go-version-file: go.mod
|
||||
cache: ${{ runner.environment == 'github-hosted' }}
|
||||
|
||||
- name: Run FoundationDB tagged tests
|
||||
env:
|
||||
|
||||
@@ -30,11 +30,6 @@ on:
|
||||
type: boolean
|
||||
default: false
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.event.pull_request.number || (github.event_name == 'push' && github.ref) || github.event.inputs.image_tag || github.run_id }}
|
||||
cancel-in-progress: false
|
||||
queue: max
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
security-events: write
|
||||
@@ -66,7 +61,7 @@ jobs:
|
||||
|
||||
# ── Pre-build the Rust binaries natively ────────────────────────────
|
||||
build-rust-binaries:
|
||||
runs-on: ubuntu-24.04
|
||||
runs-on: ubuntu-22.04
|
||||
strategy:
|
||||
matrix:
|
||||
include:
|
||||
@@ -99,7 +94,6 @@ jobs:
|
||||
echo "CFLAGS_aarch64_unknown_linux_musl=-U_FORTIFY_SOURCE" >> "$GITHUB_ENV"
|
||||
|
||||
- name: Cache cargo registry and target
|
||||
if: runner.environment == 'github-hosted'
|
||||
uses: actions/cache@v6
|
||||
with:
|
||||
path: |
|
||||
@@ -170,13 +164,10 @@ jobs:
|
||||
uses: actions/checkout@v7
|
||||
with:
|
||||
ref: ${{ github.event_name == 'workflow_dispatch' && github.event.inputs.source_ref || github.ref }}
|
||||
- name: Clean up stale FUSE mounts
|
||||
uses: ./.github/actions/cleanup-stale-fuse-mounts
|
||||
|
||||
- name: Free Disk Space
|
||||
run: |
|
||||
echo "Available disk space before cleanup:"
|
||||
timeout 15 df -h || true
|
||||
df -h
|
||||
# Remove pre-installed tools
|
||||
sudo rm -rf /usr/share/dotnet /usr/local/lib/android /opt/ghc /opt/hostedtoolcache/CodeQL
|
||||
# Clean package managers
|
||||
@@ -188,7 +179,7 @@ jobs:
|
||||
[ -d ~/.cache/go-build ] && rm -rf ~/.cache/go-build || true
|
||||
[ -d /go/pkg ] && rm -rf /go/pkg || true
|
||||
echo "Available disk space after cleanup:"
|
||||
timeout 15 df -h || true
|
||||
df -h
|
||||
|
||||
- name: Configure variant
|
||||
id: config
|
||||
|
||||
@@ -11,11 +11,6 @@ on:
|
||||
required: true
|
||||
default: ''
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.event.pull_request.number || (github.event_name == 'push' && github.ref) || format('refs/tags/{0}', github.event.inputs.release_tag) || github.run_id }}
|
||||
cancel-in-progress: false
|
||||
queue: max
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
|
||||
@@ -25,7 +25,7 @@ on:
|
||||
rocksdb_version:
|
||||
description: 'RocksDB git tag to use when variant=rocksdb'
|
||||
required: false
|
||||
default: 'v11.1.2'
|
||||
default: 'v10.10.1'
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
@@ -47,7 +47,7 @@ jobs:
|
||||
# amd64 and arm64 without QEMU, turning a 5-hour emulated cargo build into
|
||||
# ~15 minutes of native compilation.
|
||||
build-rust-binaries:
|
||||
runs-on: ubuntu-24.04
|
||||
runs-on: ubuntu-22.04
|
||||
strategy:
|
||||
matrix:
|
||||
include:
|
||||
@@ -78,7 +78,6 @@ jobs:
|
||||
echo "CFLAGS_aarch64_unknown_linux_musl=-U_FORTIFY_SOURCE" >> "$GITHUB_ENV"
|
||||
|
||||
- name: Cache cargo registry and target
|
||||
if: runner.environment == 'github-hosted'
|
||||
uses: actions/cache@v6
|
||||
with:
|
||||
path: |
|
||||
@@ -180,10 +179,6 @@ jobs:
|
||||
if: github.event_name != 'workflow_dispatch' || github.event.inputs.variant == 'all' || github.event.inputs.variant == matrix.variant
|
||||
uses: actions/checkout@v7
|
||||
|
||||
- name: Clean up stale FUSE mounts
|
||||
if: github.event_name != 'workflow_dispatch' || github.event.inputs.variant == 'all' || github.event.inputs.variant == matrix.variant
|
||||
uses: ./.github/actions/cleanup-stale-fuse-mounts
|
||||
|
||||
- name: Download pre-built Rust binaries
|
||||
if: github.event_name != 'workflow_dispatch' || github.event.inputs.variant == 'all' || github.event.inputs.variant == matrix.variant
|
||||
uses: actions/download-artifact@v8
|
||||
@@ -223,7 +218,7 @@ jobs:
|
||||
sudo docker system prune -af --volumes
|
||||
[ -d ~/.cache/go-build ] && rm -rf ~/.cache/go-build || true
|
||||
[ -d /go/pkg ] && rm -rf /go/pkg || true
|
||||
timeout 15 df -h || true
|
||||
df -h
|
||||
|
||||
- name: Docker meta
|
||||
if: github.event_name != 'workflow_dispatch' || github.event.inputs.variant == 'all' || github.event.inputs.variant == matrix.variant
|
||||
@@ -275,11 +270,8 @@ jobs:
|
||||
file: ${{ matrix.dockerfile }}
|
||||
platforms: ${{ matrix.platform }}
|
||||
labels: ${{ steps.docker_meta.outputs.labels }}
|
||||
# mode=max keeps resolvedDependencies complete and records the
|
||||
# builder identity; attestations ride inside each pushed index and
|
||||
# survive the imagetools merge into the multi-arch manifest.
|
||||
provenance: mode=max
|
||||
sbom: true
|
||||
# Flat single-platform manifest so imagetools create assembles cleanly.
|
||||
provenance: false
|
||||
outputs: type=image,name=${{ env.IMAGE }},push-by-digest=true,name-canonical=true,push=true
|
||||
cache-from: type=gha,scope=${{ matrix.variant }}-${{ matrix.arch }}
|
||||
# max only for rocksdb: its RocksDB compile is sha-independent and worth
|
||||
@@ -289,7 +281,7 @@ jobs:
|
||||
${{ matrix.build_args }}
|
||||
BUILDKIT_INLINE_CACHE=1
|
||||
BRANCH=${{ github.sha }}
|
||||
${{ matrix.variant == 'rocksdb' && format('ROCKSDB_VERSION={0}', github.event.inputs.rocksdb_version || 'v11.1.2') || '' }}
|
||||
${{ matrix.variant == 'rocksdb' && format('ROCKSDB_VERSION={0}', github.event.inputs.rocksdb_version || 'v10.10.1') || '' }}
|
||||
|
||||
- name: Export digest
|
||||
if: github.event_name != 'workflow_dispatch' || github.event.inputs.variant == 'all' || github.event.inputs.variant == matrix.variant
|
||||
@@ -315,8 +307,6 @@ jobs:
|
||||
permissions:
|
||||
contents: read
|
||||
id-token: write
|
||||
attestations: write
|
||||
packages: write
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
@@ -424,45 +414,6 @@ jobs:
|
||||
${{ env.IMAGE }}@${{ steps.manifest.outputs.digest }}
|
||||
chrislusf/seaweedfs@${{ steps.manifest.outputs.digest }}
|
||||
|
||||
# Attest the merged multi-arch digest itself, so digest-pinning consumers
|
||||
# can verify provenance against the exact index they pull. Both steps
|
||||
# reuse the GHCR login above and push as OCI referrers.
|
||||
- name: Attest build provenance for ${{ matrix.variant }}
|
||||
if: github.event_name != 'workflow_dispatch' || github.event.inputs.variant == 'all' || github.event.inputs.variant == matrix.variant
|
||||
uses: actions/attest-build-provenance@v4
|
||||
with:
|
||||
subject-name: ${{ env.IMAGE }}
|
||||
subject-digest: ${{ steps.manifest.outputs.digest }}
|
||||
push-to-registry: true
|
||||
|
||||
# SBOMs are generated and attested per platform manifest: an index-wide
|
||||
# scan only describes syft's default platform, while the manifests it
|
||||
# merges differ (some ship the real binary, others placeholders), so the
|
||||
# SBOM has to be bound to the platform digest a consumer actually pulls.
|
||||
- name: Install syft
|
||||
if: github.event_name != 'workflow_dispatch' || github.event.inputs.variant == 'all' || github.event.inputs.variant == matrix.variant
|
||||
uses: anchore/sbom-action/download-syft@v0
|
||||
|
||||
- name: Attest per-platform SBOMs for ${{ matrix.variant }}
|
||||
if: github.event_name != 'workflow_dispatch' || github.event.inputs.variant == 'all' || github.event.inputs.variant == matrix.variant
|
||||
working-directory: /tmp/digests
|
||||
run: |
|
||||
set -euo pipefail
|
||||
for f in *; do
|
||||
# Each saved digest is an OCI index (image + attestation manifests);
|
||||
# scan and attest the image manifest a same-platform pull resolves.
|
||||
ref="${{ env.IMAGE }}@sha256:${f}"
|
||||
image=$(crane manifest "$ref" | jq -r '
|
||||
[.manifests[]? | select((.annotations["vnd.docker.reference.type"] // "") != "attestation-manifest"
|
||||
and (.platform.os // "unknown") != "unknown") | .digest][0] // empty')
|
||||
ref="${{ env.IMAGE }}@${image:-sha256:${f}}"
|
||||
syft "registry:${ref}" -o "spdx-json=/tmp/sbom-${f}.spdx.json"
|
||||
cosign attest --yes \
|
||||
--predicate "/tmp/sbom-${f}.spdx.json" \
|
||||
--type spdxjson \
|
||||
"$ref"
|
||||
done
|
||||
|
||||
# Report-only trivy scan: uploads fixable HIGH/CRITICAL findings to GitHub
|
||||
# Security for visibility, but never blocks the release. Releases (including
|
||||
# `latest`) ship regardless — vulnerabilities are tracked, not gated, since
|
||||
@@ -501,9 +452,7 @@ jobs:
|
||||
scanners: vuln
|
||||
vuln-type: os,library
|
||||
severity: HIGH,CRITICAL
|
||||
# Keep unfixed findings in the report so digest-pinning consumers see
|
||||
# the complete picture, not just what a patch could resolve.
|
||||
ignore-unfixed: false
|
||||
ignore-unfixed: true
|
||||
limit-severities-for-sarif: true
|
||||
format: sarif
|
||||
output: trivy-results.sarif
|
||||
|
||||
@@ -4,9 +4,9 @@ on:
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
rocksdb_version:
|
||||
description: 'RocksDB git tag or branch to build (e.g. v11.1.2)'
|
||||
description: 'RocksDB git tag or branch to build (e.g. v10.10.1)'
|
||||
required: true
|
||||
default: 'v11.1.2'
|
||||
default: 'v10.10.1'
|
||||
seaweedfs_ref:
|
||||
description: 'SeaweedFS git tag, branch, or commit to build'
|
||||
required: true
|
||||
@@ -16,11 +16,6 @@ on:
|
||||
required: false
|
||||
default: ''
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.event.inputs.image_tag || format('rocksdb_{0}_seaweedfs_{1}', github.event.inputs.rocksdb_version, github.event.inputs.seaweedfs_ref) }}
|
||||
cancel-in-progress: false
|
||||
queue: max
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
|
||||
@@ -27,7 +27,6 @@ jobs:
|
||||
uses: actions/setup-go@v7
|
||||
with:
|
||||
go-version-file: 'telemetry/server/go.mod'
|
||||
cache: ${{ runner.environment == 'github-hosted' }}
|
||||
|
||||
- name: Build Telemetry Server
|
||||
if: github.event_name == 'workflow_dispatch' && inputs.deploy
|
||||
|
||||
@@ -1,10 +1,6 @@
|
||||
name: 'Dependency Review'
|
||||
on: [pull_request]
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.event.pull_request.number || (github.event_name == 'push' && github.ref) || github.run_id }}
|
||||
cancel-in-progress: ${{ github.event_name == 'pull_request' || (github.event_name == 'push' && !startsWith(github.ref, 'refs/tags/')) }}
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
|
||||
@@ -19,8 +19,8 @@ on:
|
||||
- '.github/workflows/e2e.yml'
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.event.pull_request.number || (github.event_name == 'push' && github.ref) || github.run_id }}
|
||||
cancel-in-progress: ${{ github.event_name == 'pull_request' || (github.event_name == 'push' && !startsWith(github.ref, 'refs/tags/')) }}
|
||||
group: ${{ github.head_ref }}/e2e
|
||||
cancel-in-progress: true
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
@@ -32,7 +32,7 @@ defaults:
|
||||
jobs:
|
||||
e2e:
|
||||
name: FUSE Mount
|
||||
runs-on: ubuntu-24.04
|
||||
runs-on: ubuntu-22.04
|
||||
timeout-minutes: 30
|
||||
steps:
|
||||
- name: Check out code into the Go module directory
|
||||
@@ -42,7 +42,6 @@ jobs:
|
||||
uses: actions/setup-go@v7
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
cache: ${{ runner.environment == 'github-hosted' }}
|
||||
|
||||
- name: Configure Docker Hub mirror
|
||||
run: |
|
||||
@@ -53,7 +52,6 @@ jobs:
|
||||
uses: docker/setup-buildx-action@v4
|
||||
|
||||
- name: Cache Docker layers
|
||||
if: runner.environment == 'github-hosted'
|
||||
uses: actions/cache@v6
|
||||
with:
|
||||
path: /tmp/.buildx-cache
|
||||
|
||||
@@ -18,23 +18,18 @@ on:
|
||||
- 'go.sum'
|
||||
- '.github/workflows/ec-integration-tests.yml'
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.event.pull_request.number || (github.event_name == 'push' && github.ref) || github.run_id }}
|
||||
cancel-in-progress: ${{ github.event_name == 'pull_request' || (github.event_name == 'push' && !startsWith(github.ref, 'refs/tags/')) }}
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
jobs:
|
||||
ec-integration-tests:
|
||||
name: EC Integration Tests
|
||||
runs-on: ubuntu-24.04
|
||||
runs-on: ubuntu-22.04
|
||||
timeout-minutes: 30
|
||||
steps:
|
||||
- name: Set up Go 1.x
|
||||
uses: actions/setup-go@v7
|
||||
with:
|
||||
cache: ${{ runner.environment == 'github-hosted' }}
|
||||
go-version: ^1.26
|
||||
id: go
|
||||
|
||||
|
||||
@@ -19,8 +19,8 @@ on:
|
||||
- '.github/actions/fix-fusermount-setuid/**'
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.event.pull_request.number || (github.event_name == 'push' && github.ref) || github.run_id }}
|
||||
cancel-in-progress: ${{ github.event_name == 'pull_request' || (github.event_name == 'push' && !startsWith(github.ref, 'refs/tags/')) }}
|
||||
group: ${{ github.head_ref || github.ref }}/fuse-dlm-integration
|
||||
cancel-in-progress: true
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
@@ -28,7 +28,7 @@ permissions:
|
||||
jobs:
|
||||
fuse-dlm-integration:
|
||||
name: FUSE DLM Integration Tests
|
||||
runs-on: ubuntu-24.04
|
||||
runs-on: ubuntu-22.04
|
||||
timeout-minutes: 30
|
||||
|
||||
steps:
|
||||
@@ -39,7 +39,6 @@ jobs:
|
||||
uses: actions/setup-go@v7
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
cache: ${{ runner.environment == 'github-hosted' }}
|
||||
|
||||
- name: Configure FUSE
|
||||
run: |
|
||||
|
||||
@@ -24,8 +24,8 @@ on:
|
||||
- '.github/actions/fix-fusermount-setuid/**'
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.event.pull_request.number || (github.event_name == 'push' && github.ref) || github.run_id }}
|
||||
cancel-in-progress: ${{ github.event_name == 'pull_request' || (github.event_name == 'push' && !startsWith(github.ref, 'refs/tags/')) }}
|
||||
group: ${{ github.head_ref || github.ref }}/fuse-failover
|
||||
cancel-in-progress: true
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
@@ -33,7 +33,7 @@ permissions:
|
||||
jobs:
|
||||
fuse-failover:
|
||||
name: FUSE Volume Server Failover
|
||||
runs-on: ubuntu-24.04
|
||||
runs-on: ubuntu-22.04
|
||||
timeout-minutes: 40
|
||||
|
||||
steps:
|
||||
@@ -42,14 +42,10 @@ jobs:
|
||||
with:
|
||||
persist-credentials: false
|
||||
|
||||
- name: Clean up stale FUSE mounts
|
||||
uses: ./.github/actions/cleanup-stale-fuse-mounts
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v7
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
cache: ${{ runner.environment == 'github-hosted' }}
|
||||
|
||||
- name: Configure FUSE
|
||||
run: |
|
||||
|
||||
@@ -17,8 +17,8 @@ on:
|
||||
- '.github/actions/fix-fusermount-setuid/**'
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.event.pull_request.number || (github.event_name == 'push' && github.ref) || github.run_id }}
|
||||
cancel-in-progress: ${{ github.event_name == 'pull_request' || (github.event_name == 'push' && !startsWith(github.ref, 'refs/tags/')) }}
|
||||
group: ${{ github.head_ref }}/fuse-integration
|
||||
cancel-in-progress: true
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
@@ -26,7 +26,7 @@ permissions:
|
||||
jobs:
|
||||
fuse-integration:
|
||||
name: FUSE Integration Testing
|
||||
runs-on: ubuntu-24.04
|
||||
runs-on: ubuntu-22.04
|
||||
timeout-minutes: 50
|
||||
|
||||
steps:
|
||||
@@ -37,7 +37,6 @@ jobs:
|
||||
uses: actions/setup-go@v7
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
cache: ${{ runner.environment == 'github-hosted' }}
|
||||
|
||||
- name: Configure FUSE
|
||||
run: |
|
||||
|
||||
@@ -25,8 +25,8 @@ on:
|
||||
- '.github/actions/fix-fusermount-setuid/**'
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.event.pull_request.number || (github.event_name == 'push' && github.ref) || github.run_id }}
|
||||
cancel-in-progress: ${{ github.event_name == 'pull_request' || (github.event_name == 'push' && !startsWith(github.ref, 'refs/tags/')) }}
|
||||
group: ${{ github.head_ref || github.ref }}/fuse-p2p-integration
|
||||
cancel-in-progress: true
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
@@ -34,21 +34,17 @@ permissions:
|
||||
jobs:
|
||||
fuse-p2p-integration:
|
||||
name: FUSE P2P Peer Chunk Sharing
|
||||
runs-on: ubuntu-24.04
|
||||
runs-on: ubuntu-22.04
|
||||
timeout-minutes: 20
|
||||
|
||||
steps:
|
||||
- name: Check out code
|
||||
uses: actions/checkout@v7
|
||||
|
||||
- name: Clean up stale FUSE mounts
|
||||
uses: ./.github/actions/cleanup-stale-fuse-mounts
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v7
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
cache: ${{ runner.environment == 'github-hosted' }}
|
||||
|
||||
- name: Configure FUSE
|
||||
run: |
|
||||
|
||||
@@ -17,8 +17,8 @@ on:
|
||||
- '.github/workflows/go.yml'
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.event.pull_request.number || (github.event_name == 'push' && github.ref) || github.run_id }}
|
||||
cancel-in-progress: ${{ github.event_name == 'pull_request' || (github.event_name == 'push' && !startsWith(github.ref, 'refs/tags/')) }}
|
||||
group: ${{ github.head_ref }}/go
|
||||
cancel-in-progress: true
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
@@ -35,7 +35,6 @@ jobs:
|
||||
uses: actions/setup-go@v7
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
cache: ${{ runner.environment == 'github-hosted' }}
|
||||
- name: Get dependencies
|
||||
run: |
|
||||
cd weed; go get -v -t -d ./...
|
||||
@@ -58,7 +57,6 @@ jobs:
|
||||
uses: actions/setup-go@v7
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
cache: ${{ runner.environment == 'github-hosted' }}
|
||||
- name: Go Vet linux/386 (type-checks code and tests for 32-bit int overflows)
|
||||
run: |
|
||||
GOOS=linux GOARCH=386 go vet ./... 2>&1 | grep -v "MessageState contains sync.Mutex" | grep -v "IdentityAccessManagement contains sync.RWMutex" | tee vet-32bit-output.txt
|
||||
@@ -74,7 +72,6 @@ jobs:
|
||||
uses: actions/setup-go@v7
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
cache: ${{ runner.environment == 'github-hosted' }}
|
||||
- name: Build
|
||||
run: cd weed; go build -tags "elastic gocdk sqlite ydb tarantool tikv rclone" -v .
|
||||
|
||||
@@ -97,7 +94,6 @@ jobs:
|
||||
uses: actions/setup-go@v7
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
cache: ${{ runner.environment == 'github-hosted' }}
|
||||
# Nothing else on a PR compiles these, so per-OS syscall constants creep
|
||||
# back into shared files unnoticed and only a release build catches it.
|
||||
- name: Cross-compile
|
||||
@@ -132,7 +128,6 @@ jobs:
|
||||
uses: actions/setup-go@v7
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
cache: ${{ runner.environment == 'github-hosted' }}
|
||||
- name: Test
|
||||
env:
|
||||
RUN_REDIS_TESTS: "1"
|
||||
@@ -148,7 +143,6 @@ jobs:
|
||||
uses: actions/setup-go@v7
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
cache: ${{ runner.environment == 'github-hosted' }}
|
||||
# 386 test binaries run natively on the amd64 runner. This catches what vet
|
||||
# can't: unaligned 64-bit atomics and arithmetic that wraps at runtime.
|
||||
# -short skips the e2e suites already covered on amd64.
|
||||
@@ -165,7 +159,6 @@ jobs:
|
||||
uses: actions/setup-go@v7
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
cache: ${{ runner.environment == 'github-hosted' }}
|
||||
# weed/kms/azure is excluded from every default build, so without this
|
||||
# nothing compiles it and tag-gated code silently rots.
|
||||
- name: Build and test with azurekms
|
||||
|
||||
@@ -8,10 +8,6 @@ on:
|
||||
branches: [ master ]
|
||||
paths: ['k8s/**', '.github/workflows/helm_ci.yml', '.github/scripts/helm_jwt_keys.py']
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.event.pull_request.number || (github.event_name == 'push' && github.ref) || github.run_id }}
|
||||
cancel-in-progress: ${{ github.event_name == 'pull_request' || (github.event_name == 'push' && !startsWith(github.ref, 'refs/tags/')) }}
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
|
||||
@@ -14,10 +14,6 @@ on:
|
||||
- 'weed/**'
|
||||
- '.github/workflows/java_integration_tests.yml'
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.event.pull_request.number || (github.event_name == 'push' && github.ref) || github.run_id }}
|
||||
cancel-in-progress: ${{ github.event_name == 'pull_request' || (github.event_name == 'push' && !startsWith(github.ref, 'refs/tags/')) }}
|
||||
|
||||
jobs:
|
||||
test:
|
||||
name: Java Integration Tests
|
||||
@@ -35,7 +31,6 @@ jobs:
|
||||
uses: actions/setup-go@v7
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
cache: ${{ runner.environment == 'github-hosted' }}
|
||||
id: go
|
||||
|
||||
- name: Set up Java
|
||||
@@ -43,7 +38,7 @@ jobs:
|
||||
with:
|
||||
java-version: ${{ matrix.java }}
|
||||
distribution: 'temurin'
|
||||
cache: ${{ runner.environment == 'github-hosted' && 'maven' || '' }}
|
||||
cache: 'maven'
|
||||
|
||||
- name: Build SeaweedFS
|
||||
run: |
|
||||
@@ -60,7 +55,6 @@ jobs:
|
||||
# Start SeaweedFS with optimized settings for CI
|
||||
# Include S3 API for s3copier integration tests
|
||||
weed server -dir="$WEED_DATA_DIR" \
|
||||
-ip=127.0.0.1 -ip.bind=127.0.0.1 \
|
||||
-master.raftHashicorp \
|
||||
-master.electionTimeout=1s \
|
||||
-master.volumeSizeLimitMB=100 \
|
||||
|
||||
@@ -77,7 +77,7 @@ jobs:
|
||||
with:
|
||||
java-version: '17'
|
||||
distribution: 'temurin'
|
||||
cache: ${{ runner.environment == 'github-hosted' && 'maven' || '' }}
|
||||
cache: 'maven'
|
||||
server-id: central
|
||||
server-username: MAVEN_CENTRAL_USERNAME
|
||||
server-password: MAVEN_CENTRAL_PASSWORD
|
||||
|
||||
@@ -12,10 +12,6 @@ on:
|
||||
- 'other/java/**'
|
||||
- '.github/workflows/java_unit_tests.yml'
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.event.pull_request.number || (github.event_name == 'push' && github.ref) || github.run_id }}
|
||||
cancel-in-progress: ${{ github.event_name == 'pull_request' || (github.event_name == 'push' && !startsWith(github.ref, 'refs/tags/')) }}
|
||||
|
||||
jobs:
|
||||
test:
|
||||
name: Java Unit Tests
|
||||
@@ -34,7 +30,7 @@ jobs:
|
||||
with:
|
||||
java-version: ${{ matrix.java }}
|
||||
distribution: 'temurin'
|
||||
cache: ${{ runner.environment == 'github-hosted' && 'maven' || '' }}
|
||||
cache: 'maven'
|
||||
|
||||
- name: Build and Install SeaweedFS Client
|
||||
working-directory: other/java/client
|
||||
|
||||
@@ -24,8 +24,8 @@ on:
|
||||
workflow_dispatch: # Allow manual trigger
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.event.pull_request.number || (github.event_name == 'push' && github.ref) || github.run_id }}
|
||||
cancel-in-progress: ${{ github.event_name == 'pull_request' || (github.event_name == 'push' && !startsWith(github.ref, 'refs/tags/')) }}
|
||||
group: ${{ github.head_ref }}/kafka-quicktest
|
||||
cancel-in-progress: true
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
@@ -43,7 +43,7 @@ jobs:
|
||||
uses: actions/setup-go@v7
|
||||
with:
|
||||
go-version: ^1.26
|
||||
cache: ${{ runner.environment == 'github-hosted' }}
|
||||
cache: true
|
||||
cache-dependency-path: |
|
||||
**/go.sum
|
||||
id: go
|
||||
|
||||
@@ -23,8 +23,8 @@ on:
|
||||
- '.github/workflows/kafka-tests.yml'
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.event.pull_request.number || (github.event_name == 'push' && github.ref) || github.run_id }}
|
||||
cancel-in-progress: ${{ github.event_name == 'pull_request' || (github.event_name == 'push' && !startsWith(github.ref, 'refs/tags/')) }}
|
||||
group: ${{ github.head_ref }}/kafka-tests
|
||||
cancel-in-progress: true
|
||||
|
||||
# Force different runners for better isolation
|
||||
env:
|
||||
@@ -53,7 +53,6 @@ jobs:
|
||||
- name: Set up Go 1.x
|
||||
uses: actions/setup-go@v7
|
||||
with:
|
||||
cache: ${{ runner.environment == 'github-hosted' }}
|
||||
go-version: ^1.26
|
||||
id: go
|
||||
|
||||
@@ -99,7 +98,6 @@ jobs:
|
||||
- name: Set up Go 1.x
|
||||
uses: actions/setup-go@v7
|
||||
with:
|
||||
cache: ${{ runner.environment == 'github-hosted' }}
|
||||
go-version: ^1.26
|
||||
id: go
|
||||
|
||||
@@ -151,7 +149,7 @@ jobs:
|
||||
uses: actions/setup-go@v7
|
||||
with:
|
||||
go-version: ^1.26
|
||||
cache: ${{ runner.environment == 'github-hosted' }}
|
||||
cache: true
|
||||
cache-dependency-path: |
|
||||
**/go.sum
|
||||
id: go
|
||||
@@ -330,7 +328,7 @@ jobs:
|
||||
uses: actions/setup-go@v7
|
||||
with:
|
||||
go-version: ^1.26
|
||||
cache: ${{ runner.environment == 'github-hosted' }}
|
||||
cache: true
|
||||
cache-dependency-path: |
|
||||
**/go.sum
|
||||
id: go
|
||||
@@ -492,7 +490,7 @@ jobs:
|
||||
uses: actions/setup-go@v7
|
||||
with:
|
||||
go-version: ^1.26
|
||||
cache: ${{ runner.environment == 'github-hosted' }}
|
||||
cache: true
|
||||
cache-dependency-path: |
|
||||
**/go.sum
|
||||
id: go
|
||||
@@ -650,7 +648,7 @@ jobs:
|
||||
uses: actions/setup-go@v7
|
||||
with:
|
||||
go-version: ^1.26
|
||||
cache: ${{ runner.environment == 'github-hosted' }}
|
||||
cache: true
|
||||
cache-dependency-path: |
|
||||
**/go.sum
|
||||
id: go
|
||||
@@ -807,7 +805,6 @@ jobs:
|
||||
- name: Set up Go 1.x
|
||||
uses: actions/setup-go@v7
|
||||
with:
|
||||
cache: ${{ runner.environment == 'github-hosted' }}
|
||||
go-version: ^1.26
|
||||
id: go
|
||||
|
||||
|
||||
@@ -19,8 +19,8 @@ on:
|
||||
- 'test/kms/**'
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.event.pull_request.number || (github.event_name == 'push' && github.ref) || github.run_id }}
|
||||
cancel-in-progress: ${{ github.event_name == 'pull_request' || (github.event_name == 'push' && !startsWith(github.ref, 'refs/tags/')) }}
|
||||
group: ${{ github.head_ref || github.ref }}-kms-tests
|
||||
cancel-in-progress: true
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
@@ -32,7 +32,7 @@ defaults:
|
||||
jobs:
|
||||
kms-provider-tests:
|
||||
name: KMS Provider Integration Tests
|
||||
runs-on: ubuntu-24.04
|
||||
runs-on: ubuntu-22.04
|
||||
timeout-minutes: 20
|
||||
|
||||
steps:
|
||||
@@ -43,7 +43,6 @@ jobs:
|
||||
uses: actions/setup-go@v7
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
cache: ${{ runner.environment == 'github-hosted' }}
|
||||
id: go
|
||||
|
||||
- name: Install SeaweedFS
|
||||
@@ -78,7 +77,7 @@ jobs:
|
||||
|
||||
s3-kms-e2e-tests:
|
||||
name: S3 KMS End-to-End Tests
|
||||
runs-on: ubuntu-24.04
|
||||
runs-on: ubuntu-22.04
|
||||
timeout-minutes: 25
|
||||
|
||||
steps:
|
||||
@@ -89,7 +88,6 @@ jobs:
|
||||
uses: actions/setup-go@v7
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
cache: ${{ runner.environment == 'github-hosted' }}
|
||||
id: go
|
||||
|
||||
- name: Install SeaweedFS
|
||||
|
||||
@@ -21,8 +21,8 @@ on:
|
||||
- '.github/workflows/master-cold-start-tests.yml'
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.event.pull_request.number || (github.event_name == 'push' && github.ref) || github.run_id }}
|
||||
cancel-in-progress: ${{ github.event_name == 'pull_request' || (github.event_name == 'push' && !startsWith(github.ref, 'refs/tags/')) }}
|
||||
group: ${{ github.head_ref || github.ref }}/master-cold-start-tests
|
||||
cancel-in-progress: true
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
@@ -30,7 +30,7 @@ permissions:
|
||||
jobs:
|
||||
master-cold-start-tests:
|
||||
name: Master Cold Start Tests
|
||||
runs-on: ubuntu-24.04
|
||||
runs-on: ubuntu-22.04
|
||||
timeout-minutes: 10
|
||||
steps:
|
||||
- name: Check out code
|
||||
@@ -42,7 +42,6 @@ jobs:
|
||||
uses: actions/setup-go@v7
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
cache: ${{ runner.environment == 'github-hosted' }}
|
||||
|
||||
- name: Build weed binary
|
||||
run: |
|
||||
|
||||
@@ -25,8 +25,8 @@ on:
|
||||
- '.github/workflows/metadata-subscribe-tests.yml'
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.event.pull_request.number || (github.event_name == 'push' && github.ref) || github.run_id }}
|
||||
cancel-in-progress: ${{ github.event_name == 'pull_request' || (github.event_name == 'push' && !startsWith(github.ref, 'refs/tags/')) }}
|
||||
group: ${{ github.head_ref }}/metadata-subscribe-tests
|
||||
cancel-in-progress: true
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
@@ -37,7 +37,7 @@ env:
|
||||
jobs:
|
||||
metadata-subscribe-integration:
|
||||
name: Metadata Subscribe Integration Tests
|
||||
runs-on: ubuntu-24.04
|
||||
runs-on: ubuntu-22.04
|
||||
timeout-minutes: 20
|
||||
|
||||
steps:
|
||||
@@ -48,7 +48,6 @@ jobs:
|
||||
uses: actions/setup-go@v7
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
cache: ${{ runner.environment == 'github-hosted' }}
|
||||
|
||||
- name: Build SeaweedFS
|
||||
run: |
|
||||
|
||||
@@ -10,8 +10,8 @@ on:
|
||||
branches: [ 'winfsp-bench**' ]
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.event.pull_request.number || (github.event_name == 'push' && github.ref) || github.run_id }}
|
||||
cancel-in-progress: ${{ github.event_name == 'pull_request' || (github.event_name == 'push' && !startsWith(github.ref, 'refs/tags/')) }}
|
||||
group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }}
|
||||
cancel-in-progress: true
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
@@ -32,7 +32,6 @@ jobs:
|
||||
- uses: actions/setup-go@v7
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
cache: ${{ runner.environment == 'github-hosted' }}
|
||||
|
||||
- name: Install WinFsp and rclone
|
||||
run: choco install winfsp rclone -y --no-progress
|
||||
@@ -151,7 +150,6 @@ jobs:
|
||||
- uses: actions/setup-go@v7
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
cache: ${{ runner.environment == 'github-hosted' }}
|
||||
|
||||
- name: Repair the fusermount3 setuid bit
|
||||
uses: ./.github/actions/fix-fusermount-setuid
|
||||
|
||||
@@ -24,8 +24,8 @@ on:
|
||||
- '.github/workflows/mount-windows-conformance.yml'
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.event.pull_request.number || (github.event_name == 'push' && github.ref) || github.run_id }}
|
||||
cancel-in-progress: ${{ github.event_name == 'pull_request' || (github.event_name == 'push' && !startsWith(github.ref, 'refs/tags/')) }}
|
||||
group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }}
|
||||
cancel-in-progress: true
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
@@ -49,7 +49,6 @@ jobs:
|
||||
- uses: actions/setup-go@v7
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
cache: ${{ runner.environment == 'github-hosted' }}
|
||||
|
||||
# cgofuse loads winfsp-x64.dll at run time, so WinFsp is needed here but
|
||||
# not to build.
|
||||
|
||||
@@ -24,8 +24,8 @@ on:
|
||||
- '.github/workflows/mount-windows.yml'
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.event.pull_request.number || (github.event_name == 'push' && github.ref) || github.run_id }}
|
||||
cancel-in-progress: ${{ github.event_name == 'pull_request' || (github.event_name == 'push' && !startsWith(github.ref, 'refs/tags/')) }}
|
||||
group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }}
|
||||
cancel-in-progress: true
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
@@ -49,7 +49,6 @@ jobs:
|
||||
- uses: actions/setup-go@v7
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
cache: ${{ runner.environment == 'github-hosted' }}
|
||||
|
||||
# cgofuse loads winfsp-x64.dll at run time, so WinFsp is needed here but
|
||||
# not to build.
|
||||
|
||||
@@ -21,8 +21,8 @@ on:
|
||||
- '.github/workflows/multi-master-tests.yml'
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.event.pull_request.number || (github.event_name == 'push' && github.ref) || github.run_id }}
|
||||
cancel-in-progress: ${{ github.event_name == 'pull_request' || (github.event_name == 'push' && !startsWith(github.ref, 'refs/tags/')) }}
|
||||
group: ${{ github.head_ref || github.ref }}/multi-master-tests
|
||||
cancel-in-progress: true
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
@@ -30,7 +30,7 @@ permissions:
|
||||
jobs:
|
||||
multi-master-failover-tests:
|
||||
name: Multi-Master Failover Tests
|
||||
runs-on: ubuntu-24.04
|
||||
runs-on: ubuntu-22.04
|
||||
timeout-minutes: 10
|
||||
steps:
|
||||
- name: Check out code
|
||||
@@ -40,7 +40,6 @@ jobs:
|
||||
uses: actions/setup-go@v7
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
cache: ${{ runner.environment == 'github-hosted' }}
|
||||
|
||||
- name: Install SeaweedFS
|
||||
run: |
|
||||
|
||||
@@ -45,8 +45,8 @@ on:
|
||||
type: string
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.event.pull_request.number || (github.event_name == 'push' && github.ref) || github.run_id }}
|
||||
cancel-in-progress: ${{ github.event_name == 'pull_request' || (github.event_name == 'push' && !startsWith(github.ref, 'refs/tags/')) }}
|
||||
group: ${{ github.head_ref || github.ref }}/performance
|
||||
cancel-in-progress: true
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
@@ -67,7 +67,6 @@ jobs:
|
||||
uses: actions/setup-go@v7
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
cache: ${{ runner.environment == 'github-hosted' }}
|
||||
|
||||
- name: Build weed
|
||||
run: go build -o weed_bin ./weed
|
||||
@@ -176,14 +175,13 @@ jobs:
|
||||
uses: actions/setup-go@v7
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
cache: ${{ runner.environment == 'github-hosted' }}
|
||||
|
||||
- name: Install Rust toolchain
|
||||
if: matrix.impl == 'rust'
|
||||
uses: dtolnay/rust-toolchain@stable
|
||||
|
||||
- name: Cache cargo registry and target
|
||||
if: matrix.impl == 'rust' && runner.environment == 'github-hosted'
|
||||
if: matrix.impl == 'rust'
|
||||
uses: actions/cache@v6
|
||||
with:
|
||||
path: |
|
||||
@@ -329,14 +327,13 @@ jobs:
|
||||
uses: actions/setup-go@v7
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
cache: ${{ runner.environment == 'github-hosted' }}
|
||||
|
||||
- name: Install Rust toolchain
|
||||
if: matrix.impl == 'rust'
|
||||
uses: dtolnay/rust-toolchain@stable
|
||||
|
||||
- name: Cache cargo registry and target
|
||||
if: matrix.impl == 'rust' && runner.environment == 'github-hosted'
|
||||
if: matrix.impl == 'rust'
|
||||
uses: actions/cache@v6
|
||||
with:
|
||||
path: |
|
||||
|
||||
@@ -27,7 +27,7 @@ permissions:
|
||||
jobs:
|
||||
pjdfstest:
|
||||
name: pjdfstest
|
||||
runs-on: ubuntu-24.04
|
||||
runs-on: ubuntu-22.04
|
||||
timeout-minutes: 60
|
||||
|
||||
steps:
|
||||
@@ -38,23 +38,15 @@ jobs:
|
||||
uses: actions/setup-go@v7
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
cache: ${{ runner.environment == 'github-hosted' }}
|
||||
|
||||
- name: Start local Docker registry
|
||||
run: |
|
||||
if ! docker start registry 2>/dev/null; then
|
||||
docker rm -f registry >/dev/null 2>&1 || true
|
||||
docker run -d --restart=always -p 5000:5000 --name registry registry:2
|
||||
fi
|
||||
run: docker run -d --restart=always -p 5000:5000 --name registry registry:2
|
||||
|
||||
- name: Set up Docker Buildx
|
||||
uses: docker/setup-buildx-action@v4
|
||||
with:
|
||||
driver-opts: network=host
|
||||
|
||||
- name: Set up Docker Compose
|
||||
uses: docker/setup-compose-action@v2
|
||||
|
||||
- name: Build weed race binary
|
||||
run: |
|
||||
cd docker
|
||||
|
||||
@@ -18,17 +18,13 @@ on:
|
||||
- 'go.sum'
|
||||
- '.github/workflows/plugin-workers.yml'
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.event.pull_request.number || (github.event_name == 'push' && github.ref) || github.run_id }}
|
||||
cancel-in-progress: ${{ github.event_name == 'pull_request' || (github.event_name == 'push' && !startsWith(github.ref, 'refs/tags/')) }}
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
jobs:
|
||||
plugin-worker:
|
||||
name: "Plugin Worker: ${{ matrix.worker }}"
|
||||
runs-on: ubuntu-24.04
|
||||
runs-on: ubuntu-22.04
|
||||
timeout-minutes: 10
|
||||
strategy:
|
||||
fail-fast: false
|
||||
@@ -45,7 +41,6 @@ jobs:
|
||||
- name: Set up Go 1.x
|
||||
uses: actions/setup-go@v7
|
||||
with:
|
||||
cache: ${{ runner.environment == 'github-hosted' }}
|
||||
go-version: ^1.26
|
||||
id: go
|
||||
|
||||
|
||||
@@ -1,791 +0,0 @@
|
||||
name: POSIX Compliance Tests
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [ main, master, develop ]
|
||||
paths:
|
||||
- 'weed/mount/**'
|
||||
- 'weed/command/mount*'
|
||||
- 'weed/command/fuse*'
|
||||
- 'test/fuse_integration/**'
|
||||
pull_request:
|
||||
branches: [ main, master, develop ]
|
||||
paths:
|
||||
- 'weed/mount/**'
|
||||
- 'weed/command/mount*'
|
||||
- 'weed/command/fuse*'
|
||||
- 'test/fuse_integration/**'
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
test_type:
|
||||
description: 'Type of POSIX tests to run (default: full for comprehensive testing)'
|
||||
required: true
|
||||
default: 'full'
|
||||
type: choice
|
||||
options:
|
||||
- critical
|
||||
- basic
|
||||
- extended
|
||||
- full
|
||||
enable_external_tests:
|
||||
description: 'Run external test suites (slower)'
|
||||
required: false
|
||||
default: false
|
||||
type: boolean
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
pull-requests: write
|
||||
issues: write
|
||||
|
||||
env:
|
||||
GO_VERSION: '1.26'
|
||||
TIMEOUT: '45m'
|
||||
|
||||
jobs:
|
||||
posix-compliance-ubuntu:
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 60
|
||||
|
||||
strategy:
|
||||
matrix:
|
||||
fuse-version: ['2.9', '3.0']
|
||||
fail-fast: false
|
||||
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v7
|
||||
with:
|
||||
fetch-depth: 1
|
||||
|
||||
- name: Clean up stale FUSE mounts
|
||||
uses: ./.github/actions/cleanup-stale-fuse-mounts
|
||||
|
||||
- name: Set up Go ${{ env.GO_VERSION }}
|
||||
uses: actions/setup-go@v7
|
||||
with:
|
||||
go-version: ${{ env.GO_VERSION }}
|
||||
cache: ${{ runner.environment == 'github-hosted' }}
|
||||
cache-dependency-path: |
|
||||
go.sum
|
||||
test/fuse_integration/go.sum
|
||||
|
||||
- name: Install system dependencies
|
||||
run: |
|
||||
sudo apt-get update
|
||||
sudo apt-get install -y \
|
||||
attr \
|
||||
acl \
|
||||
build-essential \
|
||||
git \
|
||||
python3-pip
|
||||
|
||||
# Install FUSE version specific packages
|
||||
if [ "${{ matrix.fuse-version }}" = "2.9" ]; then
|
||||
echo "Installing FUSE 2.9..."
|
||||
sudo apt-get install -y fuse libfuse-dev
|
||||
elif [ "${{ matrix.fuse-version }}" = "3.0" ]; then
|
||||
echo "Installing FUSE 3.0..."
|
||||
sudo apt-get install -y fuse3 libfuse3-dev
|
||||
# Also install fuse2 for compatibility if needed
|
||||
sudo apt-get install -y fuse libfuse-dev
|
||||
fi
|
||||
|
||||
- name: Set up user permissions for FUSE
|
||||
run: |
|
||||
# Create fuse group if it doesn't exist
|
||||
sudo groupadd -f fuse
|
||||
|
||||
# Add user to fuse group
|
||||
sudo usermod -a -G fuse $USER
|
||||
|
||||
# Set permissions on FUSE device
|
||||
sudo chmod 666 /dev/fuse
|
||||
|
||||
# Allow non-root mounts to pass -o allow_other (weed -allowOthers)
|
||||
echo 'user_allow_other' | sudo tee -a /etc/fuse.conf
|
||||
sudo chmod 644 /etc/fuse.conf
|
||||
|
||||
# Ensure fuse module is loaded
|
||||
sudo modprobe fuse || true
|
||||
|
||||
# Verify setup
|
||||
echo "FUSE setup verification:"
|
||||
ls -la /dev/fuse
|
||||
groups $USER | grep fuse || echo "User not in fuse group yet (will be effective after login)"
|
||||
lsmod | grep fuse || echo "FUSE module not loaded"
|
||||
|
||||
- name: Install external test tools
|
||||
if: ${{ github.event.inputs.enable_external_tests == 'true' }}
|
||||
run: |
|
||||
# Install nfstest for POSIX API verification
|
||||
pip3 install --user nfstest
|
||||
|
||||
# Install FIO for performance testing
|
||||
sudo apt-get install -y fio
|
||||
|
||||
# Install additional test utilities
|
||||
sudo apt-get install -y stress-ng
|
||||
|
||||
- name: Build SeaweedFS
|
||||
run: |
|
||||
make
|
||||
# Verify binary exists and is executable (it's installed to GOPATH/bin)
|
||||
weed version
|
||||
|
||||
# Make weed binary available in PATH (it's already there from go install)
|
||||
sudo cp $(which weed) /usr/local/bin/weed
|
||||
which weed
|
||||
weed version
|
||||
|
||||
- name: Set up SeaweedFS cluster
|
||||
run: |
|
||||
start_and_wait() {
|
||||
local name=$1
|
||||
local url=$2
|
||||
local pidfile=$3
|
||||
shift 3
|
||||
local cmd="$@"
|
||||
|
||||
echo "Starting $name..."
|
||||
$cmd > "/tmp/seaweedfs/$name.log" 2>&1 &
|
||||
local pid=$!
|
||||
echo $pid > "$pidfile"
|
||||
|
||||
echo "Waiting for $name to start..."
|
||||
for i in {1..30}; do
|
||||
if curl -sf "$url" > /dev/null 2>&1; then
|
||||
echo "$name is ready"
|
||||
return 0
|
||||
fi
|
||||
if [ $i -eq 30 ]; then
|
||||
echo "$name failed to start"
|
||||
cat "/tmp/seaweedfs/$name.log"
|
||||
exit 1
|
||||
fi
|
||||
sleep 2
|
||||
done
|
||||
}
|
||||
|
||||
mkdir -p /tmp/seaweedfs/{master,volume,filer,mount}
|
||||
|
||||
start_and_wait "master" "http://127.0.0.1:9333/cluster/status" "/tmp/seaweedfs/master.pid" \
|
||||
"weed master -ip=127.0.0.1 -port=9333 -mdir=/tmp/seaweedfs/master -raftBootstrap=true"
|
||||
|
||||
start_and_wait "volume" "http://127.0.0.1:8080/status" "/tmp/seaweedfs/volume.pid" \
|
||||
"weed volume -mserver=127.0.0.1:9333 -ip=127.0.0.1 -port=8080 -dir=/tmp/seaweedfs/volume -max=100"
|
||||
|
||||
start_and_wait "filer" "http://127.0.0.1:8888/" "/tmp/seaweedfs/filer.pid" \
|
||||
"weed filer -master=127.0.0.1:9333 -ip=127.0.0.1 -port=8888"
|
||||
|
||||
echo "SeaweedFS cluster status:"
|
||||
curl -s http://127.0.0.1:9333/cluster/status || true
|
||||
|
||||
- name: Set up FUSE mount
|
||||
run: |
|
||||
# Create mount point
|
||||
MOUNT_POINT="/tmp/seaweedfs/mount"
|
||||
mkdir -p $MOUNT_POINT
|
||||
|
||||
echo "Mounting SeaweedFS FUSE filesystem..."
|
||||
# Mount SeaweedFS FUSE filesystem in background
|
||||
weed mount \
|
||||
-filer=127.0.0.1:8888 \
|
||||
-dir=$MOUNT_POINT \
|
||||
-filer.path=/ \
|
||||
-dirAutoCreate=true \
|
||||
-allowOthers=true \
|
||||
-nonempty=true \
|
||||
> /tmp/seaweedfs/mount.log 2>&1 &
|
||||
MOUNT_PID=$!
|
||||
echo $MOUNT_PID > /tmp/seaweedfs/mount.pid
|
||||
|
||||
# Wait for mount to be ready. The mount point directory already
|
||||
# exists, so only a real mount counts — a directory listing would
|
||||
# pass on the local disk even if weed mount died.
|
||||
echo "Waiting for FUSE mount to be ready..."
|
||||
for i in {1..30}; do
|
||||
if mountpoint -q $MOUNT_POINT 2>/dev/null; then
|
||||
echo "FUSE mount is ready"
|
||||
break
|
||||
fi
|
||||
if ! kill -0 $MOUNT_PID 2>/dev/null; then
|
||||
echo "FUSE mount process exited"
|
||||
cat /tmp/seaweedfs/mount.log
|
||||
exit 1
|
||||
fi
|
||||
if [ $i -eq 30 ]; then
|
||||
echo "FUSE mount failed to be ready"
|
||||
cat /tmp/seaweedfs/mount.log
|
||||
echo "Mount status:"
|
||||
mount | grep fuse || echo "No FUSE mounts found"
|
||||
ps aux | grep weed || echo "No weed processes found"
|
||||
exit 1
|
||||
fi
|
||||
sleep 2
|
||||
done
|
||||
|
||||
# Verify mount is working
|
||||
echo "Testing FUSE mount functionality..."
|
||||
echo "test" > $MOUNT_POINT/test_file.txt
|
||||
if [ "$(cat $MOUNT_POINT/test_file.txt)" = "test" ]; then
|
||||
echo "FUSE mount is working correctly"
|
||||
rm $MOUNT_POINT/test_file.txt
|
||||
else
|
||||
echo "FUSE mount is not working properly"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Export mount point for tests
|
||||
echo "SEAWEEDFS_MOUNT_POINT=$MOUNT_POINT" >> $GITHUB_ENV
|
||||
|
||||
- name: Set up test environment
|
||||
run: |
|
||||
cd test/fuse_integration
|
||||
|
||||
# Initialize Go module for tests
|
||||
if [ ! -f go.mod ]; then
|
||||
go mod init seaweedfs-posix-tests
|
||||
go mod tidy
|
||||
fi
|
||||
|
||||
# Create reports directory
|
||||
mkdir -p reports
|
||||
|
||||
# Set up external tools if requested
|
||||
# Note: External tools setup is now optional and won't fail the build
|
||||
if [ "${{ github.event.inputs.enable_external_tests }}" = "true" ]; then
|
||||
echo "Setting up external POSIX testing tools..."
|
||||
make -f posix_Makefile setup-external-tools || true
|
||||
# setup-external-tools exits 0 even when a sub-target failed, so
|
||||
# require the tools only when they are actually installed
|
||||
if [ -d external_tools/pjdfstest ] && command -v nfstest_posix >/dev/null 2>&1; then
|
||||
export POSIX_REQUIRE_EXTERNAL_TOOLS="true"
|
||||
echo "POSIX_REQUIRE_EXTERNAL_TOOLS=true" >> $GITHUB_ENV
|
||||
echo "✅ External tools setup completed successfully"
|
||||
else
|
||||
echo "⚠️ External tools setup incomplete - continuing with basic tests only"
|
||||
echo "External tools will be skipped but tests will continue"
|
||||
echo "This is expected behavior when build tools are not available"
|
||||
fi
|
||||
fi
|
||||
|
||||
# Verify SeaweedFS cluster is accessible
|
||||
echo "Verifying SeaweedFS cluster accessibility..."
|
||||
curl -sf http://127.0.0.1:9333/cluster/status
|
||||
curl -sf http://127.0.0.1:8080/status
|
||||
curl -sf http://127.0.0.1:8888/
|
||||
|
||||
# Verify mount point
|
||||
echo "Mount point: $SEAWEEDFS_MOUNT_POINT"
|
||||
ls -la $SEAWEEDFS_MOUNT_POINT
|
||||
|
||||
# Verify mount is actually working by testing file operations
|
||||
echo "Verifying SeaweedFS mount functionality..."
|
||||
TEST_FILE="$SEAWEEDFS_MOUNT_POINT/.mount_verification_test"
|
||||
echo "SeaweedFS mount test" > "$TEST_FILE"
|
||||
if [ "$(cat "$TEST_FILE")" = "SeaweedFS mount test" ]; then
|
||||
echo "✅ Mount verification successful"
|
||||
rm -f "$TEST_FILE"
|
||||
else
|
||||
echo "❌ Mount verification failed - mount may not be working properly"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Check if this is actually a FUSE mount — the plain directory created
|
||||
# above passes the file test, so without this the suite can run
|
||||
# against the local disk and report a pass it never earned
|
||||
if mount | grep -q "$SEAWEEDFS_MOUNT_POINT"; then
|
||||
echo "✅ FUSE mount detected:"
|
||||
mount | grep "$SEAWEEDFS_MOUNT_POINT"
|
||||
else
|
||||
echo "❌ Mount point not found in mount table - refusing to test a local directory"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
- name: Run POSIX compliance tests
|
||||
id: posix-tests
|
||||
env:
|
||||
SEAWEEDFS_MOUNT_POINT: ${{ env.SEAWEEDFS_MOUNT_POINT }}
|
||||
POSIX_REQUIRE_EXTERNAL_TOOLS: ${{ env.POSIX_REQUIRE_EXTERNAL_TOOLS }}
|
||||
run: |
|
||||
cd test/fuse_integration
|
||||
|
||||
# Determine which tests to run
|
||||
TEST_TYPE="${{ github.event.inputs.test_type }}"
|
||||
if [ -z "$TEST_TYPE" ]; then
|
||||
TEST_TYPE="full"
|
||||
fi
|
||||
|
||||
echo "Running POSIX tests: $TEST_TYPE"
|
||||
if [ "$TEST_TYPE" = "full" ]; then
|
||||
echo "🚀 Running COMPREHENSIVE POSIX test suite (including external test suites)"
|
||||
else
|
||||
echo "Using mount point: $SEAWEEDFS_MOUNT_POINT"
|
||||
fi
|
||||
|
||||
# Set test configuration to use our mounted filesystem
|
||||
export TEST_MOUNT_POINT="$SEAWEEDFS_MOUNT_POINT"
|
||||
export TEST_SKIP_CLUSTER_SETUP="true"
|
||||
|
||||
# Debug: Show environment variables
|
||||
echo "🔍 Test Environment:"
|
||||
echo " TEST_MOUNT_POINT=$TEST_MOUNT_POINT"
|
||||
echo " TEST_SKIP_CLUSTER_SETUP=$TEST_SKIP_CLUSTER_SETUP"
|
||||
echo " SEAWEEDFS_MOUNT_POINT=$SEAWEEDFS_MOUNT_POINT"
|
||||
|
||||
case "$TEST_TYPE" in
|
||||
"critical")
|
||||
echo "Running critical POSIX tests (basic compliance only)..."
|
||||
make -f posix_Makefile test-posix-critical
|
||||
;;
|
||||
"basic")
|
||||
echo "Running basic POSIX tests (core file operations)..."
|
||||
make -f posix_Makefile test-posix-basic
|
||||
;;
|
||||
"extended")
|
||||
echo "Running extended POSIX tests (including advanced features)..."
|
||||
make -f posix_Makefile test-posix-extended
|
||||
;;
|
||||
"full")
|
||||
echo "🚀 Running COMPREHENSIVE POSIX test suite (all tests including external)..."
|
||||
echo " This includes: basic, extended, and external test suites"
|
||||
echo " External suites: pjdfstest, nfstest_posix, and custom stress tests"
|
||||
make -f posix_Makefile test-posix-full
|
||||
;;
|
||||
*)
|
||||
echo "Unknown test type: $TEST_TYPE, defaulting to full suite"
|
||||
echo "🚀 Running full POSIX test suite (all tests including external)..."
|
||||
make -f posix_Makefile test-posix-full
|
||||
;;
|
||||
esac
|
||||
|
||||
- name: Run external test suites
|
||||
if: ${{ github.event.inputs.enable_external_tests == 'true' }}
|
||||
continue-on-error: true
|
||||
run: |
|
||||
cd test/fuse_integration
|
||||
|
||||
# Point the external suites at the same mount as the Go tests;
|
||||
# without these the Makefile falls back to a local-disk path.
|
||||
export TEST_MOUNT_POINT="$SEAWEEDFS_MOUNT_POINT"
|
||||
export TEST_SKIP_CLUSTER_SETUP="true"
|
||||
|
||||
# Run external tests (may fail on some systems)
|
||||
make -f posix_Makefile test-nfstest-posix || echo "nfstest failed or not available"
|
||||
make -f posix_Makefile test-fio-posix || echo "FIO tests failed or not available"
|
||||
|
||||
- name: Generate compliance report
|
||||
if: always()
|
||||
run: |
|
||||
cd test/fuse_integration
|
||||
make -f posix_Makefile generate-report
|
||||
|
||||
- name: Collect system information for debugging
|
||||
if: failure()
|
||||
run: |
|
||||
cd test/fuse_integration
|
||||
mkdir -p reports/debug
|
||||
|
||||
echo "System Information" > reports/debug/system_info.txt
|
||||
uname -a >> reports/debug/system_info.txt
|
||||
echo "" >> reports/debug/system_info.txt
|
||||
|
||||
echo "Mount Information" >> reports/debug/system_info.txt
|
||||
mount >> reports/debug/system_info.txt
|
||||
echo "" >> reports/debug/system_info.txt
|
||||
|
||||
echo "Disk Space" >> reports/debug/system_info.txt
|
||||
timeout 15 df -h >> reports/debug/system_info.txt || true
|
||||
echo "" >> reports/debug/system_info.txt
|
||||
|
||||
echo "Memory Information" >> reports/debug/system_info.txt
|
||||
free -h >> reports/debug/system_info.txt
|
||||
echo "" >> reports/debug/system_info.txt
|
||||
|
||||
echo "FUSE Information" >> reports/debug/system_info.txt
|
||||
ls -la /dev/fuse >> reports/debug/system_info.txt
|
||||
lsmod | grep fuse >> reports/debug/system_info.txt
|
||||
echo "" >> reports/debug/system_info.txt
|
||||
|
||||
echo "SeaweedFS Version" >> reports/debug/system_info.txt
|
||||
weed version >> reports/debug/system_info.txt
|
||||
echo "" >> reports/debug/system_info.txt
|
||||
|
||||
echo "Go Version" >> reports/debug/system_info.txt
|
||||
go version >> reports/debug/system_info.txt
|
||||
echo "" >> reports/debug/system_info.txt
|
||||
|
||||
echo "Running Processes" >> reports/debug/system_info.txt
|
||||
ps aux | grep -E "(weed|fuse)" >> reports/debug/system_info.txt
|
||||
echo "" >> reports/debug/system_info.txt
|
||||
|
||||
# Collect SeaweedFS service logs
|
||||
echo "=== SeaweedFS Service Logs ===" >> reports/debug/system_info.txt
|
||||
if [ -f /tmp/seaweedfs/master.log ]; then
|
||||
echo "Master Log:" >> reports/debug/system_info.txt
|
||||
tail -50 /tmp/seaweedfs/master.log >> reports/debug/system_info.txt
|
||||
echo "" >> reports/debug/system_info.txt
|
||||
fi
|
||||
|
||||
if [ -f /tmp/seaweedfs/volume.log ]; then
|
||||
echo "Volume Log:" >> reports/debug/system_info.txt
|
||||
tail -50 /tmp/seaweedfs/volume.log >> reports/debug/system_info.txt
|
||||
echo "" >> reports/debug/system_info.txt
|
||||
fi
|
||||
|
||||
if [ -f /tmp/seaweedfs/filer.log ]; then
|
||||
echo "Filer Log:" >> reports/debug/system_info.txt
|
||||
tail -50 /tmp/seaweedfs/filer.log >> reports/debug/system_info.txt
|
||||
echo "" >> reports/debug/system_info.txt
|
||||
fi
|
||||
|
||||
if [ -f /tmp/seaweedfs/mount.log ]; then
|
||||
echo "Mount Log:" >> reports/debug/system_info.txt
|
||||
tail -50 /tmp/seaweedfs/mount.log >> reports/debug/system_info.txt
|
||||
echo "" >> reports/debug/system_info.txt
|
||||
fi
|
||||
|
||||
# Copy full logs as separate files
|
||||
if [ -f /tmp/seaweedfs/master.log ]; then
|
||||
cp /tmp/seaweedfs/master.log reports/debug/master.log
|
||||
fi
|
||||
if [ -f /tmp/seaweedfs/volume.log ]; then
|
||||
cp /tmp/seaweedfs/volume.log reports/debug/volume.log
|
||||
fi
|
||||
if [ -f /tmp/seaweedfs/filer.log ]; then
|
||||
cp /tmp/seaweedfs/filer.log reports/debug/filer.log
|
||||
fi
|
||||
if [ -f /tmp/seaweedfs/mount.log ]; then
|
||||
cp /tmp/seaweedfs/mount.log reports/debug/mount.log
|
||||
fi
|
||||
|
||||
- name: Cleanup SeaweedFS cluster and FUSE mount
|
||||
if: always()
|
||||
run: |
|
||||
stop_process() {
|
||||
local name=$1
|
||||
local pidfile=$2
|
||||
if [ -f "$pidfile" ]; then
|
||||
local pid=$(cat "$pidfile")
|
||||
if kill -0 $pid 2>/dev/null; then
|
||||
echo "Stopping $name process (PID: $pid)..."
|
||||
kill -TERM $pid || true
|
||||
sleep 2
|
||||
kill -KILL $pid 2>/dev/null || true
|
||||
fi
|
||||
fi
|
||||
}
|
||||
|
||||
echo "Cleaning up SeaweedFS cluster and FUSE mount..."
|
||||
|
||||
MOUNT_POINT="/tmp/seaweedfs/mount"
|
||||
if mountpoint -q $MOUNT_POINT 2>/dev/null; then
|
||||
echo "Unmounting FUSE filesystem..."
|
||||
fusermount -u $MOUNT_POINT || umount $MOUNT_POINT || true
|
||||
fi
|
||||
|
||||
stop_process "mount" "/tmp/seaweedfs/mount.pid"
|
||||
stop_process "filer" "/tmp/seaweedfs/filer.pid"
|
||||
stop_process "volume" "/tmp/seaweedfs/volume.pid"
|
||||
stop_process "master" "/tmp/seaweedfs/master.pid"
|
||||
|
||||
pkill -f "weed " || true
|
||||
|
||||
fusermount -u $MOUNT_POINT 2>/dev/null || true
|
||||
umount $MOUNT_POINT 2>/dev/null || true
|
||||
|
||||
rm -rf /tmp/seaweedfs || true
|
||||
|
||||
echo "Cleanup completed"
|
||||
|
||||
|
||||
- name: Upload test results
|
||||
uses: actions/upload-artifact@v7
|
||||
if: always()
|
||||
with:
|
||||
name: posix-test-results-ubuntu-fuse${{ matrix.fuse-version }}
|
||||
path: |
|
||||
test/fuse_integration/reports/
|
||||
test/fuse_integration/*.log
|
||||
retention-days: 30
|
||||
|
||||
- name: Upload test coverage
|
||||
uses: actions/upload-artifact@v7
|
||||
if: always() && (github.event.inputs.test_type == 'full' || github.event.inputs.test_type == 'extended')
|
||||
with:
|
||||
name: posix-coverage-ubuntu
|
||||
path: test/fuse_integration/reports/posix_coverage.html
|
||||
retention-days: 14
|
||||
|
||||
- name: Comment PR with results
|
||||
if: github.event_name == 'pull_request' && always()
|
||||
uses: actions/github-script@v7
|
||||
with:
|
||||
script: |
|
||||
const fs = require('fs');
|
||||
const path = require('path');
|
||||
|
||||
const reportPath = 'test/fuse_integration/reports/posix_compliance_summary.txt';
|
||||
|
||||
if (fs.existsSync(reportPath)) {
|
||||
const report = fs.readFileSync(reportPath, 'utf8');
|
||||
const comment = `## POSIX Compliance Test Results
|
||||
|
||||
**Go Version:** ${{ env.GO_VERSION }}
|
||||
**FUSE Version:** ${{ matrix.fuse-version }}
|
||||
**Test Type:** ${{ github.event.inputs.test_type || 'full' }}
|
||||
|
||||
<details>
|
||||
<summary>Test Summary</summary>
|
||||
|
||||
\`\`\`
|
||||
${report}
|
||||
\`\`\`
|
||||
</details>
|
||||
|
||||
📊 Full results available in [test artifacts](https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }})
|
||||
`;
|
||||
|
||||
try {
|
||||
await github.rest.issues.createComment({
|
||||
issue_number: context.issue.number,
|
||||
owner: context.repo.owner,
|
||||
repo: context.repo.repo,
|
||||
body: comment
|
||||
});
|
||||
} catch (err) {
|
||||
console.log('Could not post PR comment: ' + err.message);
|
||||
}
|
||||
}
|
||||
|
||||
posix-compliance-macos:
|
||||
runs-on: macos-latest
|
||||
timeout-minutes: 60
|
||||
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v7
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v7
|
||||
with:
|
||||
go-version: ${{ env.GO_VERSION }}
|
||||
cache: ${{ runner.environment == 'github-hosted' }}
|
||||
|
||||
- name: Install macFUSE
|
||||
run: |
|
||||
# Install macFUSE
|
||||
brew install macfuse
|
||||
|
||||
# Note: macFUSE may require system extension approval on macOS
|
||||
# This step may need manual intervention in some cases
|
||||
|
||||
- name: Build SeaweedFS
|
||||
run: |
|
||||
make
|
||||
# Verify binary exists and is executable (it's installed to GOPATH/bin)
|
||||
weed version
|
||||
|
||||
- name: Run critical POSIX tests (macOS)
|
||||
continue-on-error: true # macOS FUSE can be more restrictive
|
||||
run: |
|
||||
cd test/fuse_integration
|
||||
|
||||
if [ ! -f go.mod ]; then
|
||||
go mod init seaweedfs-posix-tests
|
||||
go mod tidy
|
||||
fi
|
||||
|
||||
mkdir -p reports
|
||||
|
||||
# Run basic tests only on macOS due to FUSE limitations
|
||||
make -f posix_Makefile test-posix-critical
|
||||
|
||||
- name: Upload macOS test results
|
||||
uses: actions/upload-artifact@v7
|
||||
if: always()
|
||||
with:
|
||||
name: posix-test-results-macos
|
||||
path: |
|
||||
test/fuse_integration/reports/
|
||||
test/fuse_integration/*.log
|
||||
retention-days: 30
|
||||
|
||||
security-analysis:
|
||||
runs-on: ubuntu-latest
|
||||
if: github.event_name == 'pull_request'
|
||||
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v7
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v7
|
||||
with:
|
||||
go-version: ${{ env.GO_VERSION }}
|
||||
cache: ${{ runner.environment == 'github-hosted' }}
|
||||
|
||||
- name: Install security analysis tools
|
||||
run: |
|
||||
echo "Installing staticcheck for security analysis..."
|
||||
go install honnef.co/go/tools/cmd/staticcheck@latest
|
||||
|
||||
- name: Run security analysis on FUSE code
|
||||
run: |
|
||||
# Analyze mount and FUSE-related code for security and quality issues
|
||||
echo "Running staticcheck security analysis..."
|
||||
|
||||
# Run staticcheck on FUSE-related code
|
||||
echo "Analyzing FUSE mount code..."
|
||||
staticcheck ./weed/mount/... > staticcheck-report.txt 2>&1 || true
|
||||
staticcheck ./weed/command/mount* ./weed/command/fuse* >> staticcheck-report.txt 2>&1 || true
|
||||
|
||||
# Convert to JSON format for consistency
|
||||
echo "{" > gosec-report.json
|
||||
echo " \"tool\": \"staticcheck\"," >> gosec-report.json
|
||||
echo " \"timestamp\": \"$(date -Iseconds)\"," >> gosec-report.json
|
||||
echo " \"issues\": [" >> gosec-report.json
|
||||
|
||||
if [ -s staticcheck-report.txt ]; then
|
||||
echo " \"$(cat staticcheck-report.txt | head -20 | sed 's/"/\\"/g' | tr '\n' ' ')\"" >> gosec-report.json
|
||||
fi
|
||||
|
||||
echo " ]," >> gosec-report.json
|
||||
echo " \"stats\": {" >> gosec-report.json
|
||||
echo " \"files_analyzed\": $(find ./weed/mount ./weed/command -name '*.go' | wc -l)," >> gosec-report.json
|
||||
echo " \"issues_found\": $(wc -l < staticcheck-report.txt 2>/dev/null || echo 0)" >> gosec-report.json
|
||||
echo " }" >> gosec-report.json
|
||||
echo "}" >> gosec-report.json
|
||||
|
||||
echo "Security analysis completed"
|
||||
echo "Issues found: $(wc -l < staticcheck-report.txt 2>/dev/null || echo 0)"
|
||||
|
||||
- name: Upload security analysis results
|
||||
uses: actions/upload-artifact@v7
|
||||
if: always()
|
||||
with:
|
||||
name: security-analysis-results
|
||||
path: gosec-report.json
|
||||
retention-days: 30
|
||||
|
||||
performance-baseline:
|
||||
runs-on: ubuntu-latest
|
||||
if: github.event.inputs.test_type == 'full' || github.event_name == 'schedule'
|
||||
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v7
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v7
|
||||
with:
|
||||
go-version: ${{ env.GO_VERSION }}
|
||||
cache: ${{ runner.environment == 'github-hosted' }}
|
||||
|
||||
- name: Install dependencies
|
||||
run: |
|
||||
sudo apt-get update
|
||||
sudo apt-get install -y fuse libfuse-dev fio
|
||||
sudo usermod -a -G fuse $USER
|
||||
sudo chmod 666 /dev/fuse
|
||||
# The posix suites mount with -allowOthers; allow_other needs this
|
||||
echo 'user_allow_other' | sudo tee -a /etc/fuse.conf
|
||||
|
||||
- name: Build SeaweedFS
|
||||
run: make
|
||||
|
||||
- name: Run performance baseline tests
|
||||
run: |
|
||||
cd test/fuse_integration
|
||||
|
||||
if [ ! -f go.mod ]; then
|
||||
go mod init seaweedfs-posix-tests
|
||||
go mod tidy
|
||||
fi
|
||||
|
||||
mkdir -p reports
|
||||
|
||||
# Run performance benchmarks
|
||||
make -f posix_Makefile benchmark-posix
|
||||
# This job has no FUSE mount, so the fio suite can only skip; keep the
|
||||
# run green but honest about producing no mounted-FS results
|
||||
make -f posix_Makefile test-fio-posix || echo "FIO skipped - no mounted filesystem in this job"
|
||||
|
||||
- name: Store performance baseline
|
||||
uses: actions/upload-artifact@v7
|
||||
with:
|
||||
name: performance-baseline-results
|
||||
path: |
|
||||
test/fuse_integration/reports/posix_benchmark_results.log
|
||||
test/fuse_integration/reports/fio_*_results.log
|
||||
retention-days: 90
|
||||
|
||||
# Schedule regular compliance checks
|
||||
scheduled-compliance-check:
|
||||
runs-on: ubuntu-latest
|
||||
if: github.event_name == 'schedule'
|
||||
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v7
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v7
|
||||
with:
|
||||
go-version: ${{ env.GO_VERSION }}
|
||||
cache: ${{ runner.environment == 'github-hosted' }}
|
||||
|
||||
- name: Install dependencies
|
||||
run: |
|
||||
sudo apt-get update
|
||||
sudo apt-get install -y fuse libfuse-dev
|
||||
pip3 install --user nfstest
|
||||
# The posix suites mount with -allowOthers; allow_other needs this
|
||||
echo 'user_allow_other' | sudo tee -a /etc/fuse.conf
|
||||
|
||||
- name: Build SeaweedFS
|
||||
run: make
|
||||
|
||||
- name: Run comprehensive compliance check
|
||||
run: |
|
||||
cd test/fuse_integration
|
||||
make -f posix_Makefile test-posix-full
|
||||
make -f posix_Makefile generate-report
|
||||
|
||||
- name: Create compliance issue if tests fail
|
||||
if: failure()
|
||||
uses: actions/github-script@v7
|
||||
with:
|
||||
script: |
|
||||
const issue = await github.rest.issues.create({
|
||||
owner: context.repo.owner,
|
||||
repo: context.repo.repo,
|
||||
title: 'POSIX Compliance Check Failed - ' + new Date().toISOString().split('T')[0],
|
||||
body: `## POSIX Compliance Check Failure
|
||||
|
||||
The scheduled POSIX compliance check has failed. This may indicate:
|
||||
|
||||
- Regression in FUSE mount functionality
|
||||
- Changes affecting POSIX compatibility
|
||||
- Infrastructure issues with the test environment
|
||||
|
||||
**Action Required:** Review the test results and investigate any failures.
|
||||
|
||||
**Test Run:** [Workflow Run](https://github.com/${context.repo.owner}/${context.repo.repo}/actions/runs/${context.runId})
|
||||
|
||||
**Date:** ${new Date().toISOString()}
|
||||
|
||||
---
|
||||
This issue was automatically created by the POSIX compliance workflow.
|
||||
`,
|
||||
labels: ['bug', 'posix-compliance', 'automated-issue']
|
||||
});
|
||||
|
||||
console.log('Created issue:', issue.data.number);
|
||||
|
||||
# Schedule this workflow to run weekly
|
||||
# Uncomment the following lines to enable scheduled runs:
|
||||
# schedule:
|
||||
# - cron: '0 2 * * 1' # Every Monday at 2 AM UTC
|
||||
|
||||
@@ -23,8 +23,8 @@ on:
|
||||
- '.github/workflows/postgres-tests.yml'
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.event.pull_request.number || (github.event_name == 'push' && github.ref) || github.run_id }}
|
||||
cancel-in-progress: ${{ github.event_name == 'pull_request' || (github.event_name == 'push' && !startsWith(github.ref, 'refs/tags/')) }}
|
||||
group: ${{ github.head_ref }}/postgres-tests
|
||||
cancel-in-progress: true
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
@@ -41,7 +41,6 @@ jobs:
|
||||
- name: Set up Go 1.x
|
||||
uses: actions/setup-go@v7
|
||||
with:
|
||||
cache: ${{ runner.environment == 'github-hosted' }}
|
||||
go-version: ^1.26
|
||||
id: go
|
||||
|
||||
@@ -57,7 +56,6 @@ jobs:
|
||||
uses: docker/setup-buildx-action@v4
|
||||
|
||||
- name: Cache Docker layers
|
||||
if: runner.environment == 'github-hosted'
|
||||
uses: actions/cache@v6
|
||||
with:
|
||||
path: /tmp/.buildx-cache
|
||||
|
||||
@@ -21,8 +21,8 @@ on:
|
||||
- '.github/workflows/rust-volume-server-tests.yml'
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.event.pull_request.number || (github.event_name == 'push' && github.ref) || github.run_id }}
|
||||
cancel-in-progress: ${{ github.event_name == 'pull_request' || (github.event_name == 'push' && !startsWith(github.ref, 'refs/tags/')) }}
|
||||
group: ${{ github.workflow }}-${{ github.head_ref || github.ref }}
|
||||
cancel-in-progress: true
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
@@ -52,68 +52,9 @@ jobs:
|
||||
- 'seaweed-volume/**'
|
||||
- '.github/workflows/rust-volume-server-tests.yml'
|
||||
|
||||
build-test-binaries:
|
||||
name: Build Test Binaries
|
||||
runs-on: ubuntu-24.04
|
||||
timeout-minutes: 30
|
||||
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v7
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v7
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
cache: ${{ runner.environment == 'github-hosted' }}
|
||||
|
||||
- name: Install Rust toolchain
|
||||
uses: dtolnay/rust-toolchain@stable
|
||||
|
||||
# cargo tracks its own inputs but not the runner's C toolchain, so a cached
|
||||
# target/ can carry C objects built against a different glibc than we link against.
|
||||
- name: Fingerprint build toolchain
|
||||
id: toolchain
|
||||
run: echo "fingerprint=$(getconf GNU_LIBC_VERSION | tr ' ' '-')-rustc-$(rustc -V | awk '{print $2}')" >> "$GITHUB_OUTPUT"
|
||||
|
||||
- name: Cache cargo registry and target
|
||||
if: runner.environment == 'github-hosted'
|
||||
uses: actions/cache@v6
|
||||
with:
|
||||
path: |
|
||||
~/.cargo/registry
|
||||
~/.cargo/git
|
||||
seaweed-volume/target
|
||||
key: rust-${{ steps.toolchain.outputs.fingerprint }}-${{ hashFiles('seaweed-volume/Cargo.lock', 'seaweed-common/Cargo.lock') }}
|
||||
restore-keys: |
|
||||
rust-${{ steps.toolchain.outputs.fingerprint }}-
|
||||
|
||||
- name: Build Go weed binary
|
||||
run: |
|
||||
cd weed
|
||||
go build -tags 5BytesOffset -o weed .
|
||||
chmod +x weed
|
||||
./weed version
|
||||
|
||||
- name: Build Rust volume binary
|
||||
run: cd seaweed-volume && cargo build --release
|
||||
|
||||
- name: Package binaries
|
||||
run: tar cf rust-test-binaries.tar weed/weed seaweed-volume/target/release/weed-volume
|
||||
|
||||
- name: Upload test binaries
|
||||
uses: actions/upload-artifact@v7
|
||||
with:
|
||||
# archive layout mirrors the workspace: weed/weed and
|
||||
# seaweed-volume/target/release/weed-volume
|
||||
name: rust-test-binaries
|
||||
if-no-files-found: error
|
||||
retention-days: 1
|
||||
path: rust-test-binaries.tar
|
||||
|
||||
rust-unit-tests:
|
||||
name: Rust Unit Tests
|
||||
runs-on: ubuntu-24.04
|
||||
runs-on: ubuntu-22.04
|
||||
timeout-minutes: 15
|
||||
|
||||
steps:
|
||||
@@ -130,7 +71,6 @@ jobs:
|
||||
run: echo "fingerprint=$(getconf GNU_LIBC_VERSION | tr ' ' '-')-rustc-$(rustc -V | awk '{print $2}')" >> "$GITHUB_OUTPUT"
|
||||
|
||||
- name: Cache cargo registry and target
|
||||
if: runner.environment == 'github-hosted'
|
||||
uses: actions/cache@v6
|
||||
with:
|
||||
path: |
|
||||
@@ -194,7 +134,6 @@ jobs:
|
||||
run: echo "fingerprint=windows-rustc-$(rustc -V | awk '{print $2}')" >> "$GITHUB_OUTPUT"
|
||||
|
||||
- name: Cache cargo registry and target
|
||||
if: runner.environment == 'github-hosted'
|
||||
uses: actions/cache@v6
|
||||
with:
|
||||
path: |
|
||||
@@ -213,9 +152,8 @@ jobs:
|
||||
|
||||
rust-integration-tests:
|
||||
name: Rust Integration Tests
|
||||
runs-on: ubuntu-24.04
|
||||
runs-on: ubuntu-22.04
|
||||
timeout-minutes: 30
|
||||
needs: [build-test-binaries]
|
||||
|
||||
steps:
|
||||
- name: Checkout code
|
||||
@@ -225,18 +163,36 @@ jobs:
|
||||
uses: actions/setup-go@v7
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
cache: ${{ runner.environment == 'github-hosted' }}
|
||||
|
||||
- name: Download test binaries
|
||||
uses: actions/download-artifact@v7
|
||||
- name: Install Rust toolchain
|
||||
uses: dtolnay/rust-toolchain@stable
|
||||
|
||||
# cargo tracks its own inputs but not the runner's C toolchain, so a cached
|
||||
# target/ can carry C objects built against a different glibc than we link against.
|
||||
- name: Fingerprint build toolchain
|
||||
id: toolchain
|
||||
run: echo "fingerprint=$(getconf GNU_LIBC_VERSION | tr ' ' '-')-rustc-$(rustc -V | awk '{print $2}')" >> "$GITHUB_OUTPUT"
|
||||
|
||||
- name: Cache cargo registry and target
|
||||
uses: actions/cache@v6
|
||||
with:
|
||||
name: rust-test-binaries
|
||||
path: |
|
||||
~/.cargo/registry
|
||||
~/.cargo/git
|
||||
seaweed-volume/target
|
||||
key: rust-${{ steps.toolchain.outputs.fingerprint }}-${{ hashFiles('seaweed-volume/Cargo.lock', 'seaweed-common/Cargo.lock') }}
|
||||
restore-keys: |
|
||||
rust-${{ steps.toolchain.outputs.fingerprint }}-
|
||||
|
||||
- name: Restore binaries
|
||||
- name: Build Go weed binary
|
||||
run: |
|
||||
tar xf rust-test-binaries.tar
|
||||
chmod +x weed/weed seaweed-volume/target/release/weed-volume
|
||||
./weed/weed version
|
||||
cd weed
|
||||
go build -tags 5BytesOffset -o weed .
|
||||
chmod +x weed
|
||||
./weed version
|
||||
|
||||
- name: Build Rust volume binary
|
||||
run: cd seaweed-volume && cargo build --release
|
||||
|
||||
- name: Run integration tests
|
||||
env:
|
||||
@@ -270,9 +226,8 @@ jobs:
|
||||
|
||||
rust-volume-go-tests:
|
||||
name: Go Tests with Rust Volume (${{ matrix.test-type }} - Shard ${{ matrix.shard }})
|
||||
runs-on: ubuntu-24.04
|
||||
runs-on: ubuntu-22.04
|
||||
timeout-minutes: 45
|
||||
needs: [build-test-binaries]
|
||||
env:
|
||||
# Keep in step with the length of matrix.shard below.
|
||||
SHARD_COUNT: 3
|
||||
@@ -290,19 +245,36 @@ jobs:
|
||||
uses: actions/setup-go@v7
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
cache: ${{ runner.environment == 'github-hosted' }}
|
||||
|
||||
- name: Download test binaries
|
||||
uses: actions/download-artifact@v7
|
||||
- name: Install Rust toolchain
|
||||
uses: dtolnay/rust-toolchain@stable
|
||||
|
||||
# cargo tracks its own inputs but not the runner's C toolchain, so a cached
|
||||
# target/ can carry C objects built against a different glibc than we link against.
|
||||
- name: Fingerprint build toolchain
|
||||
id: toolchain
|
||||
run: echo "fingerprint=$(getconf GNU_LIBC_VERSION | tr ' ' '-')-rustc-$(rustc -V | awk '{print $2}')" >> "$GITHUB_OUTPUT"
|
||||
|
||||
- name: Cache cargo registry and target
|
||||
uses: actions/cache@v6
|
||||
with:
|
||||
name: rust-test-binaries
|
||||
path: |
|
||||
~/.cargo/registry
|
||||
~/.cargo/git
|
||||
seaweed-volume/target
|
||||
key: rust-${{ steps.toolchain.outputs.fingerprint }}-${{ hashFiles('seaweed-volume/Cargo.lock', 'seaweed-common/Cargo.lock') }}
|
||||
restore-keys: |
|
||||
rust-${{ steps.toolchain.outputs.fingerprint }}-
|
||||
|
||||
- name: Restore binaries
|
||||
- name: Build Go weed binary
|
||||
run: |
|
||||
tar xf rust-test-binaries.tar
|
||||
chmod +x weed/weed seaweed-volume/target/release/weed-volume
|
||||
./weed/weed version
|
||||
./seaweed-volume/target/release/weed-volume --help >/dev/null || true
|
||||
cd weed
|
||||
go build -tags 5BytesOffset -o weed .
|
||||
chmod +x weed
|
||||
./weed version
|
||||
|
||||
- name: Build Rust volume binary
|
||||
run: cd seaweed-volume && cargo build --release
|
||||
|
||||
# Dealing the listed tests out one by one keeps the shards even. Bucketing
|
||||
# them by first letter did not: names cluster, so ^Test[I-S] drew 50 of
|
||||
|
||||
@@ -17,8 +17,8 @@ on:
|
||||
- '.github/workflows/rust-worker-tests.yml'
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.event.pull_request.number || (github.event_name == 'push' && github.ref) || github.run_id }}
|
||||
cancel-in-progress: ${{ github.event_name == 'pull_request' || (github.event_name == 'push' && !startsWith(github.ref, 'refs/tags/')) }}
|
||||
group: ${{ github.workflow }}-${{ github.head_ref || github.ref }}
|
||||
cancel-in-progress: true
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
@@ -26,7 +26,7 @@ permissions:
|
||||
jobs:
|
||||
rust-worker-build:
|
||||
name: Rust Plugin Worker Build and Unit Tests
|
||||
runs-on: ubuntu-24.04
|
||||
runs-on: ubuntu-22.04
|
||||
timeout-minutes: 45
|
||||
|
||||
steps:
|
||||
@@ -45,7 +45,6 @@ jobs:
|
||||
run: echo "fingerprint=$(getconf GNU_LIBC_VERSION | tr ' ' '-')-rustc-$(rustc -V | awk '{print $2}')" >> "$GITHUB_OUTPUT"
|
||||
|
||||
- name: Cache cargo registry and target
|
||||
if: runner.environment == 'github-hosted'
|
||||
uses: actions/cache@v6
|
||||
with:
|
||||
path: |
|
||||
|
||||
@@ -32,7 +32,7 @@ jobs:
|
||||
permissions:
|
||||
contents: write
|
||||
needs: cleanup
|
||||
runs-on: ubuntu-24.04
|
||||
runs-on: ubuntu-22.04
|
||||
strategy:
|
||||
matrix:
|
||||
include:
|
||||
@@ -46,7 +46,6 @@ jobs:
|
||||
uses: dtolnay/rust-toolchain@stable
|
||||
|
||||
- name: Cache cargo registry and target
|
||||
if: runner.environment == 'github-hosted'
|
||||
uses: actions/cache@v6
|
||||
with:
|
||||
path: |
|
||||
@@ -115,7 +114,6 @@ jobs:
|
||||
targets: ${{ matrix.target }}
|
||||
|
||||
- name: Cache cargo registry and target
|
||||
if: runner.environment == 'github-hosted'
|
||||
uses: actions/cache@v6
|
||||
with:
|
||||
path: |
|
||||
|
||||
@@ -15,7 +15,7 @@ jobs:
|
||||
build-rust-volume-linux:
|
||||
permissions:
|
||||
contents: write
|
||||
runs-on: ubuntu-24.04
|
||||
runs-on: ubuntu-22.04
|
||||
strategy:
|
||||
matrix:
|
||||
include:
|
||||
@@ -37,13 +37,9 @@ jobs:
|
||||
if: matrix.cross
|
||||
run: |
|
||||
sudo dpkg --add-architecture arm64
|
||||
# noble keeps apt sources in deb822 ubuntu.sources, not sources.list —
|
||||
# pin them to amd64 so apt does not probe arm64 indexes on
|
||||
# archive/security.ubuntu.com (they 404).
|
||||
sudo sed -i '/^Types: deb$/a Architectures: amd64' /etc/apt/sources.list.d/ubuntu.sources
|
||||
for suite in noble noble-updates noble-security; do
|
||||
echo "deb [arch=arm64] http://ports.ubuntu.com/ubuntu-ports $suite main restricted universe multiverse" | sudo tee -a /etc/apt/sources.list.d/arm64.list
|
||||
done
|
||||
sudo sed -i 's/^deb /deb [arch=amd64] /' /etc/apt/sources.list
|
||||
echo "deb [arch=arm64] http://ports.ubuntu.com/ jammy main restricted universe multiverse" | sudo tee /etc/apt/sources.list.d/arm64.list
|
||||
echo "deb [arch=arm64] http://ports.ubuntu.com/ jammy-updates main restricted universe multiverse" | sudo tee -a /etc/apt/sources.list.d/arm64.list
|
||||
sudo apt-get update
|
||||
sudo apt-get install -y gcc-aarch64-linux-gnu libssl-dev:arm64
|
||||
echo "CARGO_TARGET_AARCH64_UNKNOWN_LINUX_GNU_LINKER=aarch64-linux-gnu-gcc" >> "$GITHUB_ENV"
|
||||
@@ -52,7 +48,6 @@ jobs:
|
||||
echo "OPENSSL_LIB_DIR=/usr/lib/aarch64-linux-gnu" >> "$GITHUB_ENV"
|
||||
|
||||
- name: Cache cargo registry and target
|
||||
if: runner.environment == 'github-hosted'
|
||||
uses: actions/cache@v6
|
||||
with:
|
||||
path: |
|
||||
@@ -124,7 +119,7 @@ jobs:
|
||||
build-rust-worker-linux:
|
||||
permissions:
|
||||
contents: write
|
||||
runs-on: ubuntu-24.04
|
||||
runs-on: ubuntu-22.04
|
||||
strategy:
|
||||
matrix:
|
||||
include:
|
||||
@@ -150,19 +145,14 @@ jobs:
|
||||
if: matrix.cross
|
||||
run: |
|
||||
sudo dpkg --add-architecture arm64
|
||||
# noble keeps apt sources in deb822 ubuntu.sources, not sources.list —
|
||||
# pin them to amd64 so apt does not probe arm64 indexes on
|
||||
# archive/security.ubuntu.com (they 404).
|
||||
sudo sed -i '/^Types: deb$/a Architectures: amd64' /etc/apt/sources.list.d/ubuntu.sources
|
||||
for suite in noble noble-updates noble-security; do
|
||||
echo "deb [arch=arm64] http://ports.ubuntu.com/ubuntu-ports $suite main restricted universe multiverse" | sudo tee -a /etc/apt/sources.list.d/arm64.list
|
||||
done
|
||||
sudo sed -i 's/^deb /deb [arch=amd64] /' /etc/apt/sources.list
|
||||
echo "deb [arch=arm64] http://ports.ubuntu.com/ jammy main restricted universe multiverse" | sudo tee /etc/apt/sources.list.d/arm64.list
|
||||
echo "deb [arch=arm64] http://ports.ubuntu.com/ jammy-updates main restricted universe multiverse" | sudo tee -a /etc/apt/sources.list.d/arm64.list
|
||||
sudo apt-get update
|
||||
sudo apt-get install -y gcc-aarch64-linux-gnu
|
||||
echo "CARGO_TARGET_AARCH64_UNKNOWN_LINUX_GNU_LINKER=aarch64-linux-gnu-gcc" >> "$GITHUB_ENV"
|
||||
|
||||
- name: Cache cargo registry and target
|
||||
if: runner.environment == 'github-hosted'
|
||||
uses: actions/cache@v6
|
||||
with:
|
||||
path: |
|
||||
@@ -240,7 +230,6 @@ jobs:
|
||||
targets: ${{ matrix.target }}
|
||||
|
||||
- name: Cache cargo registry and target
|
||||
if: runner.environment == 'github-hosted'
|
||||
uses: actions/cache@v6
|
||||
with:
|
||||
path: |
|
||||
@@ -316,7 +305,6 @@ jobs:
|
||||
uses: dtolnay/rust-toolchain@stable
|
||||
|
||||
- name: Cache cargo registry and target
|
||||
if: runner.environment == 'github-hosted'
|
||||
uses: actions/cache@v6
|
||||
with:
|
||||
path: |
|
||||
|
||||
@@ -21,8 +21,8 @@ on:
|
||||
- '.github/workflows/s3-etag-acl-tests.yml'
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.event.pull_request.number || (github.event_name == 'push' && github.ref) || github.run_id }}
|
||||
cancel-in-progress: ${{ github.event_name == 'pull_request' || (github.event_name == 'push' && !startsWith(github.ref, 'refs/tags/')) }}
|
||||
group: ${{ github.head_ref || github.ref }}/s3-etag-acl-tests
|
||||
cancel-in-progress: true
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
@@ -30,7 +30,7 @@ permissions:
|
||||
jobs:
|
||||
s3-etag-acl-tests:
|
||||
name: S3 ETag + ACL Integration Tests
|
||||
runs-on: ubuntu-24.04
|
||||
runs-on: ubuntu-22.04
|
||||
timeout-minutes: 15
|
||||
steps:
|
||||
- name: Check out code
|
||||
@@ -40,7 +40,6 @@ jobs:
|
||||
uses: actions/setup-go@v7
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
cache: ${{ runner.environment == 'github-hosted' }}
|
||||
|
||||
- name: Install SeaweedFS
|
||||
run: |
|
||||
|
||||
@@ -13,8 +13,8 @@ on:
|
||||
- '.github/workflows/s3-example-integration-tests.yml'
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.event.pull_request.number || (github.event_name == 'push' && github.ref) || github.run_id }}
|
||||
cancel-in-progress: ${{ github.event_name == 'pull_request' || (github.event_name == 'push' && !startsWith(github.ref, 'refs/tags/')) }}
|
||||
group: ${{ github.head_ref }}/s3-integration-tests
|
||||
cancel-in-progress: true
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
@@ -22,7 +22,7 @@ permissions:
|
||||
jobs:
|
||||
tests:
|
||||
name: S3 Integration Tests
|
||||
runs-on: ubuntu-24.04
|
||||
runs-on: ubuntu-22.04
|
||||
timeout-minutes: 30
|
||||
|
||||
steps:
|
||||
@@ -33,7 +33,6 @@ jobs:
|
||||
uses: actions/setup-go@v7
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
cache: ${{ runner.environment == 'github-hosted' }}
|
||||
id: go
|
||||
|
||||
- name: Build SeaweedFS
|
||||
|
||||
@@ -12,8 +12,8 @@ on:
|
||||
- '.github/workflows/s3-filer-group-tests.yml'
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.event.pull_request.number || (github.event_name == 'push' && github.ref) || github.run_id }}
|
||||
cancel-in-progress: ${{ github.event_name == 'pull_request' || (github.event_name == 'push' && !startsWith(github.ref, 'refs/tags/')) }}
|
||||
group: ${{ github.head_ref }}/s3-filer-group-tests
|
||||
cancel-in-progress: true
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
@@ -25,21 +25,17 @@ defaults:
|
||||
jobs:
|
||||
s3-filer-group-tests:
|
||||
name: S3 Filer Group Integration Tests
|
||||
runs-on: ubuntu-24.04
|
||||
runs-on: ubuntu-22.04
|
||||
timeout-minutes: 20
|
||||
|
||||
steps:
|
||||
- name: Check out code
|
||||
uses: actions/checkout@v7
|
||||
|
||||
- name: Clean up stale FUSE mounts
|
||||
uses: ./.github/actions/cleanup-stale-fuse-mounts
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v7
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
cache: ${{ runner.environment == 'github-hosted' }}
|
||||
id: go
|
||||
|
||||
- name: Install SeaweedFS
|
||||
@@ -54,7 +50,7 @@ jobs:
|
||||
echo "=== System Information ==="
|
||||
uname -a
|
||||
free -h
|
||||
timeout 15 df -h || true
|
||||
df -h
|
||||
echo "=== Starting Filer Group Tests ==="
|
||||
|
||||
# Run tests with automatic server management
|
||||
|
||||
@@ -4,7 +4,6 @@ on:
|
||||
pull_request:
|
||||
paths:
|
||||
- 'weed/s3api/**'
|
||||
- 'weed/remote_storage/**'
|
||||
- 'weed/filer/**'
|
||||
- 'weed/server/**'
|
||||
- 'test/s3/**'
|
||||
@@ -13,8 +12,8 @@ on:
|
||||
- '.github/workflows/s3-go-tests.yml'
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.event.pull_request.number || (github.event_name == 'push' && github.ref) || github.run_id }}
|
||||
cancel-in-progress: ${{ github.event_name == 'pull_request' || (github.event_name == 'push' && !startsWith(github.ref, 'refs/tags/')) }}
|
||||
group: ${{ github.head_ref }}/s3-go-tests
|
||||
cancel-in-progress: true
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
@@ -26,7 +25,7 @@ defaults:
|
||||
jobs:
|
||||
s3-versioning-tests:
|
||||
name: S3 Versioning Tests
|
||||
runs-on: ubuntu-24.04
|
||||
runs-on: ubuntu-22.04
|
||||
timeout-minutes: 30
|
||||
strategy:
|
||||
matrix:
|
||||
@@ -36,14 +35,10 @@ jobs:
|
||||
- name: Check out code
|
||||
uses: actions/checkout@v7
|
||||
|
||||
- name: Clean up stale FUSE mounts
|
||||
uses: ./.github/actions/cleanup-stale-fuse-mounts
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v7
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
cache: ${{ runner.environment == 'github-hosted' }}
|
||||
id: go
|
||||
|
||||
- name: Install SeaweedFS
|
||||
@@ -58,7 +53,7 @@ jobs:
|
||||
echo "=== System Information ==="
|
||||
uname -a
|
||||
free -h
|
||||
timeout 15 df -h || true
|
||||
df -h
|
||||
echo "=== Starting Tests ==="
|
||||
|
||||
# Run tests with automatic server management
|
||||
@@ -97,7 +92,7 @@ jobs:
|
||||
|
||||
s3-versioning-compatibility:
|
||||
name: S3 Versioning Compatibility Test
|
||||
runs-on: ubuntu-24.04
|
||||
runs-on: ubuntu-22.04
|
||||
timeout-minutes: 20
|
||||
|
||||
steps:
|
||||
@@ -108,7 +103,6 @@ jobs:
|
||||
uses: actions/setup-go@v7
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
cache: ${{ runner.environment == 'github-hosted' }}
|
||||
id: go
|
||||
|
||||
- name: Install SeaweedFS
|
||||
@@ -146,7 +140,7 @@ jobs:
|
||||
|
||||
s3-cors-compatibility:
|
||||
name: S3 CORS Compatibility Test
|
||||
runs-on: ubuntu-24.04
|
||||
runs-on: ubuntu-22.04
|
||||
timeout-minutes: 20
|
||||
|
||||
steps:
|
||||
@@ -157,7 +151,6 @@ jobs:
|
||||
uses: actions/setup-go@v7
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
cache: ${{ runner.environment == 'github-hosted' }}
|
||||
id: go
|
||||
|
||||
- name: Install SeaweedFS
|
||||
@@ -195,7 +188,7 @@ jobs:
|
||||
|
||||
s3-retention-tests:
|
||||
name: S3 Retention Tests
|
||||
runs-on: ubuntu-24.04
|
||||
runs-on: ubuntu-22.04
|
||||
timeout-minutes: 30
|
||||
strategy:
|
||||
matrix:
|
||||
@@ -205,14 +198,10 @@ jobs:
|
||||
- name: Check out code
|
||||
uses: actions/checkout@v7
|
||||
|
||||
- name: Clean up stale FUSE mounts
|
||||
uses: ./.github/actions/cleanup-stale-fuse-mounts
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v7
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
cache: ${{ runner.environment == 'github-hosted' }}
|
||||
id: go
|
||||
|
||||
- name: Install SeaweedFS
|
||||
@@ -227,7 +216,7 @@ jobs:
|
||||
echo "=== System Information ==="
|
||||
uname -a
|
||||
free -h
|
||||
timeout 15 df -h || true
|
||||
df -h
|
||||
echo "=== Starting Tests ==="
|
||||
|
||||
# Run tests with automatic server management
|
||||
@@ -266,7 +255,7 @@ jobs:
|
||||
|
||||
s3-lifecycle-tests:
|
||||
name: S3 Lifecycle Tests
|
||||
runs-on: ubuntu-24.04
|
||||
runs-on: ubuntu-22.04
|
||||
timeout-minutes: 10
|
||||
strategy:
|
||||
fail-fast: false
|
||||
@@ -294,7 +283,6 @@ jobs:
|
||||
- TestLifecycleSuspendedVersioningExpiration
|
||||
- TestLifecycleTagFilter
|
||||
- TestLifecycleVersionedBucketCreatesDeleteMarker
|
||||
- TestLifecycleVersionedBucketExpiresOnSecondRun
|
||||
|
||||
steps:
|
||||
- name: Check out code
|
||||
@@ -304,7 +292,6 @@ jobs:
|
||||
uses: actions/setup-go@v7
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
cache: ${{ runner.environment == 'github-hosted' }}
|
||||
id: go
|
||||
|
||||
- name: Install SeaweedFS
|
||||
@@ -339,21 +326,17 @@ jobs:
|
||||
|
||||
s3-checksum-tests:
|
||||
name: S3 Checksum Tests
|
||||
runs-on: ubuntu-24.04
|
||||
runs-on: ubuntu-22.04
|
||||
timeout-minutes: 20
|
||||
|
||||
steps:
|
||||
- name: Check out code
|
||||
uses: actions/checkout@v7
|
||||
|
||||
- name: Clean up stale FUSE mounts
|
||||
uses: ./.github/actions/cleanup-stale-fuse-mounts
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v7
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
cache: ${{ runner.environment == 'github-hosted' }}
|
||||
id: go
|
||||
|
||||
- name: Install SeaweedFS
|
||||
@@ -368,7 +351,7 @@ jobs:
|
||||
echo "=== System Information ==="
|
||||
uname -a
|
||||
free -h
|
||||
timeout 15 df -h || true
|
||||
df -h
|
||||
echo "=== Starting Tests ==="
|
||||
make test-with-server
|
||||
|
||||
@@ -398,7 +381,7 @@ jobs:
|
||||
|
||||
s3-cors-tests:
|
||||
name: S3 CORS Tests
|
||||
runs-on: ubuntu-24.04
|
||||
runs-on: ubuntu-22.04
|
||||
timeout-minutes: 30
|
||||
strategy:
|
||||
matrix:
|
||||
@@ -408,14 +391,10 @@ jobs:
|
||||
- name: Check out code
|
||||
uses: actions/checkout@v7
|
||||
|
||||
- name: Clean up stale FUSE mounts
|
||||
uses: ./.github/actions/cleanup-stale-fuse-mounts
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v7
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
cache: ${{ runner.environment == 'github-hosted' }}
|
||||
id: go
|
||||
|
||||
- name: Install SeaweedFS
|
||||
@@ -430,7 +409,7 @@ jobs:
|
||||
echo "=== System Information ==="
|
||||
uname -a
|
||||
free -h
|
||||
timeout 15 df -h || true
|
||||
df -h
|
||||
echo "=== Starting Tests ==="
|
||||
|
||||
# Run tests with automatic server management
|
||||
@@ -469,7 +448,7 @@ jobs:
|
||||
|
||||
s3-retention-worm:
|
||||
name: S3 Retention WORM Integration Test
|
||||
runs-on: ubuntu-24.04
|
||||
runs-on: ubuntu-22.04
|
||||
timeout-minutes: 20
|
||||
|
||||
steps:
|
||||
@@ -480,7 +459,6 @@ jobs:
|
||||
uses: actions/setup-go@v7
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
cache: ${{ runner.environment == 'github-hosted' }}
|
||||
id: go
|
||||
|
||||
- name: Install SeaweedFS
|
||||
@@ -519,7 +497,7 @@ jobs:
|
||||
|
||||
s3-versioning-stress:
|
||||
name: S3 Versioning Stress Test
|
||||
runs-on: ubuntu-24.04
|
||||
runs-on: ubuntu-22.04
|
||||
timeout-minutes: 35
|
||||
|
||||
steps:
|
||||
@@ -530,7 +508,6 @@ jobs:
|
||||
uses: actions/setup-go@v7
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
cache: ${{ runner.environment == 'github-hosted' }}
|
||||
id: go
|
||||
|
||||
- name: Install SeaweedFS
|
||||
@@ -588,7 +565,7 @@ jobs:
|
||||
s3-tagging-tests:
|
||||
# CI job for S3 object tagging tests
|
||||
name: S3 Tagging Tests
|
||||
runs-on: ubuntu-24.04
|
||||
runs-on: ubuntu-22.04
|
||||
timeout-minutes: 20
|
||||
|
||||
steps:
|
||||
@@ -599,7 +576,6 @@ jobs:
|
||||
uses: actions/setup-go@v7
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
cache: ${{ runner.environment == 'github-hosted' }}
|
||||
id: go
|
||||
|
||||
- name: Install SeaweedFS
|
||||
@@ -644,7 +620,7 @@ jobs:
|
||||
|
||||
s3-remote-cache-tests:
|
||||
name: S3 Remote Cache Tests
|
||||
runs-on: ubuntu-24.04
|
||||
runs-on: ubuntu-22.04
|
||||
timeout-minutes: 20
|
||||
|
||||
steps:
|
||||
@@ -655,7 +631,6 @@ jobs:
|
||||
uses: actions/setup-go@v7
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
cache: ${{ runner.environment == 'github-hosted' }}
|
||||
id: go
|
||||
|
||||
- name: Install SeaweedFS
|
||||
|
||||
@@ -20,8 +20,8 @@ on:
|
||||
- '.github/workflows/s3-iam-tests.yml'
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.event.pull_request.number || (github.event_name == 'push' && github.ref) || github.run_id }}
|
||||
cancel-in-progress: ${{ github.event_name == 'pull_request' || (github.event_name == 'push' && !startsWith(github.ref, 'refs/tags/')) }}
|
||||
group: ${{ github.head_ref }}/s3-iam-tests
|
||||
cancel-in-progress: true
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
@@ -34,7 +34,7 @@ jobs:
|
||||
# Unit tests for IAM components
|
||||
iam-unit-tests:
|
||||
name: IAM Unit Tests
|
||||
runs-on: ubuntu-24.04
|
||||
runs-on: ubuntu-22.04
|
||||
timeout-minutes: 15
|
||||
|
||||
steps:
|
||||
@@ -45,7 +45,6 @@ jobs:
|
||||
uses: actions/setup-go@v7
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
cache: ${{ runner.environment == 'github-hosted' }}
|
||||
id: go
|
||||
|
||||
- name: Get dependencies
|
||||
@@ -81,7 +80,7 @@ jobs:
|
||||
# S3 IAM integration tests with SeaweedFS services
|
||||
s3-iam-integration-tests:
|
||||
name: S3 IAM Integration Tests
|
||||
runs-on: ubuntu-24.04
|
||||
runs-on: ubuntu-22.04
|
||||
timeout-minutes: 25
|
||||
strategy:
|
||||
matrix:
|
||||
@@ -91,14 +90,10 @@ jobs:
|
||||
- name: Check out code
|
||||
uses: actions/checkout@v7
|
||||
|
||||
- name: Clean up stale FUSE mounts
|
||||
uses: ./.github/actions/cleanup-stale-fuse-mounts
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v7
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
cache: ${{ runner.environment == 'github-hosted' }}
|
||||
id: go
|
||||
|
||||
- name: Install SeaweedFS
|
||||
@@ -114,7 +109,7 @@ jobs:
|
||||
echo "=== System Information ==="
|
||||
uname -a
|
||||
free -h
|
||||
timeout 15 df -h || true
|
||||
df -h
|
||||
echo "=== Starting S3 IAM Integration Tests (${{ matrix.test-type }}) ==="
|
||||
|
||||
# Set WEED_BINARY to use the installed version
|
||||
@@ -202,7 +197,7 @@ jobs:
|
||||
# Distributed IAM tests
|
||||
s3-iam-distributed-tests:
|
||||
name: S3 IAM Distributed Tests
|
||||
runs-on: ubuntu-24.04
|
||||
runs-on: ubuntu-22.04
|
||||
timeout-minutes: 25
|
||||
|
||||
steps:
|
||||
@@ -213,7 +208,6 @@ jobs:
|
||||
uses: actions/setup-go@v7
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
cache: ${{ runner.environment == 'github-hosted' }}
|
||||
id: go
|
||||
|
||||
- name: Install SeaweedFS
|
||||
@@ -263,7 +257,7 @@ jobs:
|
||||
# Performance and stress tests
|
||||
s3-iam-performance-tests:
|
||||
name: S3 IAM Performance Tests
|
||||
runs-on: ubuntu-24.04
|
||||
runs-on: ubuntu-22.04
|
||||
timeout-minutes: 30
|
||||
|
||||
steps:
|
||||
@@ -274,7 +268,6 @@ jobs:
|
||||
uses: actions/setup-go@v7
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
cache: ${{ runner.environment == 'github-hosted' }}
|
||||
id: go
|
||||
|
||||
- name: Install SeaweedFS
|
||||
|
||||
@@ -16,8 +16,8 @@ on:
|
||||
- '.github/workflows/s3-keycloak-tests.yml'
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.event.pull_request.number || (github.event_name == 'push' && github.ref) || github.run_id }}
|
||||
cancel-in-progress: ${{ github.event_name == 'pull_request' || (github.event_name == 'push' && !startsWith(github.ref, 'refs/tags/')) }}
|
||||
group: ${{ github.head_ref }}/s3-keycloak-tests
|
||||
cancel-in-progress: true
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
@@ -30,21 +30,17 @@ jobs:
|
||||
# Dedicated job for Keycloak integration tests
|
||||
s3-keycloak-integration-tests:
|
||||
name: S3 Keycloak Integration Tests
|
||||
runs-on: ubuntu-24.04
|
||||
runs-on: ubuntu-22.04
|
||||
timeout-minutes: 30
|
||||
|
||||
steps:
|
||||
- name: Check out code
|
||||
uses: actions/checkout@v7
|
||||
|
||||
- name: Clean up stale FUSE mounts
|
||||
uses: ./.github/actions/cleanup-stale-fuse-mounts
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v7
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
cache: ${{ runner.environment == 'github-hosted' }}
|
||||
id: go
|
||||
|
||||
- name: Install SeaweedFS
|
||||
@@ -60,7 +56,7 @@ jobs:
|
||||
echo "=== System Information ==="
|
||||
uname -a
|
||||
free -h
|
||||
timeout 15 df -h || true
|
||||
df -h
|
||||
echo "=== Starting S3 Keycloak Integration Tests ==="
|
||||
|
||||
# Set WEED_BINARY to use the installed version
|
||||
|
||||
@@ -12,8 +12,8 @@ on:
|
||||
- '.github/workflows/s3-mutation-regression-tests.yml'
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.event.pull_request.number || (github.event_name == 'push' && github.ref) || github.run_id }}
|
||||
cancel-in-progress: ${{ github.event_name == 'pull_request' || (github.event_name == 'push' && !startsWith(github.ref, 'refs/tags/')) }}
|
||||
group: ${{ github.head_ref }}/s3-mutation-regression-tests
|
||||
cancel-in-progress: true
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
@@ -21,7 +21,7 @@ permissions:
|
||||
jobs:
|
||||
s3-versioning-regressions:
|
||||
name: S3 Versioning Regression Tests
|
||||
runs-on: ubuntu-24.04
|
||||
runs-on: ubuntu-22.04
|
||||
timeout-minutes: 25
|
||||
|
||||
steps:
|
||||
@@ -32,7 +32,6 @@ jobs:
|
||||
uses: actions/setup-go@v7
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
cache: ${{ runner.environment == 'github-hosted' }}
|
||||
|
||||
- name: Run S3 versioning regression tests
|
||||
timeout-minutes: 20
|
||||
@@ -75,7 +74,7 @@ jobs:
|
||||
|
||||
s3-delete-regressions:
|
||||
name: S3 Delete Regression Tests
|
||||
runs-on: ubuntu-24.04
|
||||
runs-on: ubuntu-22.04
|
||||
timeout-minutes: 20
|
||||
|
||||
steps:
|
||||
@@ -86,7 +85,6 @@ jobs:
|
||||
uses: actions/setup-go@v7
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
cache: ${{ runner.environment == 'github-hosted' }}
|
||||
|
||||
- name: Run S3 delete regression tests
|
||||
timeout-minutes: 15
|
||||
@@ -114,7 +112,7 @@ jobs:
|
||||
|
||||
s3-distributed-lock-regressions:
|
||||
name: S3 Distributed Lock Regression Tests
|
||||
runs-on: ubuntu-24.04
|
||||
runs-on: ubuntu-22.04
|
||||
timeout-minutes: 30
|
||||
|
||||
steps:
|
||||
@@ -125,7 +123,6 @@ jobs:
|
||||
uses: actions/setup-go@v7
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
cache: ${{ runner.environment == 'github-hosted' }}
|
||||
|
||||
- name: Build SeaweedFS
|
||||
run: |
|
||||
|
||||
@@ -23,10 +23,6 @@ env:
|
||||
S3_ENDPOINT_URL: http://localhost:8333
|
||||
BUCKET_NAME: test-parquet-bucket
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.event.pull_request.number || (github.event_name == 'push' && github.ref) || github.run_id }}
|
||||
cancel-in-progress: ${{ github.event_name == 'pull_request' || (github.event_name == 'push' && !startsWith(github.ref, 'refs/tags/')) }}
|
||||
|
||||
jobs:
|
||||
parquet-integration-tests:
|
||||
name: PyArrow Parquet Tests (Python ${{ matrix.python-version }})
|
||||
@@ -46,13 +42,13 @@ jobs:
|
||||
uses: actions/setup-go@v7
|
||||
with:
|
||||
go-version: ^1.26
|
||||
cache: ${{ runner.environment == 'github-hosted' }}
|
||||
cache: true
|
||||
|
||||
- name: Set up Python ${{ matrix.python-version }}
|
||||
uses: actions/setup-python@v7
|
||||
with:
|
||||
python-version: ${{ matrix.python-version }}
|
||||
cache: ${{ runner.environment == 'github-hosted' && 'pip' || '' }}
|
||||
cache: 'pip'
|
||||
cache-dependency-path: 'test/s3/parquet/requirements.txt'
|
||||
|
||||
- name: Install system dependencies
|
||||
@@ -154,7 +150,7 @@ jobs:
|
||||
uses: actions/setup-go@v7
|
||||
with:
|
||||
go-version: ^1.26
|
||||
cache: ${{ runner.environment == 'github-hosted' }}
|
||||
cache: true
|
||||
|
||||
- name: Run Go unit tests
|
||||
run: |
|
||||
|
||||
@@ -24,8 +24,8 @@ on:
|
||||
- '.github/workflows/s3-policy-tests.yml'
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.event.pull_request.number || (github.event_name == 'push' && github.ref) || github.run_id }}
|
||||
cancel-in-progress: ${{ github.event_name == 'pull_request' || (github.event_name == 'push' && !startsWith(github.ref, 'refs/tags/')) }}
|
||||
group: ${{ github.head_ref }}/s3-policy-tests
|
||||
cancel-in-progress: true
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
@@ -38,7 +38,7 @@ jobs:
|
||||
# Unit tests for policy components
|
||||
policy-unit-tests:
|
||||
name: S3 Policy Unit Tests
|
||||
runs-on: ubuntu-24.04
|
||||
runs-on: ubuntu-22.04
|
||||
timeout-minutes: 15
|
||||
|
||||
steps:
|
||||
@@ -49,7 +49,6 @@ jobs:
|
||||
uses: actions/setup-go@v7
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
cache: ${{ runner.environment == 'github-hosted' }}
|
||||
id: go
|
||||
|
||||
- name: Get dependencies
|
||||
@@ -82,21 +81,17 @@ jobs:
|
||||
# S3 Policy Variables Integration Tests
|
||||
s3-policy-variables-tests:
|
||||
name: S3 Policy Variables Integration Tests
|
||||
runs-on: ubuntu-24.04
|
||||
runs-on: ubuntu-22.04
|
||||
timeout-minutes: 25
|
||||
|
||||
steps:
|
||||
- name: Check out code
|
||||
uses: actions/checkout@v7
|
||||
|
||||
- name: Clean up stale FUSE mounts
|
||||
uses: ./.github/actions/cleanup-stale-fuse-mounts
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v7
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
cache: ${{ runner.environment == 'github-hosted' }}
|
||||
id: go
|
||||
|
||||
- name: Install SeaweedFS
|
||||
@@ -111,7 +106,7 @@ jobs:
|
||||
echo "=== System Information ==="
|
||||
uname -a
|
||||
free -h
|
||||
timeout 15 df -h || true
|
||||
df -h
|
||||
|
||||
echo "=== Starting S3 Policy Variables Integration Tests ==="
|
||||
|
||||
@@ -192,7 +187,7 @@ jobs:
|
||||
# S3 Policy Enforcement Integration Tests
|
||||
s3-policy-enforcement-tests:
|
||||
name: S3 Policy Enforcement Integration Tests
|
||||
runs-on: ubuntu-24.04
|
||||
runs-on: ubuntu-22.04
|
||||
timeout-minutes: 30
|
||||
strategy:
|
||||
matrix:
|
||||
@@ -202,14 +197,10 @@ jobs:
|
||||
- name: Check out code
|
||||
uses: actions/checkout@v7
|
||||
|
||||
- name: Clean up stale FUSE mounts
|
||||
uses: ./.github/actions/cleanup-stale-fuse-mounts
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v7
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
cache: ${{ runner.environment == 'github-hosted' }}
|
||||
id: go
|
||||
|
||||
- name: Install SeaweedFS
|
||||
@@ -224,7 +215,7 @@ jobs:
|
||||
echo "=== System Information ==="
|
||||
uname -a
|
||||
free -h
|
||||
timeout 15 df -h || true
|
||||
df -h
|
||||
|
||||
echo "=== Starting S3 Policy Enforcement Tests (${{ matrix.test-case }}) ==="
|
||||
|
||||
@@ -317,7 +308,7 @@ jobs:
|
||||
# Trusted Proxy Detection Tests
|
||||
trusted-proxy-tests:
|
||||
name: Trusted Proxy Detection Tests
|
||||
runs-on: ubuntu-24.04
|
||||
runs-on: ubuntu-22.04
|
||||
timeout-minutes: 20
|
||||
|
||||
steps:
|
||||
@@ -328,7 +319,6 @@ jobs:
|
||||
uses: actions/setup-go@v7
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
cache: ${{ runner.environment == 'github-hosted' }}
|
||||
id: go
|
||||
|
||||
- name: Install SeaweedFS
|
||||
@@ -405,7 +395,7 @@ jobs:
|
||||
# S3 Policy Shell Integration Tests
|
||||
s3-policy-shell-tests:
|
||||
name: S3 Policy Shell Integration Tests
|
||||
runs-on: ubuntu-24.04
|
||||
runs-on: ubuntu-22.04
|
||||
timeout-minutes: 15
|
||||
|
||||
steps:
|
||||
@@ -416,7 +406,6 @@ jobs:
|
||||
uses: actions/setup-go@v7
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
cache: ${{ runner.environment == 'github-hosted' }}
|
||||
id: go
|
||||
|
||||
- name: Install SeaweedFS
|
||||
|
||||
@@ -21,8 +21,8 @@ on:
|
||||
- '.github/workflows/s3-proxy-signature-tests.yml'
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.event.pull_request.number || (github.event_name == 'push' && github.ref) || github.run_id }}
|
||||
cancel-in-progress: ${{ github.event_name == 'pull_request' || (github.event_name == 'push' && !startsWith(github.ref, 'refs/tags/')) }}
|
||||
group: ${{ github.head_ref || github.ref }}/s3-proxy-signature-tests
|
||||
cancel-in-progress: true
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
@@ -30,7 +30,7 @@ permissions:
|
||||
jobs:
|
||||
proxy-signature-tests:
|
||||
name: S3 Proxy Signature Verification Tests
|
||||
runs-on: ubuntu-24.04
|
||||
runs-on: ubuntu-22.04
|
||||
timeout-minutes: 15
|
||||
steps:
|
||||
- name: Check out code into the Go module directory
|
||||
@@ -40,7 +40,6 @@ jobs:
|
||||
uses: actions/setup-go@v7
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
cache: ${{ runner.environment == 'github-hosted' }}
|
||||
id: go
|
||||
|
||||
- name: Configure Docker Hub mirror
|
||||
|
||||
@@ -15,8 +15,8 @@ on:
|
||||
- '.github/workflows/s3-sdk-v2-routing-tests.yml'
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.event.pull_request.number || (github.event_name == 'push' && github.ref) || github.run_id }}
|
||||
cancel-in-progress: ${{ github.event_name == 'pull_request' || (github.event_name == 'push' && !startsWith(github.ref, 'refs/tags/')) }}
|
||||
group: ${{ github.head_ref || github.ref }}/s3-sdk-v2-routing-tests
|
||||
cancel-in-progress: true
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
@@ -24,7 +24,7 @@ permissions:
|
||||
jobs:
|
||||
s3-sdk-v2-routing-tests:
|
||||
name: S3 SDK V2 Routing Tests
|
||||
runs-on: ubuntu-24.04
|
||||
runs-on: ubuntu-22.04
|
||||
timeout-minutes: 10
|
||||
steps:
|
||||
- name: Check out code
|
||||
@@ -34,7 +34,6 @@ jobs:
|
||||
uses: actions/setup-go@v7
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
cache: ${{ runner.environment == 'github-hosted' }}
|
||||
|
||||
- name: Install SeaweedFS
|
||||
run: |
|
||||
|
||||
@@ -46,7 +46,7 @@ permissions:
|
||||
jobs:
|
||||
snowflake-s3compat-tests:
|
||||
name: Snowflake S3Compat API tests
|
||||
runs-on: ubuntu-24.04
|
||||
runs-on: ubuntu-22.04
|
||||
timeout-minutes: 30
|
||||
env:
|
||||
WORK_DIR: /tmp/seaweedfs-snowflake-tests
|
||||
@@ -60,7 +60,6 @@ jobs:
|
||||
uses: actions/setup-go@v7
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
cache: ${{ runner.environment == 'github-hosted' }}
|
||||
id: go
|
||||
|
||||
- name: Set up Java
|
||||
@@ -68,7 +67,7 @@ jobs:
|
||||
with:
|
||||
java-version: '17'
|
||||
distribution: 'temurin'
|
||||
cache: ${{ runner.environment == 'github-hosted' && 'maven' || '' }}
|
||||
cache: 'maven'
|
||||
|
||||
- name: Install SeaweedFS
|
||||
run: |
|
||||
|
||||
@@ -11,8 +11,8 @@ on:
|
||||
workflow_dispatch:
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.event.pull_request.number || (github.event_name == 'push' && github.ref) || github.run_id }}
|
||||
cancel-in-progress: ${{ github.event_name == 'pull_request' || (github.event_name == 'push' && !startsWith(github.ref, 'refs/tags/')) }}
|
||||
group: ${{ github.head_ref }}/s3-spark-tests
|
||||
cancel-in-progress: true
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
@@ -20,21 +20,17 @@ permissions:
|
||||
jobs:
|
||||
s3-spark-issue-repro-tests:
|
||||
name: S3 Spark Issue Reproduction Tests
|
||||
runs-on: ubuntu-24.04
|
||||
runs-on: ubuntu-22.04
|
||||
timeout-minutes: 45
|
||||
|
||||
steps:
|
||||
- name: Check out code
|
||||
uses: actions/checkout@v7
|
||||
|
||||
- name: Clean up stale FUSE mounts
|
||||
uses: ./.github/actions/cleanup-stale-fuse-mounts
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v7
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
cache: ${{ runner.environment == 'github-hosted' }}
|
||||
id: go
|
||||
|
||||
- name: Configure Docker Hub mirror
|
||||
@@ -60,7 +56,7 @@ jobs:
|
||||
echo "=== System Information ==="
|
||||
uname -a
|
||||
free -h
|
||||
timeout 15 df -h || true
|
||||
df -h
|
||||
echo "=== Starting S3 Spark Integration Tests ==="
|
||||
|
||||
go test -v -timeout 30m . 2>&1 | tee test-output.log || {
|
||||
|
||||
@@ -23,8 +23,8 @@ on:
|
||||
- 'test/s3/sse/**'
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.event.pull_request.number || (github.event_name == 'push' && github.ref) || github.run_id }}
|
||||
cancel-in-progress: ${{ github.event_name == 'pull_request' || (github.event_name == 'push' && !startsWith(github.ref, 'refs/tags/')) }}
|
||||
group: ${{ github.head_ref }}/s3-sse-tests
|
||||
cancel-in-progress: true
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
@@ -36,7 +36,7 @@ defaults:
|
||||
jobs:
|
||||
s3-sse-integration-tests:
|
||||
name: S3 SSE Integration Tests
|
||||
runs-on: ubuntu-24.04
|
||||
runs-on: ubuntu-22.04
|
||||
timeout-minutes: 30
|
||||
strategy:
|
||||
matrix:
|
||||
@@ -46,14 +46,10 @@ jobs:
|
||||
- name: Check out code
|
||||
uses: actions/checkout@v7
|
||||
|
||||
- name: Clean up stale FUSE mounts
|
||||
uses: ./.github/actions/cleanup-stale-fuse-mounts
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v7
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
cache: ${{ runner.environment == 'github-hosted' }}
|
||||
id: go
|
||||
|
||||
- name: Install SeaweedFS
|
||||
@@ -68,7 +64,7 @@ jobs:
|
||||
echo "=== System Information ==="
|
||||
uname -a
|
||||
free -h
|
||||
timeout 15 df -h || true
|
||||
df -h
|
||||
echo "=== Starting SSE Tests ==="
|
||||
|
||||
# Run tests with automatic server management
|
||||
@@ -111,7 +107,7 @@ jobs:
|
||||
|
||||
s3-sse-compatibility:
|
||||
name: S3 SSE Compatibility Test
|
||||
runs-on: ubuntu-24.04
|
||||
runs-on: ubuntu-22.04
|
||||
timeout-minutes: 20
|
||||
|
||||
steps:
|
||||
@@ -122,7 +118,6 @@ jobs:
|
||||
uses: actions/setup-go@v7
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
cache: ${{ runner.environment == 'github-hosted' }}
|
||||
id: go
|
||||
|
||||
- name: Install SeaweedFS
|
||||
@@ -160,7 +155,7 @@ jobs:
|
||||
|
||||
s3-sse-metadata-persistence:
|
||||
name: S3 SSE Metadata Persistence Test
|
||||
runs-on: ubuntu-24.04
|
||||
runs-on: ubuntu-22.04
|
||||
timeout-minutes: 20
|
||||
|
||||
steps:
|
||||
@@ -171,7 +166,6 @@ jobs:
|
||||
uses: actions/setup-go@v7
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
cache: ${{ runner.environment == 'github-hosted' }}
|
||||
id: go
|
||||
|
||||
- name: Install SeaweedFS
|
||||
@@ -210,7 +204,7 @@ jobs:
|
||||
|
||||
s3-sse-copy-operations:
|
||||
name: S3 SSE Copy Operations Test
|
||||
runs-on: ubuntu-24.04
|
||||
runs-on: ubuntu-22.04
|
||||
timeout-minutes: 25
|
||||
|
||||
steps:
|
||||
@@ -221,7 +215,6 @@ jobs:
|
||||
uses: actions/setup-go@v7
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
cache: ${{ runner.environment == 'github-hosted' }}
|
||||
id: go
|
||||
|
||||
- name: Install SeaweedFS
|
||||
@@ -260,7 +253,7 @@ jobs:
|
||||
|
||||
s3-sse-multipart:
|
||||
name: S3 SSE Multipart Upload Test
|
||||
runs-on: ubuntu-24.04
|
||||
runs-on: ubuntu-22.04
|
||||
timeout-minutes: 25
|
||||
|
||||
steps:
|
||||
@@ -271,7 +264,6 @@ jobs:
|
||||
uses: actions/setup-go@v7
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
cache: ${{ runner.environment == 'github-hosted' }}
|
||||
id: go
|
||||
|
||||
- name: Install SeaweedFS
|
||||
@@ -310,7 +302,7 @@ jobs:
|
||||
|
||||
s3-sse-performance:
|
||||
name: S3 SSE Performance Test
|
||||
runs-on: ubuntu-24.04
|
||||
runs-on: ubuntu-22.04
|
||||
timeout-minutes: 35
|
||||
# Only run performance tests on master branch pushes to avoid overloading PR testing
|
||||
if: github.event_name == 'push' && (github.ref == 'refs/heads/master' || github.ref == 'refs/heads/main')
|
||||
@@ -323,7 +315,6 @@ jobs:
|
||||
uses: actions/setup-go@v7
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
cache: ${{ runner.environment == 'github-hosted' }}
|
||||
id: go
|
||||
|
||||
- name: Install SeaweedFS
|
||||
@@ -361,7 +352,7 @@ jobs:
|
||||
|
||||
s3-volume-encryption:
|
||||
name: S3 Volume Encryption Test
|
||||
runs-on: ubuntu-24.04
|
||||
runs-on: ubuntu-22.04
|
||||
timeout-minutes: 20
|
||||
|
||||
steps:
|
||||
@@ -372,7 +363,6 @@ jobs:
|
||||
uses: actions/setup-go@v7
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
cache: ${{ runner.environment == 'github-hosted' }}
|
||||
id: go
|
||||
|
||||
- name: Install SeaweedFS
|
||||
|
||||
@@ -12,10 +12,6 @@ on:
|
||||
- 'go.sum'
|
||||
- '.github/workflows/s3-tables-tests.yml'
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.event.pull_request.number || (github.event_name == 'push' && github.ref) || github.run_id }}
|
||||
cancel-in-progress: ${{ github.event_name == 'pull_request' || (github.event_name == 'push' && !startsWith(github.ref, 'refs/tags/')) }}
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
@@ -23,21 +19,17 @@ permissions:
|
||||
jobs:
|
||||
s3-tables-tests:
|
||||
name: S3 Tables Integration Tests
|
||||
runs-on: ubuntu-24.04
|
||||
runs-on: ubuntu-22.04
|
||||
timeout-minutes: 30
|
||||
|
||||
steps:
|
||||
- name: Check out code
|
||||
uses: actions/checkout@v7
|
||||
|
||||
- name: Clean up stale FUSE mounts
|
||||
uses: ./.github/actions/cleanup-stale-fuse-mounts
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v7
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
cache: ${{ runner.environment == 'github-hosted' }}
|
||||
id: go
|
||||
|
||||
- name: Run go mod tidy
|
||||
@@ -56,8 +48,7 @@ jobs:
|
||||
echo "=== System Information ==="
|
||||
uname -a
|
||||
free -h
|
||||
# A stale FUSE mount can hang or fail df; keep diagnostics best effort.
|
||||
timeout 15 df -h || true
|
||||
df -h
|
||||
echo "=== Starting S3 Tables Tests ==="
|
||||
|
||||
# Run S3 Tables integration tests
|
||||
@@ -88,21 +79,17 @@ jobs:
|
||||
|
||||
iceberg-catalog-tests:
|
||||
name: Iceberg Catalog Integration Tests
|
||||
runs-on: ubuntu-24.04
|
||||
runs-on: ubuntu-22.04
|
||||
timeout-minutes: 30
|
||||
|
||||
steps:
|
||||
- name: Check out code
|
||||
uses: actions/checkout@v7
|
||||
|
||||
- name: Clean up stale FUSE mounts
|
||||
uses: ./.github/actions/cleanup-stale-fuse-mounts
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v7
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
cache: ${{ runner.environment == 'github-hosted' }}
|
||||
id: go
|
||||
|
||||
- name: Configure Docker Hub mirror
|
||||
@@ -122,7 +109,7 @@ jobs:
|
||||
echo "=== System Information ==="
|
||||
uname -a
|
||||
free -h
|
||||
timeout 15 df -h || true
|
||||
df -h
|
||||
echo "=== Starting Iceberg Catalog Tests ==="
|
||||
|
||||
# Run Iceberg catalog integration tests using Makefile (handles build)
|
||||
@@ -153,21 +140,17 @@ jobs:
|
||||
|
||||
trino-iceberg-catalog-tests:
|
||||
name: Trino Iceberg Catalog Integration Tests
|
||||
runs-on: ubuntu-24.04
|
||||
runs-on: ubuntu-22.04
|
||||
timeout-minutes: 30
|
||||
|
||||
steps:
|
||||
- name: Check out code
|
||||
uses: actions/checkout@v7
|
||||
|
||||
- name: Clean up stale FUSE mounts
|
||||
uses: ./.github/actions/cleanup-stale-fuse-mounts
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v7
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
cache: ${{ runner.environment == 'github-hosted' }}
|
||||
id: go
|
||||
|
||||
- name: Configure Docker Hub mirror
|
||||
@@ -196,7 +179,7 @@ jobs:
|
||||
echo "=== System Information ==="
|
||||
uname -a
|
||||
free -h
|
||||
timeout 15 df -h || true
|
||||
df -h
|
||||
echo "=== Starting Trino Iceberg Catalog Tests ==="
|
||||
|
||||
# Run Trino + Iceberg catalog integration tests
|
||||
@@ -227,21 +210,17 @@ jobs:
|
||||
|
||||
dremio-iceberg-catalog-tests:
|
||||
name: Dremio Iceberg Catalog Integration Tests
|
||||
runs-on: ubuntu-24.04
|
||||
runs-on: ubuntu-22.04
|
||||
timeout-minutes: 30
|
||||
|
||||
steps:
|
||||
- name: Check out code
|
||||
uses: actions/checkout@v7
|
||||
|
||||
- name: Clean up stale FUSE mounts
|
||||
uses: ./.github/actions/cleanup-stale-fuse-mounts
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v7
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
cache: ${{ runner.environment == 'github-hosted' }}
|
||||
id: go
|
||||
|
||||
- name: Configure Docker Hub mirror
|
||||
@@ -271,7 +250,7 @@ jobs:
|
||||
echo "=== System Information ==="
|
||||
uname -a
|
||||
free -h
|
||||
timeout 15 df -h || true
|
||||
df -h
|
||||
docker info
|
||||
echo "=== Starting Dremio Iceberg Catalog Tests ==="
|
||||
|
||||
@@ -304,21 +283,17 @@ jobs:
|
||||
|
||||
doris-iceberg-catalog-tests:
|
||||
name: Doris Iceberg Catalog Integration Tests
|
||||
runs-on: ubuntu-24.04
|
||||
runs-on: ubuntu-22.04
|
||||
timeout-minutes: 35
|
||||
|
||||
steps:
|
||||
- name: Check out code
|
||||
uses: actions/checkout@v7
|
||||
|
||||
- name: Clean up stale FUSE mounts
|
||||
uses: ./.github/actions/cleanup-stale-fuse-mounts
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v7
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
cache: ${{ runner.environment == 'github-hosted' }}
|
||||
id: go
|
||||
|
||||
- name: Configure Docker Hub mirror
|
||||
@@ -348,7 +323,7 @@ jobs:
|
||||
echo "=== System Information ==="
|
||||
uname -a
|
||||
free -h
|
||||
timeout 15 df -h || true
|
||||
df -h
|
||||
docker info
|
||||
echo "=== Starting Doris Iceberg Catalog Tests ==="
|
||||
|
||||
@@ -381,7 +356,7 @@ jobs:
|
||||
|
||||
clickhouse-iceberg-catalog-tests:
|
||||
name: ClickHouse Iceberg Catalog Integration Tests (${{ matrix.tag }})
|
||||
runs-on: ubuntu-24.04
|
||||
runs-on: ubuntu-22.04
|
||||
timeout-minutes: 30
|
||||
strategy:
|
||||
fail-fast: false
|
||||
@@ -398,14 +373,10 @@ jobs:
|
||||
- name: Check out code
|
||||
uses: actions/checkout@v7
|
||||
|
||||
- name: Clean up stale FUSE mounts
|
||||
uses: ./.github/actions/cleanup-stale-fuse-mounts
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v7
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
cache: ${{ runner.environment == 'github-hosted' }}
|
||||
id: go
|
||||
|
||||
- name: Configure Docker Hub mirror
|
||||
@@ -437,7 +408,7 @@ jobs:
|
||||
echo "=== System Information ==="
|
||||
uname -a
|
||||
free -h
|
||||
timeout 15 df -h || true
|
||||
df -h
|
||||
docker info
|
||||
echo "=== Starting ClickHouse Iceberg Catalog Tests ==="
|
||||
|
||||
@@ -470,7 +441,7 @@ jobs:
|
||||
|
||||
olake-iceberg-catalog-tests:
|
||||
name: OLake Iceberg Catalog Integration Tests (${{ matrix.tag }})
|
||||
runs-on: ubuntu-24.04
|
||||
runs-on: ubuntu-22.04
|
||||
timeout-minutes: 30
|
||||
strategy:
|
||||
fail-fast: false
|
||||
@@ -489,14 +460,10 @@ jobs:
|
||||
- name: Check out code
|
||||
uses: actions/checkout@v7
|
||||
|
||||
- name: Clean up stale FUSE mounts
|
||||
uses: ./.github/actions/cleanup-stale-fuse-mounts
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v7
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
cache: ${{ runner.environment == 'github-hosted' }}
|
||||
id: go
|
||||
|
||||
- name: Configure Docker Hub mirror
|
||||
@@ -529,7 +496,7 @@ jobs:
|
||||
echo "=== System Information ==="
|
||||
uname -a
|
||||
free -h
|
||||
timeout 15 df -h || true
|
||||
df -h
|
||||
docker info
|
||||
echo "=== Starting OLake Iceberg Catalog Tests ==="
|
||||
|
||||
@@ -585,21 +552,17 @@ jobs:
|
||||
|
||||
polaris-integration-tests:
|
||||
name: Polaris Integration Tests
|
||||
runs-on: ubuntu-24.04
|
||||
runs-on: ubuntu-22.04
|
||||
timeout-minutes: 30
|
||||
|
||||
steps:
|
||||
- name: Check out code
|
||||
uses: actions/checkout@v7
|
||||
|
||||
- name: Clean up stale FUSE mounts
|
||||
uses: ./.github/actions/cleanup-stale-fuse-mounts
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v7
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
cache: ${{ runner.environment == 'github-hosted' }}
|
||||
id: go
|
||||
|
||||
- name: Run go mod tidy
|
||||
@@ -627,7 +590,7 @@ jobs:
|
||||
echo "=== System Information ==="
|
||||
uname -a
|
||||
free -h
|
||||
timeout 15 df -h || true
|
||||
df -h
|
||||
echo "=== Starting Polaris Tests ==="
|
||||
|
||||
go test -v -timeout 20m ./test/s3tables/polaris 2>&1 | tee test/s3tables/polaris/test-output.log || {
|
||||
@@ -657,21 +620,17 @@ jobs:
|
||||
|
||||
spark-iceberg-catalog-tests:
|
||||
name: Spark Iceberg Catalog Integration Tests
|
||||
runs-on: ubuntu-24.04
|
||||
runs-on: ubuntu-22.04
|
||||
timeout-minutes: 30
|
||||
|
||||
steps:
|
||||
- name: Check out code
|
||||
uses: actions/checkout@v7
|
||||
|
||||
- name: Clean up stale FUSE mounts
|
||||
uses: ./.github/actions/cleanup-stale-fuse-mounts
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v7
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
cache: ${{ runner.environment == 'github-hosted' }}
|
||||
id: go
|
||||
|
||||
- name: Configure Docker Hub mirror
|
||||
@@ -700,7 +659,7 @@ jobs:
|
||||
echo "=== System Information ==="
|
||||
uname -a
|
||||
free -h
|
||||
timeout 15 df -h || true
|
||||
df -h
|
||||
echo "=== Starting Spark Iceberg Catalog Tests ==="
|
||||
|
||||
# Run Spark + Iceberg catalog integration tests
|
||||
@@ -731,21 +690,17 @@ jobs:
|
||||
|
||||
risingwave-catalog-tests:
|
||||
name: RisingWave Catalog Integration Tests
|
||||
runs-on: ubuntu-24.04
|
||||
runs-on: ubuntu-22.04
|
||||
timeout-minutes: 30
|
||||
|
||||
steps:
|
||||
- name: Check out code
|
||||
uses: actions/checkout@v7
|
||||
|
||||
- name: Clean up stale FUSE mounts
|
||||
uses: ./.github/actions/cleanup-stale-fuse-mounts
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v7
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
cache: ${{ runner.environment == 'github-hosted' }}
|
||||
id: go
|
||||
|
||||
- name: Configure Docker Hub mirror
|
||||
@@ -775,7 +730,7 @@ jobs:
|
||||
echo "=== System Information ==="
|
||||
uname -a
|
||||
free -h
|
||||
timeout 15 df -h || true
|
||||
df -h
|
||||
echo "=== Starting RisingWave Catalog Tests ==="
|
||||
|
||||
# Run RisingWave catalog integration tests
|
||||
@@ -806,21 +761,17 @@ jobs:
|
||||
|
||||
sts-integration-tests:
|
||||
name: STS Integration Tests
|
||||
runs-on: ubuntu-24.04
|
||||
runs-on: ubuntu-22.04
|
||||
timeout-minutes: 30
|
||||
|
||||
steps:
|
||||
- name: Check out code
|
||||
uses: actions/checkout@v7
|
||||
|
||||
- name: Clean up stale FUSE mounts
|
||||
uses: ./.github/actions/cleanup-stale-fuse-mounts
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v7
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
cache: ${{ runner.environment == 'github-hosted' }}
|
||||
id: go
|
||||
|
||||
- name: Configure Docker Hub mirror
|
||||
@@ -834,7 +785,6 @@ jobs:
|
||||
|
||||
- name: Restore python:3 image cache
|
||||
id: python-image
|
||||
if: runner.environment == 'github-hosted'
|
||||
uses: actions/cache@v6
|
||||
with:
|
||||
path: /tmp/python3-image.tar
|
||||
@@ -874,7 +824,7 @@ jobs:
|
||||
echo "=== System Information ==="
|
||||
uname -a
|
||||
free -h
|
||||
timeout 15 df -h || true
|
||||
df -h
|
||||
echo "=== Starting STS Integration Tests ==="
|
||||
|
||||
# Run STS integration tests
|
||||
@@ -905,21 +855,17 @@ jobs:
|
||||
|
||||
lakekeeper-integration-tests:
|
||||
name: Lakekeeper Integration Tests
|
||||
runs-on: ubuntu-24.04
|
||||
runs-on: ubuntu-22.04
|
||||
timeout-minutes: 30
|
||||
|
||||
steps:
|
||||
- name: Check out code
|
||||
uses: actions/checkout@v7
|
||||
|
||||
- name: Clean up stale FUSE mounts
|
||||
uses: ./.github/actions/cleanup-stale-fuse-mounts
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v7
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
cache: ${{ runner.environment == 'github-hosted' }}
|
||||
id: go
|
||||
|
||||
- name: Run go mod tidy
|
||||
@@ -938,7 +884,7 @@ jobs:
|
||||
echo "=== System Information ==="
|
||||
uname -a
|
||||
free -h
|
||||
timeout 15 df -h || true
|
||||
df -h
|
||||
echo "=== Starting Lakekeeper Integration Tests ==="
|
||||
|
||||
# Run Lakekeeper integration tests
|
||||
@@ -969,21 +915,17 @@ jobs:
|
||||
|
||||
unity-catalog-integration-tests:
|
||||
name: Unity Catalog Integration Tests
|
||||
runs-on: ubuntu-24.04
|
||||
runs-on: ubuntu-22.04
|
||||
timeout-minutes: 30
|
||||
|
||||
steps:
|
||||
- name: Check out code
|
||||
uses: actions/checkout@v7
|
||||
|
||||
- name: Clean up stale FUSE mounts
|
||||
uses: ./.github/actions/cleanup-stale-fuse-mounts
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v7
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
cache: ${{ runner.environment == 'github-hosted' }}
|
||||
id: go
|
||||
|
||||
- name: Configure Docker Hub mirror
|
||||
@@ -1013,7 +955,7 @@ jobs:
|
||||
echo "=== System Information ==="
|
||||
uname -a
|
||||
free -h
|
||||
timeout 15 df -h || true
|
||||
df -h
|
||||
docker info
|
||||
echo "=== Starting Unity Catalog Tests ==="
|
||||
|
||||
@@ -1046,21 +988,17 @@ jobs:
|
||||
|
||||
lancedb-namespace-tests:
|
||||
name: LanceDB Namespace Integration Tests
|
||||
runs-on: ubuntu-24.04
|
||||
runs-on: ubuntu-22.04
|
||||
timeout-minutes: 30
|
||||
|
||||
steps:
|
||||
- name: Check out code
|
||||
uses: actions/checkout@v7
|
||||
|
||||
- name: Clean up stale FUSE mounts
|
||||
uses: ./.github/actions/cleanup-stale-fuse-mounts
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v7
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
cache: ${{ runner.environment == 'github-hosted' }}
|
||||
id: go
|
||||
|
||||
- name: Configure Docker Hub mirror
|
||||
@@ -1089,7 +1027,7 @@ jobs:
|
||||
echo "=== System Information ==="
|
||||
uname -a
|
||||
free -h
|
||||
timeout 15 df -h || true
|
||||
df -h
|
||||
docker info
|
||||
echo "=== Starting LanceDB Namespace Tests ==="
|
||||
|
||||
@@ -1120,7 +1058,7 @@ jobs:
|
||||
|
||||
table-lifecycle-tests:
|
||||
name: Table Lifecycle Integration Tests
|
||||
runs-on: ubuntu-24.04
|
||||
runs-on: ubuntu-22.04
|
||||
timeout-minutes: 40
|
||||
|
||||
steps:
|
||||
@@ -1131,7 +1069,6 @@ jobs:
|
||||
uses: actions/setup-go@v7
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
cache: ${{ runner.environment == 'github-hosted' }}
|
||||
id: go
|
||||
|
||||
- name: Configure Docker Hub mirror
|
||||
@@ -1189,7 +1126,7 @@ jobs:
|
||||
|
||||
duckdb-lance-tests:
|
||||
name: DuckDB Lance Integration Tests
|
||||
runs-on: ubuntu-24.04
|
||||
runs-on: ubuntu-22.04
|
||||
timeout-minutes: 30
|
||||
|
||||
steps:
|
||||
@@ -1200,14 +1137,10 @@ jobs:
|
||||
# so do not leave a token in the checkout for it to pick up.
|
||||
persist-credentials: false
|
||||
|
||||
- name: Clean up stale FUSE mounts
|
||||
uses: ./.github/actions/cleanup-stale-fuse-mounts
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v7
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
cache: ${{ runner.environment == 'github-hosted' }}
|
||||
id: go
|
||||
|
||||
- name: Configure Docker Hub mirror
|
||||
@@ -1237,7 +1170,7 @@ jobs:
|
||||
echo "=== System Information ==="
|
||||
uname -a
|
||||
free -h
|
||||
timeout 15 df -h || true
|
||||
df -h
|
||||
docker info
|
||||
echo "=== Starting DuckDB Lance Tests ==="
|
||||
|
||||
@@ -1268,7 +1201,7 @@ jobs:
|
||||
|
||||
spark-lance-namespace-tests:
|
||||
name: Spark Lance Namespace Integration Tests
|
||||
runs-on: ubuntu-24.04
|
||||
runs-on: ubuntu-22.04
|
||||
timeout-minutes: 40
|
||||
|
||||
steps:
|
||||
@@ -1279,14 +1212,10 @@ jobs:
|
||||
# so do not leave a token in the checkout for it to pick up.
|
||||
persist-credentials: false
|
||||
|
||||
- name: Clean up stale FUSE mounts
|
||||
uses: ./.github/actions/cleanup-stale-fuse-mounts
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v7
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
cache: ${{ runner.environment == 'github-hosted' }}
|
||||
id: go
|
||||
|
||||
- name: Configure Docker Hub mirror
|
||||
@@ -1315,7 +1244,7 @@ jobs:
|
||||
echo "=== System Information ==="
|
||||
uname -a
|
||||
free -h
|
||||
timeout 15 df -h || true
|
||||
df -h
|
||||
docker info
|
||||
echo "=== Starting Spark Lance Namespace Tests ==="
|
||||
|
||||
@@ -1346,7 +1275,7 @@ jobs:
|
||||
|
||||
s3-tables-build-verification:
|
||||
name: S3 Tables Build Verification
|
||||
runs-on: ubuntu-24.04
|
||||
runs-on: ubuntu-22.04
|
||||
timeout-minutes: 15
|
||||
|
||||
steps:
|
||||
@@ -1357,7 +1286,6 @@ jobs:
|
||||
uses: actions/setup-go@v7
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
cache: ${{ runner.environment == 'github-hosted' }}
|
||||
id: go
|
||||
|
||||
- name: Verify S3 Tables Package Builds
|
||||
@@ -1412,7 +1340,7 @@ jobs:
|
||||
|
||||
s3-tables-fmt-check:
|
||||
name: S3 Tables Format Check
|
||||
runs-on: ubuntu-24.04
|
||||
runs-on: ubuntu-22.04
|
||||
timeout-minutes: 10
|
||||
|
||||
steps:
|
||||
@@ -1423,7 +1351,6 @@ jobs:
|
||||
uses: actions/setup-go@v7
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
cache: ${{ runner.environment == 'github-hosted' }}
|
||||
id: go
|
||||
|
||||
- name: Check Go Format
|
||||
@@ -1464,7 +1391,7 @@ jobs:
|
||||
|
||||
s3-tables-vet:
|
||||
name: S3 Tables Go Vet Check
|
||||
runs-on: ubuntu-24.04
|
||||
runs-on: ubuntu-22.04
|
||||
timeout-minutes: 10
|
||||
|
||||
steps:
|
||||
@@ -1475,7 +1402,6 @@ jobs:
|
||||
uses: actions/setup-go@v7
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
cache: ${{ runner.environment == 'github-hosted' }}
|
||||
id: go
|
||||
|
||||
- name: Run Go Vet
|
||||
|
||||
@@ -25,8 +25,8 @@ on:
|
||||
- '.github/workflows/s3tests.yml'
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.event.pull_request.number || (github.event_name == 'push' && github.ref) || github.run_id }}
|
||||
cancel-in-progress: ${{ github.event_name == 'pull_request' || (github.event_name == 'push' && !startsWith(github.ref, 'refs/tags/')) }}
|
||||
group: ${{ github.head_ref }}/s3tests
|
||||
cancel-in-progress: true
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
@@ -34,7 +34,7 @@ permissions:
|
||||
jobs:
|
||||
basic-s3-tests:
|
||||
name: Basic S3 tests (KV store)
|
||||
runs-on: ubuntu-24.04
|
||||
runs-on: ubuntu-22.04
|
||||
timeout-minutes: 15
|
||||
steps:
|
||||
- name: Check out code into the Go module directory
|
||||
@@ -44,7 +44,6 @@ jobs:
|
||||
uses: actions/setup-go@v7
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
cache: ${{ runner.environment == 'github-hosted' }}
|
||||
id: go
|
||||
|
||||
- name: Set up Python
|
||||
@@ -58,9 +57,9 @@ jobs:
|
||||
cd s3-tests
|
||||
sudo apt-get update -qq
|
||||
sudo apt-get install -y -qq libxml2-dev libxslt1-dev zlib1g-dev
|
||||
python3 -m pip install -r requirements.txt
|
||||
python3 -m pip install tox
|
||||
python3 -m pip install -e .
|
||||
pip install -r requirements.txt
|
||||
pip install tox
|
||||
pip install -e .
|
||||
|
||||
- name: Fix S3 tests bucket creation conflicts
|
||||
run: |
|
||||
@@ -387,7 +386,7 @@ jobs:
|
||||
|
||||
versioning-tests:
|
||||
name: S3 Versioning & Object Lock tests
|
||||
runs-on: ubuntu-24.04
|
||||
runs-on: ubuntu-22.04
|
||||
timeout-minutes: 15
|
||||
steps:
|
||||
- name: Check out code into the Go module directory
|
||||
@@ -397,7 +396,6 @@ jobs:
|
||||
uses: actions/setup-go@v7
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
cache: ${{ runner.environment == 'github-hosted' }}
|
||||
id: go
|
||||
|
||||
- name: Set up Python
|
||||
@@ -411,9 +409,9 @@ jobs:
|
||||
cd s3-tests
|
||||
sudo apt-get update -qq
|
||||
sudo apt-get install -y -qq libxml2-dev libxslt1-dev zlib1g-dev
|
||||
python3 -m pip install -r requirements.txt
|
||||
python3 -m pip install tox
|
||||
python3 -m pip install -e .
|
||||
pip install -r requirements.txt
|
||||
pip install tox
|
||||
pip install -e .
|
||||
|
||||
- name: Fix S3 tests bucket creation conflicts
|
||||
run: |
|
||||
@@ -558,7 +556,7 @@ jobs:
|
||||
|
||||
cors-tests:
|
||||
name: S3 CORS tests
|
||||
runs-on: ubuntu-24.04
|
||||
runs-on: ubuntu-22.04
|
||||
timeout-minutes: 10
|
||||
steps:
|
||||
- name: Check out code into the Go module directory
|
||||
@@ -568,7 +566,6 @@ jobs:
|
||||
uses: actions/setup-go@v7
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
cache: ${{ runner.environment == 'github-hosted' }}
|
||||
id: go
|
||||
|
||||
- name: Set up Python
|
||||
@@ -582,9 +579,9 @@ jobs:
|
||||
cd s3-tests
|
||||
sudo apt-get update -qq
|
||||
sudo apt-get install -y -qq libxml2-dev libxslt1-dev zlib1g-dev
|
||||
python3 -m pip install -r requirements.txt
|
||||
python3 -m pip install tox
|
||||
python3 -m pip install -e .
|
||||
pip install -r requirements.txt
|
||||
pip install tox
|
||||
pip install -e .
|
||||
|
||||
- name: Run S3 CORS tests
|
||||
timeout-minutes: 10
|
||||
@@ -684,7 +681,7 @@ jobs:
|
||||
|
||||
copy-tests:
|
||||
name: SeaweedFS Custom S3 Copy tests
|
||||
runs-on: ubuntu-24.04
|
||||
runs-on: ubuntu-22.04
|
||||
timeout-minutes: 10
|
||||
steps:
|
||||
- name: Check out code into the Go module directory
|
||||
@@ -694,7 +691,6 @@ jobs:
|
||||
uses: actions/setup-go@v7
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
cache: ${{ runner.environment == 'github-hosted' }}
|
||||
id: go
|
||||
|
||||
- name: Run SeaweedFS Custom S3 Copy tests
|
||||
@@ -787,7 +783,7 @@ jobs:
|
||||
|
||||
sql-store-tests:
|
||||
name: Basic S3 tests (SQL store)
|
||||
runs-on: ubuntu-24.04
|
||||
runs-on: ubuntu-22.04
|
||||
timeout-minutes: 15
|
||||
steps:
|
||||
- name: Check out code into the Go module directory
|
||||
@@ -797,7 +793,6 @@ jobs:
|
||||
uses: actions/setup-go@v7
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
cache: ${{ runner.environment == 'github-hosted' }}
|
||||
id: go
|
||||
|
||||
- name: Set up Python
|
||||
@@ -811,9 +806,9 @@ jobs:
|
||||
cd s3-tests
|
||||
sudo apt-get update -qq
|
||||
sudo apt-get install -y -qq libxml2-dev libxslt1-dev zlib1g-dev
|
||||
python3 -m pip install -r requirements.txt
|
||||
python3 -m pip install tox
|
||||
python3 -m pip install -e .
|
||||
pip install -r requirements.txt
|
||||
pip install tox
|
||||
pip install -e .
|
||||
|
||||
- name: Run Ceph S3 tests with SQL store
|
||||
timeout-minutes: 15
|
||||
|
||||
@@ -29,7 +29,7 @@ permissions:
|
||||
jobs:
|
||||
samba-integration:
|
||||
name: samba-integration
|
||||
runs-on: ubuntu-24.04
|
||||
runs-on: ubuntu-22.04
|
||||
timeout-minutes: 45
|
||||
|
||||
steps:
|
||||
@@ -40,23 +40,15 @@ jobs:
|
||||
uses: actions/setup-go@v7
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
cache: ${{ runner.environment == 'github-hosted' }}
|
||||
|
||||
- name: Start local Docker registry
|
||||
run: |
|
||||
if ! docker start registry 2>/dev/null; then
|
||||
docker rm -f registry >/dev/null 2>&1 || true
|
||||
docker run -d --restart=always -p 5000:5000 --name registry registry:2
|
||||
fi
|
||||
run: docker run -d --restart=always -p 5000:5000 --name registry registry:2
|
||||
|
||||
- name: Set up Docker Buildx
|
||||
uses: docker/setup-buildx-action@v4
|
||||
with:
|
||||
driver-opts: network=host
|
||||
|
||||
- name: Set up Docker Compose
|
||||
uses: docker/setup-compose-action@v2
|
||||
|
||||
- name: Build weed race binary
|
||||
run: |
|
||||
cd docker
|
||||
|
||||
@@ -17,8 +17,8 @@ on:
|
||||
- '.github/workflows/sftp-tests.yml'
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.event.pull_request.number || (github.event_name == 'push' && github.ref) || github.run_id }}
|
||||
cancel-in-progress: ${{ github.event_name == 'pull_request' || (github.event_name == 'push' && !startsWith(github.ref, 'refs/tags/')) }}
|
||||
group: ${{ github.head_ref }}/sftp-tests
|
||||
cancel-in-progress: true
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
@@ -29,7 +29,7 @@ env:
|
||||
jobs:
|
||||
sftp-integration:
|
||||
name: SFTP Integration Testing
|
||||
runs-on: ubuntu-24.04
|
||||
runs-on: ubuntu-22.04
|
||||
timeout-minutes: 20
|
||||
|
||||
steps:
|
||||
@@ -40,7 +40,6 @@ jobs:
|
||||
uses: actions/setup-go@v7
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
cache: ${{ runner.environment == 'github-hosted' }}
|
||||
|
||||
- name: Install dependencies
|
||||
run: |
|
||||
|
||||
@@ -15,10 +15,6 @@ on:
|
||||
- '.github/workflows/spark-integration-tests.yml'
|
||||
workflow_dispatch:
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.event.pull_request.number || (github.event_name == 'push' && github.ref) || github.run_id }}
|
||||
cancel-in-progress: ${{ github.event_name == 'pull_request' || (github.event_name == 'push' && !startsWith(github.ref, 'refs/tags/')) }}
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
checks: write
|
||||
@@ -42,13 +38,12 @@ jobs:
|
||||
with:
|
||||
java-version: '11'
|
||||
distribution: 'temurin'
|
||||
cache: ${{ runner.environment == 'github-hosted' && 'maven' || '' }}
|
||||
cache: maven
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v7
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
cache: ${{ runner.environment == 'github-hosted' }}
|
||||
|
||||
- name: Build SeaweedFS binary
|
||||
run: |
|
||||
|
||||
@@ -10,11 +10,12 @@ permissions:
|
||||
contents: write
|
||||
|
||||
concurrency:
|
||||
# Only one chart regeneration per branch at a time so overlapping runs never
|
||||
# conflict on note/star_history.svg during rebase. Scoped by ref so a manual
|
||||
# run on another branch can't cancel the daily master update.
|
||||
# Only one chart regeneration per branch at a time; a newer run on the same
|
||||
# branch cancels an in-flight one so overlapping runs never conflict on
|
||||
# note/star_history.svg during rebase. Scoped by ref so a manual run on
|
||||
# another branch can't cancel the daily master update.
|
||||
group: star-history-${{ github.ref }}
|
||||
cancel-in-progress: ${{ github.event_name == 'pull_request' || (github.event_name == 'push' && !startsWith(github.ref, 'refs/tags/')) }}
|
||||
cancel-in-progress: true
|
||||
|
||||
jobs:
|
||||
render:
|
||||
@@ -32,7 +33,7 @@ jobs:
|
||||
uses: actions/setup-python@v7
|
||||
with:
|
||||
python-version: "3.x"
|
||||
cache: ${{ runner.environment == 'github-hosted' && 'pip' || '' }}
|
||||
cache: pip
|
||||
cache-dependency-path: .github/scripts/star_history.py
|
||||
|
||||
- name: Install matplotlib
|
||||
|
||||
@@ -14,10 +14,6 @@ on:
|
||||
- 'weed/telemetry/**'
|
||||
- '.github/workflows/telemetry-integration.yml'
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.event.pull_request.number || (github.event_name == 'push' && github.ref) || github.run_id }}
|
||||
cancel-in-progress: ${{ github.event_name == 'pull_request' || (github.event_name == 'push' && !startsWith(github.ref, 'refs/tags/')) }}
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
@@ -34,7 +30,6 @@ jobs:
|
||||
uses: actions/setup-go@v7
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
cache: ${{ runner.environment == 'github-hosted' }}
|
||||
|
||||
- name: Build telemetry server
|
||||
run: cd telemetry/server && go build -o telemetry-server .
|
||||
|
||||
@@ -8,10 +8,6 @@ on:
|
||||
branches: [ master ]
|
||||
paths: ['terraform/**', '.github/workflows/terraform_ci.yml']
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.event.pull_request.number || (github.event_name == 'push' && github.ref) || github.run_id }}
|
||||
cancel-in-progress: ${{ github.event_name == 'pull_request' || (github.event_name == 'push' && !startsWith(github.ref, 'refs/tags/')) }}
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
@@ -66,7 +62,6 @@ jobs:
|
||||
uses: actions/setup-go@v7
|
||||
with:
|
||||
go-version-file: go.mod
|
||||
cache: ${{ runner.environment == 'github-hosted' }}
|
||||
|
||||
- name: Build weed
|
||||
run: go build -o "$RUNNER_TEMP/weed" ./weed
|
||||
|
||||
@@ -27,10 +27,6 @@ defaults:
|
||||
run:
|
||||
working-directory: weed
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.event.pull_request.number || (github.event_name == 'push' && github.ref) || github.run_id }}
|
||||
cancel-in-progress: ${{ github.event_name == 'pull_request' || (github.event_name == 'push' && !startsWith(github.ref, 'refs/tags/')) }}
|
||||
|
||||
jobs:
|
||||
awscli-tests:
|
||||
runs-on: ubuntu-latest
|
||||
@@ -41,7 +37,6 @@ jobs:
|
||||
- uses: actions/setup-go@v7
|
||||
with:
|
||||
go-version: ^1.26
|
||||
cache: ${{ runner.environment == 'github-hosted' }}
|
||||
|
||||
- name: Build SeaweedFS
|
||||
run: |
|
||||
|
||||
@@ -18,23 +18,18 @@ on:
|
||||
- 'go.sum'
|
||||
- '.github/workflows/tls-rotation-tests.yml'
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.event.pull_request.number || (github.event_name == 'push' && github.ref) || github.run_id }}
|
||||
cancel-in-progress: ${{ github.event_name == 'pull_request' || (github.event_name == 'push' && !startsWith(github.ref, 'refs/tags/')) }}
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
jobs:
|
||||
tls-rotation-tests:
|
||||
name: TLS Rotation Integration Tests
|
||||
runs-on: ubuntu-24.04
|
||||
runs-on: ubuntu-22.04
|
||||
timeout-minutes: 10
|
||||
steps:
|
||||
- name: Set up Go 1.x
|
||||
uses: actions/setup-go@v7
|
||||
with:
|
||||
cache: ${{ runner.environment == 'github-hosted' }}
|
||||
go-version: ^1.26
|
||||
id: go
|
||||
|
||||
|
||||
@@ -11,8 +11,8 @@ on:
|
||||
- '.github/workflows/tus-tests.yml'
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.event.pull_request.number || (github.event_name == 'push' && github.ref) || github.run_id }}
|
||||
cancel-in-progress: ${{ github.event_name == 'pull_request' || (github.event_name == 'push' && !startsWith(github.ref, 'refs/tags/')) }}
|
||||
group: ${{ github.head_ref || github.ref }}/tus-tests
|
||||
cancel-in-progress: true
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
@@ -24,21 +24,17 @@ defaults:
|
||||
jobs:
|
||||
tus-integration-tests:
|
||||
name: TUS Protocol Integration Tests
|
||||
runs-on: ubuntu-24.04
|
||||
runs-on: ubuntu-22.04
|
||||
timeout-minutes: 20
|
||||
|
||||
steps:
|
||||
- name: Check out code
|
||||
uses: actions/checkout@v7
|
||||
|
||||
- name: Clean up stale FUSE mounts
|
||||
uses: ./.github/actions/cleanup-stale-fuse-mounts
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v7
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
cache: ${{ runner.environment == 'github-hosted' }}
|
||||
id: go
|
||||
|
||||
- name: Install SeaweedFS
|
||||
@@ -53,7 +49,7 @@ jobs:
|
||||
echo "=== System Information ==="
|
||||
uname -a
|
||||
free -h
|
||||
timeout 15 df -h || true
|
||||
df -h
|
||||
echo "=== Starting TUS Tests ==="
|
||||
|
||||
# Run tests with automatic server management
|
||||
|
||||
@@ -18,23 +18,18 @@ on:
|
||||
- 'go.sum'
|
||||
- '.github/workflows/vacuum-integration-tests.yml'
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.event.pull_request.number || (github.event_name == 'push' && github.ref) || github.run_id }}
|
||||
cancel-in-progress: ${{ github.event_name == 'pull_request' || (github.event_name == 'push' && !startsWith(github.ref, 'refs/tags/')) }}
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
jobs:
|
||||
vacuum-integration-tests:
|
||||
name: Vacuum Integration Tests
|
||||
runs-on: ubuntu-24.04
|
||||
runs-on: ubuntu-22.04
|
||||
timeout-minutes: 15
|
||||
steps:
|
||||
- name: Set up Go 1.x
|
||||
uses: actions/setup-go@v7
|
||||
with:
|
||||
cache: ${{ runner.environment == 'github-hosted' }}
|
||||
go-version: ^1.26
|
||||
id: go
|
||||
|
||||
|
||||
@@ -21,8 +21,8 @@ on:
|
||||
- '.github/workflows/volume-server-integration-tests.yml'
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.event.pull_request.number || (github.event_name == 'push' && github.ref) || github.run_id }}
|
||||
cancel-in-progress: ${{ github.event_name == 'pull_request' || (github.event_name == 'push' && !startsWith(github.ref, 'refs/tags/')) }}
|
||||
group: ${{ github.head_ref || github.ref }}/volume-server-integration-tests
|
||||
cancel-in-progress: true
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
@@ -33,44 +33,10 @@ env:
|
||||
SHARD_COUNT: 3
|
||||
|
||||
jobs:
|
||||
build-weed:
|
||||
name: Build SeaweedFS Binary
|
||||
runs-on: ubuntu-24.04
|
||||
timeout-minutes: 15
|
||||
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v7
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v7
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
cache: ${{ runner.environment == 'github-hosted' }}
|
||||
|
||||
- name: Build SeaweedFS binary
|
||||
run: |
|
||||
cd weed
|
||||
go build -o weed .
|
||||
chmod +x weed
|
||||
./weed version
|
||||
|
||||
- name: Package binary
|
||||
run: tar cf weed-binary.tar weed/weed
|
||||
|
||||
- name: Upload binary
|
||||
uses: actions/upload-artifact@v7
|
||||
with:
|
||||
name: weed-binary
|
||||
if-no-files-found: error
|
||||
retention-days: 1
|
||||
path: weed-binary.tar
|
||||
|
||||
volume-server-integration-tests:
|
||||
name: Volume Server Integration Tests (${{ matrix.test-type }} - Shard ${{ matrix.shard }})
|
||||
runs-on: ubuntu-24.04
|
||||
runs-on: ubuntu-22.04
|
||||
timeout-minutes: 45
|
||||
needs: [build-weed]
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
@@ -85,18 +51,13 @@ jobs:
|
||||
uses: actions/setup-go@v7
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
cache: ${{ runner.environment == 'github-hosted' }}
|
||||
|
||||
- name: Download binary
|
||||
uses: actions/download-artifact@v7
|
||||
with:
|
||||
name: weed-binary
|
||||
|
||||
- name: Restore binary
|
||||
- name: Build SeaweedFS binary
|
||||
run: |
|
||||
tar xf weed-binary.tar
|
||||
chmod +x weed/weed
|
||||
./weed/weed version
|
||||
cd weed
|
||||
go build -o weed .
|
||||
chmod +x weed
|
||||
./weed version
|
||||
|
||||
# Dealing the listed tests out one by one keeps the shards even. Bucketing
|
||||
# them by first letter did not: names cluster, so ^Test[I-S] drew 50 of
|
||||
|
||||
@@ -144,4 +144,3 @@ weed_bin
|
||||
telemetry/server/telemetry-server
|
||||
.aider*
|
||||
/seaweed-volume/docs
|
||||
__pycache__/
|
||||
@@ -3,7 +3,7 @@ FROM golang:1.26 AS builder
|
||||
RUN apt-get update
|
||||
RUN apt-get install -y build-essential libsnappy-dev zlib1g-dev libbz2-dev libgflags-dev liblz4-dev libzstd-dev
|
||||
|
||||
ARG ROCKSDB_VERSION=v11.1.2
|
||||
ARG ROCKSDB_VERSION=v10.10.1
|
||||
ENV ROCKSDB_VERSION=${ROCKSDB_VERSION}
|
||||
|
||||
# build RocksDB
|
||||
|
||||
@@ -3,7 +3,7 @@ FROM golang:1.26 AS builder
|
||||
RUN apt-get update
|
||||
RUN apt-get install -y build-essential libsnappy-dev zlib1g-dev libbz2-dev libgflags-dev liblz4-dev libzstd-dev
|
||||
|
||||
ARG ROCKSDB_VERSION=v11.1.2
|
||||
ARG ROCKSDB_VERSION=v10.10.1
|
||||
ENV ROCKSDB_VERSION=${ROCKSDB_VERSION}
|
||||
|
||||
# build RocksDB
|
||||
|
||||
@@ -362,7 +362,7 @@ require (
|
||||
github.com/flynn/noise v1.1.0 // indirect
|
||||
github.com/gabriel-vasile/mimetype v1.4.13 // indirect
|
||||
github.com/geoffgarside/ber v1.2.0 // indirect
|
||||
github.com/go-chi/chi/v5 v5.4.0 // indirect
|
||||
github.com/go-chi/chi/v5 v5.3.1 // indirect
|
||||
github.com/go-darwin/apfs v0.0.0-20211011131704-f84b94dbf348 // indirect
|
||||
github.com/go-jose/go-jose/v4 v4.1.5 // indirect
|
||||
github.com/go-logr/logr v1.4.4 // indirect
|
||||
|
||||
@@ -483,8 +483,8 @@ github.com/gin-gonic/gin v1.11.0 h1:OW/6PLjyusp2PPXtyxKHU0RbX6I/l28FTdDlae5ueWk=
|
||||
github.com/gin-gonic/gin v1.11.0/go.mod h1:+iq/FyxlGzII0KHiBGjuNn4UNENUlKbGlNmc+W50Dls=
|
||||
github.com/go-asn1-ber/asn1-ber v1.5.8 h1:H9AZkK22UOmfX8J84ubyaZxKJZ3FMHVwn8swoMML7iQ=
|
||||
github.com/go-asn1-ber/asn1-ber v1.5.8/go.mod h1:hEBeB/ic+5LoWskz+yKT7vGhhPYkProFKoKdwZRWMe0=
|
||||
github.com/go-chi/chi/v5 v5.4.0 h1:+ufzvFTGKyin26Or5DLrlIDq4ygIDBVcj7oVNeeaYFE=
|
||||
github.com/go-chi/chi/v5 v5.4.0/go.mod h1:d1IG7GtKPSjPyBP365PH8sAGVHk3qfB/IhjFNRH1wWM=
|
||||
github.com/go-chi/chi/v5 v5.3.1 h1:3j4HZLGZQ3JpMCrPJF/Jl3mYJfWLKBfNJ6quurUGCf8=
|
||||
github.com/go-chi/chi/v5 v5.3.1/go.mod h1:R+tYY2hNuVUUjxoPtqUdgBqevM9s9njzkTLutVsOCto=
|
||||
github.com/go-darwin/apfs v0.0.0-20211011131704-f84b94dbf348 h1:JnrjqG5iR07/8k7NqrLNilRsl3s1EPRQEGvbPyOce68=
|
||||
github.com/go-darwin/apfs v0.0.0-20211011131704-f84b94dbf348/go.mod h1:Czxo/d1g948LtrALAZdL04TL/HnkopquAjxYUuI02bo=
|
||||
github.com/go-errors/errors v1.5.1 h1:ZwEMSLRCapFLflTpT7NKaAc7ukJ8ZPEjzlxt8rPN8bk=
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
apiVersion: v1
|
||||
description: SeaweedFS
|
||||
name: seaweedfs
|
||||
appVersion: "4.51"
|
||||
appVersion: "4.48"
|
||||
# Dev note: Trigger a helm chart release by `git tag -a helm-<version>`
|
||||
version: 4.51.0
|
||||
version: 4.48.2
|
||||
@@ -170,6 +170,9 @@ spec:
|
||||
{{- if .Values.filer.metricsIp }}
|
||||
-metricsIp={{ .Values.filer.metricsIp }} \
|
||||
{{- end }}
|
||||
{{- if .Values.filer.redirectOnRead }}
|
||||
-redirectOnRead \
|
||||
{{- end }}
|
||||
{{- if .Values.filer.disableHttp }}
|
||||
-disableHttp \
|
||||
{{- end }}
|
||||
|
||||
@@ -172,8 +172,7 @@ spec:
|
||||
{{- $bucketName := .name }}
|
||||
{{- $bucketLock := or .lock .objectLock .withLock }}
|
||||
bucket_list=$(/bin/echo 's3.bucket.list' | /usr/bin/weed shell) || { echo "Error listing s3 buckets"; exit 1; }
|
||||
{{- /* One awk: grep -q exits early, awk takes SIGPIPE, pipefail fails the test. */}}
|
||||
if printf '%s\n' "$bucket_list" | awk -v bucket="{{ $bucketName }}" '$1 "" == bucket { found = 1 } END { exit !found }'; then
|
||||
if echo "$bucket_list" | awk '{print $1}' | grep -Fxq "{{ $bucketName }}"; then
|
||||
echo "Bucket '{{ $bucketName }}' already exists, skipping creation."
|
||||
else
|
||||
echo "Creating bucket '{{ $bucketName }}'..."
|
||||
|
||||
@@ -740,6 +740,9 @@ filer:
|
||||
# encrypt data on volume servers
|
||||
encryptVolumeData: false
|
||||
|
||||
# Whether proxy or redirect to volume server during file GET request
|
||||
redirectOnRead: false
|
||||
|
||||
# Limit sub dir listing size (default 100000)
|
||||
dirListLimit: 100000
|
||||
|
||||
|
||||
+1099
-1034
File diff suppressed because it is too large.
Load diff
|
Before Width: | Height: | Size: 53 KiB After Width: | Height: | Size: 55 KiB |
@@ -166,10 +166,6 @@ message Entry {
|
||||
repeated FileChunk chunks = 3;
|
||||
FuseAttributes attributes = 4;
|
||||
map<string, bytes> extended = 5;
|
||||
// filer-local commit timestamp of the version stored here: the meta log
|
||||
// event's ts_ns for the mutation that produced this entry, or 0 for
|
||||
// entries written before the field existed
|
||||
int64 commit_ts_ns = 6;
|
||||
bytes hard_link_id = 7;
|
||||
int32 hard_link_counter = 8; // only exists in hard link meta data
|
||||
bytes content = 9; // if not empty, the file content
|
||||
@@ -285,7 +281,6 @@ message WriteCondition {
|
||||
IF_EXTENDED_NOT_EQUAL = 7; // fail if present and extended[ext_key] == ext_value
|
||||
IF_EXTENDED_TIME_ELAPSED = 8; // fail if present and extended[ext_key] (unix seconds) is in the future
|
||||
IF_CHUNKS_EQUAL = 9; // fail unless the stored chunk fid multiset equals fids (absent entry = no chunks)
|
||||
IF_ENTRY_EQUAL = 10; // fail unless the stored entry equals expected_entry
|
||||
}
|
||||
// Clause is one primitive comparison. IF_ETAG_MATCH holds when the current
|
||||
// entry's ETag equals any value in etags; IF_ETAG_NOT_MATCH holds when it
|
||||
@@ -314,7 +309,6 @@ message WriteCondition {
|
||||
string gate_key = 7; // IF_EXTENDED_TIME_ELAPSED: only enforce when extended[gate_key] == gate_value
|
||||
string gate_value = 8; // gate value (e.g. retention mode COMPLIANCE for governance bypass)
|
||||
repeated string fids = 9; // chunk fid strings for IF_CHUNKS_EQUAL
|
||||
Entry expected_entry = 10; // whole-entry comparison for IF_ENTRY_EQUAL
|
||||
}
|
||||
repeated Clause clauses = 1; // all must hold (logical AND)
|
||||
}
|
||||
@@ -329,7 +323,6 @@ enum FilerError {
|
||||
EXISTING_IS_FILE = 4; // cannot overwrite file with directory
|
||||
ENTRY_ALREADY_EXISTS = 5; // O_EXCL and entry already exists
|
||||
PRECONDITION_FAILED = 6; // WriteCondition not satisfied
|
||||
READ_ONLY = 7; // path matches a read-only storage rule (e.g. bucket over quota)
|
||||
}
|
||||
|
||||
// ObjectMutation is one entry-level change applied by ObjectTransaction. All
|
||||
@@ -358,7 +351,6 @@ message ObjectMutation {
|
||||
bytes content = 11; // PATCH_EXTENDED: new Entry.content when set_content
|
||||
bool touch_mtime = 12; // PATCH_EXTENDED: set the entry's Mtime to now (e.g. a metadata-replace copy)
|
||||
bool remove_empty_parent = 13; // DELETE: also remove the parent directory when the delete leaves it empty (best-effort)
|
||||
bool keep_remote_object = 14; // DELETE: leave the object on the mounted remote storage
|
||||
}
|
||||
|
||||
// Recompute re-derives a pointer entry (directory/name on the mutation) from the
|
||||
@@ -396,8 +388,6 @@ message ObjectTransactionRequest {
|
||||
string condition_key = 6; // if set, evaluate the condition against this entry instead of lock_key (still locking lock_key)
|
||||
string route_key = 7; // ring key identifying the owner filer; a non-owner forwards the whole transaction to it
|
||||
bool is_moved = 8; // set on a forwarded transaction so the receiver applies it locally instead of forwarding again
|
||||
string request_id = 9; // idempotency token: the owner records the outcome and replays it on a retried request instead of re-applying
|
||||
string condition_alt_key = 10; // if the condition's entry is absent, evaluate it against this entry instead (e.g. null slot, else latest version)
|
||||
}
|
||||
|
||||
message ObjectTransactionResponse {
|
||||
@@ -487,7 +477,6 @@ message UpdateEntryRequest {
|
||||
// the write, under the filer's per-path lock. The caller must route the
|
||||
// key's writes to this entry's owner filer for the check to be authoritative.
|
||||
WriteCondition condition = 6;
|
||||
bool is_moved = 7; // set on a forwarded request so the owner applies it locally
|
||||
}
|
||||
message UpdateEntryResponse {
|
||||
SubscribeMetadataResponse metadata_event = 1;
|
||||
@@ -525,7 +514,6 @@ message DeleteEntryRequest {
|
||||
bool is_from_other_cluster = 7;
|
||||
repeated int32 signatures = 8;
|
||||
int64 if_not_modified_after = 9;
|
||||
bool keep_remote_object = 10;
|
||||
}
|
||||
|
||||
message DeleteEntryResponse {
|
||||
@@ -585,9 +573,6 @@ message AssignVolumeResponse {
|
||||
// fsync is the storage rule's fsync decision for the assigned path, so the
|
||||
// client can carry it onto the volume server upload request.
|
||||
bool fsync = 11;
|
||||
// error_code classifies error for callers that must branch on it; error
|
||||
// keeps the human-readable text for clients that predate the field.
|
||||
FilerError error_code = 12;
|
||||
}
|
||||
|
||||
message LookupVolumeRequest {
|
||||
@@ -724,10 +709,6 @@ message TraverseBfsMetadataRequest {
|
||||
message TraverseBfsMetadataResponse {
|
||||
string directory = 1;
|
||||
Entry entry = 2;
|
||||
// The sender's per-path commit timestamp (see meta_path_ts.go): the newest
|
||||
// event time this filer committed for the path, 0 when unrecorded so
|
||||
// subscribers fall back to the entry's mtime.
|
||||
int64 entry_ts_ns = 3;
|
||||
}
|
||||
|
||||
message LogEntry {
|
||||
@@ -790,8 +771,7 @@ message FilerConf {
|
||||
string replication = 3;
|
||||
string ttl = 4;
|
||||
string disk_type = 5;
|
||||
// unset inherits from the enclosing path rule, set overrides it
|
||||
optional bool fsync = 6;
|
||||
bool fsync = 6;
|
||||
uint32 volume_growth_count = 7;
|
||||
bool read_only = 8;
|
||||
string data_center = 9;
|
||||
|
||||
Generated
+14
-40
@@ -86,7 +86,7 @@ version = "1.1.5"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "40c48f72fd53cd289104fc64099abca73db4166ad86ea0b4341abe65af83dadc"
|
||||
dependencies = [
|
||||
"windows-sys 0.60.2",
|
||||
"windows-sys 0.61.2",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -97,7 +97,7 @@ checksum = "291e6a250ff86cd4a820112fb8898808a366d8f9f58ce16d1f538353ad55747d"
|
||||
dependencies = [
|
||||
"anstyle",
|
||||
"once_cell_polyfill",
|
||||
"windows-sys 0.60.2",
|
||||
"windows-sys 0.61.2",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -510,12 +510,10 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "aws-smithy-json"
|
||||
version = "0.62.7"
|
||||
version = "0.62.5"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "701a947f4797e52a911e114a898667c746c39feea467bbd1abd7b3721f702ffa"
|
||||
checksum = "9648b0bb82a2eedd844052c6ad2a1a822d1f8e3adee5fbf668366717e428856a"
|
||||
dependencies = [
|
||||
"aws-smithy-runtime-api",
|
||||
"aws-smithy-schema",
|
||||
"aws-smithy-types",
|
||||
]
|
||||
|
||||
@@ -565,12 +563,11 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "aws-smithy-runtime-api"
|
||||
version = "1.12.3"
|
||||
version = "1.11.6"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "9db177daa6ba8afb9ee1aefcf548c907abcf52065e394ee11a92780057fe0e8c"
|
||||
checksum = "876ab3c9c29791ba4ba02b780a3049e21ec63dabda09268b175272c3733a79e6"
|
||||
dependencies = [
|
||||
"aws-smithy-async",
|
||||
"aws-smithy-runtime-api-macros",
|
||||
"aws-smithy-types",
|
||||
"bytes",
|
||||
"http 0.2.12",
|
||||
@@ -581,33 +578,11 @@ dependencies = [
|
||||
"zeroize",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "aws-smithy-runtime-api-macros"
|
||||
version = "1.0.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "8d7396fd9500589e62e460e987ecb671bad374934e55ec3b5f498cc7a8a8a7b7"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "aws-smithy-schema"
|
||||
version = "0.1.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "7442cb268338f0eb8278140a107c046756aa01093d8ef5e99628d34ae09c94f5"
|
||||
dependencies = [
|
||||
"aws-smithy-runtime-api",
|
||||
"aws-smithy-types",
|
||||
"http 1.4.0",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "aws-smithy-types"
|
||||
version = "1.5.0"
|
||||
version = "1.4.7"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "32b42fcf341259d85ca10fac9a2f6448a8ec691c6955a18e45bc3b71a85fab85"
|
||||
checksum = "9d73dbfbaa8e4bc57b9045137680b958d274823509a360abfd8e1d514d40c95c"
|
||||
dependencies = [
|
||||
"base64-simd",
|
||||
"bytes",
|
||||
@@ -1350,7 +1325,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb"
|
||||
dependencies = [
|
||||
"libc",
|
||||
"windows-sys 0.52.0",
|
||||
"windows-sys 0.61.2",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -2404,7 +2379,7 @@ version = "0.50.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "7957b9740744892f114936ab4a57b3f487491bbeafaf8083688b16841a4240e5"
|
||||
dependencies = [
|
||||
"windows-sys 0.59.0",
|
||||
"windows-sys 0.61.2",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -3416,7 +3391,7 @@ dependencies = [
|
||||
"errno 0.3.14",
|
||||
"libc",
|
||||
"linux-raw-sys 0.12.1",
|
||||
"windows-sys 0.52.0",
|
||||
"windows-sys 0.61.2",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -3701,7 +3676,7 @@ version = "1.4.8"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "c4db69cba1110affc0e9f7bcd48bbf87b3f4fc7c61fc9155afd4c469eb3d6c1b"
|
||||
dependencies = [
|
||||
"errno 0.2.8",
|
||||
"errno 0.3.14",
|
||||
"libc",
|
||||
]
|
||||
|
||||
@@ -3768,7 +3743,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "3a766e1110788c36f4fa1c2b71b387a7815aa65f88ce0229841826633d93723e"
|
||||
dependencies = [
|
||||
"libc",
|
||||
"windows-sys 0.60.2",
|
||||
"windows-sys 0.61.2",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -3902,7 +3877,7 @@ dependencies = [
|
||||
"getrandom 0.4.2",
|
||||
"once_cell",
|
||||
"rustix 1.1.4",
|
||||
"windows-sys 0.52.0",
|
||||
"windows-sys 0.61.2",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -4601,7 +4576,6 @@ dependencies = [
|
||||
"futures",
|
||||
"hex",
|
||||
"http-body 1.0.1",
|
||||
"httpdate",
|
||||
"hyper",
|
||||
"hyper-util",
|
||||
"image",
|
||||
|
||||
@@ -51,7 +51,6 @@ prost-types = "0.14"
|
||||
# HTTP server
|
||||
axum = { version = "0.8", features = ["multipart"] }
|
||||
http-body = "1"
|
||||
httpdate = "1"
|
||||
hyper = { version = "1", features = ["full"] }
|
||||
hyper-util = { version = "0.1", features = ["tokio", "service", "server-auto", "http1", "http2"] }
|
||||
tower = { version = "0.5", features = ["util"] }
|
||||
@@ -158,8 +157,6 @@ windows-sys = { version = "0.61", features = ["Win32_Storage_FileSystem"] }
|
||||
|
||||
[dev-dependencies]
|
||||
tempfile = "3"
|
||||
# Paused-time tests drive the write-queue batch window deterministically.
|
||||
tokio = { version = "1", features = ["full", "test-util"] }
|
||||
# Already a transitive dependency of aws-sdk-s3 at a single locked version;
|
||||
# needed directly only for the canned HttpClient in remote_storage::s3 tests.
|
||||
aws-smithy-runtime-api = "1"
|
||||
|
||||
@@ -233,25 +233,6 @@ pub struct Cli {
|
||||
#[arg(long = "volume.allowUntrustedRemoteEndpoints", default_value_t = false)]
|
||||
pub allow_untrusted_remote_endpoints: bool,
|
||||
|
||||
/// Minimum bytes an fsync batch gathers before a drained queue flushes
|
||||
/// without waiting.
|
||||
#[arg(long = "fsyncMinBatchBytes", default_value_t = 1024 * 1024)]
|
||||
pub fsync_min_batch_bytes: u64,
|
||||
|
||||
/// Minimum requests an fsync batch gathers before a drained queue flushes
|
||||
/// without waiting.
|
||||
#[arg(long = "fsyncMinBatchRequests", default_value_t = 128)]
|
||||
pub fsync_min_batch_requests: usize,
|
||||
|
||||
/// How long a below-minimum fsync batch waits for more requests once the
|
||||
/// queue drains; 0 flushes immediately.
|
||||
#[arg(
|
||||
long = "fsyncBatchWait",
|
||||
default_value = "500us",
|
||||
value_parser = parse_go_duration
|
||||
)]
|
||||
pub fsync_batch_wait: std::time::Duration,
|
||||
|
||||
/// A file of command line options, each line in optionName=optionValue format.
|
||||
#[arg(long = "options", default_value = "")]
|
||||
pub options: String,
|
||||
@@ -327,14 +308,6 @@ pub struct VolumeServerConfig {
|
||||
/// If true, FetchAndWriteNeedle skips remote S3 endpoint validation
|
||||
/// (allows loopback / link-local / metadata hosts).
|
||||
pub allow_untrusted_remote_endpoints: bool,
|
||||
/// Minimum bytes an fsync batch gathers before a drained queue flushes
|
||||
/// without waiting.
|
||||
pub fsync_min_batch_bytes: u64,
|
||||
/// Minimum requests an fsync batch gathers before a drained queue flushes
|
||||
/// without waiting.
|
||||
pub fsync_min_batch_requests: usize,
|
||||
/// How long a below-minimum fsync batch waits for stragglers.
|
||||
pub fsync_batch_wait: std::time::Duration,
|
||||
}
|
||||
|
||||
pub use crate::storage::needle_map::NeedleMapKind;
|
||||
@@ -965,9 +938,6 @@ fn resolve_config_with_env(cli: Cli, env: EnvLookup<'_>) -> VolumeServerConfig {
|
||||
concurrent_download_limit: cli.concurrent_download_limit_mb as i64 * 1024 * 1024,
|
||||
inflight_upload_data_timeout: cli.inflight_upload_data_timeout,
|
||||
inflight_download_data_timeout: cli.inflight_download_data_timeout,
|
||||
fsync_min_batch_bytes: cli.fsync_min_batch_bytes,
|
||||
fsync_min_batch_requests: cli.fsync_min_batch_requests,
|
||||
fsync_batch_wait: cli.fsync_batch_wait,
|
||||
has_slow_read: cli.has_slow_read,
|
||||
read_buffer_size_mb: cli.read_buffer_size_mb,
|
||||
ldb_timeout: cli.ldb_timeout,
|
||||
|
||||
@@ -23,7 +23,7 @@ use seaweed_volume::server::request_id::GrpcRequestIdLayer;
|
||||
use seaweed_volume::server::volume_server::{
|
||||
RuntimeMetricsConfig, VolumeServerState, build_metrics_router,
|
||||
};
|
||||
use seaweed_volume::server::write_queue::{self, WriteQueue};
|
||||
use seaweed_volume::server::write_queue::WriteQueue;
|
||||
use seaweed_volume::storage::store::Store;
|
||||
use seaweed_volume::storage::types::DiskType;
|
||||
|
||||
@@ -436,17 +436,7 @@ async fn run(
|
||||
// The write queue is always on: durable (fsync) uploads go through it so
|
||||
// a batch shares one .dat/.idx sync, the way Go's per-volume async writer
|
||||
// group-commits. Non-fsync writes stay on the synchronous path.
|
||||
let min_bytes = usize::try_from(config.fsync_min_batch_bytes)
|
||||
.map_err(|_| "fsyncMinBatchBytes exceeds this platform's maximum usize value")?;
|
||||
let wq = WriteQueue::new(
|
||||
state.clone(),
|
||||
128,
|
||||
write_queue::SyncBatchOption {
|
||||
min_bytes,
|
||||
min_requests: config.fsync_min_batch_requests,
|
||||
wait: config.fsync_batch_wait,
|
||||
},
|
||||
);
|
||||
let wq = WriteQueue::new(state.clone(), 128);
|
||||
let _ = state.write_queue.set(wq);
|
||||
|
||||
// Set initial metric gauges for concurrent limits and max volumes
|
||||
|
||||
@@ -393,9 +393,6 @@ mod tests {
|
||||
fix_jpg_orientation: false,
|
||||
read_mode: ReadMode::Local,
|
||||
allow_untrusted_remote_endpoints: false,
|
||||
fsync_min_batch_bytes: 1024 * 1024,
|
||||
fsync_min_batch_requests: 128,
|
||||
fsync_batch_wait: std::time::Duration::from_micros(500),
|
||||
cpu_profile: String::new(),
|
||||
mem_profile: String::new(),
|
||||
compaction_byte_per_second: 0,
|
||||
|
||||
@@ -207,15 +207,9 @@ fn read_needle_for_get(
|
||||
if request.try_meta_first(plan.size()) {
|
||||
plan.read_meta(&mut n)?;
|
||||
if n.cookie != cookie {
|
||||
// The meta read answered the request.
|
||||
plan.note_meta_only_success();
|
||||
return Ok(None);
|
||||
}
|
||||
if request.served_from_source(&n) {
|
||||
if request.is_head {
|
||||
// HEAD ends here; no payload read will record the success.
|
||||
plan.note_meta_only_success();
|
||||
}
|
||||
let info = plan.into_stream_info(&n);
|
||||
return Ok(Some((n, Some(info))));
|
||||
}
|
||||
@@ -243,9 +237,6 @@ struct StreamingBody {
|
||||
hold_read_lock_for_stream: bool,
|
||||
_held_read_lease: Option<crate::storage::volume::DataFileReadLease>,
|
||||
io_unavailable: Arc<crate::storage::volume::IoUnavailable>,
|
||||
/// Chunk read failures feed the volume's io-error streak, and finishing
|
||||
/// the stream clears it — Go readNeedleDataInto parity.
|
||||
io_errors: Arc<crate::storage::io_error::IoErrorTracker>,
|
||||
/// Pending chunk read; it hands `buf` back with the result.
|
||||
pending: Option<tokio::task::JoinHandle<(bytes::BytesMut, std::io::Result<bytes::Bytes>)>>,
|
||||
/// Chunk buffer, reclaimed once the previous frame has been written out.
|
||||
@@ -307,18 +298,12 @@ impl http_body::Body for StreamingBody {
|
||||
),
|
||||
)));
|
||||
}
|
||||
// Hyper may not poll again once Content-Length is
|
||||
// satisfied, so the read's success is recorded here.
|
||||
self.io_errors.check_read_write_error(None);
|
||||
}
|
||||
return std::task::Poll::Ready(Some(Ok(http_body::Frame::data(
|
||||
chunk,
|
||||
))));
|
||||
}
|
||||
Ok(Err(e)) => {
|
||||
self.io_errors.check_read_write_error(Some(&e));
|
||||
return std::task::Poll::Ready(Some(Err(e)));
|
||||
}
|
||||
Ok(Err(e)) => return std::task::Poll::Ready(Some(Err(e))),
|
||||
Err(e) => {
|
||||
return std::task::Poll::Ready(Some(Err(std::io::Error::other(e))));
|
||||
}
|
||||
@@ -329,7 +314,6 @@ impl http_body::Body for StreamingBody {
|
||||
|
||||
let total = self.data_size as usize;
|
||||
if self.pos >= total {
|
||||
self.io_errors.check_read_write_error(None);
|
||||
return std::task::Poll::Ready(None);
|
||||
}
|
||||
|
||||
@@ -1120,15 +1104,7 @@ async fn get_or_head_handler_inner(
|
||||
};
|
||||
|
||||
let (etag, last_modified_str) = etag_and_last_modified(&n);
|
||||
// A Range whose If-Range no longer matches is ignored and the whole
|
||||
// content is sent, so a resumed download restarts instead of mixing
|
||||
// two versions.
|
||||
let range = range.filter(|_| if_range_matches(&headers, &etag, &last_modified_str));
|
||||
if let Some(resp) = not_modified_response(&n, &headers, &etag, &last_modified_str) {
|
||||
// A request that ends here was answered by the meta read alone.
|
||||
if let Some(io_errors) = strategy.io_errors() {
|
||||
io_errors.check_read_write_error(None);
|
||||
}
|
||||
return resp;
|
||||
}
|
||||
|
||||
@@ -1171,20 +1147,14 @@ async fn get_or_head_handler_inner(
|
||||
return head_from_meta_response(&info, response_headers);
|
||||
}
|
||||
ReadStrategy::RangeFromSource(info) => {
|
||||
return match range.as_deref() {
|
||||
Some(range) => {
|
||||
range_from_source_response(
|
||||
&state,
|
||||
range,
|
||||
info,
|
||||
response_headers,
|
||||
track_download,
|
||||
)
|
||||
.await
|
||||
}
|
||||
// If-Range dropped the range; serve the whole content.
|
||||
None => stream_response(&state, info, response_headers, track_download),
|
||||
};
|
||||
return range_from_source_response(
|
||||
&state,
|
||||
range.as_deref().unwrap_or_default(),
|
||||
info,
|
||||
response_headers,
|
||||
track_download,
|
||||
)
|
||||
.await;
|
||||
}
|
||||
ReadStrategy::Buffered => {}
|
||||
}
|
||||
@@ -1228,19 +1198,6 @@ struct ReadPlan {
|
||||
strategy: ReadStrategy,
|
||||
}
|
||||
|
||||
impl ReadStrategy {
|
||||
/// The tracker a terminal meta read reports success to, when the reply
|
||||
/// is served from the data file.
|
||||
fn io_errors(&self) -> Option<&Arc<crate::storage::io_error::IoErrorTracker>> {
|
||||
match self {
|
||||
ReadStrategy::Stream(info)
|
||||
| ReadStrategy::HeadFromMeta(info)
|
||||
| ReadStrategy::RangeFromSource(info) => Some(&info.io_errors),
|
||||
ReadStrategy::Buffered => None,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// The 401 reply for a bad read JWT; Go checks it before NewVolumeId, so a
|
||||
/// bad path with JWT enabled is a 401, not a 400.
|
||||
fn reject_read_jwt(
|
||||
@@ -1529,103 +1486,41 @@ fn not_modified_response(
|
||||
etag: &str,
|
||||
last_modified_str: &Option<String>,
|
||||
) -> Option<Response> {
|
||||
let not_modified = || {
|
||||
let mut resp = StatusCode::NOT_MODIFIED.into_response();
|
||||
resp.headers_mut()
|
||||
.insert(header::ETAG, etag.parse().unwrap());
|
||||
if let Some(lm) = last_modified_str {
|
||||
resp.headers_mut()
|
||||
.insert(header::LAST_MODIFIED, lm.parse().unwrap());
|
||||
}
|
||||
resp
|
||||
};
|
||||
|
||||
// If-Modified-Since applies only when there is no If-None-Match (RFC 9110 13.2.2).
|
||||
if headers.contains_key(header::IF_NONE_MATCH) {
|
||||
let inm = headers
|
||||
.get_all(header::IF_NONE_MATCH)
|
||||
.iter()
|
||||
.map(|v| v.to_str().unwrap_or(""))
|
||||
.collect::<Vec<_>>()
|
||||
.join(",");
|
||||
return etag_list_matches(&inm, etag).then(not_modified);
|
||||
}
|
||||
|
||||
// If-Modified-Since first (Go order), then If-None-Match.
|
||||
if n.last_modified > 0
|
||||
&& let Some(ims_str) = headers
|
||||
.get(header::IF_MODIFIED_SINCE)
|
||||
.and_then(|v| v.to_str().ok())
|
||||
&& let Some(ims_header) = headers.get(header::IF_MODIFIED_SINCE)
|
||||
&& let Ok(ims_str) = ims_header.to_str()
|
||||
{
|
||||
// Parse HTTP date format: "Mon, 02 Jan 2006 15:04:05 GMT"
|
||||
if let Ok(ims_time) =
|
||||
chrono::NaiveDateTime::parse_from_str(ims_str, "%a, %d %b %Y %H:%M:%S GMT")
|
||||
&& (n.last_modified as i64) <= ims_time.and_utc().timestamp()
|
||||
{
|
||||
return Some(not_modified());
|
||||
let mut resp = StatusCode::NOT_MODIFIED.into_response();
|
||||
if let Some(lm) = last_modified_str {
|
||||
resp.headers_mut()
|
||||
.insert(header::LAST_MODIFIED, lm.parse().unwrap());
|
||||
}
|
||||
// Go sets ETag after the 304 paths, so a 304 has no ETag.
|
||||
return Some(resp);
|
||||
}
|
||||
}
|
||||
|
||||
if let Some(if_none_match) = headers.get(header::IF_NONE_MATCH)
|
||||
&& let Ok(inm) = if_none_match.to_str()
|
||||
&& inm == etag
|
||||
{
|
||||
let mut resp = StatusCode::NOT_MODIFIED.into_response();
|
||||
if let Some(lm) = last_modified_str {
|
||||
resp.headers_mut()
|
||||
.insert(header::LAST_MODIFIED, lm.parse().unwrap());
|
||||
}
|
||||
// Go sets ETag after the 304 paths, so a 304 has no ETag.
|
||||
return Some(resp);
|
||||
}
|
||||
None
|
||||
}
|
||||
|
||||
/// If-Range (RFC 9110 13.1.5): an entity-tag by strong comparison, or a date
|
||||
/// equal to Last-Modified. The response etag here is always a strong tag.
|
||||
fn if_range_matches(headers: &HeaderMap, etag: &str, last_modified_str: &Option<String>) -> bool {
|
||||
let Some(raw) = headers.get(header::IF_RANGE) else {
|
||||
return true;
|
||||
};
|
||||
let Ok(if_range) = raw.to_str() else {
|
||||
return false;
|
||||
};
|
||||
let if_range = if_range.trim();
|
||||
if if_range.is_empty() {
|
||||
return true;
|
||||
}
|
||||
if if_range.starts_with('"') || if_range.starts_with("W/") {
|
||||
return if_range == etag;
|
||||
}
|
||||
let Ok(date) = httpdate::parse_http_date(if_range) else {
|
||||
return false;
|
||||
};
|
||||
last_modified_str
|
||||
.as_deref()
|
||||
.and_then(|lm| httpdate::parse_http_date(lm).ok())
|
||||
.is_some_and(|lm| date == lm)
|
||||
}
|
||||
|
||||
/// Weak comparison against "*" or a comma-separated entity-tag list (RFC 9110 13.1.2).
|
||||
fn etag_list_matches(header: &str, etag: &str) -> bool {
|
||||
if header.trim() == "*" {
|
||||
return true;
|
||||
}
|
||||
let etag = etag.trim_matches('"');
|
||||
split_etag_list(header).into_iter().any(|tag| {
|
||||
let tag = tag.trim();
|
||||
let tag = tag.strip_prefix("W/").unwrap_or(tag);
|
||||
// Empty list elements are ignored, so padding like a trailing
|
||||
// comma cannot match an empty opaque tag.
|
||||
!tag.is_empty() && tag.trim_matches('"') == etag
|
||||
})
|
||||
}
|
||||
|
||||
/// Splits an entity-tag list on the commas between tags; a comma inside a
|
||||
/// quoted opaque tag does not split.
|
||||
fn split_etag_list(header: &str) -> Vec<&str> {
|
||||
let mut tags = Vec::new();
|
||||
let (mut start, mut quoted) = (0, false);
|
||||
for (i, b) in header.bytes().enumerate() {
|
||||
match b {
|
||||
b'"' => quoted = !quoted,
|
||||
b',' if !quoted => {
|
||||
tags.push(&header[start..i]);
|
||||
start = i + 1;
|
||||
}
|
||||
_ => {}
|
||||
}
|
||||
}
|
||||
tags.push(&header[start..]);
|
||||
tags
|
||||
}
|
||||
|
||||
/// The 200 reply headers, and the extension, which falls back to the stored name's.
|
||||
fn read_response_headers(
|
||||
n: &Needle,
|
||||
@@ -1827,7 +1722,6 @@ fn stream_response(
|
||||
data_file_access_control: info.data_file_access_control,
|
||||
hold_read_lock_for_stream: !state.has_slow_read,
|
||||
io_unavailable: info.io_unavailable,
|
||||
io_errors: info.io_errors,
|
||||
pending: None,
|
||||
buf: bytes::BytesMut::new(),
|
||||
state: tracking_state,
|
||||
@@ -2227,21 +2121,9 @@ fn handle_range_request_from_source(
|
||||
return Ok(Vec::new());
|
||||
}
|
||||
let mut buf = vec![0u8; length as usize];
|
||||
match info
|
||||
.source
|
||||
.read_exact_at(&mut buf, info.data_file_offset + start as u64)
|
||||
{
|
||||
// A finished range read clears the streak a failed meta read
|
||||
// counted, like readNeedleDataInto's terminal report.
|
||||
Ok(()) => {
|
||||
info.io_errors.check_read_write_error(None);
|
||||
Ok(buf)
|
||||
}
|
||||
Err(e) => {
|
||||
info.io_errors.check_read_write_error(Some(&e));
|
||||
Err(e)
|
||||
}
|
||||
}
|
||||
info.source
|
||||
.read_exact_at(&mut buf, info.data_file_offset + start as u64)?;
|
||||
Ok(buf)
|
||||
};
|
||||
|
||||
if ranges.len() == 1 {
|
||||
@@ -3074,62 +2956,50 @@ pub async fn post_handler(
|
||||
let needle_etag = n.etag();
|
||||
let needle_has_name = n.has_name();
|
||||
|
||||
// Replicate to remote volume servers if this volume has replicas.
|
||||
// Matches Go's GetWritableRemoteReplications: skip if copy_count == 1.
|
||||
let needs_replication = !is_replicate && {
|
||||
let store = state.store.read().unwrap();
|
||||
store
|
||||
.find_volume(vid)
|
||||
.is_some_and(|(_, v)| v.super_block().replica_placement.get_copy_count() > 1)
|
||||
};
|
||||
|
||||
// The fan-out runs concurrently with the local needle write, like Go's
|
||||
// ReplicatedWrite: it reads the already-buffered request body, so the two
|
||||
// sides share no mutable state. The ack still waits for both.
|
||||
let replication = if needs_replication {
|
||||
let state_clone = state.clone();
|
||||
let path_clone = path.clone();
|
||||
let query_clone = query.clone();
|
||||
let headers_clone = headers.clone();
|
||||
let body_clone = body.clone();
|
||||
Some(tokio::spawn(async move {
|
||||
do_replicated_request(
|
||||
&state_clone,
|
||||
vid.0,
|
||||
Method::POST,
|
||||
&path_clone,
|
||||
&query_clone,
|
||||
&headers_clone,
|
||||
Some(body_clone),
|
||||
)
|
||||
.await
|
||||
}))
|
||||
} else {
|
||||
None
|
||||
};
|
||||
|
||||
let write_result = write_upload_needle(&state, vid, n, fsync).await;
|
||||
|
||||
// Replicate to remote volume servers if this volume has replicas.
|
||||
// Matches Go's GetWritableRemoteReplications: skip if copy_count == 1.
|
||||
// No leader-known gate: when the lookup cannot run, the write must fail
|
||||
// rather than be acknowledged on fewer copies than required. The local
|
||||
// write result takes precedence, matching Go's error ordering.
|
||||
let replication_result = match replication {
|
||||
Some(task) => Some(
|
||||
task.await
|
||||
// rather than be acknowledged on fewer copies than required.
|
||||
if !is_replicate && write_result.is_ok() {
|
||||
let needs_replication = {
|
||||
let store = state.store.read().unwrap();
|
||||
store
|
||||
.find_volume(vid)
|
||||
.is_some_and(|(_, v)| v.super_block().replica_placement.get_copy_count() > 1)
|
||||
};
|
||||
if needs_replication {
|
||||
let state_clone = state.clone();
|
||||
let path_clone = path.clone();
|
||||
let query_clone = query.clone();
|
||||
let headers_clone = headers.clone();
|
||||
let body_clone = body.clone();
|
||||
let replication = tokio::spawn(async move {
|
||||
do_replicated_request(
|
||||
&state_clone,
|
||||
vid.0,
|
||||
Method::POST,
|
||||
&path_clone,
|
||||
&query_clone,
|
||||
&headers_clone,
|
||||
Some(body_clone),
|
||||
)
|
||||
.await
|
||||
});
|
||||
let replication_result = replication
|
||||
.await
|
||||
.map_err(|e| format!("replication task failed: {}", e))
|
||||
.flatten(),
|
||||
),
|
||||
None => None,
|
||||
};
|
||||
if write_result.is_ok()
|
||||
&& let Some(Err(e)) = replication_result
|
||||
{
|
||||
tracing::error!("replicated write failed: {}", e);
|
||||
return json_error_with_query(
|
||||
StatusCode::INTERNAL_SERVER_ERROR,
|
||||
format!("replication failed: {}", e),
|
||||
Some(&query),
|
||||
);
|
||||
.flatten();
|
||||
if let Err(e) = replication_result {
|
||||
tracing::error!("replicated write failed: {}", e);
|
||||
return json_error_with_query(
|
||||
StatusCode::INTERNAL_SERVER_ERROR,
|
||||
format!("replication failed: {}", e),
|
||||
Some(&query),
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
let resp = match write_result {
|
||||
@@ -5355,7 +5225,6 @@ mod tests {
|
||||
hold_read_lock_for_stream: true,
|
||||
_held_read_lease: None,
|
||||
io_unavailable: Arc::default(),
|
||||
io_errors: Arc::default(),
|
||||
pending: None,
|
||||
buf: bytes::BytesMut::new(),
|
||||
state: None,
|
||||
@@ -5728,127 +5597,6 @@ mod tests {
|
||||
delete_handler(State(state.clone()), request).await.status()
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_not_modified_checks_if_none_match_first() {
|
||||
let n = Needle {
|
||||
last_modified: 1700000000,
|
||||
..Needle::default()
|
||||
};
|
||||
let (etag, last_modified_str) = etag_and_last_modified(&n);
|
||||
let bare = n.etag();
|
||||
let before = "Tue, 14 Nov 2023 22:12:20 GMT".to_string();
|
||||
let after = "Tue, 14 Nov 2023 22:14:20 GMT".to_string();
|
||||
|
||||
let cases: Vec<(Vec<(axum::http::HeaderName, String)>, bool)> = vec![
|
||||
(vec![(header::IF_NONE_MATCH, format!("\"{bare}\""))], true),
|
||||
(vec![(header::IF_NONE_MATCH, format!("W/\"{bare}\""))], true),
|
||||
(
|
||||
vec![(header::IF_NONE_MATCH, format!("\"x\", \"{bare}\""))],
|
||||
true,
|
||||
),
|
||||
(vec![(header::IF_NONE_MATCH, "*".into())], true),
|
||||
(vec![(header::IF_NONE_MATCH, "\"x\"".into())], false),
|
||||
(
|
||||
vec![
|
||||
(header::IF_NONE_MATCH, "\"x\"".into()),
|
||||
(header::IF_NONE_MATCH, format!("\"{bare}\"")),
|
||||
],
|
||||
true,
|
||||
),
|
||||
(
|
||||
vec![(header::IF_NONE_MATCH, format!("\"{bare},old\""))],
|
||||
false,
|
||||
),
|
||||
(
|
||||
vec![(header::IF_NONE_MATCH, format!("\"x\",,\"{bare}\""))],
|
||||
true,
|
||||
),
|
||||
(
|
||||
vec![
|
||||
(header::IF_NONE_MATCH, "\"x\"".into()),
|
||||
(header::IF_MODIFIED_SINCE, after.clone()),
|
||||
],
|
||||
false,
|
||||
),
|
||||
(
|
||||
vec![
|
||||
(header::IF_NONE_MATCH, "".into()),
|
||||
(header::IF_MODIFIED_SINCE, after.clone()),
|
||||
],
|
||||
false,
|
||||
),
|
||||
(vec![(header::IF_MODIFIED_SINCE, after)], true),
|
||||
(vec![(header::IF_MODIFIED_SINCE, before)], false),
|
||||
];
|
||||
for (pairs, want) in cases {
|
||||
let mut headers = HeaderMap::new();
|
||||
for (name, value) in &pairs {
|
||||
headers.append(name, value.parse().unwrap());
|
||||
}
|
||||
assert_eq!(
|
||||
not_modified_response(&n, &headers, &etag, &last_modified_str).is_some(),
|
||||
want,
|
||||
"{pairs:?}"
|
||||
);
|
||||
}
|
||||
|
||||
let mut headers = HeaderMap::new();
|
||||
headers.insert(
|
||||
header::IF_NONE_MATCH,
|
||||
format!("\"{bare}\"").parse().unwrap(),
|
||||
);
|
||||
let resp = not_modified_response(&n, &headers, &etag, &last_modified_str).unwrap();
|
||||
assert!(resp.headers().contains_key(header::ETAG));
|
||||
assert!(resp.headers().contains_key(header::LAST_MODIFIED));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_if_range_matches() {
|
||||
let etag = "\"v2\"".to_string();
|
||||
let last_modified_str = Some("Sat, 24 Oct 2026 22:01:04 GMT".to_string());
|
||||
let cases: Vec<(Option<&str>, bool)> = vec![
|
||||
(None, true),
|
||||
(Some("\"v2\""), true),
|
||||
(Some("Sat, 24 Oct 2026 22:01:04 GMT"), true),
|
||||
(Some("Saturday, 24-Oct-26 22:01:04 GMT"), true),
|
||||
(Some("Sat Oct 24 22:01:04 2026"), true),
|
||||
(Some("\"v1\""), false),
|
||||
(Some("W/\"v2\""), false),
|
||||
(Some("Sat, 24 Oct 2026 22:01:03 GMT"), false),
|
||||
(Some("not a date"), false),
|
||||
];
|
||||
for (if_range, want) in cases {
|
||||
let mut headers = HeaderMap::new();
|
||||
if let Some(v) = if_range {
|
||||
headers.insert(header::IF_RANGE, v.parse().unwrap());
|
||||
}
|
||||
assert_eq!(
|
||||
if_range_matches(&headers, &etag, &last_modified_str),
|
||||
want,
|
||||
"{if_range:?}"
|
||||
);
|
||||
}
|
||||
|
||||
// A present value that is not valid UTF-8 fails the condition.
|
||||
let mut headers = HeaderMap::new();
|
||||
headers.insert(
|
||||
header::IF_RANGE,
|
||||
header::HeaderValue::from_bytes(b"\"\xff\xfev2\"").unwrap(),
|
||||
);
|
||||
assert!(!if_range_matches(&headers, &etag, &last_modified_str));
|
||||
|
||||
// No Last-Modified on the representation: a date validator fails,
|
||||
// an entity-tag still applies.
|
||||
let mut headers = HeaderMap::new();
|
||||
headers.insert(header::IF_RANGE, "\"v2\"".parse().unwrap());
|
||||
assert!(if_range_matches(&headers, &etag, &None));
|
||||
headers.insert(
|
||||
header::IF_RANGE,
|
||||
"Sat, 24 Oct 2026 22:01:04 GMT".parse().unwrap(),
|
||||
);
|
||||
assert!(!if_range_matches(&headers, &etag, &None));
|
||||
}
|
||||
|
||||
/// A GET whose needle read is parked must not hold the store lock: a
|
||||
/// writer, here a real append to the same volume, must get through.
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
@@ -6023,44 +5771,6 @@ mod tests {
|
||||
assert_eq!(status, StatusCode::NOT_FOUND);
|
||||
}
|
||||
|
||||
/// A ranged read served from the data file reports its outcome to the
|
||||
/// io-error streak, like readNeedleDataInto: a finished range clears the
|
||||
/// count a failed meta read left.
|
||||
#[test]
|
||||
fn test_range_from_source_reports_to_io_error_streak() {
|
||||
use std::io::Write;
|
||||
|
||||
let tmp = tempfile::TempDir::new().unwrap();
|
||||
let file = std::fs::OpenOptions::new()
|
||||
.read(true)
|
||||
.write(true)
|
||||
.create(true)
|
||||
.open(tmp.path().join("1.dat"))
|
||||
.unwrap();
|
||||
let data: Vec<u8> = (0..256u32).map(|i| i as u8).collect();
|
||||
(&file).write_all(&data).unwrap();
|
||||
|
||||
let io_errors = Arc::new(crate::storage::io_error::IoErrorTracker::default());
|
||||
let eio = crate::storage::volume::media_error_for_test();
|
||||
io_errors.check_read_write_error(Some(&eio));
|
||||
io_errors.check_read_write_error(Some(&eio));
|
||||
assert_eq!(io_errors.get_io_error_state().1, 2);
|
||||
|
||||
let info = crate::storage::volume::NeedleStreamInfo {
|
||||
source: crate::storage::volume::NeedleStreamSource::Local(file),
|
||||
data_file_offset: 0,
|
||||
data_size: data.len() as u32,
|
||||
data_file_access_control: Arc::default(),
|
||||
io_unavailable: Arc::default(),
|
||||
io_errors: io_errors.clone(),
|
||||
needle_id: NeedleId(1),
|
||||
checksum: 0,
|
||||
};
|
||||
let resp = handle_range_request_from_source("bytes=10-19", info, HeaderMap::new(), None);
|
||||
assert_eq!(resp.status(), StatusCode::PARTIAL_CONTENT);
|
||||
assert_eq!(io_errors.get_io_error_state().1, 0);
|
||||
}
|
||||
|
||||
/// Range is read as Go reads it: a byte its parseRange cannot parse is a
|
||||
/// 416, Unicode whitespace around a range is trimmed, an empty value is no
|
||||
/// range, and HEAD ignores it.
|
||||
@@ -7513,7 +7223,6 @@ mod tests {
|
||||
.set(crate::server::write_queue::WriteQueue::new(
|
||||
state.clone(),
|
||||
128,
|
||||
crate::server::write_queue::SyncBatchOption::default(),
|
||||
))
|
||||
.is_ok()
|
||||
);
|
||||
|
||||
@@ -132,9 +132,6 @@ mod tests {
|
||||
fix_jpg_orientation: false,
|
||||
read_mode: ReadMode::Local,
|
||||
allow_untrusted_remote_endpoints: false,
|
||||
fsync_min_batch_bytes: 1024 * 1024,
|
||||
fsync_min_batch_requests: 128,
|
||||
fsync_batch_wait: std::time::Duration::from_micros(500),
|
||||
cpu_profile: String::new(),
|
||||
mem_profile: String::new(),
|
||||
compaction_byte_per_second: 0,
|
||||
|
||||
@@ -56,27 +56,6 @@ const MAX_CONCURRENT_VOLUME_WRITES: usize = 8;
|
||||
/// retirement is the volume leaving the store.
|
||||
const IDLE_WORKER_POLL: std::time::Duration = std::time::Duration::from_secs(60);
|
||||
|
||||
/// Tunes the per-volume durable-write worker. Once the queue drains with the
|
||||
/// batch still below min_bytes and min_requests, the batch stays open up to
|
||||
/// wait for stragglers instead of paying an fsync per request. Mirrors Go's
|
||||
/// storage.SyncBatchOption.
|
||||
#[derive(Clone, Copy)]
|
||||
pub struct SyncBatchOption {
|
||||
pub min_bytes: usize,
|
||||
pub min_requests: usize,
|
||||
pub wait: std::time::Duration,
|
||||
}
|
||||
|
||||
impl Default for SyncBatchOption {
|
||||
fn default() -> Self {
|
||||
SyncBatchOption {
|
||||
min_bytes: 1024 * 1024,
|
||||
min_requests: MAX_BATCH_SIZE,
|
||||
wait: std::time::Duration::from_micros(500),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Handle for submitting write requests to the per-volume workers.
|
||||
#[derive(Clone)]
|
||||
pub struct WriteQueue {
|
||||
@@ -88,35 +67,23 @@ pub struct WriteQueue {
|
||||
/// alone would multiply the old single queue's capacity by volume count.
|
||||
queue_permits: Arc<Semaphore>,
|
||||
workers: Arc<Mutex<HashMap<VolumeId, mpsc::Sender<WriteRequest>>>>,
|
||||
sync_batch: SyncBatchOption,
|
||||
/// Submissions accepted, for tests asserting which writes route here.
|
||||
#[cfg(test)]
|
||||
submits: Arc<std::sync::atomic::AtomicU64>,
|
||||
/// Times a worker armed the straggler window, for tests that must land a
|
||||
/// write inside an already-open batch.
|
||||
#[cfg(test)]
|
||||
windows_armed: Arc<std::sync::atomic::AtomicU64>,
|
||||
}
|
||||
|
||||
impl WriteQueue {
|
||||
/// Create a new write queue. Per-volume workers are spawned lazily on the
|
||||
/// first submission for that volume.
|
||||
pub fn new(
|
||||
state: Arc<VolumeServerState>,
|
||||
capacity: usize,
|
||||
sync_batch: SyncBatchOption,
|
||||
) -> Self {
|
||||
pub fn new(state: Arc<VolumeServerState>, capacity: usize) -> Self {
|
||||
WriteQueue {
|
||||
state,
|
||||
capacity,
|
||||
io_permits: Arc::new(Semaphore::new(MAX_CONCURRENT_VOLUME_WRITES)),
|
||||
queue_permits: Arc::new(Semaphore::new(capacity)),
|
||||
workers: Arc::new(Mutex::new(HashMap::new())),
|
||||
sync_batch,
|
||||
#[cfg(test)]
|
||||
submits: Arc::new(std::sync::atomic::AtomicU64::new(0)),
|
||||
#[cfg(test)]
|
||||
windows_armed: Arc::new(std::sync::atomic::AtomicU64::new(0)),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -126,12 +93,6 @@ impl WriteQueue {
|
||||
self.submits.load(std::sync::atomic::Ordering::Relaxed)
|
||||
}
|
||||
|
||||
/// How many batch windows workers have armed so far.
|
||||
#[cfg(test)]
|
||||
fn windows_armed_for_test(&self) -> u64 {
|
||||
self.windows_armed.load(std::sync::atomic::Ordering::Relaxed)
|
||||
}
|
||||
|
||||
/// Sender for the volume's worker, spawning one on first use.
|
||||
fn worker_for(&self, volume_id: VolumeId) -> mpsc::Sender<WriteRequest> {
|
||||
let mut workers = self.workers.lock().unwrap();
|
||||
@@ -146,9 +107,6 @@ impl WriteQueue {
|
||||
state: self.state.clone(),
|
||||
io_permits: self.io_permits.clone(),
|
||||
workers: self.workers.clone(),
|
||||
sync_batch: self.sync_batch,
|
||||
#[cfg(test)]
|
||||
windows_armed: self.windows_armed.clone(),
|
||||
}
|
||||
.run(),
|
||||
);
|
||||
@@ -233,9 +191,6 @@ struct VolumeWriteWorker {
|
||||
state: Arc<VolumeServerState>,
|
||||
io_permits: Arc<Semaphore>,
|
||||
workers: Arc<Mutex<HashMap<VolumeId, mpsc::Sender<WriteRequest>>>>,
|
||||
sync_batch: SyncBatchOption,
|
||||
#[cfg(test)]
|
||||
windows_armed: Arc<std::sync::atomic::AtomicU64>,
|
||||
}
|
||||
|
||||
impl VolumeWriteWorker {
|
||||
@@ -283,40 +238,14 @@ impl VolumeWriteWorker {
|
||||
let mut batch = Vec::with_capacity(MAX_BATCH_SIZE);
|
||||
let mut batch_bytes: usize = first.needle.data.len();
|
||||
batch.push(first);
|
||||
// Armed once the queue first drains below the minimum batch; holds the
|
||||
// batch open for stragglers so the wait bound is per batch, not per
|
||||
// lull (Go's batchWait).
|
||||
let mut window_deadline = None;
|
||||
while batch.len() < MAX_BATCH_SIZE && batch_bytes < MAX_BATCH_BYTES {
|
||||
if window_deadline.is_some_and(|d| tokio::time::Instant::now() >= d) {
|
||||
break;
|
||||
}
|
||||
let req = match self.rx.try_recv() {
|
||||
Ok(req) => req,
|
||||
Err(mpsc::error::TryRecvError::Empty) => {
|
||||
let sync = self.sync_batch;
|
||||
if sync.wait.is_zero()
|
||||
|| batch_bytes >= sync.min_bytes
|
||||
|| batch.len() >= sync.min_requests
|
||||
{
|
||||
// a drained queue flushes right away once it holds enough
|
||||
break;
|
||||
}
|
||||
let deadline = *window_deadline.get_or_insert_with(|| {
|
||||
#[cfg(test)]
|
||||
self.windows_armed
|
||||
.fetch_add(1, std::sync::atomic::Ordering::Relaxed);
|
||||
tokio::time::Instant::now() + sync.wait
|
||||
});
|
||||
match tokio::time::timeout_at(deadline, self.rx.recv()).await {
|
||||
Ok(Some(req)) => req,
|
||||
Ok(None) | Err(_) => break,
|
||||
}
|
||||
match self.rx.try_recv() {
|
||||
Ok(req) => {
|
||||
batch_bytes += req.needle.data.len();
|
||||
batch.push(req);
|
||||
}
|
||||
Err(mpsc::error::TryRecvError::Disconnected) => break,
|
||||
};
|
||||
batch_bytes += req.needle.data.len();
|
||||
batch.push(req);
|
||||
Err(_) => break,
|
||||
}
|
||||
}
|
||||
debug!("processing write batch of {} requests", batch.len());
|
||||
|
||||
@@ -451,7 +380,7 @@ mod tests {
|
||||
async fn test_write_queue_submit_no_volume() {
|
||||
// Submit a write to a non-existent volume -- should return VolumeError::NotFound.
|
||||
let state = make_test_state();
|
||||
let queue = WriteQueue::new(state, MAX_BATCH_SIZE, SyncBatchOption::default());
|
||||
let queue = WriteQueue::new(state, MAX_BATCH_SIZE);
|
||||
|
||||
let needle = Needle {
|
||||
id: 1.into(),
|
||||
@@ -473,7 +402,7 @@ mod tests {
|
||||
async fn test_write_queue_concurrent_submissions() {
|
||||
// Submit multiple concurrent writes -- all should complete (with errors since no volume).
|
||||
let state = make_test_state();
|
||||
let queue = WriteQueue::new(state, MAX_BATCH_SIZE, SyncBatchOption::default());
|
||||
let queue = WriteQueue::new(state, MAX_BATCH_SIZE);
|
||||
|
||||
let mut handles = Vec::new();
|
||||
for i in 0..10u64 {
|
||||
@@ -501,7 +430,7 @@ mod tests {
|
||||
async fn test_write_queue_batching() {
|
||||
// Verify that many concurrent writes get processed (testing the batching path).
|
||||
let state = make_test_state();
|
||||
let queue = WriteQueue::new(state, MAX_BATCH_SIZE, SyncBatchOption::default());
|
||||
let queue = WriteQueue::new(state, MAX_BATCH_SIZE);
|
||||
|
||||
// Submit MAX_BATCH_SIZE requests concurrently
|
||||
let mut handles = Vec::new();
|
||||
@@ -593,7 +522,7 @@ mod tests {
|
||||
async fn test_write_queue_dropped_sender() {
|
||||
// When the queue is dropped, subsequent submits should fail gracefully.
|
||||
let state = make_test_state();
|
||||
let queue = WriteQueue::new(state, 1, SyncBatchOption::default());
|
||||
let queue = WriteQueue::new(state, 1);
|
||||
|
||||
// Clone then drop the original -- the worker keeps running via its rx handle.
|
||||
let queue2 = queue.clone();
|
||||
@@ -736,7 +665,7 @@ mod tests {
|
||||
.unwrap();
|
||||
}
|
||||
}
|
||||
let queue = WriteQueue::new(state, MAX_BATCH_SIZE, SyncBatchOption::default());
|
||||
let queue = WriteQueue::new(state, MAX_BATCH_SIZE);
|
||||
|
||||
let in_append = Arc::new(Barrier::new(2));
|
||||
let release = Arc::new(Barrier::new(2));
|
||||
@@ -805,85 +734,4 @@ mod tests {
|
||||
assert!(a1.await.unwrap().is_ok());
|
||||
assert!(a2.await.unwrap().is_ok());
|
||||
}
|
||||
|
||||
/// A durable write below the minimum batch rides out its window so a
|
||||
/// straggler joins the same sync instead of each paying an fsync alone —
|
||||
/// Go's batchWait. Paused time makes the hold observable: a lone write can
|
||||
/// only complete once the runtime auto-advances to the window's deadline.
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn test_batch_window_defers_flush_for_stragglers() {
|
||||
use crate::config::MinFreeSpace;
|
||||
use crate::storage::types::{DiskType, NeedleId};
|
||||
use crate::storage::volume::VolumeSpec;
|
||||
use std::time::Duration;
|
||||
|
||||
let tmp = tempfile::TempDir::new().unwrap();
|
||||
let dir = tmp.path().to_str().unwrap();
|
||||
let state = make_test_state();
|
||||
{
|
||||
let mut store = state.store.write().unwrap();
|
||||
store
|
||||
.add_location(
|
||||
dir,
|
||||
dir,
|
||||
10,
|
||||
DiskType::HardDrive,
|
||||
MinFreeSpace::Percent(1.0),
|
||||
Vec::new(),
|
||||
)
|
||||
.unwrap();
|
||||
store
|
||||
.add_volume(VolumeId(1), DiskType::HardDrive, &VolumeSpec::default())
|
||||
.unwrap();
|
||||
}
|
||||
|
||||
let queue = WriteQueue::new(
|
||||
state,
|
||||
MAX_BATCH_SIZE,
|
||||
SyncBatchOption {
|
||||
min_bytes: usize::MAX,
|
||||
min_requests: MAX_BATCH_SIZE,
|
||||
wait: Duration::from_secs(30),
|
||||
},
|
||||
);
|
||||
|
||||
let needle = |id: u64| Needle {
|
||||
id: NeedleId(id),
|
||||
cookie: 0x1111.into(),
|
||||
data: vec![id as u8; 8],
|
||||
data_size: 8,
|
||||
..Needle::default()
|
||||
};
|
||||
|
||||
let start = tokio::time::Instant::now();
|
||||
queue.submit(VolumeId(1), needle(1), true).await.unwrap();
|
||||
assert!(start.elapsed() >= Duration::from_secs(30));
|
||||
|
||||
// Wait for the worker to take the second write and open its batch
|
||||
// window, so the straggler provably lands inside an open batch
|
||||
// instead of arriving during the initial drain.
|
||||
let armed = queue.windows_armed_for_test();
|
||||
let second = tokio::spawn({
|
||||
let queue = queue.clone();
|
||||
async move { queue.submit(VolumeId(1), needle(2), true).await }
|
||||
});
|
||||
for _ in 0..10_000 {
|
||||
if queue.windows_armed_for_test() > armed {
|
||||
break;
|
||||
}
|
||||
tokio::task::yield_now().await;
|
||||
}
|
||||
assert!(queue.windows_armed_for_test() > armed);
|
||||
let third = tokio::spawn({
|
||||
let queue = queue.clone();
|
||||
async move { queue.submit(VolumeId(1), needle(3), true).await }
|
||||
});
|
||||
let (second, third) = tokio::join!(second, third);
|
||||
second.unwrap().unwrap();
|
||||
third.unwrap().unwrap();
|
||||
// One shared window ends both writes ~30s after their batch armed;
|
||||
// back-to-back windows would keep the third out to ~90s.
|
||||
let elapsed = start.elapsed();
|
||||
assert!(elapsed >= Duration::from_secs(60) && elapsed < Duration::from_secs(90));
|
||||
}
|
||||
}
|
||||
@@ -740,11 +740,7 @@ impl DiskLocation {
|
||||
if vol.is_read_only() {
|
||||
continue;
|
||||
}
|
||||
// A preallocated volume already holds its space on disk.
|
||||
let dat_size = vol
|
||||
.dat_file_size()
|
||||
.unwrap_or(0)
|
||||
.max(vol.dat_allocated_size());
|
||||
let dat_size = vol.dat_file_size().unwrap_or(0);
|
||||
let idx_size = vol.idx_file_size();
|
||||
let used = dat_size + idx_size;
|
||||
if volume_size_limit > used {
|
||||
|
||||
@@ -18,7 +18,7 @@ use crate::storage::erasure_coding::ec_volume::{
|
||||
};
|
||||
use crate::storage::needle::needle::{Needle, get_actual_size};
|
||||
use crate::storage::needle_map::NeedleMapKind;
|
||||
use crate::storage::super_block::{ReplicaPlacement, SUPER_BLOCK_SIZE, SuperBlock};
|
||||
use crate::storage::super_block::{ReplicaPlacement, SUPER_BLOCK_SIZE};
|
||||
use crate::storage::types::*;
|
||||
use crate::storage::volume::{CompactionJob, VifVolumeInfo, Volume, VolumeError, VolumeSpec};
|
||||
|
||||
@@ -762,8 +762,8 @@ impl Store {
|
||||
}
|
||||
|
||||
/// Configure a volume's replica placement on disk.
|
||||
/// The volume must already be unmounted. Rewrites the .vif and the .dat
|
||||
/// superblock so both carry the new placement.
|
||||
/// The volume must already be unmounted. This opens the .dat file directly,
|
||||
/// modifies the replica_placement byte (offset 1), and writes it back.
|
||||
pub fn configure_volume(&self, vid: VolumeId, rp: ReplicaPlacement) -> Result<(), VolumeError> {
|
||||
let (_, base_path, _) = self.find_volume_file_base(vid).ok_or_else(|| {
|
||||
VolumeError::Io(io::Error::new(
|
||||
@@ -775,7 +775,7 @@ impl Store {
|
||||
let mut vif = load_vif_volume_info(&vif_path)?;
|
||||
vif.replication = rp.to_string();
|
||||
save_vif_volume_info(&vif_path, &vif)?;
|
||||
update_dat_replica_placement(&format!("{}.dat", base_path), vid, rp)
|
||||
Ok(())
|
||||
}
|
||||
|
||||
// ---- Read / Write / Delete ----
|
||||
@@ -943,7 +943,11 @@ impl Store {
|
||||
let current = loc.max_volume_count.load(Ordering::Relaxed);
|
||||
let (_, free) = super::disk_location::get_disk_stats(&loc.directory);
|
||||
|
||||
let unused_space = loc.unused_space(volume_size_limit);
|
||||
let unused_space = if self.get_preallocate() {
|
||||
0
|
||||
} else {
|
||||
loc.unused_space(volume_size_limit)
|
||||
};
|
||||
let unclaimed = (free as i64) - (unused_space as i64);
|
||||
|
||||
let vol_count = loc.volumes_len() as i32;
|
||||
@@ -1740,61 +1744,6 @@ fn strip_volume_suffix(filename: &str) -> Option<&str> {
|
||||
.or_else(|| filename.strip_suffix(".idx"))
|
||||
}
|
||||
|
||||
/// Rewrites the replica placement in a .dat header for a volume that is not
|
||||
/// mounted; a missing .dat means a remote-only volume, where the .vif carries
|
||||
/// the placement.
|
||||
fn update_dat_replica_placement(
|
||||
dat_path: &str,
|
||||
vid: VolumeId,
|
||||
rp: ReplicaPlacement,
|
||||
) -> Result<(), VolumeError> {
|
||||
use std::io::{Read, Seek, SeekFrom, Write};
|
||||
|
||||
let mut file = match std::fs::OpenOptions::new()
|
||||
.read(true)
|
||||
.write(true)
|
||||
.open(dat_path)
|
||||
{
|
||||
Ok(f) => f,
|
||||
Err(e) if e.kind() == io::ErrorKind::NotFound => return Ok(()),
|
||||
Err(e) => {
|
||||
return Err(VolumeError::Io(io::Error::new(
|
||||
e.kind(),
|
||||
format!("volume {} failed to open {}: {}", vid, dat_path, e),
|
||||
)))
|
||||
}
|
||||
};
|
||||
let mut header = [0u8; SUPER_BLOCK_SIZE];
|
||||
file.read_exact(&mut header).map_err(|e| {
|
||||
VolumeError::Io(io::Error::new(
|
||||
e.kind(),
|
||||
format!("volume {} failed to parse superblock: {}", vid, e),
|
||||
))
|
||||
})?;
|
||||
let extra_size = u16::from_be_bytes([header[6], header[7]]) as usize;
|
||||
let mut block = header.to_vec();
|
||||
block.resize(SUPER_BLOCK_SIZE + extra_size, 0);
|
||||
if extra_size > 0 {
|
||||
file.read_exact(&mut block[SUPER_BLOCK_SIZE..]).map_err(|e| {
|
||||
VolumeError::Io(io::Error::new(
|
||||
e.kind(),
|
||||
format!("volume {} failed to parse superblock: {}", vid, e),
|
||||
))
|
||||
})?;
|
||||
}
|
||||
let mut super_block = SuperBlock::from_bytes(&block).map_err(|e| {
|
||||
VolumeError::Io(io::Error::new(
|
||||
io::ErrorKind::InvalidData,
|
||||
format!("volume {} failed to parse superblock: {}", vid, e),
|
||||
))
|
||||
})?;
|
||||
super_block.replica_placement = rp;
|
||||
file.seek(SeekFrom::Start(0))?;
|
||||
file.write_all(&super_block.to_bytes())?;
|
||||
file.sync_all()?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn load_vif_volume_info(path: &str) -> Result<VifVolumeInfo, VolumeError> {
|
||||
let content = match std::fs::read_to_string(path) {
|
||||
Ok(c) => c,
|
||||
@@ -2032,33 +1981,6 @@ mod tests {
|
||||
assert_eq!(store.total_volume_count(), 1);
|
||||
}
|
||||
|
||||
// Mirrors Go's TestConfigureVolumeUpdatesSuperblock: configure must leave
|
||||
// the .dat header and the .vif both on the new placement.
|
||||
#[test]
|
||||
fn test_configure_volume_updates_superblock() {
|
||||
let tmp = TempDir::new().unwrap();
|
||||
let dir = tmp.path().to_str().unwrap();
|
||||
let mut store = make_test_store(&[dir]);
|
||||
|
||||
store
|
||||
.add_volume(VolumeId(1), DiskType::HardDrive, &VolumeSpec::default())
|
||||
.unwrap();
|
||||
assert!(store.unmount_volume(VolumeId(1)).unwrap());
|
||||
|
||||
let rp = ReplicaPlacement::from_string("010").unwrap();
|
||||
store.configure_volume(VolumeId(1), rp).unwrap();
|
||||
|
||||
let base = volume_file_name(dir, "", VolumeId(1));
|
||||
let vif = load_vif_volume_info(&format!("{}.vif", base)).unwrap();
|
||||
assert_eq!(vif.replication, "010");
|
||||
|
||||
let dat = std::fs::read(format!("{}.dat", base)).unwrap();
|
||||
let super_block = SuperBlock::from_bytes(&dat).unwrap();
|
||||
assert_eq!(super_block.replica_placement, rp);
|
||||
|
||||
assert!(store.configure_volume(VolumeId(2), rp).is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_mount_volume_by_id_hint_skips_lone_sidecar() {
|
||||
// A lone .vif/.idx sidecar (e.g. an EC sidecar whose .ecx is on a
|
||||
@@ -2557,7 +2479,7 @@ mod tests {
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_maybe_adjust_volume_max_reserves_plain_volumes_while_preallocating() {
|
||||
fn test_maybe_adjust_volume_max_honors_preallocate_flag() {
|
||||
let tmp = TempDir::new().unwrap();
|
||||
let dir = tmp.path().to_str().unwrap();
|
||||
|
||||
@@ -2572,6 +2494,13 @@ mod tests {
|
||||
Vec::new(),
|
||||
)
|
||||
.unwrap();
|
||||
// Use a large volume_size_limit so the unused-space difference between
|
||||
// preallocate=true (0) and preallocate=false (~2 × limit) is big enough
|
||||
// that integer-division rounding and disk-free fluctuations between the
|
||||
// two get_disk_stats calls cannot make the quotients equal.
|
||||
store
|
||||
.volume_size_limit
|
||||
.store(100 * 1024 * 1024, Ordering::Relaxed);
|
||||
store
|
||||
.add_volume(
|
||||
VolumeId(61),
|
||||
@@ -2582,67 +2511,39 @@ mod tests {
|
||||
},
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
// Room for 2.5 volumes: the plain volume claims one, leaving one more slot.
|
||||
let (_, free) = crate::storage::disk_location::get_disk_stats(dir);
|
||||
let volume_size_limit = (free as f64 / 2.5) as u64;
|
||||
if volume_size_limit == 0 {
|
||||
return; // cannot determine free disk space
|
||||
}
|
||||
store
|
||||
.volume_size_limit
|
||||
.store(volume_size_limit, Ordering::Relaxed);
|
||||
store.locations[0].original_max_volume_count = 0;
|
||||
store.set_preallocate(true);
|
||||
|
||||
store.maybe_adjust_volume_max();
|
||||
assert_eq!(
|
||||
store.locations[0].max_volume_count.load(Ordering::Relaxed),
|
||||
2
|
||||
);
|
||||
}
|
||||
|
||||
#[cfg(target_os = "linux")]
|
||||
#[test]
|
||||
fn test_unused_space_counts_preallocated_volumes_as_full() {
|
||||
const LIMIT: u64 = 1 << 20;
|
||||
let tmp = TempDir::new().unwrap();
|
||||
let dir = tmp.path().to_str().unwrap();
|
||||
|
||||
let mut store = Store::new(NeedleMapKind::InMemory);
|
||||
store
|
||||
.add_location(
|
||||
dir,
|
||||
dir,
|
||||
2,
|
||||
.add_volume(
|
||||
VolumeId(62),
|
||||
DiskType::HardDrive,
|
||||
MinFreeSpace::Percent(1.0),
|
||||
Vec::new(),
|
||||
&VolumeSpec {
|
||||
collection: "preallocate_case",
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.unwrap();
|
||||
for (vid, preallocate) in [(VolumeId(71), LIMIT), (VolumeId(72), 0)] {
|
||||
store
|
||||
.add_volume(
|
||||
vid,
|
||||
DiskType::HardDrive,
|
||||
&VolumeSpec {
|
||||
collection: "preallocate_case",
|
||||
preallocate,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
for vid in [VolumeId(61), VolumeId(62)] {
|
||||
let dat_path = store.find_volume(vid).unwrap().1.dat_path();
|
||||
std::fs::OpenOptions::new()
|
||||
.write(true)
|
||||
.open(dat_path)
|
||||
.unwrap()
|
||||
.set_len((crate::storage::super_block::SUPER_BLOCK_SIZE + 1) as u64)
|
||||
.unwrap();
|
||||
}
|
||||
let preallocated = store.find_volume(VolumeId(71)).unwrap().1;
|
||||
if preallocated.dat_allocated_size() < LIMIT {
|
||||
return; // the filesystem does not support fallocate
|
||||
}
|
||||
store.locations[0].original_max_volume_count = 0;
|
||||
store.locations[0]
|
||||
.max_volume_count
|
||||
.store(0, Ordering::Relaxed);
|
||||
|
||||
let plain = store.find_volume(VolumeId(72)).unwrap().1;
|
||||
let plain_dat = plain.dat_file_size().unwrap();
|
||||
// The filesystem may round a small file up to a larger allocation unit.
|
||||
let plain_used = plain_dat.max(plain.dat_allocated_size()) + plain.idx_file_size();
|
||||
assert_eq!(store.locations[0].unused_space(LIMIT), LIMIT - plain_used);
|
||||
store.set_preallocate(false);
|
||||
assert!(store.maybe_adjust_volume_max());
|
||||
let without_preallocate = store.locations[0].max_volume_count.load(Ordering::Relaxed);
|
||||
|
||||
store.set_preallocate(true);
|
||||
assert!(store.maybe_adjust_volume_max());
|
||||
let with_preallocate = store.locations[0].max_volume_count.load(Ordering::Relaxed);
|
||||
|
||||
assert!(with_preallocate > without_preallocate);
|
||||
}
|
||||
|
||||
#[test]
|
||||
|
||||
@@ -836,7 +836,7 @@ impl NeedleReadPlan {
|
||||
pub(crate) fn read_meta(&mut self, n: &mut Needle) -> Result<(), VolumeError> {
|
||||
let (source, size, version) = (&self.source, self.size, self.version);
|
||||
let read_at = |buf: &mut [u8], at: u64| Ok(source.read_exact_at(buf, at)?);
|
||||
let result = with_offset_retry(&mut self.offset, |off| {
|
||||
with_offset_retry(&mut self.offset, |off| {
|
||||
if version == VERSION_1 {
|
||||
// A V1 body is all data, so its "meta" is the whole record.
|
||||
let mut buf = vec![0u8; get_actual_size(size, version) as usize];
|
||||
@@ -847,26 +847,13 @@ impl NeedleReadPlan {
|
||||
} else {
|
||||
Volume::read_needle_meta_blob_and_parse(read_at, n, off, size, version)
|
||||
}
|
||||
});
|
||||
// Report failures only: a successful meta read precedes the payload
|
||||
// read, so clearing the streak here would reset the count before
|
||||
// every failing download and the volume could never reach quarantine.
|
||||
if let Err(VolumeError::Io(e)) = &result {
|
||||
self.io_errors.check_read_write_error(Some(e));
|
||||
}
|
||||
result?;
|
||||
})?;
|
||||
if needle_expired(n) {
|
||||
return Err(VolumeError::NotFound);
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// A successful meta read ended the request (e.g. HEAD), so it may reset
|
||||
/// the streak — unlike a meta read that precedes a payload read.
|
||||
pub(crate) fn note_meta_only_success(&self) {
|
||||
self.io_errors.check_read_write_error(None);
|
||||
}
|
||||
|
||||
/// For a reply served from the data file, after `read_meta` filled `n`.
|
||||
pub(crate) fn into_stream_info(self, n: &Needle) -> NeedleStreamInfo {
|
||||
// V1 data starts right after the header, V2/V3 after the DataSize field.
|
||||
@@ -881,7 +868,6 @@ impl NeedleReadPlan {
|
||||
data_size: n.data_size,
|
||||
data_file_access_control: self.data_file_access_control,
|
||||
io_unavailable: self.io_unavailable,
|
||||
io_errors: self.io_errors,
|
||||
needle_id: self.needle_id,
|
||||
checksum: n.checksum.0,
|
||||
}
|
||||
@@ -1158,9 +1144,6 @@ pub struct NeedleStreamInfo {
|
||||
pub data_file_access_control: Arc<DataFileAccessControl>,
|
||||
/// Checked before each chunk: the volume can become unavailable mid-stream.
|
||||
pub(crate) io_unavailable: Arc<IoUnavailable>,
|
||||
/// Streamed chunks report read failures here and clear the streak when
|
||||
/// the whole body has been emitted, mirroring readNeedleDataInto.
|
||||
pub(crate) io_errors: Arc<IoErrorTracker>,
|
||||
pub needle_id: NeedleId,
|
||||
/// Checksum stored in the needle tail, verified once the last chunk has
|
||||
/// been read — before that frame is emitted.
|
||||
@@ -2935,52 +2918,18 @@ impl Volume {
|
||||
}
|
||||
|
||||
// Cookie validation for existing needle (matches Go: check whenever nm.Get returns ok)
|
||||
let indexed = match io.nm.as_ref() {
|
||||
Some(nm) => nm.get(n.id)?,
|
||||
None => None,
|
||||
};
|
||||
if let Some(nv) = indexed
|
||||
&& !nv.offset.is_zero()
|
||||
if let Some(nm) = &io.nm
|
||||
&& let Some(nv) = nm.get(n.id)?
|
||||
{
|
||||
let mut existing = Needle::default();
|
||||
// Read only the header to check cookie
|
||||
self.read_needle_header_unlocked(io, &mut existing, nv.offset.to_actual_offset())?;
|
||||
|
||||
if existing.id != n.id && existing.size != nv.size {
|
||||
// a 4-byte index entry can index past the 32 GiB wrap: the
|
||||
// low offset may hold another needle while the real record
|
||||
// sits at the wrapped offset, where reads already retry
|
||||
#[cfg(not(feature = "5bytes"))]
|
||||
if nv.offset.to_actual_offset() < MAX_POSSIBLE_VOLUME_SIZE as i64 {
|
||||
let mut wrapped = Needle::default();
|
||||
match self.read_needle_header_unlocked(
|
||||
io,
|
||||
&mut wrapped,
|
||||
nv.offset.to_actual_offset() + MAX_POSSIBLE_VOLUME_SIZE as i64,
|
||||
) {
|
||||
Ok(()) => existing = wrapped,
|
||||
Err(VolumeError::Io(e)) if e.kind() == io::ErrorKind::UnexpectedEof => {}
|
||||
Err(e) => return Err(e),
|
||||
}
|
||||
}
|
||||
if n.cookie.0 == 0 && !check_cookie {
|
||||
n.cookie = existing.cookie;
|
||||
}
|
||||
|
||||
if existing.id != n.id && existing.size != nv.size {
|
||||
// the entry resolves to a record that is neither the indexed
|
||||
// key nor the indexed size, so it is stale rather than a
|
||||
// duplicate — skip the cookie check and let publish_write
|
||||
// overwrite it once the append commits
|
||||
warn!(
|
||||
"volume {} index entry for needle {} resolves to needle {}; overwriting stale index entry",
|
||||
self.id.0, n.id.0, existing.id.0
|
||||
);
|
||||
} else {
|
||||
if n.cookie.0 == 0 && !check_cookie {
|
||||
n.cookie = existing.cookie;
|
||||
}
|
||||
if existing.cookie != n.cookie {
|
||||
return Err(VolumeError::CookieMismatch(n.cookie.0));
|
||||
}
|
||||
if existing.cookie != n.cookie {
|
||||
return Err(VolumeError::CookieMismatch(n.cookie.0));
|
||||
}
|
||||
}
|
||||
Ok(false)
|
||||
@@ -5036,25 +4985,6 @@ impl Volume {
|
||||
self.current_dat_file_size()
|
||||
}
|
||||
|
||||
/// Disk space held by the local .dat file, or 0 when it is unknown. It
|
||||
/// exceeds the file size when the file was preallocated with
|
||||
/// `FALLOC_FL_KEEP_SIZE`.
|
||||
pub fn dat_allocated_size(&self) -> u64 {
|
||||
#[cfg(unix)]
|
||||
{
|
||||
use std::os::unix::fs::MetadataExt;
|
||||
let io = self.io.read().unwrap();
|
||||
io.dat_file
|
||||
.as_ref()
|
||||
.and_then(|f| f.metadata().ok())
|
||||
.map_or(0, |m| m.blocks() * 512)
|
||||
}
|
||||
#[cfg(not(unix))]
|
||||
{
|
||||
0
|
||||
}
|
||||
}
|
||||
|
||||
/// Get the modification time of the .dat file as Unix seconds.
|
||||
pub fn dat_file_mod_time(&self) -> u64 {
|
||||
let io = self.io.read().unwrap();
|
||||
@@ -6118,7 +6048,7 @@ fn preallocate_file(file: &File, size: u64) {
|
||||
/// An OS error the platform reports for failing storage media, which is
|
||||
/// what counts toward the I/O error streak.
|
||||
#[cfg(test)]
|
||||
pub(crate) fn media_error_for_test() -> io::Error {
|
||||
fn media_error_for_test() -> io::Error {
|
||||
#[cfg(unix)]
|
||||
{
|
||||
io::Error::from_raw_os_error(libc::EIO)
|
||||
@@ -9095,107 +9025,6 @@ mod tests {
|
||||
assert!(matches!(err, VolumeError::CookieMismatch(_)));
|
||||
}
|
||||
|
||||
/// A damaged index entry resolving to a different needle must miss on
|
||||
/// reads and be dropped by the next write for the key, not rejected as a
|
||||
/// cookie conflict.
|
||||
#[test]
|
||||
fn test_stale_index_entry_heals_on_write() {
|
||||
let tmp = TempDir::new().unwrap();
|
||||
let dir = tmp.path().to_str().unwrap();
|
||||
let v = make_test_volume(dir);
|
||||
|
||||
let mut first = Needle {
|
||||
id: NeedleId(2),
|
||||
cookie: Cookie(0x2222),
|
||||
data: b"second".to_vec(),
|
||||
data_size: 6,
|
||||
..Needle::default()
|
||||
};
|
||||
v.write_needle(&mut first, true, false).unwrap();
|
||||
|
||||
// point key 9 at needle 2's record with a size that does not match
|
||||
// the record, as a corrupt index entry would look
|
||||
{
|
||||
let mut io = v.io.write().unwrap();
|
||||
let nm = io.nm.as_mut().unwrap();
|
||||
let nv = nm.get(NeedleId(2)).unwrap().unwrap();
|
||||
nm.put(NeedleId(9), nv.offset, Size(nv.size.0 + 1)).unwrap();
|
||||
}
|
||||
|
||||
let mut probe = Needle {
|
||||
id: NeedleId(9),
|
||||
..Needle::default()
|
||||
};
|
||||
assert!(v.read_needle(&mut probe).is_err());
|
||||
|
||||
let mut n = Needle {
|
||||
id: NeedleId(9),
|
||||
cookie: Cookie(0xabcd),
|
||||
data: b"ninth".to_vec(),
|
||||
data_size: 5,
|
||||
..Needle::default()
|
||||
};
|
||||
v.write_needle(&mut n, true, false).unwrap();
|
||||
|
||||
let mut back = Needle {
|
||||
id: NeedleId(9),
|
||||
..Needle::default()
|
||||
};
|
||||
v.read_needle(&mut back).unwrap();
|
||||
assert_eq!(back.data, b"ninth");
|
||||
assert_eq!(back.cookie, Cookie(0xabcd));
|
||||
}
|
||||
|
||||
/// A stale index entry is only replaced once the new record commits: a
|
||||
/// durable write rolled back by a failed fsync leaves the index exactly
|
||||
/// as it found it.
|
||||
#[test]
|
||||
fn test_stale_index_entry_survives_failed_write() {
|
||||
let tmp = TempDir::new().unwrap();
|
||||
let dir = tmp.path().to_str().unwrap();
|
||||
let v = make_test_volume(dir);
|
||||
|
||||
let mut existing = Needle {
|
||||
id: NeedleId(2),
|
||||
cookie: Cookie(0x2222),
|
||||
data: b"second".to_vec(),
|
||||
data_size: 6,
|
||||
..Needle::default()
|
||||
};
|
||||
v.write_needle(&mut existing, true, false).unwrap();
|
||||
|
||||
let stale_nv = {
|
||||
let mut io = v.io.write().unwrap();
|
||||
let nm = io.nm.as_mut().unwrap();
|
||||
let nv = nm.get(NeedleId(2)).unwrap().unwrap();
|
||||
nm.put(NeedleId(9), nv.offset, Size(nv.size.0 + 1)).unwrap();
|
||||
nm.get(NeedleId(9)).unwrap().unwrap()
|
||||
};
|
||||
|
||||
v.fail_next_fsync_for_test(true);
|
||||
let mut replacement = Needle {
|
||||
id: NeedleId(9),
|
||||
cookie: Cookie(0xabcd),
|
||||
data: b"ninth".to_vec(),
|
||||
data_size: 5,
|
||||
..Needle::default()
|
||||
};
|
||||
assert!(v.write_needle(&mut replacement, true, true).is_err());
|
||||
v.fail_next_fsync_for_test(false);
|
||||
|
||||
let now = {
|
||||
let io = v.io.read().unwrap();
|
||||
io.nm
|
||||
.as_ref()
|
||||
.unwrap()
|
||||
.get(NeedleId(9))
|
||||
.unwrap()
|
||||
.expect("a failed write must not drop the index entry it meant to replace")
|
||||
};
|
||||
assert_eq!(now.offset, stale_nv.offset);
|
||||
assert_eq!(now.size, stale_nv.size);
|
||||
}
|
||||
|
||||
// A size disagreeing with the blob's own header indexes the needle at the wrong
|
||||
// length and, on v3, stamps the append timestamp into the middle of the needle.
|
||||
#[test]
|
||||
|
||||
@@ -1340,11 +1340,11 @@ async fn non_ascii_fid_and_ttl_are_rejected_not_panicked() {
|
||||
|
||||
#[tokio::test]
|
||||
async fn write_queue_upload_returns_same_etag_as_direct_write() {
|
||||
use seaweed_volume::server::write_queue::{SyncBatchOption, WriteQueue};
|
||||
use seaweed_volume::server::write_queue::WriteQueue;
|
||||
|
||||
let (direct_state, _direct_tmp) = test_state();
|
||||
let (queued_state, _queued_tmp) = test_state();
|
||||
let wq = WriteQueue::new(queued_state.clone(), 128, SyncBatchOption::default());
|
||||
let wq = WriteQueue::new(queued_state.clone(), 128);
|
||||
let _ = queued_state.write_queue.set(wq);
|
||||
|
||||
let compressible = "seaweedfs ".repeat(200).into_bytes();
|
||||
|
||||
@@ -34,7 +34,6 @@ struct ListTablesResponse {
|
||||
pub struct NamespaceClient {
|
||||
base_url: String,
|
||||
http: reqwest::Client,
|
||||
api_key: Option<String>,
|
||||
}
|
||||
|
||||
/// A namespace call that has not answered by now is not going to. Without this
|
||||
@@ -42,23 +41,12 @@ pub struct NamespaceClient {
|
||||
/// slot open forever, and the sweep never finishes.
|
||||
const REQUEST_TIMEOUT: Duration = Duration::from_secs(30);
|
||||
const CONNECT_TIMEOUT: Duration = Duration::from_secs(10);
|
||||
const MAX_REDIRECT_HOPS: usize = 10;
|
||||
|
||||
impl NamespaceClient {
|
||||
pub fn new(base_url: impl Into<String>) -> Self {
|
||||
let http = reqwest::Client::builder()
|
||||
.timeout(REQUEST_TIMEOUT)
|
||||
.connect_timeout(CONNECT_TIMEOUT)
|
||||
// reqwest forwards custom headers on a follow, so only a same-origin
|
||||
// hop is allowed: anywhere else would carry x-api-key to a listener
|
||||
// the credential was never meant for.
|
||||
.redirect(reqwest::redirect::Policy::custom(|attempt| {
|
||||
if may_follow(attempt.previous(), attempt.url()) {
|
||||
attempt.follow()
|
||||
} else {
|
||||
attempt.stop()
|
||||
}
|
||||
}))
|
||||
.build()
|
||||
// The builder only fails on a bad TLS backend, which would break
|
||||
// every call anyway; a client with no timeouts is worse than a panic
|
||||
@@ -70,32 +58,6 @@ impl NamespaceClient {
|
||||
Self {
|
||||
base_url: base_url.into().trim_end_matches('/').to_string(),
|
||||
http,
|
||||
api_key: None,
|
||||
}
|
||||
}
|
||||
|
||||
/// The catalog accepts an S3 credential as `x-api-key`; where the worker
|
||||
/// already holds one for storage it authenticates the namespace calls too.
|
||||
pub fn with_fallback(mut self, fallback: &crate::dataset::FallbackOptions) -> Self {
|
||||
if let (Some(access), Some(secret)) = (
|
||||
fallback.get("aws_access_key_id"),
|
||||
fallback.get("aws_secret_access_key"),
|
||||
) {
|
||||
if !protected_transport(&self.base_url) {
|
||||
tracing::warn!(
|
||||
"catalog url {} exposes the x-api-key credential in cleartext",
|
||||
self.base_url
|
||||
);
|
||||
}
|
||||
self.api_key = Some(format!("{access}:{secret}"));
|
||||
}
|
||||
self
|
||||
}
|
||||
|
||||
fn authed(&self, request: reqwest::RequestBuilder) -> reqwest::RequestBuilder {
|
||||
match &self.api_key {
|
||||
Some(key) => request.header("x-api-key", key),
|
||||
None => request,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -103,7 +65,8 @@ impl NamespaceClient {
|
||||
pub async fn list_all_tables(&self) -> Result<Vec<String>> {
|
||||
let url = format!("{}/v1/table", self.base_url);
|
||||
let response: ListTablesResponse = self
|
||||
.authed(self.http.get(&url))
|
||||
.http
|
||||
.get(&url)
|
||||
.send()
|
||||
.await
|
||||
.context("list tables")?
|
||||
@@ -117,7 +80,8 @@ impl NamespaceClient {
|
||||
pub async fn list_namespaces(&self, id: &[String]) -> Result<Vec<String>> {
|
||||
let url = format!("{}/v1/namespace/{}/list", self.base_url, encode_id(id));
|
||||
let response: ListNamespacesResponse = self
|
||||
.authed(self.http.get(&url))
|
||||
.http
|
||||
.get(&url)
|
||||
.send()
|
||||
.await
|
||||
.context("list namespaces")?
|
||||
@@ -134,7 +98,8 @@ impl NamespaceClient {
|
||||
let url = format!("{}/v1/table/{}/describe", self.base_url, encode_id(id));
|
||||
let body = serde_json::json!({ "id": id, "vend_credentials": true });
|
||||
let description: TableDescription = self
|
||||
.authed(self.http.post(&url))
|
||||
.http
|
||||
.post(&url)
|
||||
.json(&body)
|
||||
.send()
|
||||
.await
|
||||
@@ -146,41 +111,6 @@ impl NamespaceClient {
|
||||
}
|
||||
}
|
||||
|
||||
/// Without a hop bound a catalog that redirects to itself keeps the request
|
||||
/// alive until the call timeout, so the custom policy keeps the limit the
|
||||
/// default one had.
|
||||
fn may_follow(previous: &[reqwest::Url], next: &reqwest::Url) -> bool {
|
||||
previous.len() < MAX_REDIRECT_HOPS
|
||||
&& previous.last().is_some_and(|prev| same_origin(prev, next))
|
||||
}
|
||||
|
||||
/// A redirect stays within an origin when scheme, host, and port all match, so
|
||||
/// a catalog cannot bounce the api key onto a downgraded or foreign listener.
|
||||
fn same_origin(from: &reqwest::Url, to: &reqwest::Url) -> bool {
|
||||
from.scheme() == to.scheme()
|
||||
&& from.host_str() == to.host_str()
|
||||
&& from.port_or_known_default() == to.port_or_known_default()
|
||||
}
|
||||
|
||||
/// A catalog url protects the api key when the hop is TLS or never leaves the
|
||||
/// host; anything else sends it where the network can read it.
|
||||
fn protected_transport(base_url: &str) -> bool {
|
||||
let Ok(url) = reqwest::Url::parse(base_url) else {
|
||||
return false;
|
||||
};
|
||||
if url.scheme() == "https" {
|
||||
return true;
|
||||
}
|
||||
match url.host_str() {
|
||||
Some("localhost") => true,
|
||||
Some(host) => host
|
||||
.trim_matches(['[', ']'])
|
||||
.parse::<std::net::IpAddr>()
|
||||
.is_ok_and(|ip| ip.is_loopback()),
|
||||
None => false,
|
||||
}
|
||||
}
|
||||
|
||||
fn encode_id(id: &[String]) -> String {
|
||||
if id.is_empty() {
|
||||
DELIMITER.to_string()
|
||||
@@ -197,56 +127,3 @@ pub fn parse_id(encoded: &str) -> Vec<String> {
|
||||
.map(|part| part.to_string())
|
||||
.collect()
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::{MAX_REDIRECT_HOPS, may_follow, protected_transport, same_origin};
|
||||
|
||||
#[test]
|
||||
fn may_follow_bounds_hops_within_the_origin() {
|
||||
let url = |u: &str| reqwest::Url::parse(u).unwrap();
|
||||
let first = [url("https://c.local/ns")];
|
||||
let next = url("https://c.local/other");
|
||||
assert!(may_follow(&first, &next));
|
||||
assert!(!may_follow(&[], &next));
|
||||
assert!(!may_follow(&first, &url("http://c.local/other")));
|
||||
let full: Vec<_> = (0..MAX_REDIRECT_HOPS)
|
||||
.map(|i| url(&format!("https://c.local/{i}")))
|
||||
.collect();
|
||||
assert!(!may_follow(&full, &next));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn same_origin_rejects_scheme_and_port_hops() {
|
||||
for (from, to, expected) in [
|
||||
("https://c.local/ns", "https://c.local/other", true),
|
||||
("http://c.local", "http://c.local:80/next", true),
|
||||
("https://c.local", "http://c.local/next", false),
|
||||
("https://c.local:8443", "https://c.local/next", false),
|
||||
("https://c.local", "https://other.local/next", false),
|
||||
] {
|
||||
let from = reqwest::Url::parse(from).unwrap();
|
||||
let to = reqwest::Url::parse(to).unwrap();
|
||||
assert_eq!(same_origin(&from, &to), expected, "{from} -> {to}");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn protected_transport_covers_tls_and_loopback() {
|
||||
for url in [
|
||||
"https://catalog.internal:8443",
|
||||
"http://127.0.0.1:8080",
|
||||
"http://localhost",
|
||||
"http://[::1]:8080",
|
||||
] {
|
||||
assert!(protected_transport(url), "{url} should be protected");
|
||||
}
|
||||
for url in [
|
||||
"http://catalog.internal",
|
||||
"http://127.0.0.1.evil.test",
|
||||
"not a url",
|
||||
] {
|
||||
assert!(!protected_transport(url), "{url} should warn");
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -55,7 +55,7 @@ impl CleanupVersionsHandler {
|
||||
}
|
||||
|
||||
fn client(&self) -> NamespaceClient {
|
||||
NamespaceClient::new(self.namespace_url.clone()).with_fallback(&self.fallback)
|
||||
NamespaceClient::new(self.namespace_url.clone())
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -59,7 +59,7 @@ impl CompactHandler {
|
||||
}
|
||||
|
||||
fn client(&self) -> NamespaceClient {
|
||||
NamespaceClient::new(self.namespace_url.clone()).with_fallback(&self.fallback)
|
||||
NamespaceClient::new(self.namespace_url.clone())
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -58,7 +58,7 @@ impl OptimizeIndicesHandler {
|
||||
}
|
||||
|
||||
fn client(&self) -> NamespaceClient {
|
||||
NamespaceClient::new(self.namespace_url.clone()).with_fallback(&self.fallback)
|
||||
NamespaceClient::new(self.namespace_url.clone())
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -78,7 +78,7 @@ impl SortHandler {
|
||||
}
|
||||
|
||||
fn client(&self) -> NamespaceClient {
|
||||
NamespaceClient::new(self.namespace_url.clone()).with_fallback(&self.fallback)
|
||||
NamespaceClient::new(self.namespace_url.clone())
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -35,7 +35,7 @@ impl PreviewProvider for LancePreview {
|
||||
}
|
||||
|
||||
async fn preview(&self, object_id: &[String], row_limit: usize) -> Result<Preview> {
|
||||
let client = NamespaceClient::new(self.namespace_url.clone()).with_fallback(&self.fallback);
|
||||
let client = NamespaceClient::new(self.namespace_url.clone());
|
||||
let table = dataset::open(&client, object_id, &self.fallback).await?;
|
||||
|
||||
let total_rows = table.dataset.count_rows(None).await.unwrap_or(0) as i64;
|
||||
|
||||
Loaded 100 of 528 files, more files were not shown because too many files have changed in this diff.
Show more
Reference in new issue
Block a user