mirror of
https://github.com/seaweedfs/seaweedfs.git
synced 2026-11-20 11:23:50 +01:00
Compare commits
5
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
9d84ecca2f | ||
|
|
a1d2ebbb9b | ||
|
|
371d320854 | ||
|
|
84cc179b80 | ||
|
|
c8002d63bb |
No files matched your search
@@ -1,11 +0,0 @@
|
||||
name: Clean up stale SeaweedFS test mounts
|
||||
description: Detach disconnected FUSE test mounts left by earlier jobs on a shared runner.
|
||||
runs:
|
||||
using: composite
|
||||
steps:
|
||||
- name: Unmount disconnected SeaweedFS test mounts
|
||||
if: runner.os == 'Linux' && runner.environment == 'self-hosted'
|
||||
shell: bash
|
||||
env:
|
||||
CLEANUP_SCRIPT: ${{ github.action_path }}/cleanup.py
|
||||
run: python3 "$CLEANUP_SCRIPT"
|
||||
@@ -1,89 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Detach disconnected SeaweedFS integration-test mounts without touching live ones."""
|
||||
|
||||
import errno
|
||||
import os
|
||||
from pathlib import Path
|
||||
import re
|
||||
import shutil
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
|
||||
|
||||
def test_mounts(mountinfo):
|
||||
"""Read the kernel table directly: traversing a stale mount may block."""
|
||||
roots = {tempfile.gettempdir(), "/tmp", "/var/tmp"}
|
||||
for line in mountinfo.splitlines():
|
||||
fields, filesystem = line.split(" - ", 1)
|
||||
if filesystem.split()[0].split(".")[0] != "fuse":
|
||||
continue
|
||||
fields = fields.split()
|
||||
# mountinfo escapes whitespace and backslashes with octal sequences.
|
||||
target = re.sub(r"\\([0-7]{3})", lambda m: chr(int(m[1], 8)), fields[4])
|
||||
path = Path(target)
|
||||
# Test mounts are a mount* leaf under a seaweedfs_fuse_* scratch dir
|
||||
# (seaweedfs_fuse_test_*/mount, seaweedfs_fuse_failover_*/mount0, ...)
|
||||
# or the posix-compliance workflow's fixed /tmp/seaweedfs/mount.
|
||||
if (not path.name.startswith("mount")
|
||||
or str(path.parent.parent) not in roots
|
||||
or not (path.parent.name.startswith("seaweedfs_fuse")
|
||||
or path.parent.name == "seaweedfs")):
|
||||
continue
|
||||
yield fields[0], target
|
||||
|
||||
|
||||
def mounted(mount_id, target):
|
||||
return (mount_id, target) in test_mounts(Path("/proc/self/mountinfo").read_text())
|
||||
|
||||
|
||||
def cleanup():
|
||||
failed = False
|
||||
for mount_id, target in test_mounts(Path("/proc/self/mountinfo").read_text()):
|
||||
try:
|
||||
# A separate, bounded probe prevents a wedged FUSE daemon from
|
||||
# hanging cleanup. Only ENOTCONN proves the daemon disconnected;
|
||||
# a timeout or another error is not enough to detach a live mount.
|
||||
probe = subprocess.run(
|
||||
[sys.executable, __file__, "--probe", target], timeout=5,
|
||||
capture_output=True, check=False,
|
||||
)
|
||||
except subprocess.TimeoutExpired:
|
||||
print(f"::warning::FUSE probe timed out; leaving {target} mounted")
|
||||
continue
|
||||
if probe.returncode != errno.ENOTCONN or not mounted(mount_id, target):
|
||||
continue
|
||||
|
||||
print(f"Unmounting disconnected FUSE test mount: {target}", flush=True)
|
||||
# Lazy unmount detaches a dead mount even when an old process still
|
||||
# holds it open. Prefer the unprivileged FUSE helper; sudo is fallback.
|
||||
for command in (
|
||||
["fusermount3", "-uz", "--", target],
|
||||
["fusermount", "-uz", "--", target],
|
||||
["sudo", "-n", "umount", "-l", "--", target],
|
||||
):
|
||||
try:
|
||||
subprocess.run(command, timeout=10, check=False)
|
||||
except (OSError, subprocess.TimeoutExpired) as error:
|
||||
print(f"{command[0]}: {error}", file=sys.stderr)
|
||||
if not mounted(mount_id, target):
|
||||
break
|
||||
else:
|
||||
print(f"::error::Could not unmount disconnected FUSE test mount: {target}")
|
||||
failed = True
|
||||
continue
|
||||
# The detached mount's parent still holds the crashed run's data and
|
||||
# logs; test_mounts only yields targets under a SeaweedFS test dir in
|
||||
# a temp root, so the parent is safe to reclaim here.
|
||||
shutil.rmtree(Path(target).parent, ignore_errors=True)
|
||||
return int(failed)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
if len(sys.argv) == 3 and sys.argv[1] == "--probe":
|
||||
try:
|
||||
os.stat(sys.argv[2])
|
||||
except OSError as error:
|
||||
sys.exit(error.errno)
|
||||
sys.exit(0)
|
||||
sys.exit(cleanup())
|
||||
@@ -1,137 +0,0 @@
|
||||
import errno
|
||||
import importlib.util
|
||||
from pathlib import Path
|
||||
import subprocess
|
||||
import unittest
|
||||
from unittest.mock import patch
|
||||
|
||||
spec = importlib.util.spec_from_file_location("cleanup", Path(__file__).with_name("cleanup.py"))
|
||||
cleanup = importlib.util.module_from_spec(spec)
|
||||
spec.loader.exec_module(cleanup)
|
||||
|
||||
TARGET = "/tmp/seaweedfs_fuse_test_123/mount"
|
||||
MOUNTINFO = f"42 1 0:9 / {TARGET} rw - fuse.seaweedfs seaweedfs rw\n"
|
||||
|
||||
|
||||
class CleanupTests(unittest.TestCase):
|
||||
def run_cleanup(self, results, mounted=True):
|
||||
with patch.object(cleanup.Path, "read_text", return_value=MOUNTINFO), \
|
||||
patch.object(cleanup, "mounted", side_effect=mounted if isinstance(mounted, list) else None,
|
||||
return_value=mounted) as check, \
|
||||
patch.object(cleanup.shutil, "rmtree") as rmtree, \
|
||||
patch.object(cleanup.subprocess, "run", side_effect=results) as run:
|
||||
status = cleanup.cleanup()
|
||||
return status, run, check, rmtree
|
||||
|
||||
def result(self, status):
|
||||
return subprocess.CompletedProcess([], status)
|
||||
|
||||
def test_only_fuse_integration_test_mounts_are_candidates(self):
|
||||
rows = MOUNTINFO
|
||||
rows += MOUNTINFO.replace("fuse.seaweedfs", "ext4")
|
||||
rows += MOUNTINFO.replace(TARGET, "/mnt/production")
|
||||
rows += MOUNTINFO.replace(TARGET, "/tmp/seaweedfs_fuse_test_123/data")
|
||||
rows += MOUNTINFO.replace(TARGET, "/home/user/seaweedfs_fuse_test_123/mount")
|
||||
rows += MOUNTINFO.replace(TARGET, "/tmp/seaweedfs/data")
|
||||
self.assertEqual(list(cleanup.test_mounts(rows)), [("42", TARGET)])
|
||||
|
||||
def test_all_test_mount_layouts_are_candidates(self):
|
||||
targets = (
|
||||
"/tmp/seaweedfs_fuse_test_123/mount",
|
||||
"/tmp/seaweedfs_fuse_failover_456/mount0",
|
||||
"/tmp/seaweedfs_fuse_failover_456/mount1",
|
||||
"/var/tmp/seaweedfs_fuse_p2p_test_789/mount2",
|
||||
"/tmp/seaweedfs/mount",
|
||||
)
|
||||
rows = "".join(
|
||||
MOUNTINFO.replace("42", str(40 + i)).replace(TARGET, target)
|
||||
for i, target in enumerate(targets)
|
||||
)
|
||||
self.assertEqual(
|
||||
list(cleanup.test_mounts(rows)),
|
||||
[(str(40 + i), target) for i, target in enumerate(targets)],
|
||||
)
|
||||
|
||||
def test_mountinfo_paths_are_unescaped_once(self):
|
||||
target = "/tmp/seaweedfs_fuse_test_space and\\slash/mount"
|
||||
encoded = target.replace("\\", r"\134").replace(" ", r"\040")
|
||||
rows = MOUNTINFO.replace(TARGET, encoded)
|
||||
self.assertEqual(list(cleanup.test_mounts(rows)), [("42", target)])
|
||||
|
||||
def test_no_mounts_is_a_noop(self):
|
||||
with patch.object(cleanup.Path, "read_text", return_value=""), \
|
||||
patch.object(cleanup.subprocess, "run") as run:
|
||||
self.assertEqual(cleanup.cleanup(), 0)
|
||||
run.assert_not_called()
|
||||
|
||||
def test_healthy_mount_is_untouched(self):
|
||||
status, run, _, _ = self.run_cleanup([self.result(0)])
|
||||
self.assertEqual(status, 0)
|
||||
self.assertEqual(run.call_count, 1)
|
||||
|
||||
def test_other_probe_errors_are_untouched(self):
|
||||
for error in (errno.EACCES, errno.ENOENT, errno.EIO):
|
||||
with self.subTest(error=error):
|
||||
status, run, _, _ = self.run_cleanup([self.result(error)])
|
||||
self.assertEqual(status, 0)
|
||||
self.assertEqual(run.call_count, 1)
|
||||
|
||||
def test_probe_timeout_does_not_detach_a_possibly_live_mount(self):
|
||||
status, run, _, _ = self.run_cleanup([subprocess.TimeoutExpired("probe", 5)])
|
||||
self.assertEqual(status, 0)
|
||||
self.assertEqual(run.call_count, 1)
|
||||
|
||||
def test_disconnected_mount_is_lazily_unmounted(self):
|
||||
status, run, _, _ = self.run_cleanup([self.result(errno.ENOTCONN), self.result(0)], [True, False])
|
||||
self.assertEqual(status, 0)
|
||||
self.assertEqual(run.call_args_list[1].args[0], ["fusermount3", "-uz", "--", TARGET])
|
||||
self.assertEqual(run.call_args_list[0].kwargs["timeout"], 5)
|
||||
self.assertEqual(run.call_args_list[1].kwargs["timeout"], 10)
|
||||
|
||||
def test_replaced_or_removed_mount_is_untouched(self):
|
||||
status, run, _, _ = self.run_cleanup([self.result(errno.ENOTCONN)], False)
|
||||
self.assertEqual(status, 0)
|
||||
self.assertEqual(run.call_count, 1)
|
||||
|
||||
def test_mount_identity_is_checked_without_stat(self):
|
||||
with patch.object(cleanup.Path, "read_text", return_value=MOUNTINFO):
|
||||
self.assertTrue(cleanup.mounted("42", TARGET))
|
||||
self.assertFalse(cleanup.mounted("43", TARGET))
|
||||
|
||||
def test_missing_or_timed_out_helper_uses_fallback(self):
|
||||
for error in (FileNotFoundError(), subprocess.TimeoutExpired("fusermount3", 10)):
|
||||
with self.subTest(error=error):
|
||||
status, run, _, _ = self.run_cleanup(
|
||||
[self.result(errno.ENOTCONN), error, self.result(0)], [True, True, False])
|
||||
self.assertEqual(status, 0)
|
||||
self.assertEqual(run.call_args.args[0], ["fusermount", "-uz", "--", TARGET])
|
||||
|
||||
def test_failed_helpers_fall_back_to_sudo(self):
|
||||
status, run, _, _ = self.run_cleanup(
|
||||
[self.result(errno.ENOTCONN), self.result(1), self.result(1), self.result(0)],
|
||||
[True, True, True, False])
|
||||
self.assertEqual(status, 0)
|
||||
self.assertEqual(run.call_args.args[0], ["sudo", "-n", "umount", "-l", "--", TARGET])
|
||||
|
||||
def test_cleanup_fails_when_stale_mount_remains(self):
|
||||
status, run, _, rmtree = self.run_cleanup(
|
||||
[self.result(errno.ENOTCONN), self.result(1), self.result(1), self.result(1)])
|
||||
self.assertEqual(status, 1)
|
||||
self.assertEqual(run.call_count, 4)
|
||||
rmtree.assert_not_called()
|
||||
|
||||
def test_unmounted_test_dir_is_reclaimed(self):
|
||||
status, _, _, rmtree = self.run_cleanup(
|
||||
[self.result(errno.ENOTCONN), self.result(0)], [True, False])
|
||||
self.assertEqual(status, 0)
|
||||
rmtree.assert_called_once_with(Path(TARGET).parent, ignore_errors=True)
|
||||
|
||||
def test_probe_process_reports_errno(self):
|
||||
for target, expected in ((str(Path(__file__)), 0), (str(Path(__file__) / "absent"), errno.ENOTDIR)):
|
||||
with self.subTest(target=target):
|
||||
result = subprocess.run([cleanup.sys.executable, cleanup.__file__, "--probe", target], check=False)
|
||||
self.assertEqual(result.returncode, expected)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -23,14 +23,8 @@ runs:
|
||||
|
||||
bin=$(command -v fusermount3 || true)
|
||||
if [ -z "$bin" ]; then
|
||||
# Self-hosted runners are not guaranteed to carry fuse3.
|
||||
sudo apt-get update
|
||||
sudo apt-get install -y fuse3
|
||||
bin=$(command -v fusermount3 || true)
|
||||
if [ -z "$bin" ]; then
|
||||
echo "no fusermount3 in PATH after installing fuse3" >&2
|
||||
exit 1
|
||||
fi
|
||||
echo "no fusermount3 in PATH" >&2
|
||||
exit 1
|
||||
fi
|
||||
if can_mount "$bin"; then
|
||||
ls -l "$bin"
|
||||
|
||||
@@ -1,30 +0,0 @@
|
||||
name: "actions: unit tests"
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [ master ]
|
||||
paths:
|
||||
- '.github/actions/**'
|
||||
- '.github/workflows/action-tests.yml'
|
||||
pull_request:
|
||||
branches: [ master ]
|
||||
paths:
|
||||
- '.github/actions/**'
|
||||
- '.github/workflows/action-tests.yml'
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.event.pull_request.number || (github.event_name == 'push' && github.ref) || github.run_id }}
|
||||
cancel-in-progress: ${{ github.event_name == 'pull_request' || (github.event_name == 'push' && !startsWith(github.ref, 'refs/tags/')) }}
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
jobs:
|
||||
cleanup-stale-fuse-mounts:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Check out code
|
||||
uses: actions/checkout@v7
|
||||
|
||||
- name: Run cleanup action unit tests
|
||||
run: python3 .github/actions/cleanup-stale-fuse-mounts/test_cleanup.py
|
||||
@@ -9,8 +9,8 @@ on:
|
||||
- '.github/workflows/codeql.yml'
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.event.pull_request.number || (github.event_name == 'push' && github.ref) || github.run_id }}
|
||||
cancel-in-progress: ${{ github.event_name == 'pull_request' || (github.event_name == 'push' && !startsWith(github.ref, 'refs/tags/')) }}
|
||||
group: ${{ github.head_ref }}/codeql
|
||||
cancel-in-progress: true
|
||||
|
||||
jobs:
|
||||
CodeQL-Build:
|
||||
@@ -25,12 +25,6 @@ jobs:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v7
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
cache: ${{ runner.environment == 'github-hosted' }}
|
||||
|
||||
# Initializes the CodeQL tools for scanning.
|
||||
- name: Initialize CodeQL
|
||||
uses: github/codeql-action/init@v4.39.0
|
||||
|
||||
@@ -8,10 +8,6 @@ on:
|
||||
pull_request:
|
||||
branches: [master]
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.event.pull_request.number || (github.event_name == 'push' && github.ref) || github.run_id }}
|
||||
cancel-in-progress: ${{ github.event_name == 'pull_request' || (github.event_name == 'push' && !startsWith(github.ref, 'refs/tags/')) }}
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
|
||||
@@ -13,10 +13,6 @@ on:
|
||||
- 'go.sum'
|
||||
- '.github/workflows/container_dev.yml'
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.event.pull_request.number || (github.event_name == 'push' && github.ref) || github.run_id }}
|
||||
cancel-in-progress: ${{ github.event_name == 'pull_request' || (github.event_name == 'push' && !startsWith(github.ref, 'refs/tags/')) }}
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
@@ -55,7 +51,6 @@ jobs:
|
||||
echo "CFLAGS_aarch64_unknown_linux_musl=-U_FORTIFY_SOURCE" >> "$GITHUB_ENV"
|
||||
|
||||
- name: Cache cargo registry and target
|
||||
if: runner.environment == 'github-hosted'
|
||||
uses: actions/cache@v6
|
||||
with:
|
||||
path: |
|
||||
@@ -198,12 +193,6 @@ jobs:
|
||||
platforms: linux/amd64, linux/arm64
|
||||
tags: ${{ steps.docker_meta.outputs.tags }}
|
||||
labels: ${{ steps.docker_meta.outputs.labels }}
|
||||
cache-from: type=gha
|
||||
cache-to: type=gha,mode=max
|
||||
# Key the checkout layer on the pushed sha so cached layers can't
|
||||
# serve stale code; the Dockerfile fetch fallback picks it up.
|
||||
build-args: |
|
||||
BRANCH=${{ github.sha }}
|
||||
|
||||
- name: Sign
|
||||
if: github.event_name != 'pull_request'
|
||||
|
||||
@@ -24,10 +24,6 @@ on:
|
||||
required: false
|
||||
default: ''
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.event.pull_request.number || (github.event_name == 'push' && github.ref) || github.event.inputs.image_tag || format('foundationdb_{0}_seaweedfs_{1}', github.event.inputs.fdb_version || '7.4.5', github.event.inputs.seaweedfs_ref || github.ref_name) }}
|
||||
cancel-in-progress: ${{ github.event_name == 'pull_request' || (github.event_name == 'push' && !startsWith(github.ref, 'refs/tags/')) }}
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
@@ -68,7 +64,6 @@ jobs:
|
||||
uses: actions/setup-go@v7
|
||||
with:
|
||||
go-version-file: go.mod
|
||||
cache: ${{ runner.environment == 'github-hosted' }}
|
||||
|
||||
- name: Run FoundationDB tagged tests
|
||||
env:
|
||||
|
||||
@@ -30,11 +30,6 @@ on:
|
||||
type: boolean
|
||||
default: false
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.event.pull_request.number || (github.event_name == 'push' && github.ref) || github.event.inputs.image_tag || github.run_id }}
|
||||
cancel-in-progress: false
|
||||
queue: max
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
security-events: write
|
||||
@@ -99,7 +94,6 @@ jobs:
|
||||
echo "CFLAGS_aarch64_unknown_linux_musl=-U_FORTIFY_SOURCE" >> "$GITHUB_ENV"
|
||||
|
||||
- name: Cache cargo registry and target
|
||||
if: runner.environment == 'github-hosted'
|
||||
uses: actions/cache@v6
|
||||
with:
|
||||
path: |
|
||||
@@ -170,13 +164,10 @@ jobs:
|
||||
uses: actions/checkout@v7
|
||||
with:
|
||||
ref: ${{ github.event_name == 'workflow_dispatch' && github.event.inputs.source_ref || github.ref }}
|
||||
- name: Clean up stale FUSE mounts
|
||||
uses: ./.github/actions/cleanup-stale-fuse-mounts
|
||||
|
||||
- name: Free Disk Space
|
||||
run: |
|
||||
echo "Available disk space before cleanup:"
|
||||
timeout 15 df -h || true
|
||||
df -h
|
||||
# Remove pre-installed tools
|
||||
sudo rm -rf /usr/share/dotnet /usr/local/lib/android /opt/ghc /opt/hostedtoolcache/CodeQL
|
||||
# Clean package managers
|
||||
@@ -188,7 +179,7 @@ jobs:
|
||||
[ -d ~/.cache/go-build ] && rm -rf ~/.cache/go-build || true
|
||||
[ -d /go/pkg ] && rm -rf /go/pkg || true
|
||||
echo "Available disk space after cleanup:"
|
||||
timeout 15 df -h || true
|
||||
df -h
|
||||
|
||||
- name: Configure variant
|
||||
id: config
|
||||
|
||||
@@ -11,11 +11,6 @@ on:
|
||||
required: true
|
||||
default: ''
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.event.pull_request.number || (github.event_name == 'push' && github.ref) || format('refs/tags/{0}', github.event.inputs.release_tag) || github.run_id }}
|
||||
cancel-in-progress: false
|
||||
queue: max
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
|
||||
@@ -25,7 +25,7 @@ on:
|
||||
rocksdb_version:
|
||||
description: 'RocksDB git tag to use when variant=rocksdb'
|
||||
required: false
|
||||
default: 'v11.1.2'
|
||||
default: 'v10.10.1'
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
@@ -78,7 +78,6 @@ jobs:
|
||||
echo "CFLAGS_aarch64_unknown_linux_musl=-U_FORTIFY_SOURCE" >> "$GITHUB_ENV"
|
||||
|
||||
- name: Cache cargo registry and target
|
||||
if: runner.environment == 'github-hosted'
|
||||
uses: actions/cache@v6
|
||||
with:
|
||||
path: |
|
||||
@@ -180,10 +179,6 @@ jobs:
|
||||
if: github.event_name != 'workflow_dispatch' || github.event.inputs.variant == 'all' || github.event.inputs.variant == matrix.variant
|
||||
uses: actions/checkout@v7
|
||||
|
||||
- name: Clean up stale FUSE mounts
|
||||
if: github.event_name != 'workflow_dispatch' || github.event.inputs.variant == 'all' || github.event.inputs.variant == matrix.variant
|
||||
uses: ./.github/actions/cleanup-stale-fuse-mounts
|
||||
|
||||
- name: Download pre-built Rust binaries
|
||||
if: github.event_name != 'workflow_dispatch' || github.event.inputs.variant == 'all' || github.event.inputs.variant == matrix.variant
|
||||
uses: actions/download-artifact@v8
|
||||
@@ -223,7 +218,7 @@ jobs:
|
||||
sudo docker system prune -af --volumes
|
||||
[ -d ~/.cache/go-build ] && rm -rf ~/.cache/go-build || true
|
||||
[ -d /go/pkg ] && rm -rf /go/pkg || true
|
||||
timeout 15 df -h || true
|
||||
df -h
|
||||
|
||||
- name: Docker meta
|
||||
if: github.event_name != 'workflow_dispatch' || github.event.inputs.variant == 'all' || github.event.inputs.variant == matrix.variant
|
||||
@@ -275,11 +270,8 @@ jobs:
|
||||
file: ${{ matrix.dockerfile }}
|
||||
platforms: ${{ matrix.platform }}
|
||||
labels: ${{ steps.docker_meta.outputs.labels }}
|
||||
# mode=max keeps resolvedDependencies complete and records the
|
||||
# builder identity; attestations ride inside each pushed index and
|
||||
# survive the imagetools merge into the multi-arch manifest.
|
||||
provenance: mode=max
|
||||
sbom: true
|
||||
# Flat single-platform manifest so imagetools create assembles cleanly.
|
||||
provenance: false
|
||||
outputs: type=image,name=${{ env.IMAGE }},push-by-digest=true,name-canonical=true,push=true
|
||||
cache-from: type=gha,scope=${{ matrix.variant }}-${{ matrix.arch }}
|
||||
# max only for rocksdb: its RocksDB compile is sha-independent and worth
|
||||
@@ -289,7 +281,7 @@ jobs:
|
||||
${{ matrix.build_args }}
|
||||
BUILDKIT_INLINE_CACHE=1
|
||||
BRANCH=${{ github.sha }}
|
||||
${{ matrix.variant == 'rocksdb' && format('ROCKSDB_VERSION={0}', github.event.inputs.rocksdb_version || 'v11.1.2') || '' }}
|
||||
${{ matrix.variant == 'rocksdb' && format('ROCKSDB_VERSION={0}', github.event.inputs.rocksdb_version || 'v10.10.1') || '' }}
|
||||
|
||||
- name: Export digest
|
||||
if: github.event_name != 'workflow_dispatch' || github.event.inputs.variant == 'all' || github.event.inputs.variant == matrix.variant
|
||||
@@ -315,8 +307,6 @@ jobs:
|
||||
permissions:
|
||||
contents: read
|
||||
id-token: write
|
||||
attestations: write
|
||||
packages: write
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
@@ -424,45 +414,6 @@ jobs:
|
||||
${{ env.IMAGE }}@${{ steps.manifest.outputs.digest }}
|
||||
chrislusf/seaweedfs@${{ steps.manifest.outputs.digest }}
|
||||
|
||||
# Attest the merged multi-arch digest itself, so digest-pinning consumers
|
||||
# can verify provenance against the exact index they pull. Both steps
|
||||
# reuse the GHCR login above and push as OCI referrers.
|
||||
- name: Attest build provenance for ${{ matrix.variant }}
|
||||
if: github.event_name != 'workflow_dispatch' || github.event.inputs.variant == 'all' || github.event.inputs.variant == matrix.variant
|
||||
uses: actions/attest-build-provenance@v4
|
||||
with:
|
||||
subject-name: ${{ env.IMAGE }}
|
||||
subject-digest: ${{ steps.manifest.outputs.digest }}
|
||||
push-to-registry: true
|
||||
|
||||
# SBOMs are generated and attested per platform manifest: an index-wide
|
||||
# scan only describes syft's default platform, while the manifests it
|
||||
# merges differ (some ship the real binary, others placeholders), so the
|
||||
# SBOM has to be bound to the platform digest a consumer actually pulls.
|
||||
- name: Install syft
|
||||
if: github.event_name != 'workflow_dispatch' || github.event.inputs.variant == 'all' || github.event.inputs.variant == matrix.variant
|
||||
uses: anchore/sbom-action/download-syft@v0
|
||||
|
||||
- name: Attest per-platform SBOMs for ${{ matrix.variant }}
|
||||
if: github.event_name != 'workflow_dispatch' || github.event.inputs.variant == 'all' || github.event.inputs.variant == matrix.variant
|
||||
working-directory: /tmp/digests
|
||||
run: |
|
||||
set -euo pipefail
|
||||
for f in *; do
|
||||
# Each saved digest is an OCI index (image + attestation manifests);
|
||||
# scan and attest the image manifest a same-platform pull resolves.
|
||||
ref="${{ env.IMAGE }}@sha256:${f}"
|
||||
image=$(crane manifest "$ref" | jq -r '
|
||||
[.manifests[]? | select((.annotations["vnd.docker.reference.type"] // "") != "attestation-manifest"
|
||||
and (.platform.os // "unknown") != "unknown") | .digest][0] // empty')
|
||||
ref="${{ env.IMAGE }}@${image:-sha256:${f}}"
|
||||
syft "registry:${ref}" -o "spdx-json=/tmp/sbom-${f}.spdx.json"
|
||||
cosign attest --yes \
|
||||
--predicate "/tmp/sbom-${f}.spdx.json" \
|
||||
--type spdxjson \
|
||||
"$ref"
|
||||
done
|
||||
|
||||
# Report-only trivy scan: uploads fixable HIGH/CRITICAL findings to GitHub
|
||||
# Security for visibility, but never blocks the release. Releases (including
|
||||
# `latest`) ship regardless — vulnerabilities are tracked, not gated, since
|
||||
@@ -501,9 +452,7 @@ jobs:
|
||||
scanners: vuln
|
||||
vuln-type: os,library
|
||||
severity: HIGH,CRITICAL
|
||||
# Keep unfixed findings in the report so digest-pinning consumers see
|
||||
# the complete picture, not just what a patch could resolve.
|
||||
ignore-unfixed: false
|
||||
ignore-unfixed: true
|
||||
limit-severities-for-sarif: true
|
||||
format: sarif
|
||||
output: trivy-results.sarif
|
||||
|
||||
@@ -4,9 +4,9 @@ on:
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
rocksdb_version:
|
||||
description: 'RocksDB git tag or branch to build (e.g. v11.1.2)'
|
||||
description: 'RocksDB git tag or branch to build (e.g. v10.10.1)'
|
||||
required: true
|
||||
default: 'v11.1.2'
|
||||
default: 'v10.10.1'
|
||||
seaweedfs_ref:
|
||||
description: 'SeaweedFS git tag, branch, or commit to build'
|
||||
required: true
|
||||
@@ -16,11 +16,6 @@ on:
|
||||
required: false
|
||||
default: ''
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.event.inputs.image_tag || format('rocksdb_{0}_seaweedfs_{1}', github.event.inputs.rocksdb_version, github.event.inputs.seaweedfs_ref) }}
|
||||
cancel-in-progress: false
|
||||
queue: max
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
|
||||
@@ -27,7 +27,6 @@ jobs:
|
||||
uses: actions/setup-go@v7
|
||||
with:
|
||||
go-version-file: 'telemetry/server/go.mod'
|
||||
cache: ${{ runner.environment == 'github-hosted' }}
|
||||
|
||||
- name: Build Telemetry Server
|
||||
if: github.event_name == 'workflow_dispatch' && inputs.deploy
|
||||
|
||||
@@ -1,10 +1,6 @@
|
||||
name: 'Dependency Review'
|
||||
on: [pull_request]
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.event.pull_request.number || (github.event_name == 'push' && github.ref) || github.run_id }}
|
||||
cancel-in-progress: ${{ github.event_name == 'pull_request' || (github.event_name == 'push' && !startsWith(github.ref, 'refs/tags/')) }}
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
|
||||
@@ -19,8 +19,8 @@ on:
|
||||
- '.github/workflows/e2e.yml'
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.event.pull_request.number || (github.event_name == 'push' && github.ref) || github.run_id }}
|
||||
cancel-in-progress: ${{ github.event_name == 'pull_request' || (github.event_name == 'push' && !startsWith(github.ref, 'refs/tags/')) }}
|
||||
group: ${{ github.head_ref }}/e2e
|
||||
cancel-in-progress: true
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
@@ -42,7 +42,6 @@ jobs:
|
||||
uses: actions/setup-go@v7
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
cache: ${{ runner.environment == 'github-hosted' }}
|
||||
|
||||
- name: Configure Docker Hub mirror
|
||||
run: |
|
||||
@@ -53,7 +52,6 @@ jobs:
|
||||
uses: docker/setup-buildx-action@v4
|
||||
|
||||
- name: Cache Docker layers
|
||||
if: runner.environment == 'github-hosted'
|
||||
uses: actions/cache@v6
|
||||
with:
|
||||
path: /tmp/.buildx-cache
|
||||
|
||||
@@ -18,10 +18,6 @@ on:
|
||||
- 'go.sum'
|
||||
- '.github/workflows/ec-integration-tests.yml'
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.event.pull_request.number || (github.event_name == 'push' && github.ref) || github.run_id }}
|
||||
cancel-in-progress: ${{ github.event_name == 'pull_request' || (github.event_name == 'push' && !startsWith(github.ref, 'refs/tags/')) }}
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
@@ -34,7 +30,6 @@ jobs:
|
||||
- name: Set up Go 1.x
|
||||
uses: actions/setup-go@v7
|
||||
with:
|
||||
cache: ${{ runner.environment == 'github-hosted' }}
|
||||
go-version: ^1.26
|
||||
id: go
|
||||
|
||||
|
||||
@@ -19,8 +19,8 @@ on:
|
||||
- '.github/actions/fix-fusermount-setuid/**'
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.event.pull_request.number || (github.event_name == 'push' && github.ref) || github.run_id }}
|
||||
cancel-in-progress: ${{ github.event_name == 'pull_request' || (github.event_name == 'push' && !startsWith(github.ref, 'refs/tags/')) }}
|
||||
group: ${{ github.head_ref || github.ref }}/fuse-dlm-integration
|
||||
cancel-in-progress: true
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
@@ -39,7 +39,6 @@ jobs:
|
||||
uses: actions/setup-go@v7
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
cache: ${{ runner.environment == 'github-hosted' }}
|
||||
|
||||
- name: Configure FUSE
|
||||
run: |
|
||||
|
||||
@@ -24,8 +24,8 @@ on:
|
||||
- '.github/actions/fix-fusermount-setuid/**'
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.event.pull_request.number || (github.event_name == 'push' && github.ref) || github.run_id }}
|
||||
cancel-in-progress: ${{ github.event_name == 'pull_request' || (github.event_name == 'push' && !startsWith(github.ref, 'refs/tags/')) }}
|
||||
group: ${{ github.head_ref || github.ref }}/fuse-failover
|
||||
cancel-in-progress: true
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
@@ -42,14 +42,10 @@ jobs:
|
||||
with:
|
||||
persist-credentials: false
|
||||
|
||||
- name: Clean up stale FUSE mounts
|
||||
uses: ./.github/actions/cleanup-stale-fuse-mounts
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v7
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
cache: ${{ runner.environment == 'github-hosted' }}
|
||||
|
||||
- name: Configure FUSE
|
||||
run: |
|
||||
|
||||
@@ -17,8 +17,8 @@ on:
|
||||
- '.github/actions/fix-fusermount-setuid/**'
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.event.pull_request.number || (github.event_name == 'push' && github.ref) || github.run_id }}
|
||||
cancel-in-progress: ${{ github.event_name == 'pull_request' || (github.event_name == 'push' && !startsWith(github.ref, 'refs/tags/')) }}
|
||||
group: ${{ github.head_ref }}/fuse-integration
|
||||
cancel-in-progress: true
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
@@ -37,7 +37,6 @@ jobs:
|
||||
uses: actions/setup-go@v7
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
cache: ${{ runner.environment == 'github-hosted' }}
|
||||
|
||||
- name: Configure FUSE
|
||||
run: |
|
||||
|
||||
@@ -25,8 +25,8 @@ on:
|
||||
- '.github/actions/fix-fusermount-setuid/**'
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.event.pull_request.number || (github.event_name == 'push' && github.ref) || github.run_id }}
|
||||
cancel-in-progress: ${{ github.event_name == 'pull_request' || (github.event_name == 'push' && !startsWith(github.ref, 'refs/tags/')) }}
|
||||
group: ${{ github.head_ref || github.ref }}/fuse-p2p-integration
|
||||
cancel-in-progress: true
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
@@ -41,14 +41,10 @@ jobs:
|
||||
- name: Check out code
|
||||
uses: actions/checkout@v7
|
||||
|
||||
- name: Clean up stale FUSE mounts
|
||||
uses: ./.github/actions/cleanup-stale-fuse-mounts
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v7
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
cache: ${{ runner.environment == 'github-hosted' }}
|
||||
|
||||
- name: Configure FUSE
|
||||
run: |
|
||||
|
||||
@@ -17,8 +17,8 @@ on:
|
||||
- '.github/workflows/go.yml'
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.event.pull_request.number || (github.event_name == 'push' && github.ref) || github.run_id }}
|
||||
cancel-in-progress: ${{ github.event_name == 'pull_request' || (github.event_name == 'push' && !startsWith(github.ref, 'refs/tags/')) }}
|
||||
group: ${{ github.head_ref }}/go
|
||||
cancel-in-progress: true
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
@@ -35,7 +35,6 @@ jobs:
|
||||
uses: actions/setup-go@v7
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
cache: ${{ runner.environment == 'github-hosted' }}
|
||||
- name: Get dependencies
|
||||
run: |
|
||||
cd weed; go get -v -t -d ./...
|
||||
@@ -58,7 +57,6 @@ jobs:
|
||||
uses: actions/setup-go@v7
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
cache: ${{ runner.environment == 'github-hosted' }}
|
||||
- name: Go Vet linux/386 (type-checks code and tests for 32-bit int overflows)
|
||||
run: |
|
||||
GOOS=linux GOARCH=386 go vet ./... 2>&1 | grep -v "MessageState contains sync.Mutex" | grep -v "IdentityAccessManagement contains sync.RWMutex" | tee vet-32bit-output.txt
|
||||
@@ -74,7 +72,6 @@ jobs:
|
||||
uses: actions/setup-go@v7
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
cache: ${{ runner.environment == 'github-hosted' }}
|
||||
- name: Build
|
||||
run: cd weed; go build -tags "elastic gocdk sqlite ydb tarantool tikv rclone" -v .
|
||||
|
||||
@@ -97,7 +94,6 @@ jobs:
|
||||
uses: actions/setup-go@v7
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
cache: ${{ runner.environment == 'github-hosted' }}
|
||||
# Nothing else on a PR compiles these, so per-OS syscall constants creep
|
||||
# back into shared files unnoticed and only a release build catches it.
|
||||
- name: Cross-compile
|
||||
@@ -132,7 +128,6 @@ jobs:
|
||||
uses: actions/setup-go@v7
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
cache: ${{ runner.environment == 'github-hosted' }}
|
||||
- name: Test
|
||||
env:
|
||||
RUN_REDIS_TESTS: "1"
|
||||
@@ -148,7 +143,6 @@ jobs:
|
||||
uses: actions/setup-go@v7
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
cache: ${{ runner.environment == 'github-hosted' }}
|
||||
# 386 test binaries run natively on the amd64 runner. This catches what vet
|
||||
# can't: unaligned 64-bit atomics and arithmetic that wraps at runtime.
|
||||
# -short skips the e2e suites already covered on amd64.
|
||||
@@ -165,7 +159,6 @@ jobs:
|
||||
uses: actions/setup-go@v7
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
cache: ${{ runner.environment == 'github-hosted' }}
|
||||
# weed/kms/azure is excluded from every default build, so without this
|
||||
# nothing compiles it and tag-gated code silently rots.
|
||||
- name: Build and test with azurekms
|
||||
|
||||
@@ -8,10 +8,6 @@ on:
|
||||
branches: [ master ]
|
||||
paths: ['k8s/**', '.github/workflows/helm_ci.yml', '.github/scripts/helm_jwt_keys.py']
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.event.pull_request.number || (github.event_name == 'push' && github.ref) || github.run_id }}
|
||||
cancel-in-progress: ${{ github.event_name == 'pull_request' || (github.event_name == 'push' && !startsWith(github.ref, 'refs/tags/')) }}
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
|
||||
@@ -14,10 +14,6 @@ on:
|
||||
- 'weed/**'
|
||||
- '.github/workflows/java_integration_tests.yml'
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.event.pull_request.number || (github.event_name == 'push' && github.ref) || github.run_id }}
|
||||
cancel-in-progress: ${{ github.event_name == 'pull_request' || (github.event_name == 'push' && !startsWith(github.ref, 'refs/tags/')) }}
|
||||
|
||||
jobs:
|
||||
test:
|
||||
name: Java Integration Tests
|
||||
@@ -35,7 +31,6 @@ jobs:
|
||||
uses: actions/setup-go@v7
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
cache: ${{ runner.environment == 'github-hosted' }}
|
||||
id: go
|
||||
|
||||
- name: Set up Java
|
||||
@@ -43,7 +38,7 @@ jobs:
|
||||
with:
|
||||
java-version: ${{ matrix.java }}
|
||||
distribution: 'temurin'
|
||||
cache: ${{ runner.environment == 'github-hosted' && 'maven' || '' }}
|
||||
cache: 'maven'
|
||||
|
||||
- name: Build SeaweedFS
|
||||
run: |
|
||||
@@ -60,7 +55,6 @@ jobs:
|
||||
# Start SeaweedFS with optimized settings for CI
|
||||
# Include S3 API for s3copier integration tests
|
||||
weed server -dir="$WEED_DATA_DIR" \
|
||||
-ip=127.0.0.1 -ip.bind=127.0.0.1 \
|
||||
-master.raftHashicorp \
|
||||
-master.electionTimeout=1s \
|
||||
-master.volumeSizeLimitMB=100 \
|
||||
|
||||
@@ -77,7 +77,7 @@ jobs:
|
||||
with:
|
||||
java-version: '17'
|
||||
distribution: 'temurin'
|
||||
cache: ${{ runner.environment == 'github-hosted' && 'maven' || '' }}
|
||||
cache: 'maven'
|
||||
server-id: central
|
||||
server-username: MAVEN_CENTRAL_USERNAME
|
||||
server-password: MAVEN_CENTRAL_PASSWORD
|
||||
|
||||
@@ -12,10 +12,6 @@ on:
|
||||
- 'other/java/**'
|
||||
- '.github/workflows/java_unit_tests.yml'
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.event.pull_request.number || (github.event_name == 'push' && github.ref) || github.run_id }}
|
||||
cancel-in-progress: ${{ github.event_name == 'pull_request' || (github.event_name == 'push' && !startsWith(github.ref, 'refs/tags/')) }}
|
||||
|
||||
jobs:
|
||||
test:
|
||||
name: Java Unit Tests
|
||||
@@ -34,7 +30,7 @@ jobs:
|
||||
with:
|
||||
java-version: ${{ matrix.java }}
|
||||
distribution: 'temurin'
|
||||
cache: ${{ runner.environment == 'github-hosted' && 'maven' || '' }}
|
||||
cache: 'maven'
|
||||
|
||||
- name: Build and Install SeaweedFS Client
|
||||
working-directory: other/java/client
|
||||
|
||||
@@ -24,8 +24,8 @@ on:
|
||||
workflow_dispatch: # Allow manual trigger
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.event.pull_request.number || (github.event_name == 'push' && github.ref) || github.run_id }}
|
||||
cancel-in-progress: ${{ github.event_name == 'pull_request' || (github.event_name == 'push' && !startsWith(github.ref, 'refs/tags/')) }}
|
||||
group: ${{ github.head_ref }}/kafka-quicktest
|
||||
cancel-in-progress: true
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
@@ -43,7 +43,7 @@ jobs:
|
||||
uses: actions/setup-go@v7
|
||||
with:
|
||||
go-version: ^1.26
|
||||
cache: ${{ runner.environment == 'github-hosted' }}
|
||||
cache: true
|
||||
cache-dependency-path: |
|
||||
**/go.sum
|
||||
id: go
|
||||
|
||||
@@ -23,8 +23,8 @@ on:
|
||||
- '.github/workflows/kafka-tests.yml'
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.event.pull_request.number || (github.event_name == 'push' && github.ref) || github.run_id }}
|
||||
cancel-in-progress: ${{ github.event_name == 'pull_request' || (github.event_name == 'push' && !startsWith(github.ref, 'refs/tags/')) }}
|
||||
group: ${{ github.head_ref }}/kafka-tests
|
||||
cancel-in-progress: true
|
||||
|
||||
# Force different runners for better isolation
|
||||
env:
|
||||
@@ -53,7 +53,6 @@ jobs:
|
||||
- name: Set up Go 1.x
|
||||
uses: actions/setup-go@v7
|
||||
with:
|
||||
cache: ${{ runner.environment == 'github-hosted' }}
|
||||
go-version: ^1.26
|
||||
id: go
|
||||
|
||||
@@ -99,7 +98,6 @@ jobs:
|
||||
- name: Set up Go 1.x
|
||||
uses: actions/setup-go@v7
|
||||
with:
|
||||
cache: ${{ runner.environment == 'github-hosted' }}
|
||||
go-version: ^1.26
|
||||
id: go
|
||||
|
||||
@@ -151,7 +149,7 @@ jobs:
|
||||
uses: actions/setup-go@v7
|
||||
with:
|
||||
go-version: ^1.26
|
||||
cache: ${{ runner.environment == 'github-hosted' }}
|
||||
cache: true
|
||||
cache-dependency-path: |
|
||||
**/go.sum
|
||||
id: go
|
||||
@@ -330,7 +328,7 @@ jobs:
|
||||
uses: actions/setup-go@v7
|
||||
with:
|
||||
go-version: ^1.26
|
||||
cache: ${{ runner.environment == 'github-hosted' }}
|
||||
cache: true
|
||||
cache-dependency-path: |
|
||||
**/go.sum
|
||||
id: go
|
||||
@@ -492,7 +490,7 @@ jobs:
|
||||
uses: actions/setup-go@v7
|
||||
with:
|
||||
go-version: ^1.26
|
||||
cache: ${{ runner.environment == 'github-hosted' }}
|
||||
cache: true
|
||||
cache-dependency-path: |
|
||||
**/go.sum
|
||||
id: go
|
||||
@@ -650,7 +648,7 @@ jobs:
|
||||
uses: actions/setup-go@v7
|
||||
with:
|
||||
go-version: ^1.26
|
||||
cache: ${{ runner.environment == 'github-hosted' }}
|
||||
cache: true
|
||||
cache-dependency-path: |
|
||||
**/go.sum
|
||||
id: go
|
||||
@@ -807,7 +805,6 @@ jobs:
|
||||
- name: Set up Go 1.x
|
||||
uses: actions/setup-go@v7
|
||||
with:
|
||||
cache: ${{ runner.environment == 'github-hosted' }}
|
||||
go-version: ^1.26
|
||||
id: go
|
||||
|
||||
|
||||
@@ -19,8 +19,8 @@ on:
|
||||
- 'test/kms/**'
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.event.pull_request.number || (github.event_name == 'push' && github.ref) || github.run_id }}
|
||||
cancel-in-progress: ${{ github.event_name == 'pull_request' || (github.event_name == 'push' && !startsWith(github.ref, 'refs/tags/')) }}
|
||||
group: ${{ github.head_ref || github.ref }}-kms-tests
|
||||
cancel-in-progress: true
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
@@ -43,7 +43,6 @@ jobs:
|
||||
uses: actions/setup-go@v7
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
cache: ${{ runner.environment == 'github-hosted' }}
|
||||
id: go
|
||||
|
||||
- name: Install SeaweedFS
|
||||
@@ -89,7 +88,6 @@ jobs:
|
||||
uses: actions/setup-go@v7
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
cache: ${{ runner.environment == 'github-hosted' }}
|
||||
id: go
|
||||
|
||||
- name: Install SeaweedFS
|
||||
|
||||
@@ -21,8 +21,8 @@ on:
|
||||
- '.github/workflows/master-cold-start-tests.yml'
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.event.pull_request.number || (github.event_name == 'push' && github.ref) || github.run_id }}
|
||||
cancel-in-progress: ${{ github.event_name == 'pull_request' || (github.event_name == 'push' && !startsWith(github.ref, 'refs/tags/')) }}
|
||||
group: ${{ github.head_ref || github.ref }}/master-cold-start-tests
|
||||
cancel-in-progress: true
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
@@ -42,7 +42,6 @@ jobs:
|
||||
uses: actions/setup-go@v7
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
cache: ${{ runner.environment == 'github-hosted' }}
|
||||
|
||||
- name: Build weed binary
|
||||
run: |
|
||||
|
||||
@@ -25,8 +25,8 @@ on:
|
||||
- '.github/workflows/metadata-subscribe-tests.yml'
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.event.pull_request.number || (github.event_name == 'push' && github.ref) || github.run_id }}
|
||||
cancel-in-progress: ${{ github.event_name == 'pull_request' || (github.event_name == 'push' && !startsWith(github.ref, 'refs/tags/')) }}
|
||||
group: ${{ github.head_ref }}/metadata-subscribe-tests
|
||||
cancel-in-progress: true
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
@@ -48,7 +48,6 @@ jobs:
|
||||
uses: actions/setup-go@v7
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
cache: ${{ runner.environment == 'github-hosted' }}
|
||||
|
||||
- name: Build SeaweedFS
|
||||
run: |
|
||||
|
||||
@@ -10,8 +10,8 @@ on:
|
||||
branches: [ 'winfsp-bench**' ]
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.event.pull_request.number || (github.event_name == 'push' && github.ref) || github.run_id }}
|
||||
cancel-in-progress: ${{ github.event_name == 'pull_request' || (github.event_name == 'push' && !startsWith(github.ref, 'refs/tags/')) }}
|
||||
group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }}
|
||||
cancel-in-progress: true
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
@@ -32,7 +32,6 @@ jobs:
|
||||
- uses: actions/setup-go@v7
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
cache: ${{ runner.environment == 'github-hosted' }}
|
||||
|
||||
- name: Install WinFsp and rclone
|
||||
run: choco install winfsp rclone -y --no-progress
|
||||
@@ -151,7 +150,6 @@ jobs:
|
||||
- uses: actions/setup-go@v7
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
cache: ${{ runner.environment == 'github-hosted' }}
|
||||
|
||||
- name: Repair the fusermount3 setuid bit
|
||||
uses: ./.github/actions/fix-fusermount-setuid
|
||||
|
||||
@@ -24,8 +24,8 @@ on:
|
||||
- '.github/workflows/mount-windows-conformance.yml'
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.event.pull_request.number || (github.event_name == 'push' && github.ref) || github.run_id }}
|
||||
cancel-in-progress: ${{ github.event_name == 'pull_request' || (github.event_name == 'push' && !startsWith(github.ref, 'refs/tags/')) }}
|
||||
group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }}
|
||||
cancel-in-progress: true
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
@@ -49,7 +49,6 @@ jobs:
|
||||
- uses: actions/setup-go@v7
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
cache: ${{ runner.environment == 'github-hosted' }}
|
||||
|
||||
# cgofuse loads winfsp-x64.dll at run time, so WinFsp is needed here but
|
||||
# not to build.
|
||||
|
||||
@@ -24,8 +24,8 @@ on:
|
||||
- '.github/workflows/mount-windows.yml'
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.event.pull_request.number || (github.event_name == 'push' && github.ref) || github.run_id }}
|
||||
cancel-in-progress: ${{ github.event_name == 'pull_request' || (github.event_name == 'push' && !startsWith(github.ref, 'refs/tags/')) }}
|
||||
group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }}
|
||||
cancel-in-progress: true
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
@@ -49,7 +49,6 @@ jobs:
|
||||
- uses: actions/setup-go@v7
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
cache: ${{ runner.environment == 'github-hosted' }}
|
||||
|
||||
# cgofuse loads winfsp-x64.dll at run time, so WinFsp is needed here but
|
||||
# not to build.
|
||||
|
||||
@@ -21,8 +21,8 @@ on:
|
||||
- '.github/workflows/multi-master-tests.yml'
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.event.pull_request.number || (github.event_name == 'push' && github.ref) || github.run_id }}
|
||||
cancel-in-progress: ${{ github.event_name == 'pull_request' || (github.event_name == 'push' && !startsWith(github.ref, 'refs/tags/')) }}
|
||||
group: ${{ github.head_ref || github.ref }}/multi-master-tests
|
||||
cancel-in-progress: true
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
@@ -40,7 +40,6 @@ jobs:
|
||||
uses: actions/setup-go@v7
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
cache: ${{ runner.environment == 'github-hosted' }}
|
||||
|
||||
- name: Install SeaweedFS
|
||||
run: |
|
||||
|
||||
@@ -45,8 +45,8 @@ on:
|
||||
type: string
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.event.pull_request.number || (github.event_name == 'push' && github.ref) || github.run_id }}
|
||||
cancel-in-progress: ${{ github.event_name == 'pull_request' || (github.event_name == 'push' && !startsWith(github.ref, 'refs/tags/')) }}
|
||||
group: ${{ github.head_ref || github.ref }}/performance
|
||||
cancel-in-progress: true
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
@@ -67,7 +67,6 @@ jobs:
|
||||
uses: actions/setup-go@v7
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
cache: ${{ runner.environment == 'github-hosted' }}
|
||||
|
||||
- name: Build weed
|
||||
run: go build -o weed_bin ./weed
|
||||
@@ -176,14 +175,13 @@ jobs:
|
||||
uses: actions/setup-go@v7
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
cache: ${{ runner.environment == 'github-hosted' }}
|
||||
|
||||
- name: Install Rust toolchain
|
||||
if: matrix.impl == 'rust'
|
||||
uses: dtolnay/rust-toolchain@stable
|
||||
|
||||
- name: Cache cargo registry and target
|
||||
if: matrix.impl == 'rust' && runner.environment == 'github-hosted'
|
||||
if: matrix.impl == 'rust'
|
||||
uses: actions/cache@v6
|
||||
with:
|
||||
path: |
|
||||
@@ -329,14 +327,13 @@ jobs:
|
||||
uses: actions/setup-go@v7
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
cache: ${{ runner.environment == 'github-hosted' }}
|
||||
|
||||
- name: Install Rust toolchain
|
||||
if: matrix.impl == 'rust'
|
||||
uses: dtolnay/rust-toolchain@stable
|
||||
|
||||
- name: Cache cargo registry and target
|
||||
if: matrix.impl == 'rust' && runner.environment == 'github-hosted'
|
||||
if: matrix.impl == 'rust'
|
||||
uses: actions/cache@v6
|
||||
with:
|
||||
path: |
|
||||
|
||||
@@ -38,23 +38,15 @@ jobs:
|
||||
uses: actions/setup-go@v7
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
cache: ${{ runner.environment == 'github-hosted' }}
|
||||
|
||||
- name: Start local Docker registry
|
||||
run: |
|
||||
if ! docker start registry 2>/dev/null; then
|
||||
docker rm -f registry >/dev/null 2>&1 || true
|
||||
docker run -d --restart=always -p 5000:5000 --name registry registry:2
|
||||
fi
|
||||
run: docker run -d --restart=always -p 5000:5000 --name registry registry:2
|
||||
|
||||
- name: Set up Docker Buildx
|
||||
uses: docker/setup-buildx-action@v4
|
||||
with:
|
||||
driver-opts: network=host
|
||||
|
||||
- name: Set up Docker Compose
|
||||
uses: docker/setup-compose-action@v2
|
||||
|
||||
- name: Build weed race binary
|
||||
run: |
|
||||
cd docker
|
||||
|
||||
@@ -18,10 +18,6 @@ on:
|
||||
- 'go.sum'
|
||||
- '.github/workflows/plugin-workers.yml'
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.event.pull_request.number || (github.event_name == 'push' && github.ref) || github.run_id }}
|
||||
cancel-in-progress: ${{ github.event_name == 'pull_request' || (github.event_name == 'push' && !startsWith(github.ref, 'refs/tags/')) }}
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
@@ -45,7 +41,6 @@ jobs:
|
||||
- name: Set up Go 1.x
|
||||
uses: actions/setup-go@v7
|
||||
with:
|
||||
cache: ${{ runner.environment == 'github-hosted' }}
|
||||
go-version: ^1.26
|
||||
id: go
|
||||
|
||||
|
||||
@@ -1,791 +0,0 @@
|
||||
name: POSIX Compliance Tests
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [ main, master, develop ]
|
||||
paths:
|
||||
- 'weed/mount/**'
|
||||
- 'weed/command/mount*'
|
||||
- 'weed/command/fuse*'
|
||||
- 'test/fuse_integration/**'
|
||||
pull_request:
|
||||
branches: [ main, master, develop ]
|
||||
paths:
|
||||
- 'weed/mount/**'
|
||||
- 'weed/command/mount*'
|
||||
- 'weed/command/fuse*'
|
||||
- 'test/fuse_integration/**'
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
test_type:
|
||||
description: 'Type of POSIX tests to run (default: full for comprehensive testing)'
|
||||
required: true
|
||||
default: 'full'
|
||||
type: choice
|
||||
options:
|
||||
- critical
|
||||
- basic
|
||||
- extended
|
||||
- full
|
||||
enable_external_tests:
|
||||
description: 'Run external test suites (slower)'
|
||||
required: false
|
||||
default: false
|
||||
type: boolean
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
pull-requests: write
|
||||
issues: write
|
||||
|
||||
env:
|
||||
GO_VERSION: '1.26'
|
||||
TIMEOUT: '45m'
|
||||
|
||||
jobs:
|
||||
posix-compliance-ubuntu:
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 60
|
||||
|
||||
strategy:
|
||||
matrix:
|
||||
fuse-version: ['2.9', '3.0']
|
||||
fail-fast: false
|
||||
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v7
|
||||
with:
|
||||
fetch-depth: 1
|
||||
|
||||
- name: Clean up stale FUSE mounts
|
||||
uses: ./.github/actions/cleanup-stale-fuse-mounts
|
||||
|
||||
- name: Set up Go ${{ env.GO_VERSION }}
|
||||
uses: actions/setup-go@v7
|
||||
with:
|
||||
go-version: ${{ env.GO_VERSION }}
|
||||
cache: ${{ runner.environment == 'github-hosted' }}
|
||||
cache-dependency-path: |
|
||||
go.sum
|
||||
test/fuse_integration/go.sum
|
||||
|
||||
- name: Install system dependencies
|
||||
run: |
|
||||
sudo apt-get update
|
||||
sudo apt-get install -y \
|
||||
attr \
|
||||
acl \
|
||||
build-essential \
|
||||
git \
|
||||
python3-pip
|
||||
|
||||
# Install FUSE version specific packages
|
||||
if [ "${{ matrix.fuse-version }}" = "2.9" ]; then
|
||||
echo "Installing FUSE 2.9..."
|
||||
sudo apt-get install -y fuse libfuse-dev
|
||||
elif [ "${{ matrix.fuse-version }}" = "3.0" ]; then
|
||||
echo "Installing FUSE 3.0..."
|
||||
sudo apt-get install -y fuse3 libfuse3-dev
|
||||
# Also install fuse2 for compatibility if needed
|
||||
sudo apt-get install -y fuse libfuse-dev
|
||||
fi
|
||||
|
||||
- name: Set up user permissions for FUSE
|
||||
run: |
|
||||
# Create fuse group if it doesn't exist
|
||||
sudo groupadd -f fuse
|
||||
|
||||
# Add user to fuse group
|
||||
sudo usermod -a -G fuse $USER
|
||||
|
||||
# Set permissions on FUSE device
|
||||
sudo chmod 666 /dev/fuse
|
||||
|
||||
# Allow non-root mounts to pass -o allow_other (weed -allowOthers)
|
||||
echo 'user_allow_other' | sudo tee -a /etc/fuse.conf
|
||||
sudo chmod 644 /etc/fuse.conf
|
||||
|
||||
# Ensure fuse module is loaded
|
||||
sudo modprobe fuse || true
|
||||
|
||||
# Verify setup
|
||||
echo "FUSE setup verification:"
|
||||
ls -la /dev/fuse
|
||||
groups $USER | grep fuse || echo "User not in fuse group yet (will be effective after login)"
|
||||
lsmod | grep fuse || echo "FUSE module not loaded"
|
||||
|
||||
- name: Install external test tools
|
||||
if: ${{ github.event.inputs.enable_external_tests == 'true' }}
|
||||
run: |
|
||||
# Install nfstest for POSIX API verification
|
||||
pip3 install --user nfstest
|
||||
|
||||
# Install FIO for performance testing
|
||||
sudo apt-get install -y fio
|
||||
|
||||
# Install additional test utilities
|
||||
sudo apt-get install -y stress-ng
|
||||
|
||||
- name: Build SeaweedFS
|
||||
run: |
|
||||
make
|
||||
# Verify binary exists and is executable (it's installed to GOPATH/bin)
|
||||
weed version
|
||||
|
||||
# Make weed binary available in PATH (it's already there from go install)
|
||||
sudo cp $(which weed) /usr/local/bin/weed
|
||||
which weed
|
||||
weed version
|
||||
|
||||
- name: Set up SeaweedFS cluster
|
||||
run: |
|
||||
start_and_wait() {
|
||||
local name=$1
|
||||
local url=$2
|
||||
local pidfile=$3
|
||||
shift 3
|
||||
local cmd="$@"
|
||||
|
||||
echo "Starting $name..."
|
||||
$cmd > "/tmp/seaweedfs/$name.log" 2>&1 &
|
||||
local pid=$!
|
||||
echo $pid > "$pidfile"
|
||||
|
||||
echo "Waiting for $name to start..."
|
||||
for i in {1..30}; do
|
||||
if curl -sf "$url" > /dev/null 2>&1; then
|
||||
echo "$name is ready"
|
||||
return 0
|
||||
fi
|
||||
if [ $i -eq 30 ]; then
|
||||
echo "$name failed to start"
|
||||
cat "/tmp/seaweedfs/$name.log"
|
||||
exit 1
|
||||
fi
|
||||
sleep 2
|
||||
done
|
||||
}
|
||||
|
||||
mkdir -p /tmp/seaweedfs/{master,volume,filer,mount}
|
||||
|
||||
start_and_wait "master" "http://127.0.0.1:9333/cluster/status" "/tmp/seaweedfs/master.pid" \
|
||||
"weed master -ip=127.0.0.1 -port=9333 -mdir=/tmp/seaweedfs/master -raftBootstrap=true"
|
||||
|
||||
start_and_wait "volume" "http://127.0.0.1:8080/status" "/tmp/seaweedfs/volume.pid" \
|
||||
"weed volume -mserver=127.0.0.1:9333 -ip=127.0.0.1 -port=8080 -dir=/tmp/seaweedfs/volume -max=100"
|
||||
|
||||
start_and_wait "filer" "http://127.0.0.1:8888/" "/tmp/seaweedfs/filer.pid" \
|
||||
"weed filer -master=127.0.0.1:9333 -ip=127.0.0.1 -port=8888"
|
||||
|
||||
echo "SeaweedFS cluster status:"
|
||||
curl -s http://127.0.0.1:9333/cluster/status || true
|
||||
|
||||
- name: Set up FUSE mount
|
||||
run: |
|
||||
# Create mount point
|
||||
MOUNT_POINT="/tmp/seaweedfs/mount"
|
||||
mkdir -p $MOUNT_POINT
|
||||
|
||||
echo "Mounting SeaweedFS FUSE filesystem..."
|
||||
# Mount SeaweedFS FUSE filesystem in background
|
||||
weed mount \
|
||||
-filer=127.0.0.1:8888 \
|
||||
-dir=$MOUNT_POINT \
|
||||
-filer.path=/ \
|
||||
-dirAutoCreate=true \
|
||||
-allowOthers=true \
|
||||
-nonempty=true \
|
||||
> /tmp/seaweedfs/mount.log 2>&1 &
|
||||
MOUNT_PID=$!
|
||||
echo $MOUNT_PID > /tmp/seaweedfs/mount.pid
|
||||
|
||||
# Wait for mount to be ready. The mount point directory already
|
||||
# exists, so only a real mount counts — a directory listing would
|
||||
# pass on the local disk even if weed mount died.
|
||||
echo "Waiting for FUSE mount to be ready..."
|
||||
for i in {1..30}; do
|
||||
if mountpoint -q $MOUNT_POINT 2>/dev/null; then
|
||||
echo "FUSE mount is ready"
|
||||
break
|
||||
fi
|
||||
if ! kill -0 $MOUNT_PID 2>/dev/null; then
|
||||
echo "FUSE mount process exited"
|
||||
cat /tmp/seaweedfs/mount.log
|
||||
exit 1
|
||||
fi
|
||||
if [ $i -eq 30 ]; then
|
||||
echo "FUSE mount failed to be ready"
|
||||
cat /tmp/seaweedfs/mount.log
|
||||
echo "Mount status:"
|
||||
mount | grep fuse || echo "No FUSE mounts found"
|
||||
ps aux | grep weed || echo "No weed processes found"
|
||||
exit 1
|
||||
fi
|
||||
sleep 2
|
||||
done
|
||||
|
||||
# Verify mount is working
|
||||
echo "Testing FUSE mount functionality..."
|
||||
echo "test" > $MOUNT_POINT/test_file.txt
|
||||
if [ "$(cat $MOUNT_POINT/test_file.txt)" = "test" ]; then
|
||||
echo "FUSE mount is working correctly"
|
||||
rm $MOUNT_POINT/test_file.txt
|
||||
else
|
||||
echo "FUSE mount is not working properly"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Export mount point for tests
|
||||
echo "SEAWEEDFS_MOUNT_POINT=$MOUNT_POINT" >> $GITHUB_ENV
|
||||
|
||||
- name: Set up test environment
|
||||
run: |
|
||||
cd test/fuse_integration
|
||||
|
||||
# Initialize Go module for tests
|
||||
if [ ! -f go.mod ]; then
|
||||
go mod init seaweedfs-posix-tests
|
||||
go mod tidy
|
||||
fi
|
||||
|
||||
# Create reports directory
|
||||
mkdir -p reports
|
||||
|
||||
# Set up external tools if requested
|
||||
# Note: External tools setup is now optional and won't fail the build
|
||||
if [ "${{ github.event.inputs.enable_external_tests }}" = "true" ]; then
|
||||
echo "Setting up external POSIX testing tools..."
|
||||
make -f posix_Makefile setup-external-tools || true
|
||||
# setup-external-tools exits 0 even when a sub-target failed, so
|
||||
# require the tools only when they are actually installed
|
||||
if [ -d external_tools/pjdfstest ] && command -v nfstest_posix >/dev/null 2>&1; then
|
||||
export POSIX_REQUIRE_EXTERNAL_TOOLS="true"
|
||||
echo "POSIX_REQUIRE_EXTERNAL_TOOLS=true" >> $GITHUB_ENV
|
||||
echo "✅ External tools setup completed successfully"
|
||||
else
|
||||
echo "⚠️ External tools setup incomplete - continuing with basic tests only"
|
||||
echo "External tools will be skipped but tests will continue"
|
||||
echo "This is expected behavior when build tools are not available"
|
||||
fi
|
||||
fi
|
||||
|
||||
# Verify SeaweedFS cluster is accessible
|
||||
echo "Verifying SeaweedFS cluster accessibility..."
|
||||
curl -sf http://127.0.0.1:9333/cluster/status
|
||||
curl -sf http://127.0.0.1:8080/status
|
||||
curl -sf http://127.0.0.1:8888/
|
||||
|
||||
# Verify mount point
|
||||
echo "Mount point: $SEAWEEDFS_MOUNT_POINT"
|
||||
ls -la $SEAWEEDFS_MOUNT_POINT
|
||||
|
||||
# Verify mount is actually working by testing file operations
|
||||
echo "Verifying SeaweedFS mount functionality..."
|
||||
TEST_FILE="$SEAWEEDFS_MOUNT_POINT/.mount_verification_test"
|
||||
echo "SeaweedFS mount test" > "$TEST_FILE"
|
||||
if [ "$(cat "$TEST_FILE")" = "SeaweedFS mount test" ]; then
|
||||
echo "✅ Mount verification successful"
|
||||
rm -f "$TEST_FILE"
|
||||
else
|
||||
echo "❌ Mount verification failed - mount may not be working properly"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Check if this is actually a FUSE mount — the plain directory created
|
||||
# above passes the file test, so without this the suite can run
|
||||
# against the local disk and report a pass it never earned
|
||||
if mount | grep -q "$SEAWEEDFS_MOUNT_POINT"; then
|
||||
echo "✅ FUSE mount detected:"
|
||||
mount | grep "$SEAWEEDFS_MOUNT_POINT"
|
||||
else
|
||||
echo "❌ Mount point not found in mount table - refusing to test a local directory"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
- name: Run POSIX compliance tests
|
||||
id: posix-tests
|
||||
env:
|
||||
SEAWEEDFS_MOUNT_POINT: ${{ env.SEAWEEDFS_MOUNT_POINT }}
|
||||
POSIX_REQUIRE_EXTERNAL_TOOLS: ${{ env.POSIX_REQUIRE_EXTERNAL_TOOLS }}
|
||||
run: |
|
||||
cd test/fuse_integration
|
||||
|
||||
# Determine which tests to run
|
||||
TEST_TYPE="${{ github.event.inputs.test_type }}"
|
||||
if [ -z "$TEST_TYPE" ]; then
|
||||
TEST_TYPE="full"
|
||||
fi
|
||||
|
||||
echo "Running POSIX tests: $TEST_TYPE"
|
||||
if [ "$TEST_TYPE" = "full" ]; then
|
||||
echo "🚀 Running COMPREHENSIVE POSIX test suite (including external test suites)"
|
||||
else
|
||||
echo "Using mount point: $SEAWEEDFS_MOUNT_POINT"
|
||||
fi
|
||||
|
||||
# Set test configuration to use our mounted filesystem
|
||||
export TEST_MOUNT_POINT="$SEAWEEDFS_MOUNT_POINT"
|
||||
export TEST_SKIP_CLUSTER_SETUP="true"
|
||||
|
||||
# Debug: Show environment variables
|
||||
echo "🔍 Test Environment:"
|
||||
echo " TEST_MOUNT_POINT=$TEST_MOUNT_POINT"
|
||||
echo " TEST_SKIP_CLUSTER_SETUP=$TEST_SKIP_CLUSTER_SETUP"
|
||||
echo " SEAWEEDFS_MOUNT_POINT=$SEAWEEDFS_MOUNT_POINT"
|
||||
|
||||
case "$TEST_TYPE" in
|
||||
"critical")
|
||||
echo "Running critical POSIX tests (basic compliance only)..."
|
||||
make -f posix_Makefile test-posix-critical
|
||||
;;
|
||||
"basic")
|
||||
echo "Running basic POSIX tests (core file operations)..."
|
||||
make -f posix_Makefile test-posix-basic
|
||||
;;
|
||||
"extended")
|
||||
echo "Running extended POSIX tests (including advanced features)..."
|
||||
make -f posix_Makefile test-posix-extended
|
||||
;;
|
||||
"full")
|
||||
echo "🚀 Running COMPREHENSIVE POSIX test suite (all tests including external)..."
|
||||
echo " This includes: basic, extended, and external test suites"
|
||||
echo " External suites: pjdfstest, nfstest_posix, and custom stress tests"
|
||||
make -f posix_Makefile test-posix-full
|
||||
;;
|
||||
*)
|
||||
echo "Unknown test type: $TEST_TYPE, defaulting to full suite"
|
||||
echo "🚀 Running full POSIX test suite (all tests including external)..."
|
||||
make -f posix_Makefile test-posix-full
|
||||
;;
|
||||
esac
|
||||
|
||||
- name: Run external test suites
|
||||
if: ${{ github.event.inputs.enable_external_tests == 'true' }}
|
||||
continue-on-error: true
|
||||
run: |
|
||||
cd test/fuse_integration
|
||||
|
||||
# Point the external suites at the same mount as the Go tests;
|
||||
# without these the Makefile falls back to a local-disk path.
|
||||
export TEST_MOUNT_POINT="$SEAWEEDFS_MOUNT_POINT"
|
||||
export TEST_SKIP_CLUSTER_SETUP="true"
|
||||
|
||||
# Run external tests (may fail on some systems)
|
||||
make -f posix_Makefile test-nfstest-posix || echo "nfstest failed or not available"
|
||||
make -f posix_Makefile test-fio-posix || echo "FIO tests failed or not available"
|
||||
|
||||
- name: Generate compliance report
|
||||
if: always()
|
||||
run: |
|
||||
cd test/fuse_integration
|
||||
make -f posix_Makefile generate-report
|
||||
|
||||
- name: Collect system information for debugging
|
||||
if: failure()
|
||||
run: |
|
||||
cd test/fuse_integration
|
||||
mkdir -p reports/debug
|
||||
|
||||
echo "System Information" > reports/debug/system_info.txt
|
||||
uname -a >> reports/debug/system_info.txt
|
||||
echo "" >> reports/debug/system_info.txt
|
||||
|
||||
echo "Mount Information" >> reports/debug/system_info.txt
|
||||
mount >> reports/debug/system_info.txt
|
||||
echo "" >> reports/debug/system_info.txt
|
||||
|
||||
echo "Disk Space" >> reports/debug/system_info.txt
|
||||
timeout 15 df -h >> reports/debug/system_info.txt || true
|
||||
echo "" >> reports/debug/system_info.txt
|
||||
|
||||
echo "Memory Information" >> reports/debug/system_info.txt
|
||||
free -h >> reports/debug/system_info.txt
|
||||
echo "" >> reports/debug/system_info.txt
|
||||
|
||||
echo "FUSE Information" >> reports/debug/system_info.txt
|
||||
ls -la /dev/fuse >> reports/debug/system_info.txt
|
||||
lsmod | grep fuse >> reports/debug/system_info.txt
|
||||
echo "" >> reports/debug/system_info.txt
|
||||
|
||||
echo "SeaweedFS Version" >> reports/debug/system_info.txt
|
||||
weed version >> reports/debug/system_info.txt
|
||||
echo "" >> reports/debug/system_info.txt
|
||||
|
||||
echo "Go Version" >> reports/debug/system_info.txt
|
||||
go version >> reports/debug/system_info.txt
|
||||
echo "" >> reports/debug/system_info.txt
|
||||
|
||||
echo "Running Processes" >> reports/debug/system_info.txt
|
||||
ps aux | grep -E "(weed|fuse)" >> reports/debug/system_info.txt
|
||||
echo "" >> reports/debug/system_info.txt
|
||||
|
||||
# Collect SeaweedFS service logs
|
||||
echo "=== SeaweedFS Service Logs ===" >> reports/debug/system_info.txt
|
||||
if [ -f /tmp/seaweedfs/master.log ]; then
|
||||
echo "Master Log:" >> reports/debug/system_info.txt
|
||||
tail -50 /tmp/seaweedfs/master.log >> reports/debug/system_info.txt
|
||||
echo "" >> reports/debug/system_info.txt
|
||||
fi
|
||||
|
||||
if [ -f /tmp/seaweedfs/volume.log ]; then
|
||||
echo "Volume Log:" >> reports/debug/system_info.txt
|
||||
tail -50 /tmp/seaweedfs/volume.log >> reports/debug/system_info.txt
|
||||
echo "" >> reports/debug/system_info.txt
|
||||
fi
|
||||
|
||||
if [ -f /tmp/seaweedfs/filer.log ]; then
|
||||
echo "Filer Log:" >> reports/debug/system_info.txt
|
||||
tail -50 /tmp/seaweedfs/filer.log >> reports/debug/system_info.txt
|
||||
echo "" >> reports/debug/system_info.txt
|
||||
fi
|
||||
|
||||
if [ -f /tmp/seaweedfs/mount.log ]; then
|
||||
echo "Mount Log:" >> reports/debug/system_info.txt
|
||||
tail -50 /tmp/seaweedfs/mount.log >> reports/debug/system_info.txt
|
||||
echo "" >> reports/debug/system_info.txt
|
||||
fi
|
||||
|
||||
# Copy full logs as separate files
|
||||
if [ -f /tmp/seaweedfs/master.log ]; then
|
||||
cp /tmp/seaweedfs/master.log reports/debug/master.log
|
||||
fi
|
||||
if [ -f /tmp/seaweedfs/volume.log ]; then
|
||||
cp /tmp/seaweedfs/volume.log reports/debug/volume.log
|
||||
fi
|
||||
if [ -f /tmp/seaweedfs/filer.log ]; then
|
||||
cp /tmp/seaweedfs/filer.log reports/debug/filer.log
|
||||
fi
|
||||
if [ -f /tmp/seaweedfs/mount.log ]; then
|
||||
cp /tmp/seaweedfs/mount.log reports/debug/mount.log
|
||||
fi
|
||||
|
||||
- name: Cleanup SeaweedFS cluster and FUSE mount
|
||||
if: always()
|
||||
run: |
|
||||
stop_process() {
|
||||
local name=$1
|
||||
local pidfile=$2
|
||||
if [ -f "$pidfile" ]; then
|
||||
local pid=$(cat "$pidfile")
|
||||
if kill -0 $pid 2>/dev/null; then
|
||||
echo "Stopping $name process (PID: $pid)..."
|
||||
kill -TERM $pid || true
|
||||
sleep 2
|
||||
kill -KILL $pid 2>/dev/null || true
|
||||
fi
|
||||
fi
|
||||
}
|
||||
|
||||
echo "Cleaning up SeaweedFS cluster and FUSE mount..."
|
||||
|
||||
MOUNT_POINT="/tmp/seaweedfs/mount"
|
||||
if mountpoint -q $MOUNT_POINT 2>/dev/null; then
|
||||
echo "Unmounting FUSE filesystem..."
|
||||
fusermount -u $MOUNT_POINT || umount $MOUNT_POINT || true
|
||||
fi
|
||||
|
||||
stop_process "mount" "/tmp/seaweedfs/mount.pid"
|
||||
stop_process "filer" "/tmp/seaweedfs/filer.pid"
|
||||
stop_process "volume" "/tmp/seaweedfs/volume.pid"
|
||||
stop_process "master" "/tmp/seaweedfs/master.pid"
|
||||
|
||||
pkill -f "weed " || true
|
||||
|
||||
fusermount -u $MOUNT_POINT 2>/dev/null || true
|
||||
umount $MOUNT_POINT 2>/dev/null || true
|
||||
|
||||
rm -rf /tmp/seaweedfs || true
|
||||
|
||||
echo "Cleanup completed"
|
||||
|
||||
|
||||
- name: Upload test results
|
||||
uses: actions/upload-artifact@v7
|
||||
if: always()
|
||||
with:
|
||||
name: posix-test-results-ubuntu-fuse${{ matrix.fuse-version }}
|
||||
path: |
|
||||
test/fuse_integration/reports/
|
||||
test/fuse_integration/*.log
|
||||
retention-days: 30
|
||||
|
||||
- name: Upload test coverage
|
||||
uses: actions/upload-artifact@v7
|
||||
if: always() && (github.event.inputs.test_type == 'full' || github.event.inputs.test_type == 'extended')
|
||||
with:
|
||||
name: posix-coverage-ubuntu
|
||||
path: test/fuse_integration/reports/posix_coverage.html
|
||||
retention-days: 14
|
||||
|
||||
- name: Comment PR with results
|
||||
if: github.event_name == 'pull_request' && always()
|
||||
uses: actions/github-script@v7
|
||||
with:
|
||||
script: |
|
||||
const fs = require('fs');
|
||||
const path = require('path');
|
||||
|
||||
const reportPath = 'test/fuse_integration/reports/posix_compliance_summary.txt';
|
||||
|
||||
if (fs.existsSync(reportPath)) {
|
||||
const report = fs.readFileSync(reportPath, 'utf8');
|
||||
const comment = `## POSIX Compliance Test Results
|
||||
|
||||
**Go Version:** ${{ env.GO_VERSION }}
|
||||
**FUSE Version:** ${{ matrix.fuse-version }}
|
||||
**Test Type:** ${{ github.event.inputs.test_type || 'full' }}
|
||||
|
||||
<details>
|
||||
<summary>Test Summary</summary>
|
||||
|
||||
\`\`\`
|
||||
${report}
|
||||
\`\`\`
|
||||
</details>
|
||||
|
||||
📊 Full results available in [test artifacts](https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }})
|
||||
`;
|
||||
|
||||
try {
|
||||
await github.rest.issues.createComment({
|
||||
issue_number: context.issue.number,
|
||||
owner: context.repo.owner,
|
||||
repo: context.repo.repo,
|
||||
body: comment
|
||||
});
|
||||
} catch (err) {
|
||||
console.log('Could not post PR comment: ' + err.message);
|
||||
}
|
||||
}
|
||||
|
||||
posix-compliance-macos:
|
||||
runs-on: macos-latest
|
||||
timeout-minutes: 60
|
||||
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v7
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v7
|
||||
with:
|
||||
go-version: ${{ env.GO_VERSION }}
|
||||
cache: ${{ runner.environment == 'github-hosted' }}
|
||||
|
||||
- name: Install macFUSE
|
||||
run: |
|
||||
# Install macFUSE
|
||||
brew install macfuse
|
||||
|
||||
# Note: macFUSE may require system extension approval on macOS
|
||||
# This step may need manual intervention in some cases
|
||||
|
||||
- name: Build SeaweedFS
|
||||
run: |
|
||||
make
|
||||
# Verify binary exists and is executable (it's installed to GOPATH/bin)
|
||||
weed version
|
||||
|
||||
- name: Run critical POSIX tests (macOS)
|
||||
continue-on-error: true # macOS FUSE can be more restrictive
|
||||
run: |
|
||||
cd test/fuse_integration
|
||||
|
||||
if [ ! -f go.mod ]; then
|
||||
go mod init seaweedfs-posix-tests
|
||||
go mod tidy
|
||||
fi
|
||||
|
||||
mkdir -p reports
|
||||
|
||||
# Run basic tests only on macOS due to FUSE limitations
|
||||
make -f posix_Makefile test-posix-critical
|
||||
|
||||
- name: Upload macOS test results
|
||||
uses: actions/upload-artifact@v7
|
||||
if: always()
|
||||
with:
|
||||
name: posix-test-results-macos
|
||||
path: |
|
||||
test/fuse_integration/reports/
|
||||
test/fuse_integration/*.log
|
||||
retention-days: 30
|
||||
|
||||
security-analysis:
|
||||
runs-on: ubuntu-latest
|
||||
if: github.event_name == 'pull_request'
|
||||
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v7
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v7
|
||||
with:
|
||||
go-version: ${{ env.GO_VERSION }}
|
||||
cache: ${{ runner.environment == 'github-hosted' }}
|
||||
|
||||
- name: Install security analysis tools
|
||||
run: |
|
||||
echo "Installing staticcheck for security analysis..."
|
||||
go install honnef.co/go/tools/cmd/staticcheck@latest
|
||||
|
||||
- name: Run security analysis on FUSE code
|
||||
run: |
|
||||
# Analyze mount and FUSE-related code for security and quality issues
|
||||
echo "Running staticcheck security analysis..."
|
||||
|
||||
# Run staticcheck on FUSE-related code
|
||||
echo "Analyzing FUSE mount code..."
|
||||
staticcheck ./weed/mount/... > staticcheck-report.txt 2>&1 || true
|
||||
staticcheck ./weed/command/mount* ./weed/command/fuse* >> staticcheck-report.txt 2>&1 || true
|
||||
|
||||
# Convert to JSON format for consistency
|
||||
echo "{" > gosec-report.json
|
||||
echo " \"tool\": \"staticcheck\"," >> gosec-report.json
|
||||
echo " \"timestamp\": \"$(date -Iseconds)\"," >> gosec-report.json
|
||||
echo " \"issues\": [" >> gosec-report.json
|
||||
|
||||
if [ -s staticcheck-report.txt ]; then
|
||||
echo " \"$(cat staticcheck-report.txt | head -20 | sed 's/"/\\"/g' | tr '\n' ' ')\"" >> gosec-report.json
|
||||
fi
|
||||
|
||||
echo " ]," >> gosec-report.json
|
||||
echo " \"stats\": {" >> gosec-report.json
|
||||
echo " \"files_analyzed\": $(find ./weed/mount ./weed/command -name '*.go' | wc -l)," >> gosec-report.json
|
||||
echo " \"issues_found\": $(wc -l < staticcheck-report.txt 2>/dev/null || echo 0)" >> gosec-report.json
|
||||
echo " }" >> gosec-report.json
|
||||
echo "}" >> gosec-report.json
|
||||
|
||||
echo "Security analysis completed"
|
||||
echo "Issues found: $(wc -l < staticcheck-report.txt 2>/dev/null || echo 0)"
|
||||
|
||||
- name: Upload security analysis results
|
||||
uses: actions/upload-artifact@v7
|
||||
if: always()
|
||||
with:
|
||||
name: security-analysis-results
|
||||
path: gosec-report.json
|
||||
retention-days: 30
|
||||
|
||||
performance-baseline:
|
||||
runs-on: ubuntu-latest
|
||||
if: github.event.inputs.test_type == 'full' || github.event_name == 'schedule'
|
||||
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v7
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v7
|
||||
with:
|
||||
go-version: ${{ env.GO_VERSION }}
|
||||
cache: ${{ runner.environment == 'github-hosted' }}
|
||||
|
||||
- name: Install dependencies
|
||||
run: |
|
||||
sudo apt-get update
|
||||
sudo apt-get install -y fuse libfuse-dev fio
|
||||
sudo usermod -a -G fuse $USER
|
||||
sudo chmod 666 /dev/fuse
|
||||
# The posix suites mount with -allowOthers; allow_other needs this
|
||||
echo 'user_allow_other' | sudo tee -a /etc/fuse.conf
|
||||
|
||||
- name: Build SeaweedFS
|
||||
run: make
|
||||
|
||||
- name: Run performance baseline tests
|
||||
run: |
|
||||
cd test/fuse_integration
|
||||
|
||||
if [ ! -f go.mod ]; then
|
||||
go mod init seaweedfs-posix-tests
|
||||
go mod tidy
|
||||
fi
|
||||
|
||||
mkdir -p reports
|
||||
|
||||
# Run performance benchmarks
|
||||
make -f posix_Makefile benchmark-posix
|
||||
# This job has no FUSE mount, so the fio suite can only skip; keep the
|
||||
# run green but honest about producing no mounted-FS results
|
||||
make -f posix_Makefile test-fio-posix || echo "FIO skipped - no mounted filesystem in this job"
|
||||
|
||||
- name: Store performance baseline
|
||||
uses: actions/upload-artifact@v7
|
||||
with:
|
||||
name: performance-baseline-results
|
||||
path: |
|
||||
test/fuse_integration/reports/posix_benchmark_results.log
|
||||
test/fuse_integration/reports/fio_*_results.log
|
||||
retention-days: 90
|
||||
|
||||
# Schedule regular compliance checks
|
||||
scheduled-compliance-check:
|
||||
runs-on: ubuntu-latest
|
||||
if: github.event_name == 'schedule'
|
||||
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v7
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v7
|
||||
with:
|
||||
go-version: ${{ env.GO_VERSION }}
|
||||
cache: ${{ runner.environment == 'github-hosted' }}
|
||||
|
||||
- name: Install dependencies
|
||||
run: |
|
||||
sudo apt-get update
|
||||
sudo apt-get install -y fuse libfuse-dev
|
||||
pip3 install --user nfstest
|
||||
# The posix suites mount with -allowOthers; allow_other needs this
|
||||
echo 'user_allow_other' | sudo tee -a /etc/fuse.conf
|
||||
|
||||
- name: Build SeaweedFS
|
||||
run: make
|
||||
|
||||
- name: Run comprehensive compliance check
|
||||
run: |
|
||||
cd test/fuse_integration
|
||||
make -f posix_Makefile test-posix-full
|
||||
make -f posix_Makefile generate-report
|
||||
|
||||
- name: Create compliance issue if tests fail
|
||||
if: failure()
|
||||
uses: actions/github-script@v7
|
||||
with:
|
||||
script: |
|
||||
const issue = await github.rest.issues.create({
|
||||
owner: context.repo.owner,
|
||||
repo: context.repo.repo,
|
||||
title: 'POSIX Compliance Check Failed - ' + new Date().toISOString().split('T')[0],
|
||||
body: `## POSIX Compliance Check Failure
|
||||
|
||||
The scheduled POSIX compliance check has failed. This may indicate:
|
||||
|
||||
- Regression in FUSE mount functionality
|
||||
- Changes affecting POSIX compatibility
|
||||
- Infrastructure issues with the test environment
|
||||
|
||||
**Action Required:** Review the test results and investigate any failures.
|
||||
|
||||
**Test Run:** [Workflow Run](https://github.com/${context.repo.owner}/${context.repo.repo}/actions/runs/${context.runId})
|
||||
|
||||
**Date:** ${new Date().toISOString()}
|
||||
|
||||
---
|
||||
This issue was automatically created by the POSIX compliance workflow.
|
||||
`,
|
||||
labels: ['bug', 'posix-compliance', 'automated-issue']
|
||||
});
|
||||
|
||||
console.log('Created issue:', issue.data.number);
|
||||
|
||||
# Schedule this workflow to run weekly
|
||||
# Uncomment the following lines to enable scheduled runs:
|
||||
# schedule:
|
||||
# - cron: '0 2 * * 1' # Every Monday at 2 AM UTC
|
||||
|
||||
@@ -23,8 +23,8 @@ on:
|
||||
- '.github/workflows/postgres-tests.yml'
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.event.pull_request.number || (github.event_name == 'push' && github.ref) || github.run_id }}
|
||||
cancel-in-progress: ${{ github.event_name == 'pull_request' || (github.event_name == 'push' && !startsWith(github.ref, 'refs/tags/')) }}
|
||||
group: ${{ github.head_ref }}/postgres-tests
|
||||
cancel-in-progress: true
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
@@ -41,7 +41,6 @@ jobs:
|
||||
- name: Set up Go 1.x
|
||||
uses: actions/setup-go@v7
|
||||
with:
|
||||
cache: ${{ runner.environment == 'github-hosted' }}
|
||||
go-version: ^1.26
|
||||
id: go
|
||||
|
||||
@@ -57,7 +56,6 @@ jobs:
|
||||
uses: docker/setup-buildx-action@v4
|
||||
|
||||
- name: Cache Docker layers
|
||||
if: runner.environment == 'github-hosted'
|
||||
uses: actions/cache@v6
|
||||
with:
|
||||
path: /tmp/.buildx-cache
|
||||
|
||||
@@ -21,8 +21,8 @@ on:
|
||||
- '.github/workflows/rust-volume-server-tests.yml'
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.event.pull_request.number || (github.event_name == 'push' && github.ref) || github.run_id }}
|
||||
cancel-in-progress: ${{ github.event_name == 'pull_request' || (github.event_name == 'push' && !startsWith(github.ref, 'refs/tags/')) }}
|
||||
group: ${{ github.workflow }}-${{ github.head_ref || github.ref }}
|
||||
cancel-in-progress: true
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
@@ -52,65 +52,6 @@ jobs:
|
||||
- 'seaweed-volume/**'
|
||||
- '.github/workflows/rust-volume-server-tests.yml'
|
||||
|
||||
build-test-binaries:
|
||||
name: Build Test Binaries
|
||||
runs-on: ubuntu-22.04
|
||||
timeout-minutes: 30
|
||||
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v7
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v7
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
cache: ${{ runner.environment == 'github-hosted' }}
|
||||
|
||||
- name: Install Rust toolchain
|
||||
uses: dtolnay/rust-toolchain@stable
|
||||
|
||||
# cargo tracks its own inputs but not the runner's C toolchain, so a cached
|
||||
# target/ can carry C objects built against a different glibc than we link against.
|
||||
- name: Fingerprint build toolchain
|
||||
id: toolchain
|
||||
run: echo "fingerprint=$(getconf GNU_LIBC_VERSION | tr ' ' '-')-rustc-$(rustc -V | awk '{print $2}')" >> "$GITHUB_OUTPUT"
|
||||
|
||||
- name: Cache cargo registry and target
|
||||
if: runner.environment == 'github-hosted'
|
||||
uses: actions/cache@v6
|
||||
with:
|
||||
path: |
|
||||
~/.cargo/registry
|
||||
~/.cargo/git
|
||||
seaweed-volume/target
|
||||
key: rust-${{ steps.toolchain.outputs.fingerprint }}-${{ hashFiles('seaweed-volume/Cargo.lock', 'seaweed-common/Cargo.lock') }}
|
||||
restore-keys: |
|
||||
rust-${{ steps.toolchain.outputs.fingerprint }}-
|
||||
|
||||
- name: Build Go weed binary
|
||||
run: |
|
||||
cd weed
|
||||
go build -tags 5BytesOffset -o weed .
|
||||
chmod +x weed
|
||||
./weed version
|
||||
|
||||
- name: Build Rust volume binary
|
||||
run: cd seaweed-volume && cargo build --release
|
||||
|
||||
- name: Package binaries
|
||||
run: tar cf rust-test-binaries.tar weed/weed seaweed-volume/target/release/weed-volume
|
||||
|
||||
- name: Upload test binaries
|
||||
uses: actions/upload-artifact@v7
|
||||
with:
|
||||
# archive layout mirrors the workspace: weed/weed and
|
||||
# seaweed-volume/target/release/weed-volume
|
||||
name: rust-test-binaries
|
||||
if-no-files-found: error
|
||||
retention-days: 1
|
||||
path: rust-test-binaries.tar
|
||||
|
||||
rust-unit-tests:
|
||||
name: Rust Unit Tests
|
||||
runs-on: ubuntu-22.04
|
||||
@@ -130,7 +71,6 @@ jobs:
|
||||
run: echo "fingerprint=$(getconf GNU_LIBC_VERSION | tr ' ' '-')-rustc-$(rustc -V | awk '{print $2}')" >> "$GITHUB_OUTPUT"
|
||||
|
||||
- name: Cache cargo registry and target
|
||||
if: runner.environment == 'github-hosted'
|
||||
uses: actions/cache@v6
|
||||
with:
|
||||
path: |
|
||||
@@ -194,7 +134,6 @@ jobs:
|
||||
run: echo "fingerprint=windows-rustc-$(rustc -V | awk '{print $2}')" >> "$GITHUB_OUTPUT"
|
||||
|
||||
- name: Cache cargo registry and target
|
||||
if: runner.environment == 'github-hosted'
|
||||
uses: actions/cache@v6
|
||||
with:
|
||||
path: |
|
||||
@@ -215,7 +154,6 @@ jobs:
|
||||
name: Rust Integration Tests
|
||||
runs-on: ubuntu-22.04
|
||||
timeout-minutes: 30
|
||||
needs: [build-test-binaries]
|
||||
|
||||
steps:
|
||||
- name: Checkout code
|
||||
@@ -225,18 +163,36 @@ jobs:
|
||||
uses: actions/setup-go@v7
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
cache: ${{ runner.environment == 'github-hosted' }}
|
||||
|
||||
- name: Download test binaries
|
||||
uses: actions/download-artifact@v7
|
||||
- name: Install Rust toolchain
|
||||
uses: dtolnay/rust-toolchain@stable
|
||||
|
||||
# cargo tracks its own inputs but not the runner's C toolchain, so a cached
|
||||
# target/ can carry C objects built against a different glibc than we link against.
|
||||
- name: Fingerprint build toolchain
|
||||
id: toolchain
|
||||
run: echo "fingerprint=$(getconf GNU_LIBC_VERSION | tr ' ' '-')-rustc-$(rustc -V | awk '{print $2}')" >> "$GITHUB_OUTPUT"
|
||||
|
||||
- name: Cache cargo registry and target
|
||||
uses: actions/cache@v6
|
||||
with:
|
||||
name: rust-test-binaries
|
||||
path: |
|
||||
~/.cargo/registry
|
||||
~/.cargo/git
|
||||
seaweed-volume/target
|
||||
key: rust-${{ steps.toolchain.outputs.fingerprint }}-${{ hashFiles('seaweed-volume/Cargo.lock', 'seaweed-common/Cargo.lock') }}
|
||||
restore-keys: |
|
||||
rust-${{ steps.toolchain.outputs.fingerprint }}-
|
||||
|
||||
- name: Restore binaries
|
||||
- name: Build Go weed binary
|
||||
run: |
|
||||
tar xf rust-test-binaries.tar
|
||||
chmod +x weed/weed seaweed-volume/target/release/weed-volume
|
||||
./weed/weed version
|
||||
cd weed
|
||||
go build -tags 5BytesOffset -o weed .
|
||||
chmod +x weed
|
||||
./weed version
|
||||
|
||||
- name: Build Rust volume binary
|
||||
run: cd seaweed-volume && cargo build --release
|
||||
|
||||
- name: Run integration tests
|
||||
env:
|
||||
@@ -272,7 +228,6 @@ jobs:
|
||||
name: Go Tests with Rust Volume (${{ matrix.test-type }} - Shard ${{ matrix.shard }})
|
||||
runs-on: ubuntu-22.04
|
||||
timeout-minutes: 45
|
||||
needs: [build-test-binaries]
|
||||
env:
|
||||
# Keep in step with the length of matrix.shard below.
|
||||
SHARD_COUNT: 3
|
||||
@@ -290,19 +245,36 @@ jobs:
|
||||
uses: actions/setup-go@v7
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
cache: ${{ runner.environment == 'github-hosted' }}
|
||||
|
||||
- name: Download test binaries
|
||||
uses: actions/download-artifact@v7
|
||||
- name: Install Rust toolchain
|
||||
uses: dtolnay/rust-toolchain@stable
|
||||
|
||||
# cargo tracks its own inputs but not the runner's C toolchain, so a cached
|
||||
# target/ can carry C objects built against a different glibc than we link against.
|
||||
- name: Fingerprint build toolchain
|
||||
id: toolchain
|
||||
run: echo "fingerprint=$(getconf GNU_LIBC_VERSION | tr ' ' '-')-rustc-$(rustc -V | awk '{print $2}')" >> "$GITHUB_OUTPUT"
|
||||
|
||||
- name: Cache cargo registry and target
|
||||
uses: actions/cache@v6
|
||||
with:
|
||||
name: rust-test-binaries
|
||||
path: |
|
||||
~/.cargo/registry
|
||||
~/.cargo/git
|
||||
seaweed-volume/target
|
||||
key: rust-${{ steps.toolchain.outputs.fingerprint }}-${{ hashFiles('seaweed-volume/Cargo.lock', 'seaweed-common/Cargo.lock') }}
|
||||
restore-keys: |
|
||||
rust-${{ steps.toolchain.outputs.fingerprint }}-
|
||||
|
||||
- name: Restore binaries
|
||||
- name: Build Go weed binary
|
||||
run: |
|
||||
tar xf rust-test-binaries.tar
|
||||
chmod +x weed/weed seaweed-volume/target/release/weed-volume
|
||||
./weed/weed version
|
||||
./seaweed-volume/target/release/weed-volume --help >/dev/null || true
|
||||
cd weed
|
||||
go build -tags 5BytesOffset -o weed .
|
||||
chmod +x weed
|
||||
./weed version
|
||||
|
||||
- name: Build Rust volume binary
|
||||
run: cd seaweed-volume && cargo build --release
|
||||
|
||||
# Dealing the listed tests out one by one keeps the shards even. Bucketing
|
||||
# them by first letter did not: names cluster, so ^Test[I-S] drew 50 of
|
||||
|
||||
@@ -17,8 +17,8 @@ on:
|
||||
- '.github/workflows/rust-worker-tests.yml'
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.event.pull_request.number || (github.event_name == 'push' && github.ref) || github.run_id }}
|
||||
cancel-in-progress: ${{ github.event_name == 'pull_request' || (github.event_name == 'push' && !startsWith(github.ref, 'refs/tags/')) }}
|
||||
group: ${{ github.workflow }}-${{ github.head_ref || github.ref }}
|
||||
cancel-in-progress: true
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
@@ -45,7 +45,6 @@ jobs:
|
||||
run: echo "fingerprint=$(getconf GNU_LIBC_VERSION | tr ' ' '-')-rustc-$(rustc -V | awk '{print $2}')" >> "$GITHUB_OUTPUT"
|
||||
|
||||
- name: Cache cargo registry and target
|
||||
if: runner.environment == 'github-hosted'
|
||||
uses: actions/cache@v6
|
||||
with:
|
||||
path: |
|
||||
|
||||
@@ -46,7 +46,6 @@ jobs:
|
||||
uses: dtolnay/rust-toolchain@stable
|
||||
|
||||
- name: Cache cargo registry and target
|
||||
if: runner.environment == 'github-hosted'
|
||||
uses: actions/cache@v6
|
||||
with:
|
||||
path: |
|
||||
@@ -115,7 +114,6 @@ jobs:
|
||||
targets: ${{ matrix.target }}
|
||||
|
||||
- name: Cache cargo registry and target
|
||||
if: runner.environment == 'github-hosted'
|
||||
uses: actions/cache@v6
|
||||
with:
|
||||
path: |
|
||||
|
||||
@@ -48,7 +48,6 @@ jobs:
|
||||
echo "OPENSSL_LIB_DIR=/usr/lib/aarch64-linux-gnu" >> "$GITHUB_ENV"
|
||||
|
||||
- name: Cache cargo registry and target
|
||||
if: runner.environment == 'github-hosted'
|
||||
uses: actions/cache@v6
|
||||
with:
|
||||
path: |
|
||||
@@ -154,7 +153,6 @@ jobs:
|
||||
echo "CARGO_TARGET_AARCH64_UNKNOWN_LINUX_GNU_LINKER=aarch64-linux-gnu-gcc" >> "$GITHUB_ENV"
|
||||
|
||||
- name: Cache cargo registry and target
|
||||
if: runner.environment == 'github-hosted'
|
||||
uses: actions/cache@v6
|
||||
with:
|
||||
path: |
|
||||
@@ -232,7 +230,6 @@ jobs:
|
||||
targets: ${{ matrix.target }}
|
||||
|
||||
- name: Cache cargo registry and target
|
||||
if: runner.environment == 'github-hosted'
|
||||
uses: actions/cache@v6
|
||||
with:
|
||||
path: |
|
||||
@@ -308,7 +305,6 @@ jobs:
|
||||
uses: dtolnay/rust-toolchain@stable
|
||||
|
||||
- name: Cache cargo registry and target
|
||||
if: runner.environment == 'github-hosted'
|
||||
uses: actions/cache@v6
|
||||
with:
|
||||
path: |
|
||||
|
||||
@@ -21,8 +21,8 @@ on:
|
||||
- '.github/workflows/s3-etag-acl-tests.yml'
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.event.pull_request.number || (github.event_name == 'push' && github.ref) || github.run_id }}
|
||||
cancel-in-progress: ${{ github.event_name == 'pull_request' || (github.event_name == 'push' && !startsWith(github.ref, 'refs/tags/')) }}
|
||||
group: ${{ github.head_ref || github.ref }}/s3-etag-acl-tests
|
||||
cancel-in-progress: true
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
@@ -40,7 +40,6 @@ jobs:
|
||||
uses: actions/setup-go@v7
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
cache: ${{ runner.environment == 'github-hosted' }}
|
||||
|
||||
- name: Install SeaweedFS
|
||||
run: |
|
||||
|
||||
@@ -13,8 +13,8 @@ on:
|
||||
- '.github/workflows/s3-example-integration-tests.yml'
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.event.pull_request.number || (github.event_name == 'push' && github.ref) || github.run_id }}
|
||||
cancel-in-progress: ${{ github.event_name == 'pull_request' || (github.event_name == 'push' && !startsWith(github.ref, 'refs/tags/')) }}
|
||||
group: ${{ github.head_ref }}/s3-integration-tests
|
||||
cancel-in-progress: true
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
@@ -33,7 +33,6 @@ jobs:
|
||||
uses: actions/setup-go@v7
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
cache: ${{ runner.environment == 'github-hosted' }}
|
||||
id: go
|
||||
|
||||
- name: Build SeaweedFS
|
||||
|
||||
@@ -12,8 +12,8 @@ on:
|
||||
- '.github/workflows/s3-filer-group-tests.yml'
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.event.pull_request.number || (github.event_name == 'push' && github.ref) || github.run_id }}
|
||||
cancel-in-progress: ${{ github.event_name == 'pull_request' || (github.event_name == 'push' && !startsWith(github.ref, 'refs/tags/')) }}
|
||||
group: ${{ github.head_ref }}/s3-filer-group-tests
|
||||
cancel-in-progress: true
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
@@ -32,14 +32,10 @@ jobs:
|
||||
- name: Check out code
|
||||
uses: actions/checkout@v7
|
||||
|
||||
- name: Clean up stale FUSE mounts
|
||||
uses: ./.github/actions/cleanup-stale-fuse-mounts
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v7
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
cache: ${{ runner.environment == 'github-hosted' }}
|
||||
id: go
|
||||
|
||||
- name: Install SeaweedFS
|
||||
@@ -54,7 +50,7 @@ jobs:
|
||||
echo "=== System Information ==="
|
||||
uname -a
|
||||
free -h
|
||||
timeout 15 df -h || true
|
||||
df -h
|
||||
echo "=== Starting Filer Group Tests ==="
|
||||
|
||||
# Run tests with automatic server management
|
||||
|
||||
@@ -4,7 +4,6 @@ on:
|
||||
pull_request:
|
||||
paths:
|
||||
- 'weed/s3api/**'
|
||||
- 'weed/remote_storage/**'
|
||||
- 'weed/filer/**'
|
||||
- 'weed/server/**'
|
||||
- 'test/s3/**'
|
||||
@@ -13,8 +12,8 @@ on:
|
||||
- '.github/workflows/s3-go-tests.yml'
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.event.pull_request.number || (github.event_name == 'push' && github.ref) || github.run_id }}
|
||||
cancel-in-progress: ${{ github.event_name == 'pull_request' || (github.event_name == 'push' && !startsWith(github.ref, 'refs/tags/')) }}
|
||||
group: ${{ github.head_ref }}/s3-go-tests
|
||||
cancel-in-progress: true
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
@@ -36,14 +35,10 @@ jobs:
|
||||
- name: Check out code
|
||||
uses: actions/checkout@v7
|
||||
|
||||
- name: Clean up stale FUSE mounts
|
||||
uses: ./.github/actions/cleanup-stale-fuse-mounts
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v7
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
cache: ${{ runner.environment == 'github-hosted' }}
|
||||
id: go
|
||||
|
||||
- name: Install SeaweedFS
|
||||
@@ -58,7 +53,7 @@ jobs:
|
||||
echo "=== System Information ==="
|
||||
uname -a
|
||||
free -h
|
||||
timeout 15 df -h || true
|
||||
df -h
|
||||
echo "=== Starting Tests ==="
|
||||
|
||||
# Run tests with automatic server management
|
||||
@@ -108,7 +103,6 @@ jobs:
|
||||
uses: actions/setup-go@v7
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
cache: ${{ runner.environment == 'github-hosted' }}
|
||||
id: go
|
||||
|
||||
- name: Install SeaweedFS
|
||||
@@ -157,7 +151,6 @@ jobs:
|
||||
uses: actions/setup-go@v7
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
cache: ${{ runner.environment == 'github-hosted' }}
|
||||
id: go
|
||||
|
||||
- name: Install SeaweedFS
|
||||
@@ -205,14 +198,10 @@ jobs:
|
||||
- name: Check out code
|
||||
uses: actions/checkout@v7
|
||||
|
||||
- name: Clean up stale FUSE mounts
|
||||
uses: ./.github/actions/cleanup-stale-fuse-mounts
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v7
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
cache: ${{ runner.environment == 'github-hosted' }}
|
||||
id: go
|
||||
|
||||
- name: Install SeaweedFS
|
||||
@@ -227,7 +216,7 @@ jobs:
|
||||
echo "=== System Information ==="
|
||||
uname -a
|
||||
free -h
|
||||
timeout 15 df -h || true
|
||||
df -h
|
||||
echo "=== Starting Tests ==="
|
||||
|
||||
# Run tests with automatic server management
|
||||
@@ -294,7 +283,6 @@ jobs:
|
||||
- TestLifecycleSuspendedVersioningExpiration
|
||||
- TestLifecycleTagFilter
|
||||
- TestLifecycleVersionedBucketCreatesDeleteMarker
|
||||
- TestLifecycleVersionedBucketExpiresOnSecondRun
|
||||
|
||||
steps:
|
||||
- name: Check out code
|
||||
@@ -304,7 +292,6 @@ jobs:
|
||||
uses: actions/setup-go@v7
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
cache: ${{ runner.environment == 'github-hosted' }}
|
||||
id: go
|
||||
|
||||
- name: Install SeaweedFS
|
||||
@@ -346,14 +333,10 @@ jobs:
|
||||
- name: Check out code
|
||||
uses: actions/checkout@v7
|
||||
|
||||
- name: Clean up stale FUSE mounts
|
||||
uses: ./.github/actions/cleanup-stale-fuse-mounts
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v7
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
cache: ${{ runner.environment == 'github-hosted' }}
|
||||
id: go
|
||||
|
||||
- name: Install SeaweedFS
|
||||
@@ -368,7 +351,7 @@ jobs:
|
||||
echo "=== System Information ==="
|
||||
uname -a
|
||||
free -h
|
||||
timeout 15 df -h || true
|
||||
df -h
|
||||
echo "=== Starting Tests ==="
|
||||
make test-with-server
|
||||
|
||||
@@ -408,14 +391,10 @@ jobs:
|
||||
- name: Check out code
|
||||
uses: actions/checkout@v7
|
||||
|
||||
- name: Clean up stale FUSE mounts
|
||||
uses: ./.github/actions/cleanup-stale-fuse-mounts
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v7
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
cache: ${{ runner.environment == 'github-hosted' }}
|
||||
id: go
|
||||
|
||||
- name: Install SeaweedFS
|
||||
@@ -430,7 +409,7 @@ jobs:
|
||||
echo "=== System Information ==="
|
||||
uname -a
|
||||
free -h
|
||||
timeout 15 df -h || true
|
||||
df -h
|
||||
echo "=== Starting Tests ==="
|
||||
|
||||
# Run tests with automatic server management
|
||||
@@ -480,7 +459,6 @@ jobs:
|
||||
uses: actions/setup-go@v7
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
cache: ${{ runner.environment == 'github-hosted' }}
|
||||
id: go
|
||||
|
||||
- name: Install SeaweedFS
|
||||
@@ -530,7 +508,6 @@ jobs:
|
||||
uses: actions/setup-go@v7
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
cache: ${{ runner.environment == 'github-hosted' }}
|
||||
id: go
|
||||
|
||||
- name: Install SeaweedFS
|
||||
@@ -599,7 +576,6 @@ jobs:
|
||||
uses: actions/setup-go@v7
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
cache: ${{ runner.environment == 'github-hosted' }}
|
||||
id: go
|
||||
|
||||
- name: Install SeaweedFS
|
||||
@@ -655,7 +631,6 @@ jobs:
|
||||
uses: actions/setup-go@v7
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
cache: ${{ runner.environment == 'github-hosted' }}
|
||||
id: go
|
||||
|
||||
- name: Install SeaweedFS
|
||||
|
||||
@@ -20,8 +20,8 @@ on:
|
||||
- '.github/workflows/s3-iam-tests.yml'
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.event.pull_request.number || (github.event_name == 'push' && github.ref) || github.run_id }}
|
||||
cancel-in-progress: ${{ github.event_name == 'pull_request' || (github.event_name == 'push' && !startsWith(github.ref, 'refs/tags/')) }}
|
||||
group: ${{ github.head_ref }}/s3-iam-tests
|
||||
cancel-in-progress: true
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
@@ -45,7 +45,6 @@ jobs:
|
||||
uses: actions/setup-go@v7
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
cache: ${{ runner.environment == 'github-hosted' }}
|
||||
id: go
|
||||
|
||||
- name: Get dependencies
|
||||
@@ -91,14 +90,10 @@ jobs:
|
||||
- name: Check out code
|
||||
uses: actions/checkout@v7
|
||||
|
||||
- name: Clean up stale FUSE mounts
|
||||
uses: ./.github/actions/cleanup-stale-fuse-mounts
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v7
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
cache: ${{ runner.environment == 'github-hosted' }}
|
||||
id: go
|
||||
|
||||
- name: Install SeaweedFS
|
||||
@@ -114,7 +109,7 @@ jobs:
|
||||
echo "=== System Information ==="
|
||||
uname -a
|
||||
free -h
|
||||
timeout 15 df -h || true
|
||||
df -h
|
||||
echo "=== Starting S3 IAM Integration Tests (${{ matrix.test-type }}) ==="
|
||||
|
||||
# Set WEED_BINARY to use the installed version
|
||||
@@ -213,7 +208,6 @@ jobs:
|
||||
uses: actions/setup-go@v7
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
cache: ${{ runner.environment == 'github-hosted' }}
|
||||
id: go
|
||||
|
||||
- name: Install SeaweedFS
|
||||
@@ -274,7 +268,6 @@ jobs:
|
||||
uses: actions/setup-go@v7
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
cache: ${{ runner.environment == 'github-hosted' }}
|
||||
id: go
|
||||
|
||||
- name: Install SeaweedFS
|
||||
|
||||
@@ -16,8 +16,8 @@ on:
|
||||
- '.github/workflows/s3-keycloak-tests.yml'
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.event.pull_request.number || (github.event_name == 'push' && github.ref) || github.run_id }}
|
||||
cancel-in-progress: ${{ github.event_name == 'pull_request' || (github.event_name == 'push' && !startsWith(github.ref, 'refs/tags/')) }}
|
||||
group: ${{ github.head_ref }}/s3-keycloak-tests
|
||||
cancel-in-progress: true
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
@@ -37,14 +37,10 @@ jobs:
|
||||
- name: Check out code
|
||||
uses: actions/checkout@v7
|
||||
|
||||
- name: Clean up stale FUSE mounts
|
||||
uses: ./.github/actions/cleanup-stale-fuse-mounts
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v7
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
cache: ${{ runner.environment == 'github-hosted' }}
|
||||
id: go
|
||||
|
||||
- name: Install SeaweedFS
|
||||
@@ -60,7 +56,7 @@ jobs:
|
||||
echo "=== System Information ==="
|
||||
uname -a
|
||||
free -h
|
||||
timeout 15 df -h || true
|
||||
df -h
|
||||
echo "=== Starting S3 Keycloak Integration Tests ==="
|
||||
|
||||
# Set WEED_BINARY to use the installed version
|
||||
|
||||
@@ -12,8 +12,8 @@ on:
|
||||
- '.github/workflows/s3-mutation-regression-tests.yml'
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.event.pull_request.number || (github.event_name == 'push' && github.ref) || github.run_id }}
|
||||
cancel-in-progress: ${{ github.event_name == 'pull_request' || (github.event_name == 'push' && !startsWith(github.ref, 'refs/tags/')) }}
|
||||
group: ${{ github.head_ref }}/s3-mutation-regression-tests
|
||||
cancel-in-progress: true
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
@@ -32,7 +32,6 @@ jobs:
|
||||
uses: actions/setup-go@v7
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
cache: ${{ runner.environment == 'github-hosted' }}
|
||||
|
||||
- name: Run S3 versioning regression tests
|
||||
timeout-minutes: 20
|
||||
@@ -86,7 +85,6 @@ jobs:
|
||||
uses: actions/setup-go@v7
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
cache: ${{ runner.environment == 'github-hosted' }}
|
||||
|
||||
- name: Run S3 delete regression tests
|
||||
timeout-minutes: 15
|
||||
@@ -125,7 +123,6 @@ jobs:
|
||||
uses: actions/setup-go@v7
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
cache: ${{ runner.environment == 'github-hosted' }}
|
||||
|
||||
- name: Build SeaweedFS
|
||||
run: |
|
||||
|
||||
@@ -23,10 +23,6 @@ env:
|
||||
S3_ENDPOINT_URL: http://localhost:8333
|
||||
BUCKET_NAME: test-parquet-bucket
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.event.pull_request.number || (github.event_name == 'push' && github.ref) || github.run_id }}
|
||||
cancel-in-progress: ${{ github.event_name == 'pull_request' || (github.event_name == 'push' && !startsWith(github.ref, 'refs/tags/')) }}
|
||||
|
||||
jobs:
|
||||
parquet-integration-tests:
|
||||
name: PyArrow Parquet Tests (Python ${{ matrix.python-version }})
|
||||
@@ -46,13 +42,13 @@ jobs:
|
||||
uses: actions/setup-go@v7
|
||||
with:
|
||||
go-version: ^1.26
|
||||
cache: ${{ runner.environment == 'github-hosted' }}
|
||||
cache: true
|
||||
|
||||
- name: Set up Python ${{ matrix.python-version }}
|
||||
uses: actions/setup-python@v7
|
||||
with:
|
||||
python-version: ${{ matrix.python-version }}
|
||||
cache: ${{ runner.environment == 'github-hosted' && 'pip' || '' }}
|
||||
cache: 'pip'
|
||||
cache-dependency-path: 'test/s3/parquet/requirements.txt'
|
||||
|
||||
- name: Install system dependencies
|
||||
@@ -154,7 +150,7 @@ jobs:
|
||||
uses: actions/setup-go@v7
|
||||
with:
|
||||
go-version: ^1.26
|
||||
cache: ${{ runner.environment == 'github-hosted' }}
|
||||
cache: true
|
||||
|
||||
- name: Run Go unit tests
|
||||
run: |
|
||||
|
||||
@@ -24,8 +24,8 @@ on:
|
||||
- '.github/workflows/s3-policy-tests.yml'
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.event.pull_request.number || (github.event_name == 'push' && github.ref) || github.run_id }}
|
||||
cancel-in-progress: ${{ github.event_name == 'pull_request' || (github.event_name == 'push' && !startsWith(github.ref, 'refs/tags/')) }}
|
||||
group: ${{ github.head_ref }}/s3-policy-tests
|
||||
cancel-in-progress: true
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
@@ -49,7 +49,6 @@ jobs:
|
||||
uses: actions/setup-go@v7
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
cache: ${{ runner.environment == 'github-hosted' }}
|
||||
id: go
|
||||
|
||||
- name: Get dependencies
|
||||
@@ -89,14 +88,10 @@ jobs:
|
||||
- name: Check out code
|
||||
uses: actions/checkout@v7
|
||||
|
||||
- name: Clean up stale FUSE mounts
|
||||
uses: ./.github/actions/cleanup-stale-fuse-mounts
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v7
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
cache: ${{ runner.environment == 'github-hosted' }}
|
||||
id: go
|
||||
|
||||
- name: Install SeaweedFS
|
||||
@@ -111,7 +106,7 @@ jobs:
|
||||
echo "=== System Information ==="
|
||||
uname -a
|
||||
free -h
|
||||
timeout 15 df -h || true
|
||||
df -h
|
||||
|
||||
echo "=== Starting S3 Policy Variables Integration Tests ==="
|
||||
|
||||
@@ -202,14 +197,10 @@ jobs:
|
||||
- name: Check out code
|
||||
uses: actions/checkout@v7
|
||||
|
||||
- name: Clean up stale FUSE mounts
|
||||
uses: ./.github/actions/cleanup-stale-fuse-mounts
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v7
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
cache: ${{ runner.environment == 'github-hosted' }}
|
||||
id: go
|
||||
|
||||
- name: Install SeaweedFS
|
||||
@@ -224,7 +215,7 @@ jobs:
|
||||
echo "=== System Information ==="
|
||||
uname -a
|
||||
free -h
|
||||
timeout 15 df -h || true
|
||||
df -h
|
||||
|
||||
echo "=== Starting S3 Policy Enforcement Tests (${{ matrix.test-case }}) ==="
|
||||
|
||||
@@ -328,7 +319,6 @@ jobs:
|
||||
uses: actions/setup-go@v7
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
cache: ${{ runner.environment == 'github-hosted' }}
|
||||
id: go
|
||||
|
||||
- name: Install SeaweedFS
|
||||
@@ -416,7 +406,6 @@ jobs:
|
||||
uses: actions/setup-go@v7
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
cache: ${{ runner.environment == 'github-hosted' }}
|
||||
id: go
|
||||
|
||||
- name: Install SeaweedFS
|
||||
|
||||
@@ -21,8 +21,8 @@ on:
|
||||
- '.github/workflows/s3-proxy-signature-tests.yml'
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.event.pull_request.number || (github.event_name == 'push' && github.ref) || github.run_id }}
|
||||
cancel-in-progress: ${{ github.event_name == 'pull_request' || (github.event_name == 'push' && !startsWith(github.ref, 'refs/tags/')) }}
|
||||
group: ${{ github.head_ref || github.ref }}/s3-proxy-signature-tests
|
||||
cancel-in-progress: true
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
@@ -40,7 +40,6 @@ jobs:
|
||||
uses: actions/setup-go@v7
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
cache: ${{ runner.environment == 'github-hosted' }}
|
||||
id: go
|
||||
|
||||
- name: Configure Docker Hub mirror
|
||||
|
||||
@@ -15,8 +15,8 @@ on:
|
||||
- '.github/workflows/s3-sdk-v2-routing-tests.yml'
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.event.pull_request.number || (github.event_name == 'push' && github.ref) || github.run_id }}
|
||||
cancel-in-progress: ${{ github.event_name == 'pull_request' || (github.event_name == 'push' && !startsWith(github.ref, 'refs/tags/')) }}
|
||||
group: ${{ github.head_ref || github.ref }}/s3-sdk-v2-routing-tests
|
||||
cancel-in-progress: true
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
@@ -34,7 +34,6 @@ jobs:
|
||||
uses: actions/setup-go@v7
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
cache: ${{ runner.environment == 'github-hosted' }}
|
||||
|
||||
- name: Install SeaweedFS
|
||||
run: |
|
||||
|
||||
@@ -60,7 +60,6 @@ jobs:
|
||||
uses: actions/setup-go@v7
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
cache: ${{ runner.environment == 'github-hosted' }}
|
||||
id: go
|
||||
|
||||
- name: Set up Java
|
||||
@@ -68,7 +67,7 @@ jobs:
|
||||
with:
|
||||
java-version: '17'
|
||||
distribution: 'temurin'
|
||||
cache: ${{ runner.environment == 'github-hosted' && 'maven' || '' }}
|
||||
cache: 'maven'
|
||||
|
||||
- name: Install SeaweedFS
|
||||
run: |
|
||||
|
||||
@@ -11,8 +11,8 @@ on:
|
||||
workflow_dispatch:
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.event.pull_request.number || (github.event_name == 'push' && github.ref) || github.run_id }}
|
||||
cancel-in-progress: ${{ github.event_name == 'pull_request' || (github.event_name == 'push' && !startsWith(github.ref, 'refs/tags/')) }}
|
||||
group: ${{ github.head_ref }}/s3-spark-tests
|
||||
cancel-in-progress: true
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
@@ -27,14 +27,10 @@ jobs:
|
||||
- name: Check out code
|
||||
uses: actions/checkout@v7
|
||||
|
||||
- name: Clean up stale FUSE mounts
|
||||
uses: ./.github/actions/cleanup-stale-fuse-mounts
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v7
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
cache: ${{ runner.environment == 'github-hosted' }}
|
||||
id: go
|
||||
|
||||
- name: Configure Docker Hub mirror
|
||||
@@ -60,7 +56,7 @@ jobs:
|
||||
echo "=== System Information ==="
|
||||
uname -a
|
||||
free -h
|
||||
timeout 15 df -h || true
|
||||
df -h
|
||||
echo "=== Starting S3 Spark Integration Tests ==="
|
||||
|
||||
go test -v -timeout 30m . 2>&1 | tee test-output.log || {
|
||||
|
||||
@@ -23,8 +23,8 @@ on:
|
||||
- 'test/s3/sse/**'
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.event.pull_request.number || (github.event_name == 'push' && github.ref) || github.run_id }}
|
||||
cancel-in-progress: ${{ github.event_name == 'pull_request' || (github.event_name == 'push' && !startsWith(github.ref, 'refs/tags/')) }}
|
||||
group: ${{ github.head_ref }}/s3-sse-tests
|
||||
cancel-in-progress: true
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
@@ -46,14 +46,10 @@ jobs:
|
||||
- name: Check out code
|
||||
uses: actions/checkout@v7
|
||||
|
||||
- name: Clean up stale FUSE mounts
|
||||
uses: ./.github/actions/cleanup-stale-fuse-mounts
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v7
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
cache: ${{ runner.environment == 'github-hosted' }}
|
||||
id: go
|
||||
|
||||
- name: Install SeaweedFS
|
||||
@@ -68,7 +64,7 @@ jobs:
|
||||
echo "=== System Information ==="
|
||||
uname -a
|
||||
free -h
|
||||
timeout 15 df -h || true
|
||||
df -h
|
||||
echo "=== Starting SSE Tests ==="
|
||||
|
||||
# Run tests with automatic server management
|
||||
@@ -122,7 +118,6 @@ jobs:
|
||||
uses: actions/setup-go@v7
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
cache: ${{ runner.environment == 'github-hosted' }}
|
||||
id: go
|
||||
|
||||
- name: Install SeaweedFS
|
||||
@@ -171,7 +166,6 @@ jobs:
|
||||
uses: actions/setup-go@v7
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
cache: ${{ runner.environment == 'github-hosted' }}
|
||||
id: go
|
||||
|
||||
- name: Install SeaweedFS
|
||||
@@ -221,7 +215,6 @@ jobs:
|
||||
uses: actions/setup-go@v7
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
cache: ${{ runner.environment == 'github-hosted' }}
|
||||
id: go
|
||||
|
||||
- name: Install SeaweedFS
|
||||
@@ -271,7 +264,6 @@ jobs:
|
||||
uses: actions/setup-go@v7
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
cache: ${{ runner.environment == 'github-hosted' }}
|
||||
id: go
|
||||
|
||||
- name: Install SeaweedFS
|
||||
@@ -323,7 +315,6 @@ jobs:
|
||||
uses: actions/setup-go@v7
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
cache: ${{ runner.environment == 'github-hosted' }}
|
||||
id: go
|
||||
|
||||
- name: Install SeaweedFS
|
||||
@@ -372,7 +363,6 @@ jobs:
|
||||
uses: actions/setup-go@v7
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
cache: ${{ runner.environment == 'github-hosted' }}
|
||||
id: go
|
||||
|
||||
- name: Install SeaweedFS
|
||||
|
||||
@@ -12,10 +12,6 @@ on:
|
||||
- 'go.sum'
|
||||
- '.github/workflows/s3-tables-tests.yml'
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.event.pull_request.number || (github.event_name == 'push' && github.ref) || github.run_id }}
|
||||
cancel-in-progress: ${{ github.event_name == 'pull_request' || (github.event_name == 'push' && !startsWith(github.ref, 'refs/tags/')) }}
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
@@ -30,14 +26,10 @@ jobs:
|
||||
- name: Check out code
|
||||
uses: actions/checkout@v7
|
||||
|
||||
- name: Clean up stale FUSE mounts
|
||||
uses: ./.github/actions/cleanup-stale-fuse-mounts
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v7
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
cache: ${{ runner.environment == 'github-hosted' }}
|
||||
id: go
|
||||
|
||||
- name: Run go mod tidy
|
||||
@@ -56,8 +48,7 @@ jobs:
|
||||
echo "=== System Information ==="
|
||||
uname -a
|
||||
free -h
|
||||
# A stale FUSE mount can hang or fail df; keep diagnostics best effort.
|
||||
timeout 15 df -h || true
|
||||
df -h
|
||||
echo "=== Starting S3 Tables Tests ==="
|
||||
|
||||
# Run S3 Tables integration tests
|
||||
@@ -95,14 +86,10 @@ jobs:
|
||||
- name: Check out code
|
||||
uses: actions/checkout@v7
|
||||
|
||||
- name: Clean up stale FUSE mounts
|
||||
uses: ./.github/actions/cleanup-stale-fuse-mounts
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v7
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
cache: ${{ runner.environment == 'github-hosted' }}
|
||||
id: go
|
||||
|
||||
- name: Configure Docker Hub mirror
|
||||
@@ -122,7 +109,7 @@ jobs:
|
||||
echo "=== System Information ==="
|
||||
uname -a
|
||||
free -h
|
||||
timeout 15 df -h || true
|
||||
df -h
|
||||
echo "=== Starting Iceberg Catalog Tests ==="
|
||||
|
||||
# Run Iceberg catalog integration tests using Makefile (handles build)
|
||||
@@ -160,14 +147,10 @@ jobs:
|
||||
- name: Check out code
|
||||
uses: actions/checkout@v7
|
||||
|
||||
- name: Clean up stale FUSE mounts
|
||||
uses: ./.github/actions/cleanup-stale-fuse-mounts
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v7
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
cache: ${{ runner.environment == 'github-hosted' }}
|
||||
id: go
|
||||
|
||||
- name: Configure Docker Hub mirror
|
||||
@@ -196,7 +179,7 @@ jobs:
|
||||
echo "=== System Information ==="
|
||||
uname -a
|
||||
free -h
|
||||
timeout 15 df -h || true
|
||||
df -h
|
||||
echo "=== Starting Trino Iceberg Catalog Tests ==="
|
||||
|
||||
# Run Trino + Iceberg catalog integration tests
|
||||
@@ -234,14 +217,10 @@ jobs:
|
||||
- name: Check out code
|
||||
uses: actions/checkout@v7
|
||||
|
||||
- name: Clean up stale FUSE mounts
|
||||
uses: ./.github/actions/cleanup-stale-fuse-mounts
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v7
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
cache: ${{ runner.environment == 'github-hosted' }}
|
||||
id: go
|
||||
|
||||
- name: Configure Docker Hub mirror
|
||||
@@ -271,7 +250,7 @@ jobs:
|
||||
echo "=== System Information ==="
|
||||
uname -a
|
||||
free -h
|
||||
timeout 15 df -h || true
|
||||
df -h
|
||||
docker info
|
||||
echo "=== Starting Dremio Iceberg Catalog Tests ==="
|
||||
|
||||
@@ -311,14 +290,10 @@ jobs:
|
||||
- name: Check out code
|
||||
uses: actions/checkout@v7
|
||||
|
||||
- name: Clean up stale FUSE mounts
|
||||
uses: ./.github/actions/cleanup-stale-fuse-mounts
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v7
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
cache: ${{ runner.environment == 'github-hosted' }}
|
||||
id: go
|
||||
|
||||
- name: Configure Docker Hub mirror
|
||||
@@ -348,7 +323,7 @@ jobs:
|
||||
echo "=== System Information ==="
|
||||
uname -a
|
||||
free -h
|
||||
timeout 15 df -h || true
|
||||
df -h
|
||||
docker info
|
||||
echo "=== Starting Doris Iceberg Catalog Tests ==="
|
||||
|
||||
@@ -398,14 +373,10 @@ jobs:
|
||||
- name: Check out code
|
||||
uses: actions/checkout@v7
|
||||
|
||||
- name: Clean up stale FUSE mounts
|
||||
uses: ./.github/actions/cleanup-stale-fuse-mounts
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v7
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
cache: ${{ runner.environment == 'github-hosted' }}
|
||||
id: go
|
||||
|
||||
- name: Configure Docker Hub mirror
|
||||
@@ -437,7 +408,7 @@ jobs:
|
||||
echo "=== System Information ==="
|
||||
uname -a
|
||||
free -h
|
||||
timeout 15 df -h || true
|
||||
df -h
|
||||
docker info
|
||||
echo "=== Starting ClickHouse Iceberg Catalog Tests ==="
|
||||
|
||||
@@ -489,14 +460,10 @@ jobs:
|
||||
- name: Check out code
|
||||
uses: actions/checkout@v7
|
||||
|
||||
- name: Clean up stale FUSE mounts
|
||||
uses: ./.github/actions/cleanup-stale-fuse-mounts
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v7
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
cache: ${{ runner.environment == 'github-hosted' }}
|
||||
id: go
|
||||
|
||||
- name: Configure Docker Hub mirror
|
||||
@@ -529,7 +496,7 @@ jobs:
|
||||
echo "=== System Information ==="
|
||||
uname -a
|
||||
free -h
|
||||
timeout 15 df -h || true
|
||||
df -h
|
||||
docker info
|
||||
echo "=== Starting OLake Iceberg Catalog Tests ==="
|
||||
|
||||
@@ -592,14 +559,10 @@ jobs:
|
||||
- name: Check out code
|
||||
uses: actions/checkout@v7
|
||||
|
||||
- name: Clean up stale FUSE mounts
|
||||
uses: ./.github/actions/cleanup-stale-fuse-mounts
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v7
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
cache: ${{ runner.environment == 'github-hosted' }}
|
||||
id: go
|
||||
|
||||
- name: Run go mod tidy
|
||||
@@ -627,7 +590,7 @@ jobs:
|
||||
echo "=== System Information ==="
|
||||
uname -a
|
||||
free -h
|
||||
timeout 15 df -h || true
|
||||
df -h
|
||||
echo "=== Starting Polaris Tests ==="
|
||||
|
||||
go test -v -timeout 20m ./test/s3tables/polaris 2>&1 | tee test/s3tables/polaris/test-output.log || {
|
||||
@@ -664,14 +627,10 @@ jobs:
|
||||
- name: Check out code
|
||||
uses: actions/checkout@v7
|
||||
|
||||
- name: Clean up stale FUSE mounts
|
||||
uses: ./.github/actions/cleanup-stale-fuse-mounts
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v7
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
cache: ${{ runner.environment == 'github-hosted' }}
|
||||
id: go
|
||||
|
||||
- name: Configure Docker Hub mirror
|
||||
@@ -700,7 +659,7 @@ jobs:
|
||||
echo "=== System Information ==="
|
||||
uname -a
|
||||
free -h
|
||||
timeout 15 df -h || true
|
||||
df -h
|
||||
echo "=== Starting Spark Iceberg Catalog Tests ==="
|
||||
|
||||
# Run Spark + Iceberg catalog integration tests
|
||||
@@ -738,14 +697,10 @@ jobs:
|
||||
- name: Check out code
|
||||
uses: actions/checkout@v7
|
||||
|
||||
- name: Clean up stale FUSE mounts
|
||||
uses: ./.github/actions/cleanup-stale-fuse-mounts
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v7
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
cache: ${{ runner.environment == 'github-hosted' }}
|
||||
id: go
|
||||
|
||||
- name: Configure Docker Hub mirror
|
||||
@@ -775,7 +730,7 @@ jobs:
|
||||
echo "=== System Information ==="
|
||||
uname -a
|
||||
free -h
|
||||
timeout 15 df -h || true
|
||||
df -h
|
||||
echo "=== Starting RisingWave Catalog Tests ==="
|
||||
|
||||
# Run RisingWave catalog integration tests
|
||||
@@ -813,14 +768,10 @@ jobs:
|
||||
- name: Check out code
|
||||
uses: actions/checkout@v7
|
||||
|
||||
- name: Clean up stale FUSE mounts
|
||||
uses: ./.github/actions/cleanup-stale-fuse-mounts
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v7
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
cache: ${{ runner.environment == 'github-hosted' }}
|
||||
id: go
|
||||
|
||||
- name: Configure Docker Hub mirror
|
||||
@@ -834,7 +785,6 @@ jobs:
|
||||
|
||||
- name: Restore python:3 image cache
|
||||
id: python-image
|
||||
if: runner.environment == 'github-hosted'
|
||||
uses: actions/cache@v6
|
||||
with:
|
||||
path: /tmp/python3-image.tar
|
||||
@@ -874,7 +824,7 @@ jobs:
|
||||
echo "=== System Information ==="
|
||||
uname -a
|
||||
free -h
|
||||
timeout 15 df -h || true
|
||||
df -h
|
||||
echo "=== Starting STS Integration Tests ==="
|
||||
|
||||
# Run STS integration tests
|
||||
@@ -912,14 +862,10 @@ jobs:
|
||||
- name: Check out code
|
||||
uses: actions/checkout@v7
|
||||
|
||||
- name: Clean up stale FUSE mounts
|
||||
uses: ./.github/actions/cleanup-stale-fuse-mounts
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v7
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
cache: ${{ runner.environment == 'github-hosted' }}
|
||||
id: go
|
||||
|
||||
- name: Run go mod tidy
|
||||
@@ -938,7 +884,7 @@ jobs:
|
||||
echo "=== System Information ==="
|
||||
uname -a
|
||||
free -h
|
||||
timeout 15 df -h || true
|
||||
df -h
|
||||
echo "=== Starting Lakekeeper Integration Tests ==="
|
||||
|
||||
# Run Lakekeeper integration tests
|
||||
@@ -976,14 +922,10 @@ jobs:
|
||||
- name: Check out code
|
||||
uses: actions/checkout@v7
|
||||
|
||||
- name: Clean up stale FUSE mounts
|
||||
uses: ./.github/actions/cleanup-stale-fuse-mounts
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v7
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
cache: ${{ runner.environment == 'github-hosted' }}
|
||||
id: go
|
||||
|
||||
- name: Configure Docker Hub mirror
|
||||
@@ -1013,7 +955,7 @@ jobs:
|
||||
echo "=== System Information ==="
|
||||
uname -a
|
||||
free -h
|
||||
timeout 15 df -h || true
|
||||
df -h
|
||||
docker info
|
||||
echo "=== Starting Unity Catalog Tests ==="
|
||||
|
||||
@@ -1053,14 +995,10 @@ jobs:
|
||||
- name: Check out code
|
||||
uses: actions/checkout@v7
|
||||
|
||||
- name: Clean up stale FUSE mounts
|
||||
uses: ./.github/actions/cleanup-stale-fuse-mounts
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v7
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
cache: ${{ runner.environment == 'github-hosted' }}
|
||||
id: go
|
||||
|
||||
- name: Configure Docker Hub mirror
|
||||
@@ -1089,7 +1027,7 @@ jobs:
|
||||
echo "=== System Information ==="
|
||||
uname -a
|
||||
free -h
|
||||
timeout 15 df -h || true
|
||||
df -h
|
||||
docker info
|
||||
echo "=== Starting LanceDB Namespace Tests ==="
|
||||
|
||||
@@ -1131,7 +1069,6 @@ jobs:
|
||||
uses: actions/setup-go@v7
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
cache: ${{ runner.environment == 'github-hosted' }}
|
||||
id: go
|
||||
|
||||
- name: Configure Docker Hub mirror
|
||||
@@ -1200,14 +1137,10 @@ jobs:
|
||||
# so do not leave a token in the checkout for it to pick up.
|
||||
persist-credentials: false
|
||||
|
||||
- name: Clean up stale FUSE mounts
|
||||
uses: ./.github/actions/cleanup-stale-fuse-mounts
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v7
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
cache: ${{ runner.environment == 'github-hosted' }}
|
||||
id: go
|
||||
|
||||
- name: Configure Docker Hub mirror
|
||||
@@ -1237,7 +1170,7 @@ jobs:
|
||||
echo "=== System Information ==="
|
||||
uname -a
|
||||
free -h
|
||||
timeout 15 df -h || true
|
||||
df -h
|
||||
docker info
|
||||
echo "=== Starting DuckDB Lance Tests ==="
|
||||
|
||||
@@ -1279,14 +1212,10 @@ jobs:
|
||||
# so do not leave a token in the checkout for it to pick up.
|
||||
persist-credentials: false
|
||||
|
||||
- name: Clean up stale FUSE mounts
|
||||
uses: ./.github/actions/cleanup-stale-fuse-mounts
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v7
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
cache: ${{ runner.environment == 'github-hosted' }}
|
||||
id: go
|
||||
|
||||
- name: Configure Docker Hub mirror
|
||||
@@ -1315,7 +1244,7 @@ jobs:
|
||||
echo "=== System Information ==="
|
||||
uname -a
|
||||
free -h
|
||||
timeout 15 df -h || true
|
||||
df -h
|
||||
docker info
|
||||
echo "=== Starting Spark Lance Namespace Tests ==="
|
||||
|
||||
@@ -1357,7 +1286,6 @@ jobs:
|
||||
uses: actions/setup-go@v7
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
cache: ${{ runner.environment == 'github-hosted' }}
|
||||
id: go
|
||||
|
||||
- name: Verify S3 Tables Package Builds
|
||||
@@ -1423,7 +1351,6 @@ jobs:
|
||||
uses: actions/setup-go@v7
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
cache: ${{ runner.environment == 'github-hosted' }}
|
||||
id: go
|
||||
|
||||
- name: Check Go Format
|
||||
@@ -1475,7 +1402,6 @@ jobs:
|
||||
uses: actions/setup-go@v7
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
cache: ${{ runner.environment == 'github-hosted' }}
|
||||
id: go
|
||||
|
||||
- name: Run Go Vet
|
||||
|
||||
@@ -25,8 +25,8 @@ on:
|
||||
- '.github/workflows/s3tests.yml'
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.event.pull_request.number || (github.event_name == 'push' && github.ref) || github.run_id }}
|
||||
cancel-in-progress: ${{ github.event_name == 'pull_request' || (github.event_name == 'push' && !startsWith(github.ref, 'refs/tags/')) }}
|
||||
group: ${{ github.head_ref }}/s3tests
|
||||
cancel-in-progress: true
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
@@ -44,7 +44,6 @@ jobs:
|
||||
uses: actions/setup-go@v7
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
cache: ${{ runner.environment == 'github-hosted' }}
|
||||
id: go
|
||||
|
||||
- name: Set up Python
|
||||
@@ -58,9 +57,9 @@ jobs:
|
||||
cd s3-tests
|
||||
sudo apt-get update -qq
|
||||
sudo apt-get install -y -qq libxml2-dev libxslt1-dev zlib1g-dev
|
||||
python3 -m pip install -r requirements.txt
|
||||
python3 -m pip install tox
|
||||
python3 -m pip install -e .
|
||||
pip install -r requirements.txt
|
||||
pip install tox
|
||||
pip install -e .
|
||||
|
||||
- name: Fix S3 tests bucket creation conflicts
|
||||
run: |
|
||||
@@ -397,7 +396,6 @@ jobs:
|
||||
uses: actions/setup-go@v7
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
cache: ${{ runner.environment == 'github-hosted' }}
|
||||
id: go
|
||||
|
||||
- name: Set up Python
|
||||
@@ -411,9 +409,9 @@ jobs:
|
||||
cd s3-tests
|
||||
sudo apt-get update -qq
|
||||
sudo apt-get install -y -qq libxml2-dev libxslt1-dev zlib1g-dev
|
||||
python3 -m pip install -r requirements.txt
|
||||
python3 -m pip install tox
|
||||
python3 -m pip install -e .
|
||||
pip install -r requirements.txt
|
||||
pip install tox
|
||||
pip install -e .
|
||||
|
||||
- name: Fix S3 tests bucket creation conflicts
|
||||
run: |
|
||||
@@ -568,7 +566,6 @@ jobs:
|
||||
uses: actions/setup-go@v7
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
cache: ${{ runner.environment == 'github-hosted' }}
|
||||
id: go
|
||||
|
||||
- name: Set up Python
|
||||
@@ -582,9 +579,9 @@ jobs:
|
||||
cd s3-tests
|
||||
sudo apt-get update -qq
|
||||
sudo apt-get install -y -qq libxml2-dev libxslt1-dev zlib1g-dev
|
||||
python3 -m pip install -r requirements.txt
|
||||
python3 -m pip install tox
|
||||
python3 -m pip install -e .
|
||||
pip install -r requirements.txt
|
||||
pip install tox
|
||||
pip install -e .
|
||||
|
||||
- name: Run S3 CORS tests
|
||||
timeout-minutes: 10
|
||||
@@ -694,7 +691,6 @@ jobs:
|
||||
uses: actions/setup-go@v7
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
cache: ${{ runner.environment == 'github-hosted' }}
|
||||
id: go
|
||||
|
||||
- name: Run SeaweedFS Custom S3 Copy tests
|
||||
@@ -797,7 +793,6 @@ jobs:
|
||||
uses: actions/setup-go@v7
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
cache: ${{ runner.environment == 'github-hosted' }}
|
||||
id: go
|
||||
|
||||
- name: Set up Python
|
||||
@@ -811,9 +806,9 @@ jobs:
|
||||
cd s3-tests
|
||||
sudo apt-get update -qq
|
||||
sudo apt-get install -y -qq libxml2-dev libxslt1-dev zlib1g-dev
|
||||
python3 -m pip install -r requirements.txt
|
||||
python3 -m pip install tox
|
||||
python3 -m pip install -e .
|
||||
pip install -r requirements.txt
|
||||
pip install tox
|
||||
pip install -e .
|
||||
|
||||
- name: Run Ceph S3 tests with SQL store
|
||||
timeout-minutes: 15
|
||||
|
||||
@@ -40,23 +40,15 @@ jobs:
|
||||
uses: actions/setup-go@v7
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
cache: ${{ runner.environment == 'github-hosted' }}
|
||||
|
||||
- name: Start local Docker registry
|
||||
run: |
|
||||
if ! docker start registry 2>/dev/null; then
|
||||
docker rm -f registry >/dev/null 2>&1 || true
|
||||
docker run -d --restart=always -p 5000:5000 --name registry registry:2
|
||||
fi
|
||||
run: docker run -d --restart=always -p 5000:5000 --name registry registry:2
|
||||
|
||||
- name: Set up Docker Buildx
|
||||
uses: docker/setup-buildx-action@v4
|
||||
with:
|
||||
driver-opts: network=host
|
||||
|
||||
- name: Set up Docker Compose
|
||||
uses: docker/setup-compose-action@v2
|
||||
|
||||
- name: Build weed race binary
|
||||
run: |
|
||||
cd docker
|
||||
|
||||
@@ -17,8 +17,8 @@ on:
|
||||
- '.github/workflows/sftp-tests.yml'
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.event.pull_request.number || (github.event_name == 'push' && github.ref) || github.run_id }}
|
||||
cancel-in-progress: ${{ github.event_name == 'pull_request' || (github.event_name == 'push' && !startsWith(github.ref, 'refs/tags/')) }}
|
||||
group: ${{ github.head_ref }}/sftp-tests
|
||||
cancel-in-progress: true
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
@@ -40,7 +40,6 @@ jobs:
|
||||
uses: actions/setup-go@v7
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
cache: ${{ runner.environment == 'github-hosted' }}
|
||||
|
||||
- name: Install dependencies
|
||||
run: |
|
||||
|
||||
@@ -15,10 +15,6 @@ on:
|
||||
- '.github/workflows/spark-integration-tests.yml'
|
||||
workflow_dispatch:
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.event.pull_request.number || (github.event_name == 'push' && github.ref) || github.run_id }}
|
||||
cancel-in-progress: ${{ github.event_name == 'pull_request' || (github.event_name == 'push' && !startsWith(github.ref, 'refs/tags/')) }}
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
checks: write
|
||||
@@ -42,13 +38,12 @@ jobs:
|
||||
with:
|
||||
java-version: '11'
|
||||
distribution: 'temurin'
|
||||
cache: ${{ runner.environment == 'github-hosted' && 'maven' || '' }}
|
||||
cache: maven
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v7
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
cache: ${{ runner.environment == 'github-hosted' }}
|
||||
|
||||
- name: Build SeaweedFS binary
|
||||
run: |
|
||||
|
||||
@@ -10,11 +10,12 @@ permissions:
|
||||
contents: write
|
||||
|
||||
concurrency:
|
||||
# Only one chart regeneration per branch at a time so overlapping runs never
|
||||
# conflict on note/star_history.svg during rebase. Scoped by ref so a manual
|
||||
# run on another branch can't cancel the daily master update.
|
||||
# Only one chart regeneration per branch at a time; a newer run on the same
|
||||
# branch cancels an in-flight one so overlapping runs never conflict on
|
||||
# note/star_history.svg during rebase. Scoped by ref so a manual run on
|
||||
# another branch can't cancel the daily master update.
|
||||
group: star-history-${{ github.ref }}
|
||||
cancel-in-progress: ${{ github.event_name == 'pull_request' || (github.event_name == 'push' && !startsWith(github.ref, 'refs/tags/')) }}
|
||||
cancel-in-progress: true
|
||||
|
||||
jobs:
|
||||
render:
|
||||
@@ -32,7 +33,7 @@ jobs:
|
||||
uses: actions/setup-python@v7
|
||||
with:
|
||||
python-version: "3.x"
|
||||
cache: ${{ runner.environment == 'github-hosted' && 'pip' || '' }}
|
||||
cache: pip
|
||||
cache-dependency-path: .github/scripts/star_history.py
|
||||
|
||||
- name: Install matplotlib
|
||||
|
||||
@@ -14,10 +14,6 @@ on:
|
||||
- 'weed/telemetry/**'
|
||||
- '.github/workflows/telemetry-integration.yml'
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.event.pull_request.number || (github.event_name == 'push' && github.ref) || github.run_id }}
|
||||
cancel-in-progress: ${{ github.event_name == 'pull_request' || (github.event_name == 'push' && !startsWith(github.ref, 'refs/tags/')) }}
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
@@ -34,7 +30,6 @@ jobs:
|
||||
uses: actions/setup-go@v7
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
cache: ${{ runner.environment == 'github-hosted' }}
|
||||
|
||||
- name: Build telemetry server
|
||||
run: cd telemetry/server && go build -o telemetry-server .
|
||||
|
||||
@@ -8,10 +8,6 @@ on:
|
||||
branches: [ master ]
|
||||
paths: ['terraform/**', '.github/workflows/terraform_ci.yml']
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.event.pull_request.number || (github.event_name == 'push' && github.ref) || github.run_id }}
|
||||
cancel-in-progress: ${{ github.event_name == 'pull_request' || (github.event_name == 'push' && !startsWith(github.ref, 'refs/tags/')) }}
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
@@ -66,7 +62,6 @@ jobs:
|
||||
uses: actions/setup-go@v7
|
||||
with:
|
||||
go-version-file: go.mod
|
||||
cache: ${{ runner.environment == 'github-hosted' }}
|
||||
|
||||
- name: Build weed
|
||||
run: go build -o "$RUNNER_TEMP/weed" ./weed
|
||||
|
||||
@@ -27,10 +27,6 @@ defaults:
|
||||
run:
|
||||
working-directory: weed
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.event.pull_request.number || (github.event_name == 'push' && github.ref) || github.run_id }}
|
||||
cancel-in-progress: ${{ github.event_name == 'pull_request' || (github.event_name == 'push' && !startsWith(github.ref, 'refs/tags/')) }}
|
||||
|
||||
jobs:
|
||||
awscli-tests:
|
||||
runs-on: ubuntu-latest
|
||||
@@ -41,7 +37,6 @@ jobs:
|
||||
- uses: actions/setup-go@v7
|
||||
with:
|
||||
go-version: ^1.26
|
||||
cache: ${{ runner.environment == 'github-hosted' }}
|
||||
|
||||
- name: Build SeaweedFS
|
||||
run: |
|
||||
|
||||
@@ -18,10 +18,6 @@ on:
|
||||
- 'go.sum'
|
||||
- '.github/workflows/tls-rotation-tests.yml'
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.event.pull_request.number || (github.event_name == 'push' && github.ref) || github.run_id }}
|
||||
cancel-in-progress: ${{ github.event_name == 'pull_request' || (github.event_name == 'push' && !startsWith(github.ref, 'refs/tags/')) }}
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
@@ -34,7 +30,6 @@ jobs:
|
||||
- name: Set up Go 1.x
|
||||
uses: actions/setup-go@v7
|
||||
with:
|
||||
cache: ${{ runner.environment == 'github-hosted' }}
|
||||
go-version: ^1.26
|
||||
id: go
|
||||
|
||||
|
||||
@@ -11,8 +11,8 @@ on:
|
||||
- '.github/workflows/tus-tests.yml'
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.event.pull_request.number || (github.event_name == 'push' && github.ref) || github.run_id }}
|
||||
cancel-in-progress: ${{ github.event_name == 'pull_request' || (github.event_name == 'push' && !startsWith(github.ref, 'refs/tags/')) }}
|
||||
group: ${{ github.head_ref || github.ref }}/tus-tests
|
||||
cancel-in-progress: true
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
@@ -31,14 +31,10 @@ jobs:
|
||||
- name: Check out code
|
||||
uses: actions/checkout@v7
|
||||
|
||||
- name: Clean up stale FUSE mounts
|
||||
uses: ./.github/actions/cleanup-stale-fuse-mounts
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v7
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
cache: ${{ runner.environment == 'github-hosted' }}
|
||||
id: go
|
||||
|
||||
- name: Install SeaweedFS
|
||||
@@ -53,7 +49,7 @@ jobs:
|
||||
echo "=== System Information ==="
|
||||
uname -a
|
||||
free -h
|
||||
timeout 15 df -h || true
|
||||
df -h
|
||||
echo "=== Starting TUS Tests ==="
|
||||
|
||||
# Run tests with automatic server management
|
||||
|
||||
@@ -18,10 +18,6 @@ on:
|
||||
- 'go.sum'
|
||||
- '.github/workflows/vacuum-integration-tests.yml'
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.event.pull_request.number || (github.event_name == 'push' && github.ref) || github.run_id }}
|
||||
cancel-in-progress: ${{ github.event_name == 'pull_request' || (github.event_name == 'push' && !startsWith(github.ref, 'refs/tags/')) }}
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
@@ -34,7 +30,6 @@ jobs:
|
||||
- name: Set up Go 1.x
|
||||
uses: actions/setup-go@v7
|
||||
with:
|
||||
cache: ${{ runner.environment == 'github-hosted' }}
|
||||
go-version: ^1.26
|
||||
id: go
|
||||
|
||||
|
||||
@@ -9,7 +9,6 @@ on:
|
||||
- 'weed/storage/**'
|
||||
- 'weed/pb/volume_server.proto'
|
||||
- 'weed/pb/volume_server_pb/**'
|
||||
- 'rust/volume_server/**'
|
||||
- '.github/workflows/volume-server-integration-tests.yml'
|
||||
push:
|
||||
branches: [ master, main ]
|
||||
@@ -19,12 +18,11 @@ on:
|
||||
- 'weed/storage/**'
|
||||
- 'weed/pb/volume_server.proto'
|
||||
- 'weed/pb/volume_server_pb/**'
|
||||
- 'rust/volume_server/**'
|
||||
- '.github/workflows/volume-server-integration-tests.yml'
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.event.pull_request.number || (github.event_name == 'push' && github.ref) || github.run_id }}
|
||||
cancel-in-progress: ${{ github.event_name == 'pull_request' || (github.event_name == 'push' && !startsWith(github.ref, 'refs/tags/')) }}
|
||||
group: ${{ github.head_ref || github.ref }}/volume-server-integration-tests
|
||||
cancel-in-progress: true
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
@@ -35,44 +33,10 @@ env:
|
||||
SHARD_COUNT: 3
|
||||
|
||||
jobs:
|
||||
build-weed:
|
||||
name: Build SeaweedFS Binary
|
||||
runs-on: ubuntu-22.04
|
||||
timeout-minutes: 15
|
||||
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v7
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v7
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
cache: ${{ runner.environment == 'github-hosted' }}
|
||||
|
||||
- name: Build SeaweedFS binary
|
||||
run: |
|
||||
cd weed
|
||||
go build -o weed .
|
||||
chmod +x weed
|
||||
./weed version
|
||||
|
||||
- name: Package binary
|
||||
run: tar cf weed-binary.tar weed/weed
|
||||
|
||||
- name: Upload binary
|
||||
uses: actions/upload-artifact@v7
|
||||
with:
|
||||
name: weed-binary
|
||||
if-no-files-found: error
|
||||
retention-days: 1
|
||||
path: weed-binary.tar
|
||||
|
||||
volume-server-integration-tests:
|
||||
name: Volume Server Integration Tests (${{ matrix.test-type }} - Shard ${{ matrix.shard }})
|
||||
runs-on: ubuntu-22.04
|
||||
timeout-minutes: 45
|
||||
needs: [build-weed]
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
@@ -87,18 +51,13 @@ jobs:
|
||||
uses: actions/setup-go@v7
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
cache: ${{ runner.environment == 'github-hosted' }}
|
||||
|
||||
- name: Download binary
|
||||
uses: actions/download-artifact@v7
|
||||
with:
|
||||
name: weed-binary
|
||||
|
||||
- name: Restore binary
|
||||
- name: Build SeaweedFS binary
|
||||
run: |
|
||||
tar xf weed-binary.tar
|
||||
chmod +x weed/weed
|
||||
./weed/weed version
|
||||
cd weed
|
||||
go build -o weed .
|
||||
chmod +x weed
|
||||
./weed version
|
||||
|
||||
# Dealing the listed tests out one by one keeps the shards even. Bucketing
|
||||
# them by first letter did not: names cluster, so ^Test[I-S] drew 50 of
|
||||
@@ -143,77 +102,3 @@ jobs:
|
||||
echo "## Volume Server Integration Test Summary (${{ matrix.test-type }} - Shard ${{ matrix.shard }})" >> "$GITHUB_STEP_SUMMARY"
|
||||
echo "- Suite: test/volume_server/${{ matrix.test-type }} (shard ${{ matrix.shard }} of ${SHARD_COUNT}, see 'Select this shard's tests' for the split)" >> "$GITHUB_STEP_SUMMARY"
|
||||
echo "- Command: go test -v -count=1 -timeout=${{ env.TEST_TIMEOUT }} ./test/volume_server/${{ matrix.test-type }}/... -run \"\${TEST_PATTERN}\"" >> "$GITHUB_STEP_SUMMARY"
|
||||
|
||||
volume-server-rust-smoke:
|
||||
name: Volume Server Rust Smoke (${{ matrix.mode }})
|
||||
runs-on: ubuntu-22.04
|
||||
timeout-minutes: 40
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
mode: [exec, proxy, native]
|
||||
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v6
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
|
||||
- name: Set up Rust toolchain
|
||||
uses: dtolnay/rust-toolchain@stable
|
||||
|
||||
- name: Build SeaweedFS binary
|
||||
run: |
|
||||
cd weed
|
||||
go build -o weed .
|
||||
chmod +x weed
|
||||
./weed version
|
||||
|
||||
- name: Build Rust volume launcher
|
||||
run: |
|
||||
cd rust/volume_server
|
||||
cargo build --release
|
||||
|
||||
- name: Run Rust-mode HTTP smoke tests
|
||||
env:
|
||||
WEED_BINARY: ${{ github.workspace }}/weed/weed
|
||||
VOLUME_SERVER_IMPL: rust
|
||||
RUST_VOLUME_BINARY: ${{ github.workspace }}/rust/volume_server/target/release/weed-volume-rs
|
||||
VOLUME_SERVER_RUST_MODE: ${{ matrix.mode }}
|
||||
run: |
|
||||
go test -v -count=1 -timeout=25m ./test/volume_server/http/... -run "TestAdminStatusAndHealthz|TestUploadReadRangeHeadDeleteRoundTrip"
|
||||
|
||||
- name: Run Rust-mode gRPC smoke tests
|
||||
env:
|
||||
WEED_BINARY: ${{ github.workspace }}/weed/weed
|
||||
VOLUME_SERVER_IMPL: rust
|
||||
RUST_VOLUME_BINARY: ${{ github.workspace }}/rust/volume_server/target/release/weed-volume-rs
|
||||
VOLUME_SERVER_RUST_MODE: ${{ matrix.mode }}
|
||||
run: |
|
||||
go test -v -count=1 -timeout=25m ./test/volume_server/grpc/... -run "TestStateAndStatusRPCs|TestVolumeSyncStatusAndReadVolumeFileStatus"
|
||||
|
||||
- name: Collect logs on failure
|
||||
if: failure()
|
||||
run: |
|
||||
mkdir -p /tmp/volume-server-rust-smoke-logs
|
||||
find /tmp -maxdepth 1 -type d -name "seaweedfs_volume_server_it_*" -print -exec cp -r {} /tmp/volume-server-rust-smoke-logs/ \; || true
|
||||
|
||||
- name: Archive logs on failure
|
||||
if: failure()
|
||||
uses: actions/upload-artifact@v7
|
||||
with:
|
||||
name: volume-server-rust-smoke-logs-${{ matrix.mode }}
|
||||
path: /tmp/volume-server-rust-smoke-logs/
|
||||
if-no-files-found: warn
|
||||
retention-days: 7
|
||||
|
||||
- name: Rust smoke summary
|
||||
if: always()
|
||||
run: |
|
||||
echo "## Volume Server Rust Smoke Summary (${{ matrix.mode }})" >> "$GITHUB_STEP_SUMMARY"
|
||||
echo "- Mode: Go master + Rust volume server launcher (VOLUME_SERVER_IMPL=rust, VOLUME_SERVER_RUST_MODE=${{ matrix.mode }})" >> "$GITHUB_STEP_SUMMARY"
|
||||
echo "- HTTP Command: go test -v -count=1 -timeout=25m ./test/volume_server/http/... -run \"TestAdminStatusAndHealthz|TestUploadReadRangeHeadDeleteRoundTrip\"" >> "$GITHUB_STEP_SUMMARY"
|
||||
echo "- gRPC Command: go test -v -count=1 -timeout=25m ./test/volume_server/grpc/... -run \"TestStateAndStatusRPCs|TestVolumeSyncStatusAndReadVolumeFileStatus\"" >> "$GITHUB_STEP_SUMMARY"
|
||||
@@ -144,4 +144,3 @@ weed_bin
|
||||
telemetry/server/telemetry-server
|
||||
.aider*
|
||||
/seaweed-volume/docs
|
||||
__pycache__/
|
||||
@@ -3,7 +3,7 @@ FROM golang:1.26 AS builder
|
||||
RUN apt-get update
|
||||
RUN apt-get install -y build-essential libsnappy-dev zlib1g-dev libbz2-dev libgflags-dev liblz4-dev libzstd-dev
|
||||
|
||||
ARG ROCKSDB_VERSION=v11.1.2
|
||||
ARG ROCKSDB_VERSION=v10.10.1
|
||||
ENV ROCKSDB_VERSION=${ROCKSDB_VERSION}
|
||||
|
||||
# build RocksDB
|
||||
|
||||
@@ -3,7 +3,7 @@ FROM golang:1.26 AS builder
|
||||
RUN apt-get update
|
||||
RUN apt-get install -y build-essential libsnappy-dev zlib1g-dev libbz2-dev libgflags-dev liblz4-dev libzstd-dev
|
||||
|
||||
ARG ROCKSDB_VERSION=v11.1.2
|
||||
ARG ROCKSDB_VERSION=v10.10.1
|
||||
ENV ROCKSDB_VERSION=${ROCKSDB_VERSION}
|
||||
|
||||
# build RocksDB
|
||||
|
||||
@@ -362,7 +362,7 @@ require (
|
||||
github.com/flynn/noise v1.1.0 // indirect
|
||||
github.com/gabriel-vasile/mimetype v1.4.13 // indirect
|
||||
github.com/geoffgarside/ber v1.2.0 // indirect
|
||||
github.com/go-chi/chi/v5 v5.4.0 // indirect
|
||||
github.com/go-chi/chi/v5 v5.3.1 // indirect
|
||||
github.com/go-darwin/apfs v0.0.0-20211011131704-f84b94dbf348 // indirect
|
||||
github.com/go-jose/go-jose/v4 v4.1.5 // indirect
|
||||
github.com/go-logr/logr v1.4.4 // indirect
|
||||
|
||||
@@ -483,8 +483,8 @@ github.com/gin-gonic/gin v1.11.0 h1:OW/6PLjyusp2PPXtyxKHU0RbX6I/l28FTdDlae5ueWk=
|
||||
github.com/gin-gonic/gin v1.11.0/go.mod h1:+iq/FyxlGzII0KHiBGjuNn4UNENUlKbGlNmc+W50Dls=
|
||||
github.com/go-asn1-ber/asn1-ber v1.5.8 h1:H9AZkK22UOmfX8J84ubyaZxKJZ3FMHVwn8swoMML7iQ=
|
||||
github.com/go-asn1-ber/asn1-ber v1.5.8/go.mod h1:hEBeB/ic+5LoWskz+yKT7vGhhPYkProFKoKdwZRWMe0=
|
||||
github.com/go-chi/chi/v5 v5.4.0 h1:+ufzvFTGKyin26Or5DLrlIDq4ygIDBVcj7oVNeeaYFE=
|
||||
github.com/go-chi/chi/v5 v5.4.0/go.mod h1:d1IG7GtKPSjPyBP365PH8sAGVHk3qfB/IhjFNRH1wWM=
|
||||
github.com/go-chi/chi/v5 v5.3.1 h1:3j4HZLGZQ3JpMCrPJF/Jl3mYJfWLKBfNJ6quurUGCf8=
|
||||
github.com/go-chi/chi/v5 v5.3.1/go.mod h1:R+tYY2hNuVUUjxoPtqUdgBqevM9s9njzkTLutVsOCto=
|
||||
github.com/go-darwin/apfs v0.0.0-20211011131704-f84b94dbf348 h1:JnrjqG5iR07/8k7NqrLNilRsl3s1EPRQEGvbPyOce68=
|
||||
github.com/go-darwin/apfs v0.0.0-20211011131704-f84b94dbf348/go.mod h1:Czxo/d1g948LtrALAZdL04TL/HnkopquAjxYUuI02bo=
|
||||
github.com/go-errors/errors v1.5.1 h1:ZwEMSLRCapFLflTpT7NKaAc7ukJ8ZPEjzlxt8rPN8bk=
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
apiVersion: v1
|
||||
description: SeaweedFS
|
||||
name: seaweedfs
|
||||
appVersion: "4.50"
|
||||
appVersion: "4.48"
|
||||
# Dev note: Trigger a helm chart release by `git tag -a helm-<version>`
|
||||
version: 4.50.0
|
||||
version: 4.48.2
|
||||
@@ -170,6 +170,9 @@ spec:
|
||||
{{- if .Values.filer.metricsIp }}
|
||||
-metricsIp={{ .Values.filer.metricsIp }} \
|
||||
{{- end }}
|
||||
{{- if .Values.filer.redirectOnRead }}
|
||||
-redirectOnRead \
|
||||
{{- end }}
|
||||
{{- if .Values.filer.disableHttp }}
|
||||
-disableHttp \
|
||||
{{- end }}
|
||||
|
||||
@@ -172,8 +172,7 @@ spec:
|
||||
{{- $bucketName := .name }}
|
||||
{{- $bucketLock := or .lock .objectLock .withLock }}
|
||||
bucket_list=$(/bin/echo 's3.bucket.list' | /usr/bin/weed shell) || { echo "Error listing s3 buckets"; exit 1; }
|
||||
{{- /* One awk: grep -q exits early, awk takes SIGPIPE, pipefail fails the test. */}}
|
||||
if printf '%s\n' "$bucket_list" | awk -v bucket="{{ $bucketName }}" '$1 "" == bucket { found = 1 } END { exit !found }'; then
|
||||
if echo "$bucket_list" | awk '{print $1}' | grep -Fxq "{{ $bucketName }}"; then
|
||||
echo "Bucket '{{ $bucketName }}' already exists, skipping creation."
|
||||
else
|
||||
echo "Creating bucket '{{ $bucketName }}'..."
|
||||
|
||||
@@ -740,6 +740,9 @@ filer:
|
||||
# encrypt data on volume servers
|
||||
encryptVolumeData: false
|
||||
|
||||
# Whether proxy or redirect to volume server during file GET request
|
||||
redirectOnRead: false
|
||||
|
||||
# Limit sub dir listing size (default 100000)
|
||||
dirListLimit: 100000
|
||||
|
||||
|
||||
+1099
-1034
File diff suppressed because it is too large.
Load diff
|
Before Width: | Height: | Size: 53 KiB After Width: | Height: | Size: 55 KiB |
@@ -771,8 +771,7 @@ message FilerConf {
|
||||
string replication = 3;
|
||||
string ttl = 4;
|
||||
string disk_type = 5;
|
||||
// unset inherits from the enclosing path rule, set overrides it
|
||||
optional bool fsync = 6;
|
||||
bool fsync = 6;
|
||||
uint32 volume_growth_count = 7;
|
||||
bool read_only = 8;
|
||||
string data_center = 9;
|
||||
|
||||
Generated
-7
@@ -1,7 +0,0 @@
|
||||
# This file is automatically @generated by Cargo.
|
||||
# It is not intended for manual editing.
|
||||
version = 4
|
||||
|
||||
[[package]]
|
||||
name = "weed-volume-rs"
|
||||
version = "0.1.0"
|
||||
@@ -1,6 +0,0 @@
|
||||
[package]
|
||||
name = "weed-volume-rs"
|
||||
version = "0.1.0"
|
||||
edition = "2021"
|
||||
|
||||
[dependencies]
|
||||
@@ -1,245 +0,0 @@
|
||||
# Rust Volume Server Parity Implementation Plan
|
||||
|
||||
## Objective
|
||||
Implement a native Rust volume server that replicates Go volume-server behavior for HTTP and gRPC APIs, so it can become a drop-in replacement validated by existing integration suites.
|
||||
|
||||
## Current Focus (2026-02-16)
|
||||
- Program focus is now Rust implementation parity, not broad test expansion.
|
||||
- `test/volume_server` is treated as the parity gate.
|
||||
- Existing Rust launcher modes (`exec`, `proxy`) are transition tools; they are not the final target.
|
||||
|
||||
## Current Status
|
||||
- Rust crate and launcher are in place.
|
||||
- Integration harness can run:
|
||||
- Go master + Go volume (default)
|
||||
- Go master + Rust launcher (`VOLUME_SERVER_IMPL=rust`)
|
||||
- Rust launcher `proxy` mode has full-suite integration pass while delegating backend handlers to Go.
|
||||
- Rust launcher `native` mode is wired as the default Rust entrypoint and currently bootstraps via supervised Go backend delegation.
|
||||
- Native Rust HTTP handlers now serve control/surface paths in `native` mode:
|
||||
- `/status`, `/healthz`
|
||||
- `OPTIONS` admin/public behavior (method allow-list + CORS preflight headers)
|
||||
- `/ui/index.html` with config-driven JWT/access-ui gating
|
||||
- `/favicon.ico` and `/seaweedfsstatic/*`
|
||||
- public non-read methods (`POST`/`PUT`/`DELETE`/unsupported verbs) as `200` no-op passthrough parity
|
||||
- admin unsupported verbs as `400` parity
|
||||
- Native Rust control-surface parity now also includes:
|
||||
- `/healthz` status mirroring from backend state transitions (e.g. leave/stopping => `503`)
|
||||
- absolute-form HTTP request-target normalization before native route matching
|
||||
- Native Rust HTTP data-path prevalidation now includes:
|
||||
- early malformed vid/fid rejection (`400`) for GET/HEAD/POST/PUT fid-route shapes before delegation
|
||||
- slash-form fid-route parsing for `/{vid}/{fid}` and `/{vid}/{fid}/{filename}` with reserved-path exclusions (`/status`, `/healthz`, `/ui/index.html`, `/stats/*`, static assets)
|
||||
- write-path error prevalidation for fid routes:
|
||||
- malformed multipart form-data requests without a boundary => `400`
|
||||
- requests containing `Content-MD5` header => `400` mismatch parity branch
|
||||
- request bodies exceeding configured `-fileSizeLimitMB` (by `Content-Length`) => `400`
|
||||
- Native Rust API/storage handlers are not implemented yet.
|
||||
|
||||
## Parity Exit Criteria
|
||||
1. Native mode passes:
|
||||
- `env VOLUME_SERVER_IMPL=rust VOLUME_SERVER_RUST_MODE=native go test -count=1 ./test/volume_server/http`
|
||||
- `env VOLUME_SERVER_IMPL=rust VOLUME_SERVER_RUST_MODE=native go test -count=1 ./test/volume_server/grpc`
|
||||
2. CI runs native Rust mode integration coverage (at least smoke, then expanded shards).
|
||||
3. Rust mode defaults to native behavior for integration harness.
|
||||
4. Go-backend delegation is removed (or retained only as explicit fallback mode).
|
||||
|
||||
## Architecture Workstreams
|
||||
|
||||
### A. Runtime and Configuration Parity
|
||||
- [x] Add `native` runtime mode in `weed-volume-rs` (bootstrap delegation path).
|
||||
- [ ] Parse and honor volume-server CLI/config flags used by integration harness:
|
||||
- [ ] network/bind ports (`-ip`, `-port`, `-port.grpc`, `-port.public`)
|
||||
- [ ] master target/config dir/read mode/throttling/JWT-related config
|
||||
- [ ] size/timeout controls and maintenance state defaults
|
||||
- [ ] Implement graceful lifecycle behavior (signals, shutdown, readiness).
|
||||
|
||||
### B. Native HTTP Surface
|
||||
- [ ] Admin/control endpoints:
|
||||
- [x] `GET /status` (native Rust in `native` mode)
|
||||
- [x] `GET /healthz` (native Rust in `native` mode)
|
||||
- [x] `OPTIONS` admin/public method+CORS control behavior (native Rust in `native` mode)
|
||||
- [x] static/UI endpoints currently exercised
|
||||
- [ ] Data read path parity:
|
||||
- [ ] fid parsing/path variants
|
||||
- [ ] conditional headers (`If-Modified-Since`, `If-None-Match`)
|
||||
- [ ] range handling (single/multi/invalid)
|
||||
- [ ] deleted reads, auth checks, read-mode branches
|
||||
- [ ] chunk-manifest and compression/image transformation branches
|
||||
- [ ] Data write/delete parity:
|
||||
- [ ] write success/unchanged/error paths
|
||||
- [ ] replication and file-size-limit paths
|
||||
- [ ] delete and chunk-manifest delete branches
|
||||
- [ ] Method/CORS/public-port parity for split admin/public behavior.
|
||||
|
||||
### C. Native gRPC Surface
|
||||
- [ ] Control-plane RPCs:
|
||||
- [ ] `GetState`, `SetState`, `VolumeServerStatus`, `Ping`, `VolumeServerLeave`
|
||||
- [ ] admin lifecycle: allocate/mount/unmount/delete/configure/readonly/writable
|
||||
- [ ] Data RPCs:
|
||||
- [ ] `ReadNeedleBlob`, `ReadNeedleMeta`, `WriteNeedleBlob`
|
||||
- [ ] `BatchDelete`, `ReadAllNeedles`
|
||||
- [ ] sync/copy/receive and status endpoints
|
||||
- [ ] Stream RPCs:
|
||||
- [ ] tail sender/receiver
|
||||
- [ ] vacuum streams
|
||||
- [ ] query streams
|
||||
- [ ] Advanced families:
|
||||
- [ ] erasure coding RPC set
|
||||
- [ ] tiering/remote fetch
|
||||
- [ ] scrub/query mode matrix
|
||||
|
||||
### D. Storage Compatibility Layer
|
||||
- [ ] Implement volume data/index handling compatible with Go on-disk format.
|
||||
- [ ] Preserve cookie/checksum/timestamp semantics used by tests.
|
||||
- [ ] Match read/write/delete consistency and error mapping behavior.
|
||||
- [ ] Ensure EC metadata/data-path compatibility with existing files.
|
||||
|
||||
### E. Operational Hardening
|
||||
- [ ] Deterministic startup/readiness and shutdown semantics.
|
||||
- [ ] Log/error parity sufficient for debugging and CI triage.
|
||||
- [ ] Concurrency/timeout behavior alignment for throttling and streams.
|
||||
- [ ] Performance baseline checks vs Go for key flows.
|
||||
|
||||
## Milestone Plan
|
||||
|
||||
### M0 (Completed): Harness + Launcher Transition
|
||||
- [x] Rust launcher integrated into harness.
|
||||
- [x] Proxy mode full-suite validation with Go backend delegation.
|
||||
|
||||
### M1: Native Skeleton (Control Plane First)
|
||||
- [ ] `native` mode boots and serves:
|
||||
- [x] `/status`, `/healthz`
|
||||
- [ ] `GetState`, `SetState`, `VolumeServerStatus`, `Ping`, `VolumeServerLeave`
|
||||
- Gate:
|
||||
- [x] targeted HTTP/grpc control tests pass in `native` mode (delegated backend path).
|
||||
|
||||
### M2: Native Core Data Paths
|
||||
- [ ] Native HTTP read/write/delete baseline parity.
|
||||
- [ ] Native gRPC data baseline parity (`Read/WriteNeedle*`, `BatchDelete`, `ReadAllNeedles`).
|
||||
- Gate:
|
||||
- core HTTP and gRPC data suites pass in `native` mode.
|
||||
|
||||
### M3: Native Stream + Copy/Sync
|
||||
- [ ] Tail/copy/receive/sync paths in native mode.
|
||||
- Gate:
|
||||
- stream/copy families pass in `native` mode.
|
||||
|
||||
### M4: Native Advanced Feature Families
|
||||
- [ ] EC, tiering, scrub/query advanced branches.
|
||||
- Gate:
|
||||
- full `/test/volume_server/http` and `/test/volume_server/grpc` pass in `native` mode.
|
||||
|
||||
### M5: CI/Cutover
|
||||
- [x] Add/expand native-mode CI jobs (smoke matrix includes `native`).
|
||||
- [x] Make native mode default for Rust integration runs.
|
||||
- [ ] Keep `exec`/`proxy` only as explicit fallback modes during rollout.
|
||||
|
||||
## Immediate Next Steps
|
||||
1. Implement minimal native gRPC state/ping RPC handlers (`GetState`, `SetState`, `VolumeServerStatus`, `Ping`, `VolumeServerLeave`).
|
||||
2. Expand native HTTP parity from control/static surface into data-path handlers (read/write/delete variants) while preserving current native control behavior.
|
||||
3. Keep rerunning native-mode integration suites as each delegated branch is replaced.
|
||||
4. Add mismatch triage notes for each API moved from delegation to native implementation.
|
||||
|
||||
## Risk Register
|
||||
- On-disk format mismatch risk:
|
||||
- Mitigation: implement format-level compatibility tests early (idx/dat/needle encoding).
|
||||
- Behavioral drift in edge branches:
|
||||
- Mitigation: use integration suite failures as primary truth; only add tests for newly discovered untracked branches.
|
||||
- Stream/concurrency semantic mismatch:
|
||||
- Mitigation: stabilize with focused interruption/timeout parity tests.
|
||||
|
||||
## Progress Log
|
||||
- Date: 2026-02-15
|
||||
- Change: Added Rust launcher integration (`exec`) and harness wiring.
|
||||
- Validation: Rust launcher mode passed smoke and full integration suites while delegating to Go backend.
|
||||
- Commits: `7beab85c2`, `880c2e1da`, `63d08e8a9`, `d402573ea`, `3bd20e6a1`, `6ce4d7ede`
|
||||
|
||||
- Date: 2026-02-15
|
||||
- Change: Added Rust proxy supervisor mode and validated full integration suite.
|
||||
- Validation:
|
||||
- `env VOLUME_SERVER_IMPL=rust VOLUME_SERVER_RUST_MODE=proxy go test -count=1 ./test/volume_server/http`
|
||||
- `env VOLUME_SERVER_IMPL=rust VOLUME_SERVER_RUST_MODE=proxy go test -count=1 ./test/volume_server/grpc`
|
||||
- Commits: `a7f50d23b`, `548b3d9a3`
|
||||
|
||||
- Date: 2026-02-16
|
||||
- Change: Re-focused plan from test expansion to native Rust implementation parity.
|
||||
- Validation basis: latest Rust proxy full-suite pass keeps regression baseline stable while native implementation starts.
|
||||
- Commits: `14c863dbf`
|
||||
|
||||
- Date: 2026-02-16
|
||||
- Change: Added native Rust launcher mode bootstrap, set Rust launcher default mode to `native`, and expanded CI Rust smoke matrix to include `native`.
|
||||
- Validation:
|
||||
- `env VOLUME_SERVER_IMPL=rust VOLUME_SERVER_RUST_MODE=native go test -count=1 ./test/volume_server/http`
|
||||
- `env VOLUME_SERVER_IMPL=rust VOLUME_SERVER_RUST_MODE=native go test -count=1 ./test/volume_server/grpc`
|
||||
- `env VOLUME_SERVER_IMPL=rust go test -count=1 ./test/volume_server/http -run '^TestAdminStatusAndHealthz$'`
|
||||
- `env VOLUME_SERVER_IMPL=rust go test -count=1 ./test/volume_server/grpc -run '^TestStateAndStatusRPCs$'`
|
||||
- Commits: `70ddbee37`, `61befd10f`, `2e65966c0`
|
||||
|
||||
- Date: 2026-02-16
|
||||
- Change: Implemented first native Rust HTTP handlers in `native` mode for `/status` and `/healthz` on the admin listener while preserving proxy delegation for other APIs.
|
||||
- Validation:
|
||||
- `env VOLUME_SERVER_IMPL=rust VOLUME_SERVER_RUST_MODE=native go test -count=1 ./test/volume_server/http -run '^TestAdminStatusAndHealthz$'`
|
||||
- `env VOLUME_SERVER_IMPL=rust VOLUME_SERVER_RUST_MODE=native go test -count=1 ./test/volume_server/http`
|
||||
- `env VOLUME_SERVER_IMPL=rust VOLUME_SERVER_RUST_MODE=native go test -count=1 ./test/volume_server/grpc`
|
||||
- Commits: `7e6e0261a`
|
||||
|
||||
- Date: 2026-02-16
|
||||
- Change: Implemented native Rust `OPTIONS` handling for admin and public listeners in `native` mode, including method allow-list and origin-driven CORS response parity used by integration tests.
|
||||
- Validation:
|
||||
- `env VOLUME_SERVER_IMPL=rust VOLUME_SERVER_RUST_MODE=native go test -count=1 ./test/volume_server/http -run '^TestOptionsMethodsByPort$|^TestOptionsWithOriginIncludesCorsHeaders$'`
|
||||
- `env VOLUME_SERVER_IMPL=rust VOLUME_SERVER_RUST_MODE=native go test -count=1 ./test/volume_server/http`
|
||||
- `env VOLUME_SERVER_IMPL=rust VOLUME_SERVER_RUST_MODE=native go test -count=1 ./test/volume_server/grpc`
|
||||
- Commits: `fbff2cb39`
|
||||
|
||||
- Date: 2026-02-16
|
||||
- Change: Added native Rust UI/static/public-surface handlers in `native` mode:
|
||||
- `/ui/index.html` with config-driven JWT/access-ui gating parity
|
||||
- `/favicon.ico` and `/seaweedfsstatic/*` static endpoint parity
|
||||
- public non-read methods as `200` no-op passthrough parity
|
||||
- admin unsupported methods as `400` parity
|
||||
- Validation:
|
||||
- `env VOLUME_SERVER_IMPL=rust VOLUME_SERVER_RUST_MODE=native go test -count=1 ./test/volume_server/http -run '^TestAdminStatusAndHealthz$|^TestUiIndexNotExposedWhenJwtSigningEnabled$|^TestUiIndexExposedWhenJwtSigningEnabledAndAccessUITrue$|^TestStaticAssetEndpoints$|^TestStaticAssetEndpointsOnPublicPort$'`
|
||||
- `env VOLUME_SERVER_IMPL=rust VOLUME_SERVER_RUST_MODE=native go test -count=1 ./test/volume_server/http -run '^TestOptionsMethodsByPort$|^TestOptionsWithOriginIncludesCorsHeaders$|^TestPublicPortReadOnlyMethodBehavior$|^TestCorsAndUnsupportedMethodBehavior$|^TestUnsupportedMethodTraceParity$|^TestUnsupportedMethodPropfindParity$|^TestUnsupportedMethodConnectParity$|^TestUnsupportedMethodMkcolParity$'`
|
||||
- `env VOLUME_SERVER_IMPL=rust VOLUME_SERVER_RUST_MODE=native go test -count=1 ./test/volume_server/http`
|
||||
- `env VOLUME_SERVER_IMPL=rust VOLUME_SERVER_RUST_MODE=native go test -count=1 ./test/volume_server/grpc`
|
||||
- Commits: `23e4497b2`
|
||||
|
||||
- Date: 2026-02-16
|
||||
- Change: Hardened native Rust HTTP control-path parity:
|
||||
- `/healthz` now mirrors backend health status (`200`/`503`) in native mode
|
||||
- absolute-form request-target parsing (e.g. `GET http://host/path HTTP/1.1`) now normalizes to route path parity
|
||||
- Validation:
|
||||
- `env VOLUME_SERVER_IMPL=rust VOLUME_SERVER_RUST_MODE=native go test -count=1 ./test/volume_server/http -run '^TestAdminStatusAndHealthz$'`
|
||||
- `env VOLUME_SERVER_IMPL=rust VOLUME_SERVER_RUST_MODE=native go test -count=1 ./test/volume_server/http/...`
|
||||
- `env VOLUME_SERVER_IMPL=rust VOLUME_SERVER_RUST_MODE=native go test -count=1 ./test/volume_server/grpc/...`
|
||||
- Commits: `d6ff6ed6d`
|
||||
|
||||
- Date: 2026-02-16
|
||||
- Change: Added native malformed fid-route validation in `native` mode:
|
||||
- GET/HEAD/POST/PUT requests matching fid URL shapes now return Rust-native `400 Bad Request` for invalid vid/fid tokens
|
||||
- valid fid-route requests still delegate to backend handlers for storage/data execution paths
|
||||
- Validation:
|
||||
- `env VOLUME_SERVER_IMPL=rust VOLUME_SERVER_RUST_MODE=native go test -count=1 ./test/volume_server/http -run '^TestInvalidReadPathReturnsBadRequest$|^TestWriteInvalidVidAndFidReturnBadRequest$|^TestMalformedVidFidPathReturnsBadRequest$|^TestDownloadLimitInvalidVidWhileOverLimitReturnsBadRequest$'`
|
||||
- `env VOLUME_SERVER_IMPL=rust VOLUME_SERVER_RUST_MODE=native go test -count=1 ./test/volume_server/http/...`
|
||||
- `env VOLUME_SERVER_IMPL=rust VOLUME_SERVER_RUST_MODE=native go test -count=1 ./test/volume_server/grpc/...`
|
||||
- Commits: `1ce0174b2`
|
||||
|
||||
- Date: 2026-02-16
|
||||
- Change: Added native Rust write-path prevalidation branches for fid routes in `native` mode:
|
||||
- multipart form uploads without boundary now return native `400`
|
||||
- `Content-MD5` requests now return native `400` mismatch branch
|
||||
- `Content-Length` over configured `-fileSizeLimitMB` now returns native `400` limit branch
|
||||
- Validation:
|
||||
- `env VOLUME_SERVER_IMPL=rust VOLUME_SERVER_RUST_MODE=native go test -count=1 ./test/volume_server/http -run '^TestUploadReadRangeHeadDeleteRoundTrip$|^TestWriteMalformedMultipartAndMD5Mismatch$|^TestWriteRejectsPayloadOverFileSizeLimit$'`
|
||||
- `env VOLUME_SERVER_IMPL=rust VOLUME_SERVER_RUST_MODE=native go test -count=1 ./test/volume_server/http/...`
|
||||
- `env VOLUME_SERVER_IMPL=rust VOLUME_SERVER_RUST_MODE=native go test -count=1 ./test/volume_server/grpc/...`
|
||||
- Commits: `94cefd6f4`
|
||||
|
||||
- Date: 2026-02-16
|
||||
- Change: Broadened native fid-route parser to cover slash-form URLs:
|
||||
- `/{vid}/{fid}` and `/{vid}/{fid}/{filename}` now flow through native malformed vid/fid validation
|
||||
- added explicit non-fid exclusions for control/stats/static paths to preserve route parity
|
||||
- Validation:
|
||||
- `env VOLUME_SERVER_IMPL=rust VOLUME_SERVER_RUST_MODE=native go test -count=1 ./test/volume_server/http -run '^TestAdminStatusAndHealthz$|^TestReadPathShapesAndIfModifiedSince$|^TestMalformedVidFidPathReturnsBadRequest$'`
|
||||
- `env VOLUME_SERVER_IMPL=rust VOLUME_SERVER_RUST_MODE=native go test -count=1 ./test/volume_server/http/...`
|
||||
- `env VOLUME_SERVER_IMPL=rust VOLUME_SERVER_RUST_MODE=native go test -count=1 ./test/volume_server/grpc/...`
|
||||
- Commits: `e66983e9b`
|
||||
File diff suppressed because it is too large.
Load diff
Generated
+14
-39
@@ -86,7 +86,7 @@ version = "1.1.5"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "40c48f72fd53cd289104fc64099abca73db4166ad86ea0b4341abe65af83dadc"
|
||||
dependencies = [
|
||||
"windows-sys 0.60.2",
|
||||
"windows-sys 0.61.2",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -97,7 +97,7 @@ checksum = "291e6a250ff86cd4a820112fb8898808a366d8f9f58ce16d1f538353ad55747d"
|
||||
dependencies = [
|
||||
"anstyle",
|
||||
"once_cell_polyfill",
|
||||
"windows-sys 0.60.2",
|
||||
"windows-sys 0.61.2",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -510,12 +510,10 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "aws-smithy-json"
|
||||
version = "0.62.7"
|
||||
version = "0.62.5"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "701a947f4797e52a911e114a898667c746c39feea467bbd1abd7b3721f702ffa"
|
||||
checksum = "9648b0bb82a2eedd844052c6ad2a1a822d1f8e3adee5fbf668366717e428856a"
|
||||
dependencies = [
|
||||
"aws-smithy-runtime-api",
|
||||
"aws-smithy-schema",
|
||||
"aws-smithy-types",
|
||||
]
|
||||
|
||||
@@ -565,12 +563,11 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "aws-smithy-runtime-api"
|
||||
version = "1.12.3"
|
||||
version = "1.11.6"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "9db177daa6ba8afb9ee1aefcf548c907abcf52065e394ee11a92780057fe0e8c"
|
||||
checksum = "876ab3c9c29791ba4ba02b780a3049e21ec63dabda09268b175272c3733a79e6"
|
||||
dependencies = [
|
||||
"aws-smithy-async",
|
||||
"aws-smithy-runtime-api-macros",
|
||||
"aws-smithy-types",
|
||||
"bytes",
|
||||
"http 0.2.12",
|
||||
@@ -581,33 +578,11 @@ dependencies = [
|
||||
"zeroize",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "aws-smithy-runtime-api-macros"
|
||||
version = "1.0.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "8d7396fd9500589e62e460e987ecb671bad374934e55ec3b5f498cc7a8a8a7b7"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "aws-smithy-schema"
|
||||
version = "0.1.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "7442cb268338f0eb8278140a107c046756aa01093d8ef5e99628d34ae09c94f5"
|
||||
dependencies = [
|
||||
"aws-smithy-runtime-api",
|
||||
"aws-smithy-types",
|
||||
"http 1.4.0",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "aws-smithy-types"
|
||||
version = "1.5.0"
|
||||
version = "1.4.7"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "32b42fcf341259d85ca10fac9a2f6448a8ec691c6955a18e45bc3b71a85fab85"
|
||||
checksum = "9d73dbfbaa8e4bc57b9045137680b958d274823509a360abfd8e1d514d40c95c"
|
||||
dependencies = [
|
||||
"base64-simd",
|
||||
"bytes",
|
||||
@@ -1350,7 +1325,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb"
|
||||
dependencies = [
|
||||
"libc",
|
||||
"windows-sys 0.52.0",
|
||||
"windows-sys 0.61.2",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -2404,7 +2379,7 @@ version = "0.50.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "7957b9740744892f114936ab4a57b3f487491bbeafaf8083688b16841a4240e5"
|
||||
dependencies = [
|
||||
"windows-sys 0.59.0",
|
||||
"windows-sys 0.61.2",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -3416,7 +3391,7 @@ dependencies = [
|
||||
"errno 0.3.14",
|
||||
"libc",
|
||||
"linux-raw-sys 0.12.1",
|
||||
"windows-sys 0.52.0",
|
||||
"windows-sys 0.61.2",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -3701,7 +3676,7 @@ version = "1.4.8"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "c4db69cba1110affc0e9f7bcd48bbf87b3f4fc7c61fc9155afd4c469eb3d6c1b"
|
||||
dependencies = [
|
||||
"errno 0.2.8",
|
||||
"errno 0.3.14",
|
||||
"libc",
|
||||
]
|
||||
|
||||
@@ -3768,7 +3743,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "3a766e1110788c36f4fa1c2b71b387a7815aa65f88ce0229841826633d93723e"
|
||||
dependencies = [
|
||||
"libc",
|
||||
"windows-sys 0.60.2",
|
||||
"windows-sys 0.61.2",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -3902,7 +3877,7 @@ dependencies = [
|
||||
"getrandom 0.4.2",
|
||||
"once_cell",
|
||||
"rustix 1.1.4",
|
||||
"windows-sys 0.52.0",
|
||||
"windows-sys 0.61.2",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
|
||||
@@ -157,8 +157,6 @@ windows-sys = { version = "0.61", features = ["Win32_Storage_FileSystem"] }
|
||||
|
||||
[dev-dependencies]
|
||||
tempfile = "3"
|
||||
# Paused-time tests drive the write-queue batch window deterministically.
|
||||
tokio = { version = "1", features = ["full", "test-util"] }
|
||||
# Already a transitive dependency of aws-sdk-s3 at a single locked version;
|
||||
# needed directly only for the canned HttpClient in remote_storage::s3 tests.
|
||||
aws-smithy-runtime-api = "1"
|
||||
|
||||
@@ -233,25 +233,6 @@ pub struct Cli {
|
||||
#[arg(long = "volume.allowUntrustedRemoteEndpoints", default_value_t = false)]
|
||||
pub allow_untrusted_remote_endpoints: bool,
|
||||
|
||||
/// Minimum bytes an fsync batch gathers before a drained queue flushes
|
||||
/// without waiting.
|
||||
#[arg(long = "fsyncMinBatchBytes", default_value_t = 1024 * 1024)]
|
||||
pub fsync_min_batch_bytes: u64,
|
||||
|
||||
/// Minimum requests an fsync batch gathers before a drained queue flushes
|
||||
/// without waiting.
|
||||
#[arg(long = "fsyncMinBatchRequests", default_value_t = 128)]
|
||||
pub fsync_min_batch_requests: usize,
|
||||
|
||||
/// How long a below-minimum fsync batch waits for more requests once the
|
||||
/// queue drains; 0 flushes immediately.
|
||||
#[arg(
|
||||
long = "fsyncBatchWait",
|
||||
default_value = "500us",
|
||||
value_parser = parse_go_duration
|
||||
)]
|
||||
pub fsync_batch_wait: std::time::Duration,
|
||||
|
||||
/// A file of command line options, each line in optionName=optionValue format.
|
||||
#[arg(long = "options", default_value = "")]
|
||||
pub options: String,
|
||||
@@ -327,14 +308,6 @@ pub struct VolumeServerConfig {
|
||||
/// If true, FetchAndWriteNeedle skips remote S3 endpoint validation
|
||||
/// (allows loopback / link-local / metadata hosts).
|
||||
pub allow_untrusted_remote_endpoints: bool,
|
||||
/// Minimum bytes an fsync batch gathers before a drained queue flushes
|
||||
/// without waiting.
|
||||
pub fsync_min_batch_bytes: u64,
|
||||
/// Minimum requests an fsync batch gathers before a drained queue flushes
|
||||
/// without waiting.
|
||||
pub fsync_min_batch_requests: usize,
|
||||
/// How long a below-minimum fsync batch waits for stragglers.
|
||||
pub fsync_batch_wait: std::time::Duration,
|
||||
}
|
||||
|
||||
pub use crate::storage::needle_map::NeedleMapKind;
|
||||
@@ -965,9 +938,6 @@ fn resolve_config_with_env(cli: Cli, env: EnvLookup<'_>) -> VolumeServerConfig {
|
||||
concurrent_download_limit: cli.concurrent_download_limit_mb as i64 * 1024 * 1024,
|
||||
inflight_upload_data_timeout: cli.inflight_upload_data_timeout,
|
||||
inflight_download_data_timeout: cli.inflight_download_data_timeout,
|
||||
fsync_min_batch_bytes: cli.fsync_min_batch_bytes,
|
||||
fsync_min_batch_requests: cli.fsync_min_batch_requests,
|
||||
fsync_batch_wait: cli.fsync_batch_wait,
|
||||
has_slow_read: cli.has_slow_read,
|
||||
read_buffer_size_mb: cli.read_buffer_size_mb,
|
||||
ldb_timeout: cli.ldb_timeout,
|
||||
|
||||
@@ -23,7 +23,7 @@ use seaweed_volume::server::request_id::GrpcRequestIdLayer;
|
||||
use seaweed_volume::server::volume_server::{
|
||||
RuntimeMetricsConfig, VolumeServerState, build_metrics_router,
|
||||
};
|
||||
use seaweed_volume::server::write_queue::{self, WriteQueue};
|
||||
use seaweed_volume::server::write_queue::WriteQueue;
|
||||
use seaweed_volume::storage::store::Store;
|
||||
use seaweed_volume::storage::types::DiskType;
|
||||
|
||||
@@ -436,17 +436,7 @@ async fn run(
|
||||
// The write queue is always on: durable (fsync) uploads go through it so
|
||||
// a batch shares one .dat/.idx sync, the way Go's per-volume async writer
|
||||
// group-commits. Non-fsync writes stay on the synchronous path.
|
||||
let min_bytes = usize::try_from(config.fsync_min_batch_bytes)
|
||||
.map_err(|_| "fsyncMinBatchBytes exceeds this platform's maximum usize value")?;
|
||||
let wq = WriteQueue::new(
|
||||
state.clone(),
|
||||
128,
|
||||
write_queue::SyncBatchOption {
|
||||
min_bytes,
|
||||
min_requests: config.fsync_min_batch_requests,
|
||||
wait: config.fsync_batch_wait,
|
||||
},
|
||||
);
|
||||
let wq = WriteQueue::new(state.clone(), 128);
|
||||
let _ = state.write_queue.set(wq);
|
||||
|
||||
// Set initial metric gauges for concurrent limits and max volumes
|
||||
|
||||
@@ -393,9 +393,6 @@ mod tests {
|
||||
fix_jpg_orientation: false,
|
||||
read_mode: ReadMode::Local,
|
||||
allow_untrusted_remote_endpoints: false,
|
||||
fsync_min_batch_bytes: 1024 * 1024,
|
||||
fsync_min_batch_requests: 128,
|
||||
fsync_batch_wait: std::time::Duration::from_micros(500),
|
||||
cpu_profile: String::new(),
|
||||
mem_profile: String::new(),
|
||||
compaction_byte_per_second: 0,
|
||||
|
||||
@@ -207,15 +207,9 @@ fn read_needle_for_get(
|
||||
if request.try_meta_first(plan.size()) {
|
||||
plan.read_meta(&mut n)?;
|
||||
if n.cookie != cookie {
|
||||
// The meta read answered the request.
|
||||
plan.note_meta_only_success();
|
||||
return Ok(None);
|
||||
}
|
||||
if request.served_from_source(&n) {
|
||||
if request.is_head {
|
||||
// HEAD ends here; no payload read will record the success.
|
||||
plan.note_meta_only_success();
|
||||
}
|
||||
let info = plan.into_stream_info(&n);
|
||||
return Ok(Some((n, Some(info))));
|
||||
}
|
||||
@@ -243,9 +237,6 @@ struct StreamingBody {
|
||||
hold_read_lock_for_stream: bool,
|
||||
_held_read_lease: Option<crate::storage::volume::DataFileReadLease>,
|
||||
io_unavailable: Arc<crate::storage::volume::IoUnavailable>,
|
||||
/// Chunk read failures feed the volume's io-error streak, and finishing
|
||||
/// the stream clears it — Go readNeedleDataInto parity.
|
||||
io_errors: Arc<crate::storage::io_error::IoErrorTracker>,
|
||||
/// Pending chunk read; it hands `buf` back with the result.
|
||||
pending: Option<tokio::task::JoinHandle<(bytes::BytesMut, std::io::Result<bytes::Bytes>)>>,
|
||||
/// Chunk buffer, reclaimed once the previous frame has been written out.
|
||||
@@ -307,18 +298,12 @@ impl http_body::Body for StreamingBody {
|
||||
),
|
||||
)));
|
||||
}
|
||||
// Hyper may not poll again once Content-Length is
|
||||
// satisfied, so the read's success is recorded here.
|
||||
self.io_errors.check_read_write_error(None);
|
||||
}
|
||||
return std::task::Poll::Ready(Some(Ok(http_body::Frame::data(
|
||||
chunk,
|
||||
))));
|
||||
}
|
||||
Ok(Err(e)) => {
|
||||
self.io_errors.check_read_write_error(Some(&e));
|
||||
return std::task::Poll::Ready(Some(Err(e)));
|
||||
}
|
||||
Ok(Err(e)) => return std::task::Poll::Ready(Some(Err(e))),
|
||||
Err(e) => {
|
||||
return std::task::Poll::Ready(Some(Err(std::io::Error::other(e))));
|
||||
}
|
||||
@@ -329,7 +314,6 @@ impl http_body::Body for StreamingBody {
|
||||
|
||||
let total = self.data_size as usize;
|
||||
if self.pos >= total {
|
||||
self.io_errors.check_read_write_error(None);
|
||||
return std::task::Poll::Ready(None);
|
||||
}
|
||||
|
||||
@@ -1121,10 +1105,6 @@ async fn get_or_head_handler_inner(
|
||||
|
||||
let (etag, last_modified_str) = etag_and_last_modified(&n);
|
||||
if let Some(resp) = not_modified_response(&n, &headers, &etag, &last_modified_str) {
|
||||
// A request that ends here was answered by the meta read alone.
|
||||
if let Some(io_errors) = strategy.io_errors() {
|
||||
io_errors.check_read_write_error(None);
|
||||
}
|
||||
return resp;
|
||||
}
|
||||
|
||||
@@ -1218,19 +1198,6 @@ struct ReadPlan {
|
||||
strategy: ReadStrategy,
|
||||
}
|
||||
|
||||
impl ReadStrategy {
|
||||
/// The tracker a terminal meta read reports success to, when the reply
|
||||
/// is served from the data file.
|
||||
fn io_errors(&self) -> Option<&Arc<crate::storage::io_error::IoErrorTracker>> {
|
||||
match self {
|
||||
ReadStrategy::Stream(info)
|
||||
| ReadStrategy::HeadFromMeta(info)
|
||||
| ReadStrategy::RangeFromSource(info) => Some(&info.io_errors),
|
||||
ReadStrategy::Buffered => None,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// The 401 reply for a bad read JWT; Go checks it before NewVolumeId, so a
|
||||
/// bad path with JWT enabled is a 401, not a 400.
|
||||
fn reject_read_jwt(
|
||||
@@ -1519,78 +1486,41 @@ fn not_modified_response(
|
||||
etag: &str,
|
||||
last_modified_str: &Option<String>,
|
||||
) -> Option<Response> {
|
||||
let not_modified = || {
|
||||
let mut resp = StatusCode::NOT_MODIFIED.into_response();
|
||||
resp.headers_mut()
|
||||
.insert(header::ETAG, etag.parse().unwrap());
|
||||
if let Some(lm) = last_modified_str {
|
||||
resp.headers_mut()
|
||||
.insert(header::LAST_MODIFIED, lm.parse().unwrap());
|
||||
}
|
||||
resp
|
||||
};
|
||||
|
||||
// If-Modified-Since applies only when there is no If-None-Match (RFC 9110 13.2.2).
|
||||
if headers.contains_key(header::IF_NONE_MATCH) {
|
||||
let inm = headers
|
||||
.get_all(header::IF_NONE_MATCH)
|
||||
.iter()
|
||||
.map(|v| v.to_str().unwrap_or(""))
|
||||
.collect::<Vec<_>>()
|
||||
.join(",");
|
||||
return etag_list_matches(&inm, etag).then(not_modified);
|
||||
}
|
||||
|
||||
// If-Modified-Since first (Go order), then If-None-Match.
|
||||
if n.last_modified > 0
|
||||
&& let Some(ims_str) = headers
|
||||
.get(header::IF_MODIFIED_SINCE)
|
||||
.and_then(|v| v.to_str().ok())
|
||||
&& let Some(ims_header) = headers.get(header::IF_MODIFIED_SINCE)
|
||||
&& let Ok(ims_str) = ims_header.to_str()
|
||||
{
|
||||
// Parse HTTP date format: "Mon, 02 Jan 2006 15:04:05 GMT"
|
||||
if let Ok(ims_time) =
|
||||
chrono::NaiveDateTime::parse_from_str(ims_str, "%a, %d %b %Y %H:%M:%S GMT")
|
||||
&& (n.last_modified as i64) <= ims_time.and_utc().timestamp()
|
||||
{
|
||||
return Some(not_modified());
|
||||
let mut resp = StatusCode::NOT_MODIFIED.into_response();
|
||||
if let Some(lm) = last_modified_str {
|
||||
resp.headers_mut()
|
||||
.insert(header::LAST_MODIFIED, lm.parse().unwrap());
|
||||
}
|
||||
// Go sets ETag after the 304 paths, so a 304 has no ETag.
|
||||
return Some(resp);
|
||||
}
|
||||
}
|
||||
|
||||
if let Some(if_none_match) = headers.get(header::IF_NONE_MATCH)
|
||||
&& let Ok(inm) = if_none_match.to_str()
|
||||
&& inm == etag
|
||||
{
|
||||
let mut resp = StatusCode::NOT_MODIFIED.into_response();
|
||||
if let Some(lm) = last_modified_str {
|
||||
resp.headers_mut()
|
||||
.insert(header::LAST_MODIFIED, lm.parse().unwrap());
|
||||
}
|
||||
// Go sets ETag after the 304 paths, so a 304 has no ETag.
|
||||
return Some(resp);
|
||||
}
|
||||
None
|
||||
}
|
||||
|
||||
/// Weak comparison against "*" or a comma-separated entity-tag list (RFC 9110 13.1.2).
|
||||
fn etag_list_matches(header: &str, etag: &str) -> bool {
|
||||
if header.trim() == "*" {
|
||||
return true;
|
||||
}
|
||||
let etag = etag.trim_matches('"');
|
||||
split_etag_list(header).into_iter().any(|tag| {
|
||||
let tag = tag.trim();
|
||||
let tag = tag.strip_prefix("W/").unwrap_or(tag);
|
||||
// Empty list elements are ignored, so padding like a trailing
|
||||
// comma cannot match an empty opaque tag.
|
||||
!tag.is_empty() && tag.trim_matches('"') == etag
|
||||
})
|
||||
}
|
||||
|
||||
/// Splits an entity-tag list on the commas between tags; a comma inside a
|
||||
/// quoted opaque tag does not split.
|
||||
fn split_etag_list(header: &str) -> Vec<&str> {
|
||||
let mut tags = Vec::new();
|
||||
let (mut start, mut quoted) = (0, false);
|
||||
for (i, b) in header.bytes().enumerate() {
|
||||
match b {
|
||||
b'"' => quoted = !quoted,
|
||||
b',' if !quoted => {
|
||||
tags.push(&header[start..i]);
|
||||
start = i + 1;
|
||||
}
|
||||
_ => {}
|
||||
}
|
||||
}
|
||||
tags.push(&header[start..]);
|
||||
tags
|
||||
}
|
||||
|
||||
/// The 200 reply headers, and the extension, which falls back to the stored name's.
|
||||
fn read_response_headers(
|
||||
n: &Needle,
|
||||
@@ -1792,7 +1722,6 @@ fn stream_response(
|
||||
data_file_access_control: info.data_file_access_control,
|
||||
hold_read_lock_for_stream: !state.has_slow_read,
|
||||
io_unavailable: info.io_unavailable,
|
||||
io_errors: info.io_errors,
|
||||
pending: None,
|
||||
buf: bytes::BytesMut::new(),
|
||||
state: tracking_state,
|
||||
@@ -2192,21 +2121,9 @@ fn handle_range_request_from_source(
|
||||
return Ok(Vec::new());
|
||||
}
|
||||
let mut buf = vec![0u8; length as usize];
|
||||
match info
|
||||
.source
|
||||
.read_exact_at(&mut buf, info.data_file_offset + start as u64)
|
||||
{
|
||||
// A finished range read clears the streak a failed meta read
|
||||
// counted, like readNeedleDataInto's terminal report.
|
||||
Ok(()) => {
|
||||
info.io_errors.check_read_write_error(None);
|
||||
Ok(buf)
|
||||
}
|
||||
Err(e) => {
|
||||
info.io_errors.check_read_write_error(Some(&e));
|
||||
Err(e)
|
||||
}
|
||||
}
|
||||
info.source
|
||||
.read_exact_at(&mut buf, info.data_file_offset + start as u64)?;
|
||||
Ok(buf)
|
||||
};
|
||||
|
||||
if ranges.len() == 1 {
|
||||
@@ -3039,62 +2956,50 @@ pub async fn post_handler(
|
||||
let needle_etag = n.etag();
|
||||
let needle_has_name = n.has_name();
|
||||
|
||||
// Replicate to remote volume servers if this volume has replicas.
|
||||
// Matches Go's GetWritableRemoteReplications: skip if copy_count == 1.
|
||||
let needs_replication = !is_replicate && {
|
||||
let store = state.store.read().unwrap();
|
||||
store
|
||||
.find_volume(vid)
|
||||
.is_some_and(|(_, v)| v.super_block().replica_placement.get_copy_count() > 1)
|
||||
};
|
||||
|
||||
// The fan-out runs concurrently with the local needle write, like Go's
|
||||
// ReplicatedWrite: it reads the already-buffered request body, so the two
|
||||
// sides share no mutable state. The ack still waits for both.
|
||||
let replication = if needs_replication {
|
||||
let state_clone = state.clone();
|
||||
let path_clone = path.clone();
|
||||
let query_clone = query.clone();
|
||||
let headers_clone = headers.clone();
|
||||
let body_clone = body.clone();
|
||||
Some(tokio::spawn(async move {
|
||||
do_replicated_request(
|
||||
&state_clone,
|
||||
vid.0,
|
||||
Method::POST,
|
||||
&path_clone,
|
||||
&query_clone,
|
||||
&headers_clone,
|
||||
Some(body_clone),
|
||||
)
|
||||
.await
|
||||
}))
|
||||
} else {
|
||||
None
|
||||
};
|
||||
|
||||
let write_result = write_upload_needle(&state, vid, n, fsync).await;
|
||||
|
||||
// Replicate to remote volume servers if this volume has replicas.
|
||||
// Matches Go's GetWritableRemoteReplications: skip if copy_count == 1.
|
||||
// No leader-known gate: when the lookup cannot run, the write must fail
|
||||
// rather than be acknowledged on fewer copies than required. The local
|
||||
// write result takes precedence, matching Go's error ordering.
|
||||
let replication_result = match replication {
|
||||
Some(task) => Some(
|
||||
task.await
|
||||
// rather than be acknowledged on fewer copies than required.
|
||||
if !is_replicate && write_result.is_ok() {
|
||||
let needs_replication = {
|
||||
let store = state.store.read().unwrap();
|
||||
store
|
||||
.find_volume(vid)
|
||||
.is_some_and(|(_, v)| v.super_block().replica_placement.get_copy_count() > 1)
|
||||
};
|
||||
if needs_replication {
|
||||
let state_clone = state.clone();
|
||||
let path_clone = path.clone();
|
||||
let query_clone = query.clone();
|
||||
let headers_clone = headers.clone();
|
||||
let body_clone = body.clone();
|
||||
let replication = tokio::spawn(async move {
|
||||
do_replicated_request(
|
||||
&state_clone,
|
||||
vid.0,
|
||||
Method::POST,
|
||||
&path_clone,
|
||||
&query_clone,
|
||||
&headers_clone,
|
||||
Some(body_clone),
|
||||
)
|
||||
.await
|
||||
});
|
||||
let replication_result = replication
|
||||
.await
|
||||
.map_err(|e| format!("replication task failed: {}", e))
|
||||
.flatten(),
|
||||
),
|
||||
None => None,
|
||||
};
|
||||
if write_result.is_ok()
|
||||
&& let Some(Err(e)) = replication_result
|
||||
{
|
||||
tracing::error!("replicated write failed: {}", e);
|
||||
return json_error_with_query(
|
||||
StatusCode::INTERNAL_SERVER_ERROR,
|
||||
format!("replication failed: {}", e),
|
||||
Some(&query),
|
||||
);
|
||||
.flatten();
|
||||
if let Err(e) = replication_result {
|
||||
tracing::error!("replicated write failed: {}", e);
|
||||
return json_error_with_query(
|
||||
StatusCode::INTERNAL_SERVER_ERROR,
|
||||
format!("replication failed: {}", e),
|
||||
Some(&query),
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
let resp = match write_result {
|
||||
@@ -5320,7 +5225,6 @@ mod tests {
|
||||
hold_read_lock_for_stream: true,
|
||||
_held_read_lease: None,
|
||||
io_unavailable: Arc::default(),
|
||||
io_errors: Arc::default(),
|
||||
pending: None,
|
||||
buf: bytes::BytesMut::new(),
|
||||
state: None,
|
||||
@@ -5693,80 +5597,6 @@ mod tests {
|
||||
delete_handler(State(state.clone()), request).await.status()
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_not_modified_checks_if_none_match_first() {
|
||||
let n = Needle {
|
||||
last_modified: 1700000000,
|
||||
..Needle::default()
|
||||
};
|
||||
let (etag, last_modified_str) = etag_and_last_modified(&n);
|
||||
let bare = n.etag();
|
||||
let before = "Tue, 14 Nov 2023 22:12:20 GMT".to_string();
|
||||
let after = "Tue, 14 Nov 2023 22:14:20 GMT".to_string();
|
||||
|
||||
let cases: Vec<(Vec<(axum::http::HeaderName, String)>, bool)> = vec![
|
||||
(vec![(header::IF_NONE_MATCH, format!("\"{bare}\""))], true),
|
||||
(vec![(header::IF_NONE_MATCH, format!("W/\"{bare}\""))], true),
|
||||
(
|
||||
vec![(header::IF_NONE_MATCH, format!("\"x\", \"{bare}\""))],
|
||||
true,
|
||||
),
|
||||
(vec![(header::IF_NONE_MATCH, "*".into())], true),
|
||||
(vec![(header::IF_NONE_MATCH, "\"x\"".into())], false),
|
||||
(
|
||||
vec![
|
||||
(header::IF_NONE_MATCH, "\"x\"".into()),
|
||||
(header::IF_NONE_MATCH, format!("\"{bare}\"")),
|
||||
],
|
||||
true,
|
||||
),
|
||||
(
|
||||
vec![(header::IF_NONE_MATCH, format!("\"{bare},old\""))],
|
||||
false,
|
||||
),
|
||||
(
|
||||
vec![(header::IF_NONE_MATCH, format!("\"x\",,\"{bare}\""))],
|
||||
true,
|
||||
),
|
||||
(
|
||||
vec![
|
||||
(header::IF_NONE_MATCH, "\"x\"".into()),
|
||||
(header::IF_MODIFIED_SINCE, after.clone()),
|
||||
],
|
||||
false,
|
||||
),
|
||||
(
|
||||
vec![
|
||||
(header::IF_NONE_MATCH, "".into()),
|
||||
(header::IF_MODIFIED_SINCE, after.clone()),
|
||||
],
|
||||
false,
|
||||
),
|
||||
(vec![(header::IF_MODIFIED_SINCE, after)], true),
|
||||
(vec![(header::IF_MODIFIED_SINCE, before)], false),
|
||||
];
|
||||
for (pairs, want) in cases {
|
||||
let mut headers = HeaderMap::new();
|
||||
for (name, value) in &pairs {
|
||||
headers.append(name, value.parse().unwrap());
|
||||
}
|
||||
assert_eq!(
|
||||
not_modified_response(&n, &headers, &etag, &last_modified_str).is_some(),
|
||||
want,
|
||||
"{pairs:?}"
|
||||
);
|
||||
}
|
||||
|
||||
let mut headers = HeaderMap::new();
|
||||
headers.insert(
|
||||
header::IF_NONE_MATCH,
|
||||
format!("\"{bare}\"").parse().unwrap(),
|
||||
);
|
||||
let resp = not_modified_response(&n, &headers, &etag, &last_modified_str).unwrap();
|
||||
assert!(resp.headers().contains_key(header::ETAG));
|
||||
assert!(resp.headers().contains_key(header::LAST_MODIFIED));
|
||||
}
|
||||
|
||||
/// A GET whose needle read is parked must not hold the store lock: a
|
||||
/// writer, here a real append to the same volume, must get through.
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
@@ -5941,44 +5771,6 @@ mod tests {
|
||||
assert_eq!(status, StatusCode::NOT_FOUND);
|
||||
}
|
||||
|
||||
/// A ranged read served from the data file reports its outcome to the
|
||||
/// io-error streak, like readNeedleDataInto: a finished range clears the
|
||||
/// count a failed meta read left.
|
||||
#[test]
|
||||
fn test_range_from_source_reports_to_io_error_streak() {
|
||||
use std::io::Write;
|
||||
|
||||
let tmp = tempfile::TempDir::new().unwrap();
|
||||
let file = std::fs::OpenOptions::new()
|
||||
.read(true)
|
||||
.write(true)
|
||||
.create(true)
|
||||
.open(tmp.path().join("1.dat"))
|
||||
.unwrap();
|
||||
let data: Vec<u8> = (0..256u32).map(|i| i as u8).collect();
|
||||
(&file).write_all(&data).unwrap();
|
||||
|
||||
let io_errors = Arc::new(crate::storage::io_error::IoErrorTracker::default());
|
||||
let eio = crate::storage::volume::media_error_for_test();
|
||||
io_errors.check_read_write_error(Some(&eio));
|
||||
io_errors.check_read_write_error(Some(&eio));
|
||||
assert_eq!(io_errors.get_io_error_state().1, 2);
|
||||
|
||||
let info = crate::storage::volume::NeedleStreamInfo {
|
||||
source: crate::storage::volume::NeedleStreamSource::Local(file),
|
||||
data_file_offset: 0,
|
||||
data_size: data.len() as u32,
|
||||
data_file_access_control: Arc::default(),
|
||||
io_unavailable: Arc::default(),
|
||||
io_errors: io_errors.clone(),
|
||||
needle_id: NeedleId(1),
|
||||
checksum: 0,
|
||||
};
|
||||
let resp = handle_range_request_from_source("bytes=10-19", info, HeaderMap::new(), None);
|
||||
assert_eq!(resp.status(), StatusCode::PARTIAL_CONTENT);
|
||||
assert_eq!(io_errors.get_io_error_state().1, 0);
|
||||
}
|
||||
|
||||
/// Range is read as Go reads it: a byte its parseRange cannot parse is a
|
||||
/// 416, Unicode whitespace around a range is trimmed, an empty value is no
|
||||
/// range, and HEAD ignores it.
|
||||
@@ -7431,7 +7223,6 @@ mod tests {
|
||||
.set(crate::server::write_queue::WriteQueue::new(
|
||||
state.clone(),
|
||||
128,
|
||||
crate::server::write_queue::SyncBatchOption::default(),
|
||||
))
|
||||
.is_ok()
|
||||
);
|
||||
|
||||
@@ -132,9 +132,6 @@ mod tests {
|
||||
fix_jpg_orientation: false,
|
||||
read_mode: ReadMode::Local,
|
||||
allow_untrusted_remote_endpoints: false,
|
||||
fsync_min_batch_bytes: 1024 * 1024,
|
||||
fsync_min_batch_requests: 128,
|
||||
fsync_batch_wait: std::time::Duration::from_micros(500),
|
||||
cpu_profile: String::new(),
|
||||
mem_profile: String::new(),
|
||||
compaction_byte_per_second: 0,
|
||||
|
||||
@@ -56,27 +56,6 @@ const MAX_CONCURRENT_VOLUME_WRITES: usize = 8;
|
||||
/// retirement is the volume leaving the store.
|
||||
const IDLE_WORKER_POLL: std::time::Duration = std::time::Duration::from_secs(60);
|
||||
|
||||
/// Tunes the per-volume durable-write worker. Once the queue drains with the
|
||||
/// batch still below min_bytes and min_requests, the batch stays open up to
|
||||
/// wait for stragglers instead of paying an fsync per request. Mirrors Go's
|
||||
/// storage.SyncBatchOption.
|
||||
#[derive(Clone, Copy)]
|
||||
pub struct SyncBatchOption {
|
||||
pub min_bytes: usize,
|
||||
pub min_requests: usize,
|
||||
pub wait: std::time::Duration,
|
||||
}
|
||||
|
||||
impl Default for SyncBatchOption {
|
||||
fn default() -> Self {
|
||||
SyncBatchOption {
|
||||
min_bytes: 1024 * 1024,
|
||||
min_requests: MAX_BATCH_SIZE,
|
||||
wait: std::time::Duration::from_micros(500),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Handle for submitting write requests to the per-volume workers.
|
||||
#[derive(Clone)]
|
||||
pub struct WriteQueue {
|
||||
@@ -88,35 +67,23 @@ pub struct WriteQueue {
|
||||
/// alone would multiply the old single queue's capacity by volume count.
|
||||
queue_permits: Arc<Semaphore>,
|
||||
workers: Arc<Mutex<HashMap<VolumeId, mpsc::Sender<WriteRequest>>>>,
|
||||
sync_batch: SyncBatchOption,
|
||||
/// Submissions accepted, for tests asserting which writes route here.
|
||||
#[cfg(test)]
|
||||
submits: Arc<std::sync::atomic::AtomicU64>,
|
||||
/// Times a worker armed the straggler window, for tests that must land a
|
||||
/// write inside an already-open batch.
|
||||
#[cfg(test)]
|
||||
windows_armed: Arc<std::sync::atomic::AtomicU64>,
|
||||
}
|
||||
|
||||
impl WriteQueue {
|
||||
/// Create a new write queue. Per-volume workers are spawned lazily on the
|
||||
/// first submission for that volume.
|
||||
pub fn new(
|
||||
state: Arc<VolumeServerState>,
|
||||
capacity: usize,
|
||||
sync_batch: SyncBatchOption,
|
||||
) -> Self {
|
||||
pub fn new(state: Arc<VolumeServerState>, capacity: usize) -> Self {
|
||||
WriteQueue {
|
||||
state,
|
||||
capacity,
|
||||
io_permits: Arc::new(Semaphore::new(MAX_CONCURRENT_VOLUME_WRITES)),
|
||||
queue_permits: Arc::new(Semaphore::new(capacity)),
|
||||
workers: Arc::new(Mutex::new(HashMap::new())),
|
||||
sync_batch,
|
||||
#[cfg(test)]
|
||||
submits: Arc::new(std::sync::atomic::AtomicU64::new(0)),
|
||||
#[cfg(test)]
|
||||
windows_armed: Arc::new(std::sync::atomic::AtomicU64::new(0)),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -126,12 +93,6 @@ impl WriteQueue {
|
||||
self.submits.load(std::sync::atomic::Ordering::Relaxed)
|
||||
}
|
||||
|
||||
/// How many batch windows workers have armed so far.
|
||||
#[cfg(test)]
|
||||
fn windows_armed_for_test(&self) -> u64 {
|
||||
self.windows_armed.load(std::sync::atomic::Ordering::Relaxed)
|
||||
}
|
||||
|
||||
/// Sender for the volume's worker, spawning one on first use.
|
||||
fn worker_for(&self, volume_id: VolumeId) -> mpsc::Sender<WriteRequest> {
|
||||
let mut workers = self.workers.lock().unwrap();
|
||||
@@ -146,9 +107,6 @@ impl WriteQueue {
|
||||
state: self.state.clone(),
|
||||
io_permits: self.io_permits.clone(),
|
||||
workers: self.workers.clone(),
|
||||
sync_batch: self.sync_batch,
|
||||
#[cfg(test)]
|
||||
windows_armed: self.windows_armed.clone(),
|
||||
}
|
||||
.run(),
|
||||
);
|
||||
@@ -233,9 +191,6 @@ struct VolumeWriteWorker {
|
||||
state: Arc<VolumeServerState>,
|
||||
io_permits: Arc<Semaphore>,
|
||||
workers: Arc<Mutex<HashMap<VolumeId, mpsc::Sender<WriteRequest>>>>,
|
||||
sync_batch: SyncBatchOption,
|
||||
#[cfg(test)]
|
||||
windows_armed: Arc<std::sync::atomic::AtomicU64>,
|
||||
}
|
||||
|
||||
impl VolumeWriteWorker {
|
||||
@@ -283,40 +238,14 @@ impl VolumeWriteWorker {
|
||||
let mut batch = Vec::with_capacity(MAX_BATCH_SIZE);
|
||||
let mut batch_bytes: usize = first.needle.data.len();
|
||||
batch.push(first);
|
||||
// Armed once the queue first drains below the minimum batch; holds the
|
||||
// batch open for stragglers so the wait bound is per batch, not per
|
||||
// lull (Go's batchWait).
|
||||
let mut window_deadline = None;
|
||||
while batch.len() < MAX_BATCH_SIZE && batch_bytes < MAX_BATCH_BYTES {
|
||||
if window_deadline.is_some_and(|d| tokio::time::Instant::now() >= d) {
|
||||
break;
|
||||
}
|
||||
let req = match self.rx.try_recv() {
|
||||
Ok(req) => req,
|
||||
Err(mpsc::error::TryRecvError::Empty) => {
|
||||
let sync = self.sync_batch;
|
||||
if sync.wait.is_zero()
|
||||
|| batch_bytes >= sync.min_bytes
|
||||
|| batch.len() >= sync.min_requests
|
||||
{
|
||||
// a drained queue flushes right away once it holds enough
|
||||
break;
|
||||
}
|
||||
let deadline = *window_deadline.get_or_insert_with(|| {
|
||||
#[cfg(test)]
|
||||
self.windows_armed
|
||||
.fetch_add(1, std::sync::atomic::Ordering::Relaxed);
|
||||
tokio::time::Instant::now() + sync.wait
|
||||
});
|
||||
match tokio::time::timeout_at(deadline, self.rx.recv()).await {
|
||||
Ok(Some(req)) => req,
|
||||
Ok(None) | Err(_) => break,
|
||||
}
|
||||
match self.rx.try_recv() {
|
||||
Ok(req) => {
|
||||
batch_bytes += req.needle.data.len();
|
||||
batch.push(req);
|
||||
}
|
||||
Err(mpsc::error::TryRecvError::Disconnected) => break,
|
||||
};
|
||||
batch_bytes += req.needle.data.len();
|
||||
batch.push(req);
|
||||
Err(_) => break,
|
||||
}
|
||||
}
|
||||
debug!("processing write batch of {} requests", batch.len());
|
||||
|
||||
@@ -451,7 +380,7 @@ mod tests {
|
||||
async fn test_write_queue_submit_no_volume() {
|
||||
// Submit a write to a non-existent volume -- should return VolumeError::NotFound.
|
||||
let state = make_test_state();
|
||||
let queue = WriteQueue::new(state, MAX_BATCH_SIZE, SyncBatchOption::default());
|
||||
let queue = WriteQueue::new(state, MAX_BATCH_SIZE);
|
||||
|
||||
let needle = Needle {
|
||||
id: 1.into(),
|
||||
@@ -473,7 +402,7 @@ mod tests {
|
||||
async fn test_write_queue_concurrent_submissions() {
|
||||
// Submit multiple concurrent writes -- all should complete (with errors since no volume).
|
||||
let state = make_test_state();
|
||||
let queue = WriteQueue::new(state, MAX_BATCH_SIZE, SyncBatchOption::default());
|
||||
let queue = WriteQueue::new(state, MAX_BATCH_SIZE);
|
||||
|
||||
let mut handles = Vec::new();
|
||||
for i in 0..10u64 {
|
||||
@@ -501,7 +430,7 @@ mod tests {
|
||||
async fn test_write_queue_batching() {
|
||||
// Verify that many concurrent writes get processed (testing the batching path).
|
||||
let state = make_test_state();
|
||||
let queue = WriteQueue::new(state, MAX_BATCH_SIZE, SyncBatchOption::default());
|
||||
let queue = WriteQueue::new(state, MAX_BATCH_SIZE);
|
||||
|
||||
// Submit MAX_BATCH_SIZE requests concurrently
|
||||
let mut handles = Vec::new();
|
||||
@@ -593,7 +522,7 @@ mod tests {
|
||||
async fn test_write_queue_dropped_sender() {
|
||||
// When the queue is dropped, subsequent submits should fail gracefully.
|
||||
let state = make_test_state();
|
||||
let queue = WriteQueue::new(state, 1, SyncBatchOption::default());
|
||||
let queue = WriteQueue::new(state, 1);
|
||||
|
||||
// Clone then drop the original -- the worker keeps running via its rx handle.
|
||||
let queue2 = queue.clone();
|
||||
@@ -736,7 +665,7 @@ mod tests {
|
||||
.unwrap();
|
||||
}
|
||||
}
|
||||
let queue = WriteQueue::new(state, MAX_BATCH_SIZE, SyncBatchOption::default());
|
||||
let queue = WriteQueue::new(state, MAX_BATCH_SIZE);
|
||||
|
||||
let in_append = Arc::new(Barrier::new(2));
|
||||
let release = Arc::new(Barrier::new(2));
|
||||
@@ -805,85 +734,4 @@ mod tests {
|
||||
assert!(a1.await.unwrap().is_ok());
|
||||
assert!(a2.await.unwrap().is_ok());
|
||||
}
|
||||
|
||||
/// A durable write below the minimum batch rides out its window so a
|
||||
/// straggler joins the same sync instead of each paying an fsync alone —
|
||||
/// Go's batchWait. Paused time makes the hold observable: a lone write can
|
||||
/// only complete once the runtime auto-advances to the window's deadline.
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn test_batch_window_defers_flush_for_stragglers() {
|
||||
use crate::config::MinFreeSpace;
|
||||
use crate::storage::types::{DiskType, NeedleId};
|
||||
use crate::storage::volume::VolumeSpec;
|
||||
use std::time::Duration;
|
||||
|
||||
let tmp = tempfile::TempDir::new().unwrap();
|
||||
let dir = tmp.path().to_str().unwrap();
|
||||
let state = make_test_state();
|
||||
{
|
||||
let mut store = state.store.write().unwrap();
|
||||
store
|
||||
.add_location(
|
||||
dir,
|
||||
dir,
|
||||
10,
|
||||
DiskType::HardDrive,
|
||||
MinFreeSpace::Percent(1.0),
|
||||
Vec::new(),
|
||||
)
|
||||
.unwrap();
|
||||
store
|
||||
.add_volume(VolumeId(1), DiskType::HardDrive, &VolumeSpec::default())
|
||||
.unwrap();
|
||||
}
|
||||
|
||||
let queue = WriteQueue::new(
|
||||
state,
|
||||
MAX_BATCH_SIZE,
|
||||
SyncBatchOption {
|
||||
min_bytes: usize::MAX,
|
||||
min_requests: MAX_BATCH_SIZE,
|
||||
wait: Duration::from_secs(30),
|
||||
},
|
||||
);
|
||||
|
||||
let needle = |id: u64| Needle {
|
||||
id: NeedleId(id),
|
||||
cookie: 0x1111.into(),
|
||||
data: vec![id as u8; 8],
|
||||
data_size: 8,
|
||||
..Needle::default()
|
||||
};
|
||||
|
||||
let start = tokio::time::Instant::now();
|
||||
queue.submit(VolumeId(1), needle(1), true).await.unwrap();
|
||||
assert!(start.elapsed() >= Duration::from_secs(30));
|
||||
|
||||
// Wait for the worker to take the second write and open its batch
|
||||
// window, so the straggler provably lands inside an open batch
|
||||
// instead of arriving during the initial drain.
|
||||
let armed = queue.windows_armed_for_test();
|
||||
let second = tokio::spawn({
|
||||
let queue = queue.clone();
|
||||
async move { queue.submit(VolumeId(1), needle(2), true).await }
|
||||
});
|
||||
for _ in 0..10_000 {
|
||||
if queue.windows_armed_for_test() > armed {
|
||||
break;
|
||||
}
|
||||
tokio::task::yield_now().await;
|
||||
}
|
||||
assert!(queue.windows_armed_for_test() > armed);
|
||||
let third = tokio::spawn({
|
||||
let queue = queue.clone();
|
||||
async move { queue.submit(VolumeId(1), needle(3), true).await }
|
||||
});
|
||||
let (second, third) = tokio::join!(second, third);
|
||||
second.unwrap().unwrap();
|
||||
third.unwrap().unwrap();
|
||||
// One shared window ends both writes ~30s after their batch armed;
|
||||
// back-to-back windows would keep the third out to ~90s.
|
||||
let elapsed = start.elapsed();
|
||||
assert!(elapsed >= Duration::from_secs(60) && elapsed < Duration::from_secs(90));
|
||||
}
|
||||
}
|
||||
@@ -740,11 +740,7 @@ impl DiskLocation {
|
||||
if vol.is_read_only() {
|
||||
continue;
|
||||
}
|
||||
// A preallocated volume already holds its space on disk.
|
||||
let dat_size = vol
|
||||
.dat_file_size()
|
||||
.unwrap_or(0)
|
||||
.max(vol.dat_allocated_size());
|
||||
let dat_size = vol.dat_file_size().unwrap_or(0);
|
||||
let idx_size = vol.idx_file_size();
|
||||
let used = dat_size + idx_size;
|
||||
if volume_size_limit > used {
|
||||
|
||||
@@ -18,7 +18,7 @@ use crate::storage::erasure_coding::ec_volume::{
|
||||
};
|
||||
use crate::storage::needle::needle::{Needle, get_actual_size};
|
||||
use crate::storage::needle_map::NeedleMapKind;
|
||||
use crate::storage::super_block::{ReplicaPlacement, SUPER_BLOCK_SIZE, SuperBlock};
|
||||
use crate::storage::super_block::{ReplicaPlacement, SUPER_BLOCK_SIZE};
|
||||
use crate::storage::types::*;
|
||||
use crate::storage::volume::{CompactionJob, VifVolumeInfo, Volume, VolumeError, VolumeSpec};
|
||||
|
||||
@@ -762,8 +762,8 @@ impl Store {
|
||||
}
|
||||
|
||||
/// Configure a volume's replica placement on disk.
|
||||
/// The volume must already be unmounted. Rewrites the .vif and the .dat
|
||||
/// superblock so both carry the new placement.
|
||||
/// The volume must already be unmounted. This opens the .dat file directly,
|
||||
/// modifies the replica_placement byte (offset 1), and writes it back.
|
||||
pub fn configure_volume(&self, vid: VolumeId, rp: ReplicaPlacement) -> Result<(), VolumeError> {
|
||||
let (_, base_path, _) = self.find_volume_file_base(vid).ok_or_else(|| {
|
||||
VolumeError::Io(io::Error::new(
|
||||
@@ -775,7 +775,7 @@ impl Store {
|
||||
let mut vif = load_vif_volume_info(&vif_path)?;
|
||||
vif.replication = rp.to_string();
|
||||
save_vif_volume_info(&vif_path, &vif)?;
|
||||
update_dat_replica_placement(&format!("{}.dat", base_path), vid, rp)
|
||||
Ok(())
|
||||
}
|
||||
|
||||
// ---- Read / Write / Delete ----
|
||||
@@ -943,7 +943,11 @@ impl Store {
|
||||
let current = loc.max_volume_count.load(Ordering::Relaxed);
|
||||
let (_, free) = super::disk_location::get_disk_stats(&loc.directory);
|
||||
|
||||
let unused_space = loc.unused_space(volume_size_limit);
|
||||
let unused_space = if self.get_preallocate() {
|
||||
0
|
||||
} else {
|
||||
loc.unused_space(volume_size_limit)
|
||||
};
|
||||
let unclaimed = (free as i64) - (unused_space as i64);
|
||||
|
||||
let vol_count = loc.volumes_len() as i32;
|
||||
@@ -1740,61 +1744,6 @@ fn strip_volume_suffix(filename: &str) -> Option<&str> {
|
||||
.or_else(|| filename.strip_suffix(".idx"))
|
||||
}
|
||||
|
||||
/// Rewrites the replica placement in a .dat header for a volume that is not
|
||||
/// mounted; a missing .dat means a remote-only volume, where the .vif carries
|
||||
/// the placement.
|
||||
fn update_dat_replica_placement(
|
||||
dat_path: &str,
|
||||
vid: VolumeId,
|
||||
rp: ReplicaPlacement,
|
||||
) -> Result<(), VolumeError> {
|
||||
use std::io::{Read, Seek, SeekFrom, Write};
|
||||
|
||||
let mut file = match std::fs::OpenOptions::new()
|
||||
.read(true)
|
||||
.write(true)
|
||||
.open(dat_path)
|
||||
{
|
||||
Ok(f) => f,
|
||||
Err(e) if e.kind() == io::ErrorKind::NotFound => return Ok(()),
|
||||
Err(e) => {
|
||||
return Err(VolumeError::Io(io::Error::new(
|
||||
e.kind(),
|
||||
format!("volume {} failed to open {}: {}", vid, dat_path, e),
|
||||
)))
|
||||
}
|
||||
};
|
||||
let mut header = [0u8; SUPER_BLOCK_SIZE];
|
||||
file.read_exact(&mut header).map_err(|e| {
|
||||
VolumeError::Io(io::Error::new(
|
||||
e.kind(),
|
||||
format!("volume {} failed to parse superblock: {}", vid, e),
|
||||
))
|
||||
})?;
|
||||
let extra_size = u16::from_be_bytes([header[6], header[7]]) as usize;
|
||||
let mut block = header.to_vec();
|
||||
block.resize(SUPER_BLOCK_SIZE + extra_size, 0);
|
||||
if extra_size > 0 {
|
||||
file.read_exact(&mut block[SUPER_BLOCK_SIZE..]).map_err(|e| {
|
||||
VolumeError::Io(io::Error::new(
|
||||
e.kind(),
|
||||
format!("volume {} failed to parse superblock: {}", vid, e),
|
||||
))
|
||||
})?;
|
||||
}
|
||||
let mut super_block = SuperBlock::from_bytes(&block).map_err(|e| {
|
||||
VolumeError::Io(io::Error::new(
|
||||
io::ErrorKind::InvalidData,
|
||||
format!("volume {} failed to parse superblock: {}", vid, e),
|
||||
))
|
||||
})?;
|
||||
super_block.replica_placement = rp;
|
||||
file.seek(SeekFrom::Start(0))?;
|
||||
file.write_all(&super_block.to_bytes())?;
|
||||
file.sync_all()?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn load_vif_volume_info(path: &str) -> Result<VifVolumeInfo, VolumeError> {
|
||||
let content = match std::fs::read_to_string(path) {
|
||||
Ok(c) => c,
|
||||
@@ -2032,33 +1981,6 @@ mod tests {
|
||||
assert_eq!(store.total_volume_count(), 1);
|
||||
}
|
||||
|
||||
// Mirrors Go's TestConfigureVolumeUpdatesSuperblock: configure must leave
|
||||
// the .dat header and the .vif both on the new placement.
|
||||
#[test]
|
||||
fn test_configure_volume_updates_superblock() {
|
||||
let tmp = TempDir::new().unwrap();
|
||||
let dir = tmp.path().to_str().unwrap();
|
||||
let mut store = make_test_store(&[dir]);
|
||||
|
||||
store
|
||||
.add_volume(VolumeId(1), DiskType::HardDrive, &VolumeSpec::default())
|
||||
.unwrap();
|
||||
assert!(store.unmount_volume(VolumeId(1)).unwrap());
|
||||
|
||||
let rp = ReplicaPlacement::from_string("010").unwrap();
|
||||
store.configure_volume(VolumeId(1), rp).unwrap();
|
||||
|
||||
let base = volume_file_name(dir, "", VolumeId(1));
|
||||
let vif = load_vif_volume_info(&format!("{}.vif", base)).unwrap();
|
||||
assert_eq!(vif.replication, "010");
|
||||
|
||||
let dat = std::fs::read(format!("{}.dat", base)).unwrap();
|
||||
let super_block = SuperBlock::from_bytes(&dat).unwrap();
|
||||
assert_eq!(super_block.replica_placement, rp);
|
||||
|
||||
assert!(store.configure_volume(VolumeId(2), rp).is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_mount_volume_by_id_hint_skips_lone_sidecar() {
|
||||
// A lone .vif/.idx sidecar (e.g. an EC sidecar whose .ecx is on a
|
||||
@@ -2557,7 +2479,7 @@ mod tests {
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_maybe_adjust_volume_max_reserves_plain_volumes_while_preallocating() {
|
||||
fn test_maybe_adjust_volume_max_honors_preallocate_flag() {
|
||||
let tmp = TempDir::new().unwrap();
|
||||
let dir = tmp.path().to_str().unwrap();
|
||||
|
||||
@@ -2572,6 +2494,13 @@ mod tests {
|
||||
Vec::new(),
|
||||
)
|
||||
.unwrap();
|
||||
// Use a large volume_size_limit so the unused-space difference between
|
||||
// preallocate=true (0) and preallocate=false (~2 × limit) is big enough
|
||||
// that integer-division rounding and disk-free fluctuations between the
|
||||
// two get_disk_stats calls cannot make the quotients equal.
|
||||
store
|
||||
.volume_size_limit
|
||||
.store(100 * 1024 * 1024, Ordering::Relaxed);
|
||||
store
|
||||
.add_volume(
|
||||
VolumeId(61),
|
||||
@@ -2582,67 +2511,39 @@ mod tests {
|
||||
},
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
// Room for 2.5 volumes: the plain volume claims one, leaving one more slot.
|
||||
let (_, free) = crate::storage::disk_location::get_disk_stats(dir);
|
||||
let volume_size_limit = (free as f64 / 2.5) as u64;
|
||||
if volume_size_limit == 0 {
|
||||
return; // cannot determine free disk space
|
||||
}
|
||||
store
|
||||
.volume_size_limit
|
||||
.store(volume_size_limit, Ordering::Relaxed);
|
||||
store.locations[0].original_max_volume_count = 0;
|
||||
store.set_preallocate(true);
|
||||
|
||||
store.maybe_adjust_volume_max();
|
||||
assert_eq!(
|
||||
store.locations[0].max_volume_count.load(Ordering::Relaxed),
|
||||
2
|
||||
);
|
||||
}
|
||||
|
||||
#[cfg(target_os = "linux")]
|
||||
#[test]
|
||||
fn test_unused_space_counts_preallocated_volumes_as_full() {
|
||||
const LIMIT: u64 = 1 << 20;
|
||||
let tmp = TempDir::new().unwrap();
|
||||
let dir = tmp.path().to_str().unwrap();
|
||||
|
||||
let mut store = Store::new(NeedleMapKind::InMemory);
|
||||
store
|
||||
.add_location(
|
||||
dir,
|
||||
dir,
|
||||
2,
|
||||
.add_volume(
|
||||
VolumeId(62),
|
||||
DiskType::HardDrive,
|
||||
MinFreeSpace::Percent(1.0),
|
||||
Vec::new(),
|
||||
&VolumeSpec {
|
||||
collection: "preallocate_case",
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.unwrap();
|
||||
for (vid, preallocate) in [(VolumeId(71), LIMIT), (VolumeId(72), 0)] {
|
||||
store
|
||||
.add_volume(
|
||||
vid,
|
||||
DiskType::HardDrive,
|
||||
&VolumeSpec {
|
||||
collection: "preallocate_case",
|
||||
preallocate,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
for vid in [VolumeId(61), VolumeId(62)] {
|
||||
let dat_path = store.find_volume(vid).unwrap().1.dat_path();
|
||||
std::fs::OpenOptions::new()
|
||||
.write(true)
|
||||
.open(dat_path)
|
||||
.unwrap()
|
||||
.set_len((crate::storage::super_block::SUPER_BLOCK_SIZE + 1) as u64)
|
||||
.unwrap();
|
||||
}
|
||||
let preallocated = store.find_volume(VolumeId(71)).unwrap().1;
|
||||
if preallocated.dat_allocated_size() < LIMIT {
|
||||
return; // the filesystem does not support fallocate
|
||||
}
|
||||
store.locations[0].original_max_volume_count = 0;
|
||||
store.locations[0]
|
||||
.max_volume_count
|
||||
.store(0, Ordering::Relaxed);
|
||||
|
||||
let plain = store.find_volume(VolumeId(72)).unwrap().1;
|
||||
let plain_dat = plain.dat_file_size().unwrap();
|
||||
// The filesystem may round a small file up to a larger allocation unit.
|
||||
let plain_used = plain_dat.max(plain.dat_allocated_size()) + plain.idx_file_size();
|
||||
assert_eq!(store.locations[0].unused_space(LIMIT), LIMIT - plain_used);
|
||||
store.set_preallocate(false);
|
||||
assert!(store.maybe_adjust_volume_max());
|
||||
let without_preallocate = store.locations[0].max_volume_count.load(Ordering::Relaxed);
|
||||
|
||||
store.set_preallocate(true);
|
||||
assert!(store.maybe_adjust_volume_max());
|
||||
let with_preallocate = store.locations[0].max_volume_count.load(Ordering::Relaxed);
|
||||
|
||||
assert!(with_preallocate > without_preallocate);
|
||||
}
|
||||
|
||||
#[test]
|
||||
|
||||
@@ -836,7 +836,7 @@ impl NeedleReadPlan {
|
||||
pub(crate) fn read_meta(&mut self, n: &mut Needle) -> Result<(), VolumeError> {
|
||||
let (source, size, version) = (&self.source, self.size, self.version);
|
||||
let read_at = |buf: &mut [u8], at: u64| Ok(source.read_exact_at(buf, at)?);
|
||||
let result = with_offset_retry(&mut self.offset, |off| {
|
||||
with_offset_retry(&mut self.offset, |off| {
|
||||
if version == VERSION_1 {
|
||||
// A V1 body is all data, so its "meta" is the whole record.
|
||||
let mut buf = vec![0u8; get_actual_size(size, version) as usize];
|
||||
@@ -847,26 +847,13 @@ impl NeedleReadPlan {
|
||||
} else {
|
||||
Volume::read_needle_meta_blob_and_parse(read_at, n, off, size, version)
|
||||
}
|
||||
});
|
||||
// Report failures only: a successful meta read precedes the payload
|
||||
// read, so clearing the streak here would reset the count before
|
||||
// every failing download and the volume could never reach quarantine.
|
||||
if let Err(VolumeError::Io(e)) = &result {
|
||||
self.io_errors.check_read_write_error(Some(e));
|
||||
}
|
||||
result?;
|
||||
})?;
|
||||
if needle_expired(n) {
|
||||
return Err(VolumeError::NotFound);
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// A successful meta read ended the request (e.g. HEAD), so it may reset
|
||||
/// the streak — unlike a meta read that precedes a payload read.
|
||||
pub(crate) fn note_meta_only_success(&self) {
|
||||
self.io_errors.check_read_write_error(None);
|
||||
}
|
||||
|
||||
/// For a reply served from the data file, after `read_meta` filled `n`.
|
||||
pub(crate) fn into_stream_info(self, n: &Needle) -> NeedleStreamInfo {
|
||||
// V1 data starts right after the header, V2/V3 after the DataSize field.
|
||||
@@ -881,7 +868,6 @@ impl NeedleReadPlan {
|
||||
data_size: n.data_size,
|
||||
data_file_access_control: self.data_file_access_control,
|
||||
io_unavailable: self.io_unavailable,
|
||||
io_errors: self.io_errors,
|
||||
needle_id: self.needle_id,
|
||||
checksum: n.checksum.0,
|
||||
}
|
||||
@@ -1158,9 +1144,6 @@ pub struct NeedleStreamInfo {
|
||||
pub data_file_access_control: Arc<DataFileAccessControl>,
|
||||
/// Checked before each chunk: the volume can become unavailable mid-stream.
|
||||
pub(crate) io_unavailable: Arc<IoUnavailable>,
|
||||
/// Streamed chunks report read failures here and clear the streak when
|
||||
/// the whole body has been emitted, mirroring readNeedleDataInto.
|
||||
pub(crate) io_errors: Arc<IoErrorTracker>,
|
||||
pub needle_id: NeedleId,
|
||||
/// Checksum stored in the needle tail, verified once the last chunk has
|
||||
/// been read — before that frame is emitted.
|
||||
@@ -2935,52 +2918,18 @@ impl Volume {
|
||||
}
|
||||
|
||||
// Cookie validation for existing needle (matches Go: check whenever nm.Get returns ok)
|
||||
let indexed = match io.nm.as_ref() {
|
||||
Some(nm) => nm.get(n.id)?,
|
||||
None => None,
|
||||
};
|
||||
if let Some(nv) = indexed
|
||||
&& !nv.offset.is_zero()
|
||||
if let Some(nm) = &io.nm
|
||||
&& let Some(nv) = nm.get(n.id)?
|
||||
{
|
||||
let mut existing = Needle::default();
|
||||
// Read only the header to check cookie
|
||||
self.read_needle_header_unlocked(io, &mut existing, nv.offset.to_actual_offset())?;
|
||||
|
||||
if existing.id != n.id && existing.size != nv.size {
|
||||
// a 4-byte index entry can index past the 32 GiB wrap: the
|
||||
// low offset may hold another needle while the real record
|
||||
// sits at the wrapped offset, where reads already retry
|
||||
#[cfg(not(feature = "5bytes"))]
|
||||
if nv.offset.to_actual_offset() < MAX_POSSIBLE_VOLUME_SIZE as i64 {
|
||||
let mut wrapped = Needle::default();
|
||||
match self.read_needle_header_unlocked(
|
||||
io,
|
||||
&mut wrapped,
|
||||
nv.offset.to_actual_offset() + MAX_POSSIBLE_VOLUME_SIZE as i64,
|
||||
) {
|
||||
Ok(()) => existing = wrapped,
|
||||
Err(VolumeError::Io(e)) if e.kind() == io::ErrorKind::UnexpectedEof => {}
|
||||
Err(e) => return Err(e),
|
||||
}
|
||||
}
|
||||
if n.cookie.0 == 0 && !check_cookie {
|
||||
n.cookie = existing.cookie;
|
||||
}
|
||||
|
||||
if existing.id != n.id && existing.size != nv.size {
|
||||
// the entry resolves to a record that is neither the indexed
|
||||
// key nor the indexed size, so it is stale rather than a
|
||||
// duplicate — skip the cookie check and let publish_write
|
||||
// overwrite it once the append commits
|
||||
warn!(
|
||||
"volume {} index entry for needle {} resolves to needle {}; overwriting stale index entry",
|
||||
self.id.0, n.id.0, existing.id.0
|
||||
);
|
||||
} else {
|
||||
if n.cookie.0 == 0 && !check_cookie {
|
||||
n.cookie = existing.cookie;
|
||||
}
|
||||
if existing.cookie != n.cookie {
|
||||
return Err(VolumeError::CookieMismatch(n.cookie.0));
|
||||
}
|
||||
if existing.cookie != n.cookie {
|
||||
return Err(VolumeError::CookieMismatch(n.cookie.0));
|
||||
}
|
||||
}
|
||||
Ok(false)
|
||||
@@ -5036,25 +4985,6 @@ impl Volume {
|
||||
self.current_dat_file_size()
|
||||
}
|
||||
|
||||
/// Disk space held by the local .dat file, or 0 when it is unknown. It
|
||||
/// exceeds the file size when the file was preallocated with
|
||||
/// `FALLOC_FL_KEEP_SIZE`.
|
||||
pub fn dat_allocated_size(&self) -> u64 {
|
||||
#[cfg(unix)]
|
||||
{
|
||||
use std::os::unix::fs::MetadataExt;
|
||||
let io = self.io.read().unwrap();
|
||||
io.dat_file
|
||||
.as_ref()
|
||||
.and_then(|f| f.metadata().ok())
|
||||
.map_or(0, |m| m.blocks() * 512)
|
||||
}
|
||||
#[cfg(not(unix))]
|
||||
{
|
||||
0
|
||||
}
|
||||
}
|
||||
|
||||
/// Get the modification time of the .dat file as Unix seconds.
|
||||
pub fn dat_file_mod_time(&self) -> u64 {
|
||||
let io = self.io.read().unwrap();
|
||||
@@ -6118,7 +6048,7 @@ fn preallocate_file(file: &File, size: u64) {
|
||||
/// An OS error the platform reports for failing storage media, which is
|
||||
/// what counts toward the I/O error streak.
|
||||
#[cfg(test)]
|
||||
pub(crate) fn media_error_for_test() -> io::Error {
|
||||
fn media_error_for_test() -> io::Error {
|
||||
#[cfg(unix)]
|
||||
{
|
||||
io::Error::from_raw_os_error(libc::EIO)
|
||||
@@ -9095,107 +9025,6 @@ mod tests {
|
||||
assert!(matches!(err, VolumeError::CookieMismatch(_)));
|
||||
}
|
||||
|
||||
/// A damaged index entry resolving to a different needle must miss on
|
||||
/// reads and be dropped by the next write for the key, not rejected as a
|
||||
/// cookie conflict.
|
||||
#[test]
|
||||
fn test_stale_index_entry_heals_on_write() {
|
||||
let tmp = TempDir::new().unwrap();
|
||||
let dir = tmp.path().to_str().unwrap();
|
||||
let v = make_test_volume(dir);
|
||||
|
||||
let mut first = Needle {
|
||||
id: NeedleId(2),
|
||||
cookie: Cookie(0x2222),
|
||||
data: b"second".to_vec(),
|
||||
data_size: 6,
|
||||
..Needle::default()
|
||||
};
|
||||
v.write_needle(&mut first, true, false).unwrap();
|
||||
|
||||
// point key 9 at needle 2's record with a size that does not match
|
||||
// the record, as a corrupt index entry would look
|
||||
{
|
||||
let mut io = v.io.write().unwrap();
|
||||
let nm = io.nm.as_mut().unwrap();
|
||||
let nv = nm.get(NeedleId(2)).unwrap().unwrap();
|
||||
nm.put(NeedleId(9), nv.offset, Size(nv.size.0 + 1)).unwrap();
|
||||
}
|
||||
|
||||
let mut probe = Needle {
|
||||
id: NeedleId(9),
|
||||
..Needle::default()
|
||||
};
|
||||
assert!(v.read_needle(&mut probe).is_err());
|
||||
|
||||
let mut n = Needle {
|
||||
id: NeedleId(9),
|
||||
cookie: Cookie(0xabcd),
|
||||
data: b"ninth".to_vec(),
|
||||
data_size: 5,
|
||||
..Needle::default()
|
||||
};
|
||||
v.write_needle(&mut n, true, false).unwrap();
|
||||
|
||||
let mut back = Needle {
|
||||
id: NeedleId(9),
|
||||
..Needle::default()
|
||||
};
|
||||
v.read_needle(&mut back).unwrap();
|
||||
assert_eq!(back.data, b"ninth");
|
||||
assert_eq!(back.cookie, Cookie(0xabcd));
|
||||
}
|
||||
|
||||
/// A stale index entry is only replaced once the new record commits: a
|
||||
/// durable write rolled back by a failed fsync leaves the index exactly
|
||||
/// as it found it.
|
||||
#[test]
|
||||
fn test_stale_index_entry_survives_failed_write() {
|
||||
let tmp = TempDir::new().unwrap();
|
||||
let dir = tmp.path().to_str().unwrap();
|
||||
let v = make_test_volume(dir);
|
||||
|
||||
let mut existing = Needle {
|
||||
id: NeedleId(2),
|
||||
cookie: Cookie(0x2222),
|
||||
data: b"second".to_vec(),
|
||||
data_size: 6,
|
||||
..Needle::default()
|
||||
};
|
||||
v.write_needle(&mut existing, true, false).unwrap();
|
||||
|
||||
let stale_nv = {
|
||||
let mut io = v.io.write().unwrap();
|
||||
let nm = io.nm.as_mut().unwrap();
|
||||
let nv = nm.get(NeedleId(2)).unwrap().unwrap();
|
||||
nm.put(NeedleId(9), nv.offset, Size(nv.size.0 + 1)).unwrap();
|
||||
nm.get(NeedleId(9)).unwrap().unwrap()
|
||||
};
|
||||
|
||||
v.fail_next_fsync_for_test(true);
|
||||
let mut replacement = Needle {
|
||||
id: NeedleId(9),
|
||||
cookie: Cookie(0xabcd),
|
||||
data: b"ninth".to_vec(),
|
||||
data_size: 5,
|
||||
..Needle::default()
|
||||
};
|
||||
assert!(v.write_needle(&mut replacement, true, true).is_err());
|
||||
v.fail_next_fsync_for_test(false);
|
||||
|
||||
let now = {
|
||||
let io = v.io.read().unwrap();
|
||||
io.nm
|
||||
.as_ref()
|
||||
.unwrap()
|
||||
.get(NeedleId(9))
|
||||
.unwrap()
|
||||
.expect("a failed write must not drop the index entry it meant to replace")
|
||||
};
|
||||
assert_eq!(now.offset, stale_nv.offset);
|
||||
assert_eq!(now.size, stale_nv.size);
|
||||
}
|
||||
|
||||
// A size disagreeing with the blob's own header indexes the needle at the wrong
|
||||
// length and, on v3, stamps the append timestamp into the middle of the needle.
|
||||
#[test]
|
||||
|
||||
@@ -1340,11 +1340,11 @@ async fn non_ascii_fid_and_ttl_are_rejected_not_panicked() {
|
||||
|
||||
#[tokio::test]
|
||||
async fn write_queue_upload_returns_same_etag_as_direct_write() {
|
||||
use seaweed_volume::server::write_queue::{SyncBatchOption, WriteQueue};
|
||||
use seaweed_volume::server::write_queue::WriteQueue;
|
||||
|
||||
let (direct_state, _direct_tmp) = test_state();
|
||||
let (queued_state, _queued_tmp) = test_state();
|
||||
let wq = WriteQueue::new(queued_state.clone(), 128, SyncBatchOption::default());
|
||||
let wq = WriteQueue::new(queued_state.clone(), 128);
|
||||
let _ = queued_state.write_queue.set(wq);
|
||||
|
||||
let compressible = "seaweedfs ".repeat(200).into_bytes();
|
||||
|
||||
@@ -3,12 +3,9 @@ package erasure_coding
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"math"
|
||||
"net"
|
||||
"net/http"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
@@ -16,7 +13,6 @@ import (
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/seaweedfs/seaweedfs/test/testutil"
|
||||
"github.com/seaweedfs/seaweedfs/weed/operation"
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb"
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb/volume_server_pb"
|
||||
@@ -51,15 +47,17 @@ func TestECEncodingVolumeLocationTimingBug(t *testing.T) {
|
||||
defer cluster.Stop()
|
||||
|
||||
// Wait for servers to be ready
|
||||
require.NoError(t, waitForServer(cluster.masterAddr, 30*time.Second))
|
||||
for _, addr := range cluster.volumeAddrs {
|
||||
require.NoError(t, waitForServer(addr, 30*time.Second))
|
||||
}
|
||||
require.NoError(t, waitForMasterLeader(cluster.masterAddr, 30*time.Second))
|
||||
require.NoError(t, waitForServer("127.0.0.1:9333", 30*time.Second))
|
||||
require.NoError(t, waitForServer("127.0.0.1:8080", 30*time.Second))
|
||||
require.NoError(t, waitForServer("127.0.0.1:8081", 30*time.Second))
|
||||
require.NoError(t, waitForServer("127.0.0.1:8082", 30*time.Second))
|
||||
require.NoError(t, waitForServer("127.0.0.1:8083", 30*time.Second))
|
||||
require.NoError(t, waitForServer("127.0.0.1:8084", 30*time.Second))
|
||||
require.NoError(t, waitForServer("127.0.0.1:8085", 30*time.Second))
|
||||
|
||||
// Create command environment
|
||||
options := &shell.ShellOptions{
|
||||
Masters: stringPtr(cluster.masterAddr),
|
||||
Masters: stringPtr("127.0.0.1:9333"),
|
||||
GrpcDialOption: grpc.WithInsecure(),
|
||||
FilerGroup: stringPtr("default"),
|
||||
}
|
||||
@@ -73,7 +71,7 @@ func TestECEncodingVolumeLocationTimingBug(t *testing.T) {
|
||||
|
||||
// Upload some test data to create volumes
|
||||
testData := []byte("This is test data for EC encoding integration test")
|
||||
volumeId, err := uploadTestData(testData, cluster.masterAddr)
|
||||
volumeId, err := uploadTestData(testData, "127.0.0.1:9333")
|
||||
require.NoError(t, err)
|
||||
t.Logf("Created volume %d with test data", volumeId)
|
||||
|
||||
@@ -259,13 +257,12 @@ func TestECEncodingMasterTimingRaceCondition(t *testing.T) {
|
||||
defer cluster.Stop()
|
||||
|
||||
// Wait for servers to be ready
|
||||
require.NoError(t, waitForServer(cluster.masterAddr, 30*time.Second))
|
||||
require.NoError(t, waitForServer(cluster.volumeAddrs[0], 30*time.Second))
|
||||
require.NoError(t, waitForMasterLeader(cluster.masterAddr, 30*time.Second))
|
||||
require.NoError(t, waitForServer("127.0.0.1:9333", 30*time.Second))
|
||||
require.NoError(t, waitForServer("127.0.0.1:8080", 30*time.Second))
|
||||
|
||||
// Create command environment
|
||||
options := &shell.ShellOptions{
|
||||
Masters: stringPtr(cluster.masterAddr),
|
||||
Masters: stringPtr("127.0.0.1:9333"),
|
||||
GrpcDialOption: grpc.WithInsecure(),
|
||||
FilerGroup: stringPtr("default"),
|
||||
}
|
||||
@@ -279,7 +276,7 @@ func TestECEncodingMasterTimingRaceCondition(t *testing.T) {
|
||||
|
||||
// Upload test data
|
||||
testData := []byte("Race condition test data")
|
||||
volumeId, err := uploadTestData(testData, cluster.masterAddr)
|
||||
volumeId, err := uploadTestData(testData, "127.0.0.1:9333")
|
||||
require.NoError(t, err)
|
||||
t.Logf("Created volume %d for race condition test", volumeId)
|
||||
|
||||
@@ -369,8 +366,6 @@ func TestECEncodingMasterTimingRaceCondition(t *testing.T) {
|
||||
type TestCluster struct {
|
||||
masterCmd *exec.Cmd
|
||||
volumeServers []*exec.Cmd
|
||||
masterAddr string
|
||||
volumeAddrs []string
|
||||
}
|
||||
|
||||
func (c *TestCluster) Stop() {
|
||||
@@ -407,23 +402,9 @@ func startSeaweedFSCluster(ctx context.Context, dataDir string) (*TestCluster, e
|
||||
securityToml := filepath.Join(dataDir, "security.toml")
|
||||
os.WriteFile(securityToml, []byte("# Empty security config for testing\n"), 0644)
|
||||
|
||||
// Allocate ports instead of using the defaults: on a shared or reused
|
||||
// runner a stale weed process can still hold 9333/808x, which kills the
|
||||
// freshly started master at bind time and leaves the test talking to a
|
||||
// raft follower that answers every assign with "not current leader".
|
||||
// Mini-safe ports guarantee each HTTP port's derived gRPC port
|
||||
// (port + 10000) is free as well.
|
||||
ports, err := testutil.AllocateMiniPorts(7)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to allocate ports: %v", err)
|
||||
}
|
||||
masterPort := ports[0]
|
||||
masterAddr := fmt.Sprintf("127.0.0.1:%d", masterPort)
|
||||
cluster.masterAddr = masterAddr
|
||||
|
||||
// Start master server
|
||||
masterCmd := exec.CommandContext(ctx, weedBinary, "master",
|
||||
"-port", fmt.Sprintf("%d", masterPort),
|
||||
"-port", "9333",
|
||||
"-mdir", masterDir,
|
||||
"-volumeSizeLimitMB", "10", // Small volumes for testing
|
||||
"-ip", "127.0.0.1",
|
||||
@@ -451,14 +432,13 @@ func startSeaweedFSCluster(ctx context.Context, dataDir string) (*TestCluster, e
|
||||
volumeDir := filepath.Join(dataDir, fmt.Sprintf("volume%d", i))
|
||||
os.MkdirAll(volumeDir, 0755)
|
||||
|
||||
port := fmt.Sprintf("%d", ports[1+i])
|
||||
cluster.volumeAddrs = append(cluster.volumeAddrs, fmt.Sprintf("127.0.0.1:%d", ports[1+i]))
|
||||
port := fmt.Sprintf("808%d", i)
|
||||
rack := fmt.Sprintf("rack%d", i)
|
||||
volumeCmd := exec.CommandContext(ctx, weedBinary, "volume",
|
||||
"-port", port,
|
||||
"-dir", volumeDir,
|
||||
"-max", "10",
|
||||
"-master", masterAddr,
|
||||
"-master", "127.0.0.1:9333",
|
||||
"-volume.allowUntrustedRemoteEndpoints",
|
||||
"-ip", "127.0.0.1",
|
||||
"-dataCenter", "dc1",
|
||||
@@ -518,8 +498,7 @@ func findWeedBinary() string {
|
||||
func waitForServer(address string, timeout time.Duration) error {
|
||||
start := time.Now()
|
||||
for time.Since(start) < timeout {
|
||||
conn, err := net.DialTimeout("tcp", address, time.Second)
|
||||
if err == nil {
|
||||
if conn, err := grpc.NewClient(address, grpc.WithInsecure()); err == nil {
|
||||
conn.Close()
|
||||
return nil
|
||||
}
|
||||
@@ -528,29 +507,6 @@ func waitForServer(address string, timeout time.Duration) error {
|
||||
return fmt.Errorf("timeout waiting for server %s", address)
|
||||
}
|
||||
|
||||
// waitForMasterLeader polls the master's /cluster/status endpoint until it
|
||||
// reports itself as the raft leader. A goraft master can take up to the
|
||||
// configured election timeout (10s by default) before it is ready to serve
|
||||
// raft-gated requests like volume assign, so a bare port check is not enough.
|
||||
func waitForMasterLeader(masterAddr string, timeout time.Duration) error {
|
||||
start := time.Now()
|
||||
for time.Since(start) < timeout {
|
||||
resp, err := http.Get(fmt.Sprintf("http://%s/cluster/status", masterAddr))
|
||||
if err == nil {
|
||||
var status struct {
|
||||
IsLeader bool `json:"IsLeader"`
|
||||
}
|
||||
err = json.NewDecoder(resp.Body).Decode(&status)
|
||||
resp.Body.Close()
|
||||
if err == nil && status.IsLeader {
|
||||
return nil
|
||||
}
|
||||
}
|
||||
time.Sleep(500 * time.Millisecond)
|
||||
}
|
||||
return fmt.Errorf("timeout waiting for master %s to become raft leader", masterAddr)
|
||||
}
|
||||
|
||||
func uploadTestData(data []byte, masterAddress string) (needle.VolumeId, error) {
|
||||
// Upload data to get a file ID
|
||||
assignResult, err := operation.Assign(context.Background(), func(ctx context.Context) pb.ServerAddress {
|
||||
@@ -2253,15 +2209,14 @@ func TestECEncodeReplicatedVolumeSync(t *testing.T) {
|
||||
defer cluster.Stop()
|
||||
|
||||
// Wait for servers to be ready
|
||||
require.NoError(t, waitForServer(cluster.masterAddr, 30*time.Second))
|
||||
for _, addr := range cluster.volumeAddrs {
|
||||
require.NoError(t, waitForServer(addr, 30*time.Second))
|
||||
require.NoError(t, waitForServer("127.0.0.1:9333", 30*time.Second))
|
||||
for i := 0; i < 6; i++ {
|
||||
require.NoError(t, waitForServer(fmt.Sprintf("127.0.0.1:%d", 8080+i), 30*time.Second))
|
||||
}
|
||||
require.NoError(t, waitForMasterLeader(cluster.masterAddr, 30*time.Second))
|
||||
|
||||
// Create command environment
|
||||
options := &shell.ShellOptions{
|
||||
Masters: stringPtr(cluster.masterAddr),
|
||||
Masters: stringPtr("127.0.0.1:9333"),
|
||||
GrpcDialOption: grpc.WithInsecure(),
|
||||
FilerGroup: stringPtr("default"),
|
||||
}
|
||||
@@ -2285,7 +2240,7 @@ func TestECEncodeReplicatedVolumeSync(t *testing.T) {
|
||||
var volumeId needle.VolumeId
|
||||
var uploadErr error
|
||||
for retry := 0; retry < 5; retry++ {
|
||||
volumeId, uploadErr = uploadTestDataWithReplication(t, cluster.masterAddr, testCollection, testReplication)
|
||||
volumeId, uploadErr = uploadTestDataWithReplication(t, "127.0.0.1:9333", testCollection, testReplication)
|
||||
if uploadErr == nil {
|
||||
break
|
||||
}
|
||||
@@ -2335,7 +2290,7 @@ func TestECEncodeReplicatedVolumeSync(t *testing.T) {
|
||||
var volumeId needle.VolumeId
|
||||
var uploadErr error
|
||||
for retry := 0; retry < 5; retry++ {
|
||||
volumeId, uploadErr = uploadTestDataWithReplication(t, cluster.masterAddr, testCollection, testReplication)
|
||||
volumeId, uploadErr = uploadTestDataWithReplication(t, "127.0.0.1:9333", testCollection, testReplication)
|
||||
if uploadErr == nil {
|
||||
break
|
||||
}
|
||||
|
||||
@@ -110,21 +110,6 @@ func startFailoverCluster(t testing.TB, numVolumes, numMounts int) *failoverClus
|
||||
"mount %d not ready\n%s", i, c.tailLog(fmt.Sprintf("mount%d", i)))
|
||||
}
|
||||
require.NoError(t, c.WaitForVolumeServers(60*time.Second))
|
||||
|
||||
// A mounted filesystem and registered volume servers do not yet guarantee
|
||||
// that the lock ring has settled or the first replicated volume is ready.
|
||||
// Exercise the write/read path during setup so the append latency checks
|
||||
// measure failover on a running cluster rather than its cold start.
|
||||
probeName := ".failover-ready"
|
||||
probeData := []byte("ready\n")
|
||||
probePath := filepath.Join(c.MountDir(0), probeName)
|
||||
require.NoError(t, os.WriteFile(probePath, probeData, 0644),
|
||||
"readiness write failed\n%s", c.tailLog("mount0"))
|
||||
for i := 0; i < numMounts; i++ {
|
||||
_, ready := waitForContent(filepath.Join(c.MountDir(i), probeName), probeData, convergeTimeout)
|
||||
require.True(t, ready, "mount %d did not observe readiness write\n%s", i, c.tailLog(fmt.Sprintf("mount%d", i)))
|
||||
}
|
||||
require.NoError(t, os.Remove(probePath))
|
||||
return c
|
||||
}
|
||||
|
||||
|
||||
Loaded 100 of 513 files, more files were not shown because too many files have changed in this diff.
Show more
Reference in new issue
Block a user