Compare commits

..
Author SHA1 Message Date
Devin 9d84ecca2f upload -dir: finish every file before reporting a failed upload
A failed batch stopped all remaining batches, and the file-list path
returned before printing results, hiding ids of files that did upload.
Now every file is still attempted, results print for each batch, and
the command fails at the end if any upload failed.
2026-10-21 15:20:40 -07:00
Devin a1d2ebbb9b operation: warm lazy log fds in the file-part leak test 2026-10-21 15:15:01 -07:00
Devin 371d320854 operation: close open file parts when a batch fails to build
Batched -dir uploads now open up to 32 files at once; a mid-batch error
left every earlier handle open until process exit. Close the parts
already built, and close the handle when Stat fails after Open.
2026-10-21 15:14:12 -07:00
Chris LuandDevin 84cc179b80 upload -dir: fetch file ids in batches instead of per file
Directory upload submitted every file through its own SubmitFiles
call, so each file paid a separate serial assign RPC. Files are now
collected during the walk and submitted in batches, sharing one batch
assign and the parallel uploads.

Generated with [Devin](https://devin.ai)

Co-Authored-By: Devin <158243242+devin-ai-integration[bot]@users.noreply.github.com>
2026-10-21 15:10:30 -07:00
Chris LuandDevin c8002d63bb upload: submit batched files concurrently
SubmitFiles fetched all file ids in one assign RPC but then uploaded
the files one after another, so a multi-file upload spent its time in
serial per-file round trips. Uploads now run through a small worker
pool while results keep their input order.

Generated with [Devin](https://devin.ai)

Co-Authored-By: Devin <158243242+devin-ai-integration[bot]@users.noreply.github.com>
2026-10-21 15:10:30 -07:00
513 changed files with 3304 additions and 32105 deletions

No files matched your search

@@ -1,11 +0,0 @@
name: Clean up stale SeaweedFS test mounts
description: Detach disconnected FUSE test mounts left by earlier jobs on a shared runner.
runs:
using: composite
steps:
- name: Unmount disconnected SeaweedFS test mounts
if: runner.os == 'Linux' && runner.environment == 'self-hosted'
shell: bash
env:
CLEANUP_SCRIPT: ${{ github.action_path }}/cleanup.py
run: python3 "$CLEANUP_SCRIPT"
@@ -1,89 +0,0 @@
#!/usr/bin/env python3
"""Detach disconnected SeaweedFS integration-test mounts without touching live ones."""
import errno
import os
from pathlib import Path
import re
import shutil
import subprocess
import sys
import tempfile
def test_mounts(mountinfo):
"""Read the kernel table directly: traversing a stale mount may block."""
roots = {tempfile.gettempdir(), "/tmp", "/var/tmp"}
for line in mountinfo.splitlines():
fields, filesystem = line.split(" - ", 1)
if filesystem.split()[0].split(".")[0] != "fuse":
continue
fields = fields.split()
# mountinfo escapes whitespace and backslashes with octal sequences.
target = re.sub(r"\\([0-7]{3})", lambda m: chr(int(m[1], 8)), fields[4])
path = Path(target)
# Test mounts are a mount* leaf under a seaweedfs_fuse_* scratch dir
# (seaweedfs_fuse_test_*/mount, seaweedfs_fuse_failover_*/mount0, ...)
# or the posix-compliance workflow's fixed /tmp/seaweedfs/mount.
if (not path.name.startswith("mount")
or str(path.parent.parent) not in roots
or not (path.parent.name.startswith("seaweedfs_fuse")
or path.parent.name == "seaweedfs")):
continue
yield fields[0], target
def mounted(mount_id, target):
return (mount_id, target) in test_mounts(Path("/proc/self/mountinfo").read_text())
def cleanup():
failed = False
for mount_id, target in test_mounts(Path("/proc/self/mountinfo").read_text()):
try:
# A separate, bounded probe prevents a wedged FUSE daemon from
# hanging cleanup. Only ENOTCONN proves the daemon disconnected;
# a timeout or another error is not enough to detach a live mount.
probe = subprocess.run(
[sys.executable, __file__, "--probe", target], timeout=5,
capture_output=True, check=False,
)
except subprocess.TimeoutExpired:
print(f"::warning::FUSE probe timed out; leaving {target} mounted")
continue
if probe.returncode != errno.ENOTCONN or not mounted(mount_id, target):
continue
print(f"Unmounting disconnected FUSE test mount: {target}", flush=True)
# Lazy unmount detaches a dead mount even when an old process still
# holds it open. Prefer the unprivileged FUSE helper; sudo is fallback.
for command in (
["fusermount3", "-uz", "--", target],
["fusermount", "-uz", "--", target],
["sudo", "-n", "umount", "-l", "--", target],
):
try:
subprocess.run(command, timeout=10, check=False)
except (OSError, subprocess.TimeoutExpired) as error:
print(f"{command[0]}: {error}", file=sys.stderr)
if not mounted(mount_id, target):
break
else:
print(f"::error::Could not unmount disconnected FUSE test mount: {target}")
failed = True
continue
# The detached mount's parent still holds the crashed run's data and
# logs; test_mounts only yields targets under a SeaweedFS test dir in
# a temp root, so the parent is safe to reclaim here.
shutil.rmtree(Path(target).parent, ignore_errors=True)
return int(failed)
if __name__ == "__main__":
if len(sys.argv) == 3 and sys.argv[1] == "--probe":
try:
os.stat(sys.argv[2])
except OSError as error:
sys.exit(error.errno)
sys.exit(0)
sys.exit(cleanup())
@@ -1,137 +0,0 @@
import errno
import importlib.util
from pathlib import Path
import subprocess
import unittest
from unittest.mock import patch
spec = importlib.util.spec_from_file_location("cleanup", Path(__file__).with_name("cleanup.py"))
cleanup = importlib.util.module_from_spec(spec)
spec.loader.exec_module(cleanup)
TARGET = "/tmp/seaweedfs_fuse_test_123/mount"
MOUNTINFO = f"42 1 0:9 / {TARGET} rw - fuse.seaweedfs seaweedfs rw\n"
class CleanupTests(unittest.TestCase):
def run_cleanup(self, results, mounted=True):
with patch.object(cleanup.Path, "read_text", return_value=MOUNTINFO), \
patch.object(cleanup, "mounted", side_effect=mounted if isinstance(mounted, list) else None,
return_value=mounted) as check, \
patch.object(cleanup.shutil, "rmtree") as rmtree, \
patch.object(cleanup.subprocess, "run", side_effect=results) as run:
status = cleanup.cleanup()
return status, run, check, rmtree
def result(self, status):
return subprocess.CompletedProcess([], status)
def test_only_fuse_integration_test_mounts_are_candidates(self):
rows = MOUNTINFO
rows += MOUNTINFO.replace("fuse.seaweedfs", "ext4")
rows += MOUNTINFO.replace(TARGET, "/mnt/production")
rows += MOUNTINFO.replace(TARGET, "/tmp/seaweedfs_fuse_test_123/data")
rows += MOUNTINFO.replace(TARGET, "/home/user/seaweedfs_fuse_test_123/mount")
rows += MOUNTINFO.replace(TARGET, "/tmp/seaweedfs/data")
self.assertEqual(list(cleanup.test_mounts(rows)), [("42", TARGET)])
def test_all_test_mount_layouts_are_candidates(self):
targets = (
"/tmp/seaweedfs_fuse_test_123/mount",
"/tmp/seaweedfs_fuse_failover_456/mount0",
"/tmp/seaweedfs_fuse_failover_456/mount1",
"/var/tmp/seaweedfs_fuse_p2p_test_789/mount2",
"/tmp/seaweedfs/mount",
)
rows = "".join(
MOUNTINFO.replace("42", str(40 + i)).replace(TARGET, target)
for i, target in enumerate(targets)
)
self.assertEqual(
list(cleanup.test_mounts(rows)),
[(str(40 + i), target) for i, target in enumerate(targets)],
)
def test_mountinfo_paths_are_unescaped_once(self):
target = "/tmp/seaweedfs_fuse_test_space and\\slash/mount"
encoded = target.replace("\\", r"\134").replace(" ", r"\040")
rows = MOUNTINFO.replace(TARGET, encoded)
self.assertEqual(list(cleanup.test_mounts(rows)), [("42", target)])
def test_no_mounts_is_a_noop(self):
with patch.object(cleanup.Path, "read_text", return_value=""), \
patch.object(cleanup.subprocess, "run") as run:
self.assertEqual(cleanup.cleanup(), 0)
run.assert_not_called()
def test_healthy_mount_is_untouched(self):
status, run, _, _ = self.run_cleanup([self.result(0)])
self.assertEqual(status, 0)
self.assertEqual(run.call_count, 1)
def test_other_probe_errors_are_untouched(self):
for error in (errno.EACCES, errno.ENOENT, errno.EIO):
with self.subTest(error=error):
status, run, _, _ = self.run_cleanup([self.result(error)])
self.assertEqual(status, 0)
self.assertEqual(run.call_count, 1)
def test_probe_timeout_does_not_detach_a_possibly_live_mount(self):
status, run, _, _ = self.run_cleanup([subprocess.TimeoutExpired("probe", 5)])
self.assertEqual(status, 0)
self.assertEqual(run.call_count, 1)
def test_disconnected_mount_is_lazily_unmounted(self):
status, run, _, _ = self.run_cleanup([self.result(errno.ENOTCONN), self.result(0)], [True, False])
self.assertEqual(status, 0)
self.assertEqual(run.call_args_list[1].args[0], ["fusermount3", "-uz", "--", TARGET])
self.assertEqual(run.call_args_list[0].kwargs["timeout"], 5)
self.assertEqual(run.call_args_list[1].kwargs["timeout"], 10)
def test_replaced_or_removed_mount_is_untouched(self):
status, run, _, _ = self.run_cleanup([self.result(errno.ENOTCONN)], False)
self.assertEqual(status, 0)
self.assertEqual(run.call_count, 1)
def test_mount_identity_is_checked_without_stat(self):
with patch.object(cleanup.Path, "read_text", return_value=MOUNTINFO):
self.assertTrue(cleanup.mounted("42", TARGET))
self.assertFalse(cleanup.mounted("43", TARGET))
def test_missing_or_timed_out_helper_uses_fallback(self):
for error in (FileNotFoundError(), subprocess.TimeoutExpired("fusermount3", 10)):
with self.subTest(error=error):
status, run, _, _ = self.run_cleanup(
[self.result(errno.ENOTCONN), error, self.result(0)], [True, True, False])
self.assertEqual(status, 0)
self.assertEqual(run.call_args.args[0], ["fusermount", "-uz", "--", TARGET])
def test_failed_helpers_fall_back_to_sudo(self):
status, run, _, _ = self.run_cleanup(
[self.result(errno.ENOTCONN), self.result(1), self.result(1), self.result(0)],
[True, True, True, False])
self.assertEqual(status, 0)
self.assertEqual(run.call_args.args[0], ["sudo", "-n", "umount", "-l", "--", TARGET])
def test_cleanup_fails_when_stale_mount_remains(self):
status, run, _, rmtree = self.run_cleanup(
[self.result(errno.ENOTCONN), self.result(1), self.result(1), self.result(1)])
self.assertEqual(status, 1)
self.assertEqual(run.call_count, 4)
rmtree.assert_not_called()
def test_unmounted_test_dir_is_reclaimed(self):
status, _, _, rmtree = self.run_cleanup(
[self.result(errno.ENOTCONN), self.result(0)], [True, False])
self.assertEqual(status, 0)
rmtree.assert_called_once_with(Path(TARGET).parent, ignore_errors=True)
def test_probe_process_reports_errno(self):
for target, expected in ((str(Path(__file__)), 0), (str(Path(__file__) / "absent"), errno.ENOTDIR)):
with self.subTest(target=target):
result = subprocess.run([cleanup.sys.executable, cleanup.__file__, "--probe", target], check=False)
self.assertEqual(result.returncode, expected)
if __name__ == "__main__":
unittest.main()
@@ -23,14 +23,8 @@ runs:
bin=$(command -v fusermount3 || true)
if [ -z "$bin" ]; then
# Self-hosted runners are not guaranteed to carry fuse3.
sudo apt-get update
sudo apt-get install -y fuse3
bin=$(command -v fusermount3 || true)
if [ -z "$bin" ]; then
echo "no fusermount3 in PATH after installing fuse3" >&2
exit 1
fi
echo "no fusermount3 in PATH" >&2
exit 1
fi
if can_mount "$bin"; then
ls -l "$bin"
-30
View File
@@ -1,30 +0,0 @@
name: "actions: unit tests"
on:
push:
branches: [ master ]
paths:
- '.github/actions/**'
- '.github/workflows/action-tests.yml'
pull_request:
branches: [ master ]
paths:
- '.github/actions/**'
- '.github/workflows/action-tests.yml'
concurrency:
group: ${{ github.workflow }}-${{ github.event.pull_request.number || (github.event_name == 'push' && github.ref) || github.run_id }}
cancel-in-progress: ${{ github.event_name == 'pull_request' || (github.event_name == 'push' && !startsWith(github.ref, 'refs/tags/')) }}
permissions:
contents: read
jobs:
cleanup-stale-fuse-mounts:
runs-on: ubuntu-latest
steps:
- name: Check out code
uses: actions/checkout@v7
- name: Run cleanup action unit tests
run: python3 .github/actions/cleanup-stale-fuse-mounts/test_cleanup.py
+2 -8
View File
@@ -9,8 +9,8 @@ on:
- '.github/workflows/codeql.yml'
concurrency:
group: ${{ github.workflow }}-${{ github.event.pull_request.number || (github.event_name == 'push' && github.ref) || github.run_id }}
cancel-in-progress: ${{ github.event_name == 'pull_request' || (github.event_name == 'push' && !startsWith(github.ref, 'refs/tags/')) }}
group: ${{ github.head_ref }}/codeql
cancel-in-progress: true
jobs:
CodeQL-Build:
@@ -25,12 +25,6 @@ jobs:
- name: Checkout repository
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1
- name: Set up Go
uses: actions/setup-go@v7
with:
go-version-file: 'go.mod'
cache: ${{ runner.environment == 'github-hosted' }}
# Initializes the CodeQL tools for scanning.
- name: Initialize CodeQL
uses: github/codeql-action/init@v4.39.0
-4
View File
@@ -8,10 +8,6 @@ on:
pull_request:
branches: [master]
concurrency:
group: ${{ github.workflow }}-${{ github.event.pull_request.number || (github.event_name == 'push' && github.ref) || github.run_id }}
cancel-in-progress: ${{ github.event_name == 'pull_request' || (github.event_name == 'push' && !startsWith(github.ref, 'refs/tags/')) }}
permissions:
contents: read
-11
View File
@@ -13,10 +13,6 @@ on:
- 'go.sum'
- '.github/workflows/container_dev.yml'
concurrency:
group: ${{ github.workflow }}-${{ github.event.pull_request.number || (github.event_name == 'push' && github.ref) || github.run_id }}
cancel-in-progress: ${{ github.event_name == 'pull_request' || (github.event_name == 'push' && !startsWith(github.ref, 'refs/tags/')) }}
permissions:
contents: read
@@ -55,7 +51,6 @@ jobs:
echo "CFLAGS_aarch64_unknown_linux_musl=-U_FORTIFY_SOURCE" >> "$GITHUB_ENV"
- name: Cache cargo registry and target
if: runner.environment == 'github-hosted'
uses: actions/cache@v6
with:
path: |
@@ -198,12 +193,6 @@ jobs:
platforms: linux/amd64, linux/arm64
tags: ${{ steps.docker_meta.outputs.tags }}
labels: ${{ steps.docker_meta.outputs.labels }}
cache-from: type=gha
cache-to: type=gha,mode=max
# Key the checkout layer on the pushed sha so cached layers can't
# serve stale code; the Dockerfile fetch fallback picks it up.
build-args: |
BRANCH=${{ github.sha }}
- name: Sign
if: github.event_name != 'pull_request'
@@ -24,10 +24,6 @@ on:
required: false
default: ''
concurrency:
group: ${{ github.workflow }}-${{ github.event.pull_request.number || (github.event_name == 'push' && github.ref) || github.event.inputs.image_tag || format('foundationdb_{0}_seaweedfs_{1}', github.event.inputs.fdb_version || '7.4.5', github.event.inputs.seaweedfs_ref || github.ref_name) }}
cancel-in-progress: ${{ github.event_name == 'pull_request' || (github.event_name == 'push' && !startsWith(github.ref, 'refs/tags/')) }}
permissions:
contents: read
@@ -68,7 +64,6 @@ jobs:
uses: actions/setup-go@v7
with:
go-version-file: go.mod
cache: ${{ runner.environment == 'github-hosted' }}
- name: Run FoundationDB tagged tests
env:
+2 -11
View File
@@ -30,11 +30,6 @@ on:
type: boolean
default: false
concurrency:
group: ${{ github.workflow }}-${{ github.event.pull_request.number || (github.event_name == 'push' && github.ref) || github.event.inputs.image_tag || github.run_id }}
cancel-in-progress: false
queue: max
permissions:
contents: read
security-events: write
@@ -99,7 +94,6 @@ jobs:
echo "CFLAGS_aarch64_unknown_linux_musl=-U_FORTIFY_SOURCE" >> "$GITHUB_ENV"
- name: Cache cargo registry and target
if: runner.environment == 'github-hosted'
uses: actions/cache@v6
with:
path: |
@@ -170,13 +164,10 @@ jobs:
uses: actions/checkout@v7
with:
ref: ${{ github.event_name == 'workflow_dispatch' && github.event.inputs.source_ref || github.ref }}
- name: Clean up stale FUSE mounts
uses: ./.github/actions/cleanup-stale-fuse-mounts
- name: Free Disk Space
run: |
echo "Available disk space before cleanup:"
timeout 15 df -h || true
df -h
# Remove pre-installed tools
sudo rm -rf /usr/share/dotnet /usr/local/lib/android /opt/ghc /opt/hostedtoolcache/CodeQL
# Clean package managers
@@ -188,7 +179,7 @@ jobs:
[ -d ~/.cache/go-build ] && rm -rf ~/.cache/go-build || true
[ -d /go/pkg ] && rm -rf /go/pkg || true
echo "Available disk space after cleanup:"
timeout 15 df -h || true
df -h
- name: Configure variant
id: config
@@ -11,11 +11,6 @@ on:
required: true
default: ''
concurrency:
group: ${{ github.workflow }}-${{ github.event.pull_request.number || (github.event_name == 'push' && github.ref) || format('refs/tags/{0}', github.event.inputs.release_tag) || github.run_id }}
cancel-in-progress: false
queue: max
permissions:
contents: read
@@ -25,7 +25,7 @@ on:
rocksdb_version:
description: 'RocksDB git tag to use when variant=rocksdb'
required: false
default: 'v11.1.2'
default: 'v10.10.1'
permissions:
contents: read
@@ -78,7 +78,6 @@ jobs:
echo "CFLAGS_aarch64_unknown_linux_musl=-U_FORTIFY_SOURCE" >> "$GITHUB_ENV"
- name: Cache cargo registry and target
if: runner.environment == 'github-hosted'
uses: actions/cache@v6
with:
path: |
@@ -180,10 +179,6 @@ jobs:
if: github.event_name != 'workflow_dispatch' || github.event.inputs.variant == 'all' || github.event.inputs.variant == matrix.variant
uses: actions/checkout@v7
- name: Clean up stale FUSE mounts
if: github.event_name != 'workflow_dispatch' || github.event.inputs.variant == 'all' || github.event.inputs.variant == matrix.variant
uses: ./.github/actions/cleanup-stale-fuse-mounts
- name: Download pre-built Rust binaries
if: github.event_name != 'workflow_dispatch' || github.event.inputs.variant == 'all' || github.event.inputs.variant == matrix.variant
uses: actions/download-artifact@v8
@@ -223,7 +218,7 @@ jobs:
sudo docker system prune -af --volumes
[ -d ~/.cache/go-build ] && rm -rf ~/.cache/go-build || true
[ -d /go/pkg ] && rm -rf /go/pkg || true
timeout 15 df -h || true
df -h
- name: Docker meta
if: github.event_name != 'workflow_dispatch' || github.event.inputs.variant == 'all' || github.event.inputs.variant == matrix.variant
@@ -275,11 +270,8 @@ jobs:
file: ${{ matrix.dockerfile }}
platforms: ${{ matrix.platform }}
labels: ${{ steps.docker_meta.outputs.labels }}
# mode=max keeps resolvedDependencies complete and records the
# builder identity; attestations ride inside each pushed index and
# survive the imagetools merge into the multi-arch manifest.
provenance: mode=max
sbom: true
# Flat single-platform manifest so imagetools create assembles cleanly.
provenance: false
outputs: type=image,name=${{ env.IMAGE }},push-by-digest=true,name-canonical=true,push=true
cache-from: type=gha,scope=${{ matrix.variant }}-${{ matrix.arch }}
# max only for rocksdb: its RocksDB compile is sha-independent and worth
@@ -289,7 +281,7 @@ jobs:
${{ matrix.build_args }}
BUILDKIT_INLINE_CACHE=1
BRANCH=${{ github.sha }}
${{ matrix.variant == 'rocksdb' && format('ROCKSDB_VERSION={0}', github.event.inputs.rocksdb_version || 'v11.1.2') || '' }}
${{ matrix.variant == 'rocksdb' && format('ROCKSDB_VERSION={0}', github.event.inputs.rocksdb_version || 'v10.10.1') || '' }}
- name: Export digest
if: github.event_name != 'workflow_dispatch' || github.event.inputs.variant == 'all' || github.event.inputs.variant == matrix.variant
@@ -315,8 +307,6 @@ jobs:
permissions:
contents: read
id-token: write
attestations: write
packages: write
strategy:
fail-fast: false
matrix:
@@ -424,45 +414,6 @@ jobs:
${{ env.IMAGE }}@${{ steps.manifest.outputs.digest }}
chrislusf/seaweedfs@${{ steps.manifest.outputs.digest }}
# Attest the merged multi-arch digest itself, so digest-pinning consumers
# can verify provenance against the exact index they pull. Both steps
# reuse the GHCR login above and push as OCI referrers.
- name: Attest build provenance for ${{ matrix.variant }}
if: github.event_name != 'workflow_dispatch' || github.event.inputs.variant == 'all' || github.event.inputs.variant == matrix.variant
uses: actions/attest-build-provenance@v4
with:
subject-name: ${{ env.IMAGE }}
subject-digest: ${{ steps.manifest.outputs.digest }}
push-to-registry: true
# SBOMs are generated and attested per platform manifest: an index-wide
# scan only describes syft's default platform, while the manifests it
# merges differ (some ship the real binary, others placeholders), so the
# SBOM has to be bound to the platform digest a consumer actually pulls.
- name: Install syft
if: github.event_name != 'workflow_dispatch' || github.event.inputs.variant == 'all' || github.event.inputs.variant == matrix.variant
uses: anchore/sbom-action/download-syft@v0
- name: Attest per-platform SBOMs for ${{ matrix.variant }}
if: github.event_name != 'workflow_dispatch' || github.event.inputs.variant == 'all' || github.event.inputs.variant == matrix.variant
working-directory: /tmp/digests
run: |
set -euo pipefail
for f in *; do
# Each saved digest is an OCI index (image + attestation manifests);
# scan and attest the image manifest a same-platform pull resolves.
ref="${{ env.IMAGE }}@sha256:${f}"
image=$(crane manifest "$ref" | jq -r '
[.manifests[]? | select((.annotations["vnd.docker.reference.type"] // "") != "attestation-manifest"
and (.platform.os // "unknown") != "unknown") | .digest][0] // empty')
ref="${{ env.IMAGE }}@${image:-sha256:${f}}"
syft "registry:${ref}" -o "spdx-json=/tmp/sbom-${f}.spdx.json"
cosign attest --yes \
--predicate "/tmp/sbom-${f}.spdx.json" \
--type spdxjson \
"$ref"
done
# Report-only trivy scan: uploads fixable HIGH/CRITICAL findings to GitHub
# Security for visibility, but never blocks the release. Releases (including
# `latest`) ship regardless — vulnerabilities are tracked, not gated, since
@@ -501,9 +452,7 @@ jobs:
scanners: vuln
vuln-type: os,library
severity: HIGH,CRITICAL
# Keep unfixed findings in the report so digest-pinning consumers see
# the complete picture, not just what a patch could resolve.
ignore-unfixed: false
ignore-unfixed: true
limit-severities-for-sarif: true
format: sarif
output: trivy-results.sarif
@@ -4,9 +4,9 @@ on:
workflow_dispatch:
inputs:
rocksdb_version:
description: 'RocksDB git tag or branch to build (e.g. v11.1.2)'
description: 'RocksDB git tag or branch to build (e.g. v10.10.1)'
required: true
default: 'v11.1.2'
default: 'v10.10.1'
seaweedfs_ref:
description: 'SeaweedFS git tag, branch, or commit to build'
required: true
@@ -16,11 +16,6 @@ on:
required: false
default: ''
concurrency:
group: ${{ github.workflow }}-${{ github.event.inputs.image_tag || format('rocksdb_{0}_seaweedfs_{1}', github.event.inputs.rocksdb_version, github.event.inputs.seaweedfs_ref) }}
cancel-in-progress: false
queue: max
permissions:
contents: read
-1
View File
@@ -27,7 +27,6 @@ jobs:
uses: actions/setup-go@v7
with:
go-version-file: 'telemetry/server/go.mod'
cache: ${{ runner.environment == 'github-hosted' }}
- name: Build Telemetry Server
if: github.event_name == 'workflow_dispatch' && inputs.deploy
-4
View File
@@ -1,10 +1,6 @@
name: 'Dependency Review'
on: [pull_request]
concurrency:
group: ${{ github.workflow }}-${{ github.event.pull_request.number || (github.event_name == 'push' && github.ref) || github.run_id }}
cancel-in-progress: ${{ github.event_name == 'pull_request' || (github.event_name == 'push' && !startsWith(github.ref, 'refs/tags/')) }}
permissions:
contents: read
+2 -4
View File
@@ -19,8 +19,8 @@ on:
- '.github/workflows/e2e.yml'
concurrency:
group: ${{ github.workflow }}-${{ github.event.pull_request.number || (github.event_name == 'push' && github.ref) || github.run_id }}
cancel-in-progress: ${{ github.event_name == 'pull_request' || (github.event_name == 'push' && !startsWith(github.ref, 'refs/tags/')) }}
group: ${{ github.head_ref }}/e2e
cancel-in-progress: true
permissions:
contents: read
@@ -42,7 +42,6 @@ jobs:
uses: actions/setup-go@v7
with:
go-version-file: 'go.mod'
cache: ${{ runner.environment == 'github-hosted' }}
- name: Configure Docker Hub mirror
run: |
@@ -53,7 +52,6 @@ jobs:
uses: docker/setup-buildx-action@v4
- name: Cache Docker layers
if: runner.environment == 'github-hosted'
uses: actions/cache@v6
with:
path: /tmp/.buildx-cache
@@ -18,10 +18,6 @@ on:
- 'go.sum'
- '.github/workflows/ec-integration-tests.yml'
concurrency:
group: ${{ github.workflow }}-${{ github.event.pull_request.number || (github.event_name == 'push' && github.ref) || github.run_id }}
cancel-in-progress: ${{ github.event_name == 'pull_request' || (github.event_name == 'push' && !startsWith(github.ref, 'refs/tags/')) }}
permissions:
contents: read
@@ -34,7 +30,6 @@ jobs:
- name: Set up Go 1.x
uses: actions/setup-go@v7
with:
cache: ${{ runner.environment == 'github-hosted' }}
go-version: ^1.26
id: go
+2 -3
View File
@@ -19,8 +19,8 @@ on:
- '.github/actions/fix-fusermount-setuid/**'
concurrency:
group: ${{ github.workflow }}-${{ github.event.pull_request.number || (github.event_name == 'push' && github.ref) || github.run_id }}
cancel-in-progress: ${{ github.event_name == 'pull_request' || (github.event_name == 'push' && !startsWith(github.ref, 'refs/tags/')) }}
group: ${{ github.head_ref || github.ref }}/fuse-dlm-integration
cancel-in-progress: true
permissions:
contents: read
@@ -39,7 +39,6 @@ jobs:
uses: actions/setup-go@v7
with:
go-version-file: 'go.mod'
cache: ${{ runner.environment == 'github-hosted' }}
- name: Configure FUSE
run: |
+2 -6
View File
@@ -24,8 +24,8 @@ on:
- '.github/actions/fix-fusermount-setuid/**'
concurrency:
group: ${{ github.workflow }}-${{ github.event.pull_request.number || (github.event_name == 'push' && github.ref) || github.run_id }}
cancel-in-progress: ${{ github.event_name == 'pull_request' || (github.event_name == 'push' && !startsWith(github.ref, 'refs/tags/')) }}
group: ${{ github.head_ref || github.ref }}/fuse-failover
cancel-in-progress: true
permissions:
contents: read
@@ -42,14 +42,10 @@ jobs:
with:
persist-credentials: false
- name: Clean up stale FUSE mounts
uses: ./.github/actions/cleanup-stale-fuse-mounts
- name: Set up Go
uses: actions/setup-go@v7
with:
go-version-file: 'go.mod'
cache: ${{ runner.environment == 'github-hosted' }}
- name: Configure FUSE
run: |
+2 -3
View File
@@ -17,8 +17,8 @@ on:
- '.github/actions/fix-fusermount-setuid/**'
concurrency:
group: ${{ github.workflow }}-${{ github.event.pull_request.number || (github.event_name == 'push' && github.ref) || github.run_id }}
cancel-in-progress: ${{ github.event_name == 'pull_request' || (github.event_name == 'push' && !startsWith(github.ref, 'refs/tags/')) }}
group: ${{ github.head_ref }}/fuse-integration
cancel-in-progress: true
permissions:
contents: read
@@ -37,7 +37,6 @@ jobs:
uses: actions/setup-go@v7
with:
go-version-file: 'go.mod'
cache: ${{ runner.environment == 'github-hosted' }}
- name: Configure FUSE
run: |
+2 -6
View File
@@ -25,8 +25,8 @@ on:
- '.github/actions/fix-fusermount-setuid/**'
concurrency:
group: ${{ github.workflow }}-${{ github.event.pull_request.number || (github.event_name == 'push' && github.ref) || github.run_id }}
cancel-in-progress: ${{ github.event_name == 'pull_request' || (github.event_name == 'push' && !startsWith(github.ref, 'refs/tags/')) }}
group: ${{ github.head_ref || github.ref }}/fuse-p2p-integration
cancel-in-progress: true
permissions:
contents: read
@@ -41,14 +41,10 @@ jobs:
- name: Check out code
uses: actions/checkout@v7
- name: Clean up stale FUSE mounts
uses: ./.github/actions/cleanup-stale-fuse-mounts
- name: Set up Go
uses: actions/setup-go@v7
with:
go-version-file: 'go.mod'
cache: ${{ runner.environment == 'github-hosted' }}
- name: Configure FUSE
run: |
+2 -9
View File
@@ -17,8 +17,8 @@ on:
- '.github/workflows/go.yml'
concurrency:
group: ${{ github.workflow }}-${{ github.event.pull_request.number || (github.event_name == 'push' && github.ref) || github.run_id }}
cancel-in-progress: ${{ github.event_name == 'pull_request' || (github.event_name == 'push' && !startsWith(github.ref, 'refs/tags/')) }}
group: ${{ github.head_ref }}/go
cancel-in-progress: true
permissions:
contents: read
@@ -35,7 +35,6 @@ jobs:
uses: actions/setup-go@v7
with:
go-version-file: 'go.mod'
cache: ${{ runner.environment == 'github-hosted' }}
- name: Get dependencies
run: |
cd weed; go get -v -t -d ./...
@@ -58,7 +57,6 @@ jobs:
uses: actions/setup-go@v7
with:
go-version-file: 'go.mod'
cache: ${{ runner.environment == 'github-hosted' }}
- name: Go Vet linux/386 (type-checks code and tests for 32-bit int overflows)
run: |
GOOS=linux GOARCH=386 go vet ./... 2>&1 | grep -v "MessageState contains sync.Mutex" | grep -v "IdentityAccessManagement contains sync.RWMutex" | tee vet-32bit-output.txt
@@ -74,7 +72,6 @@ jobs:
uses: actions/setup-go@v7
with:
go-version-file: 'go.mod'
cache: ${{ runner.environment == 'github-hosted' }}
- name: Build
run: cd weed; go build -tags "elastic gocdk sqlite ydb tarantool tikv rclone" -v .
@@ -97,7 +94,6 @@ jobs:
uses: actions/setup-go@v7
with:
go-version-file: 'go.mod'
cache: ${{ runner.environment == 'github-hosted' }}
# Nothing else on a PR compiles these, so per-OS syscall constants creep
# back into shared files unnoticed and only a release build catches it.
- name: Cross-compile
@@ -132,7 +128,6 @@ jobs:
uses: actions/setup-go@v7
with:
go-version-file: 'go.mod'
cache: ${{ runner.environment == 'github-hosted' }}
- name: Test
env:
RUN_REDIS_TESTS: "1"
@@ -148,7 +143,6 @@ jobs:
uses: actions/setup-go@v7
with:
go-version-file: 'go.mod'
cache: ${{ runner.environment == 'github-hosted' }}
# 386 test binaries run natively on the amd64 runner. This catches what vet
# can't: unaligned 64-bit atomics and arithmetic that wraps at runtime.
# -short skips the e2e suites already covered on amd64.
@@ -165,7 +159,6 @@ jobs:
uses: actions/setup-go@v7
with:
go-version-file: 'go.mod'
cache: ${{ runner.environment == 'github-hosted' }}
# weed/kms/azure is excluded from every default build, so without this
# nothing compiles it and tag-gated code silently rots.
- name: Build and test with azurekms
-4
View File
@@ -8,10 +8,6 @@ on:
branches: [ master ]
paths: ['k8s/**', '.github/workflows/helm_ci.yml', '.github/scripts/helm_jwt_keys.py']
concurrency:
group: ${{ github.workflow }}-${{ github.event.pull_request.number || (github.event_name == 'push' && github.ref) || github.run_id }}
cancel-in-progress: ${{ github.event_name == 'pull_request' || (github.event_name == 'push' && !startsWith(github.ref, 'refs/tags/')) }}
permissions:
contents: read
+1 -7
View File
@@ -14,10 +14,6 @@ on:
- 'weed/**'
- '.github/workflows/java_integration_tests.yml'
concurrency:
group: ${{ github.workflow }}-${{ github.event.pull_request.number || (github.event_name == 'push' && github.ref) || github.run_id }}
cancel-in-progress: ${{ github.event_name == 'pull_request' || (github.event_name == 'push' && !startsWith(github.ref, 'refs/tags/')) }}
jobs:
test:
name: Java Integration Tests
@@ -35,7 +31,6 @@ jobs:
uses: actions/setup-go@v7
with:
go-version-file: 'go.mod'
cache: ${{ runner.environment == 'github-hosted' }}
id: go
- name: Set up Java
@@ -43,7 +38,7 @@ jobs:
with:
java-version: ${{ matrix.java }}
distribution: 'temurin'
cache: ${{ runner.environment == 'github-hosted' && 'maven' || '' }}
cache: 'maven'
- name: Build SeaweedFS
run: |
@@ -60,7 +55,6 @@ jobs:
# Start SeaweedFS with optimized settings for CI
# Include S3 API for s3copier integration tests
weed server -dir="$WEED_DATA_DIR" \
-ip=127.0.0.1 -ip.bind=127.0.0.1 \
-master.raftHashicorp \
-master.electionTimeout=1s \
-master.volumeSizeLimitMB=100 \
+1 -1
View File
@@ -77,7 +77,7 @@ jobs:
with:
java-version: '17'
distribution: 'temurin'
cache: ${{ runner.environment == 'github-hosted' && 'maven' || '' }}
cache: 'maven'
server-id: central
server-username: MAVEN_CENTRAL_USERNAME
server-password: MAVEN_CENTRAL_PASSWORD
+1 -5
View File
@@ -12,10 +12,6 @@ on:
- 'other/java/**'
- '.github/workflows/java_unit_tests.yml'
concurrency:
group: ${{ github.workflow }}-${{ github.event.pull_request.number || (github.event_name == 'push' && github.ref) || github.run_id }}
cancel-in-progress: ${{ github.event_name == 'pull_request' || (github.event_name == 'push' && !startsWith(github.ref, 'refs/tags/')) }}
jobs:
test:
name: Java Unit Tests
@@ -34,7 +30,7 @@ jobs:
with:
java-version: ${{ matrix.java }}
distribution: 'temurin'
cache: ${{ runner.environment == 'github-hosted' && 'maven' || '' }}
cache: 'maven'
- name: Build and Install SeaweedFS Client
working-directory: other/java/client
+3 -3
View File
@@ -24,8 +24,8 @@ on:
workflow_dispatch: # Allow manual trigger
concurrency:
group: ${{ github.workflow }}-${{ github.event.pull_request.number || (github.event_name == 'push' && github.ref) || github.run_id }}
cancel-in-progress: ${{ github.event_name == 'pull_request' || (github.event_name == 'push' && !startsWith(github.ref, 'refs/tags/')) }}
group: ${{ github.head_ref }}/kafka-quicktest
cancel-in-progress: true
permissions:
contents: read
@@ -43,7 +43,7 @@ jobs:
uses: actions/setup-go@v7
with:
go-version: ^1.26
cache: ${{ runner.environment == 'github-hosted' }}
cache: true
cache-dependency-path: |
**/go.sum
id: go
+6 -9
View File
@@ -23,8 +23,8 @@ on:
- '.github/workflows/kafka-tests.yml'
concurrency:
group: ${{ github.workflow }}-${{ github.event.pull_request.number || (github.event_name == 'push' && github.ref) || github.run_id }}
cancel-in-progress: ${{ github.event_name == 'pull_request' || (github.event_name == 'push' && !startsWith(github.ref, 'refs/tags/')) }}
group: ${{ github.head_ref }}/kafka-tests
cancel-in-progress: true
# Force different runners for better isolation
env:
@@ -53,7 +53,6 @@ jobs:
- name: Set up Go 1.x
uses: actions/setup-go@v7
with:
cache: ${{ runner.environment == 'github-hosted' }}
go-version: ^1.26
id: go
@@ -99,7 +98,6 @@ jobs:
- name: Set up Go 1.x
uses: actions/setup-go@v7
with:
cache: ${{ runner.environment == 'github-hosted' }}
go-version: ^1.26
id: go
@@ -151,7 +149,7 @@ jobs:
uses: actions/setup-go@v7
with:
go-version: ^1.26
cache: ${{ runner.environment == 'github-hosted' }}
cache: true
cache-dependency-path: |
**/go.sum
id: go
@@ -330,7 +328,7 @@ jobs:
uses: actions/setup-go@v7
with:
go-version: ^1.26
cache: ${{ runner.environment == 'github-hosted' }}
cache: true
cache-dependency-path: |
**/go.sum
id: go
@@ -492,7 +490,7 @@ jobs:
uses: actions/setup-go@v7
with:
go-version: ^1.26
cache: ${{ runner.environment == 'github-hosted' }}
cache: true
cache-dependency-path: |
**/go.sum
id: go
@@ -650,7 +648,7 @@ jobs:
uses: actions/setup-go@v7
with:
go-version: ^1.26
cache: ${{ runner.environment == 'github-hosted' }}
cache: true
cache-dependency-path: |
**/go.sum
id: go
@@ -807,7 +805,6 @@ jobs:
- name: Set up Go 1.x
uses: actions/setup-go@v7
with:
cache: ${{ runner.environment == 'github-hosted' }}
go-version: ^1.26
id: go
+2 -4
View File
@@ -19,8 +19,8 @@ on:
- 'test/kms/**'
concurrency:
group: ${{ github.workflow }}-${{ github.event.pull_request.number || (github.event_name == 'push' && github.ref) || github.run_id }}
cancel-in-progress: ${{ github.event_name == 'pull_request' || (github.event_name == 'push' && !startsWith(github.ref, 'refs/tags/')) }}
group: ${{ github.head_ref || github.ref }}-kms-tests
cancel-in-progress: true
permissions:
contents: read
@@ -43,7 +43,6 @@ jobs:
uses: actions/setup-go@v7
with:
go-version-file: 'go.mod'
cache: ${{ runner.environment == 'github-hosted' }}
id: go
- name: Install SeaweedFS
@@ -89,7 +88,6 @@ jobs:
uses: actions/setup-go@v7
with:
go-version-file: 'go.mod'
cache: ${{ runner.environment == 'github-hosted' }}
id: go
- name: Install SeaweedFS
@@ -21,8 +21,8 @@ on:
- '.github/workflows/master-cold-start-tests.yml'
concurrency:
group: ${{ github.workflow }}-${{ github.event.pull_request.number || (github.event_name == 'push' && github.ref) || github.run_id }}
cancel-in-progress: ${{ github.event_name == 'pull_request' || (github.event_name == 'push' && !startsWith(github.ref, 'refs/tags/')) }}
group: ${{ github.head_ref || github.ref }}/master-cold-start-tests
cancel-in-progress: true
permissions:
contents: read
@@ -42,7 +42,6 @@ jobs:
uses: actions/setup-go@v7
with:
go-version-file: 'go.mod'
cache: ${{ runner.environment == 'github-hosted' }}
- name: Build weed binary
run: |
@@ -25,8 +25,8 @@ on:
- '.github/workflows/metadata-subscribe-tests.yml'
concurrency:
group: ${{ github.workflow }}-${{ github.event.pull_request.number || (github.event_name == 'push' && github.ref) || github.run_id }}
cancel-in-progress: ${{ github.event_name == 'pull_request' || (github.event_name == 'push' && !startsWith(github.ref, 'refs/tags/')) }}
group: ${{ github.head_ref }}/metadata-subscribe-tests
cancel-in-progress: true
permissions:
contents: read
@@ -48,7 +48,6 @@ jobs:
uses: actions/setup-go@v7
with:
go-version-file: 'go.mod'
cache: ${{ runner.environment == 'github-hosted' }}
- name: Build SeaweedFS
run: |
+2 -4
View File
@@ -10,8 +10,8 @@ on:
branches: [ 'winfsp-bench**' ]
concurrency:
group: ${{ github.workflow }}-${{ github.event.pull_request.number || (github.event_name == 'push' && github.ref) || github.run_id }}
cancel-in-progress: ${{ github.event_name == 'pull_request' || (github.event_name == 'push' && !startsWith(github.ref, 'refs/tags/')) }}
group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }}
cancel-in-progress: true
permissions:
contents: read
@@ -32,7 +32,6 @@ jobs:
- uses: actions/setup-go@v7
with:
go-version-file: 'go.mod'
cache: ${{ runner.environment == 'github-hosted' }}
- name: Install WinFsp and rclone
run: choco install winfsp rclone -y --no-progress
@@ -151,7 +150,6 @@ jobs:
- uses: actions/setup-go@v7
with:
go-version-file: 'go.mod'
cache: ${{ runner.environment == 'github-hosted' }}
- name: Repair the fusermount3 setuid bit
uses: ./.github/actions/fix-fusermount-setuid
@@ -24,8 +24,8 @@ on:
- '.github/workflows/mount-windows-conformance.yml'
concurrency:
group: ${{ github.workflow }}-${{ github.event.pull_request.number || (github.event_name == 'push' && github.ref) || github.run_id }}
cancel-in-progress: ${{ github.event_name == 'pull_request' || (github.event_name == 'push' && !startsWith(github.ref, 'refs/tags/')) }}
group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }}
cancel-in-progress: true
permissions:
contents: read
@@ -49,7 +49,6 @@ jobs:
- uses: actions/setup-go@v7
with:
go-version-file: 'go.mod'
cache: ${{ runner.environment == 'github-hosted' }}
# cgofuse loads winfsp-x64.dll at run time, so WinFsp is needed here but
# not to build.
+2 -3
View File
@@ -24,8 +24,8 @@ on:
- '.github/workflows/mount-windows.yml'
concurrency:
group: ${{ github.workflow }}-${{ github.event.pull_request.number || (github.event_name == 'push' && github.ref) || github.run_id }}
cancel-in-progress: ${{ github.event_name == 'pull_request' || (github.event_name == 'push' && !startsWith(github.ref, 'refs/tags/')) }}
group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }}
cancel-in-progress: true
permissions:
contents: read
@@ -49,7 +49,6 @@ jobs:
- uses: actions/setup-go@v7
with:
go-version-file: 'go.mod'
cache: ${{ runner.environment == 'github-hosted' }}
# cgofuse loads winfsp-x64.dll at run time, so WinFsp is needed here but
# not to build.
+2 -3
View File
@@ -21,8 +21,8 @@ on:
- '.github/workflows/multi-master-tests.yml'
concurrency:
group: ${{ github.workflow }}-${{ github.event.pull_request.number || (github.event_name == 'push' && github.ref) || github.run_id }}
cancel-in-progress: ${{ github.event_name == 'pull_request' || (github.event_name == 'push' && !startsWith(github.ref, 'refs/tags/')) }}
group: ${{ github.head_ref || github.ref }}/multi-master-tests
cancel-in-progress: true
permissions:
contents: read
@@ -40,7 +40,6 @@ jobs:
uses: actions/setup-go@v7
with:
go-version-file: 'go.mod'
cache: ${{ runner.environment == 'github-hosted' }}
- name: Install SeaweedFS
run: |
+4 -7
View File
@@ -45,8 +45,8 @@ on:
type: string
concurrency:
group: ${{ github.workflow }}-${{ github.event.pull_request.number || (github.event_name == 'push' && github.ref) || github.run_id }}
cancel-in-progress: ${{ github.event_name == 'pull_request' || (github.event_name == 'push' && !startsWith(github.ref, 'refs/tags/')) }}
group: ${{ github.head_ref || github.ref }}/performance
cancel-in-progress: true
permissions:
contents: read
@@ -67,7 +67,6 @@ jobs:
uses: actions/setup-go@v7
with:
go-version-file: 'go.mod'
cache: ${{ runner.environment == 'github-hosted' }}
- name: Build weed
run: go build -o weed_bin ./weed
@@ -176,14 +175,13 @@ jobs:
uses: actions/setup-go@v7
with:
go-version-file: 'go.mod'
cache: ${{ runner.environment == 'github-hosted' }}
- name: Install Rust toolchain
if: matrix.impl == 'rust'
uses: dtolnay/rust-toolchain@stable
- name: Cache cargo registry and target
if: matrix.impl == 'rust' && runner.environment == 'github-hosted'
if: matrix.impl == 'rust'
uses: actions/cache@v6
with:
path: |
@@ -329,14 +327,13 @@ jobs:
uses: actions/setup-go@v7
with:
go-version-file: 'go.mod'
cache: ${{ runner.environment == 'github-hosted' }}
- name: Install Rust toolchain
if: matrix.impl == 'rust'
uses: dtolnay/rust-toolchain@stable
- name: Cache cargo registry and target
if: matrix.impl == 'rust' && runner.environment == 'github-hosted'
if: matrix.impl == 'rust'
uses: actions/cache@v6
with:
path: |
+1 -9
View File
@@ -38,23 +38,15 @@ jobs:
uses: actions/setup-go@v7
with:
go-version-file: 'go.mod'
cache: ${{ runner.environment == 'github-hosted' }}
- name: Start local Docker registry
run: |
if ! docker start registry 2>/dev/null; then
docker rm -f registry >/dev/null 2>&1 || true
docker run -d --restart=always -p 5000:5000 --name registry registry:2
fi
run: docker run -d --restart=always -p 5000:5000 --name registry registry:2
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v4
with:
driver-opts: network=host
- name: Set up Docker Compose
uses: docker/setup-compose-action@v2
- name: Build weed race binary
run: |
cd docker
-5
View File
@@ -18,10 +18,6 @@ on:
- 'go.sum'
- '.github/workflows/plugin-workers.yml'
concurrency:
group: ${{ github.workflow }}-${{ github.event.pull_request.number || (github.event_name == 'push' && github.ref) || github.run_id }}
cancel-in-progress: ${{ github.event_name == 'pull_request' || (github.event_name == 'push' && !startsWith(github.ref, 'refs/tags/')) }}
permissions:
contents: read
@@ -45,7 +41,6 @@ jobs:
- name: Set up Go 1.x
uses: actions/setup-go@v7
with:
cache: ${{ runner.environment == 'github-hosted' }}
go-version: ^1.26
id: go
-791
View File
@@ -1,791 +0,0 @@
name: POSIX Compliance Tests
on:
push:
branches: [ main, master, develop ]
paths:
- 'weed/mount/**'
- 'weed/command/mount*'
- 'weed/command/fuse*'
- 'test/fuse_integration/**'
pull_request:
branches: [ main, master, develop ]
paths:
- 'weed/mount/**'
- 'weed/command/mount*'
- 'weed/command/fuse*'
- 'test/fuse_integration/**'
workflow_dispatch:
inputs:
test_type:
description: 'Type of POSIX tests to run (default: full for comprehensive testing)'
required: true
default: 'full'
type: choice
options:
- critical
- basic
- extended
- full
enable_external_tests:
description: 'Run external test suites (slower)'
required: false
default: false
type: boolean
permissions:
contents: read
pull-requests: write
issues: write
env:
GO_VERSION: '1.26'
TIMEOUT: '45m'
jobs:
posix-compliance-ubuntu:
runs-on: ubuntu-latest
timeout-minutes: 60
strategy:
matrix:
fuse-version: ['2.9', '3.0']
fail-fast: false
steps:
- name: Checkout code
uses: actions/checkout@v7
with:
fetch-depth: 1
- name: Clean up stale FUSE mounts
uses: ./.github/actions/cleanup-stale-fuse-mounts
- name: Set up Go ${{ env.GO_VERSION }}
uses: actions/setup-go@v7
with:
go-version: ${{ env.GO_VERSION }}
cache: ${{ runner.environment == 'github-hosted' }}
cache-dependency-path: |
go.sum
test/fuse_integration/go.sum
- name: Install system dependencies
run: |
sudo apt-get update
sudo apt-get install -y \
attr \
acl \
build-essential \
git \
python3-pip
# Install FUSE version specific packages
if [ "${{ matrix.fuse-version }}" = "2.9" ]; then
echo "Installing FUSE 2.9..."
sudo apt-get install -y fuse libfuse-dev
elif [ "${{ matrix.fuse-version }}" = "3.0" ]; then
echo "Installing FUSE 3.0..."
sudo apt-get install -y fuse3 libfuse3-dev
# Also install fuse2 for compatibility if needed
sudo apt-get install -y fuse libfuse-dev
fi
- name: Set up user permissions for FUSE
run: |
# Create fuse group if it doesn't exist
sudo groupadd -f fuse
# Add user to fuse group
sudo usermod -a -G fuse $USER
# Set permissions on FUSE device
sudo chmod 666 /dev/fuse
# Allow non-root mounts to pass -o allow_other (weed -allowOthers)
echo 'user_allow_other' | sudo tee -a /etc/fuse.conf
sudo chmod 644 /etc/fuse.conf
# Ensure fuse module is loaded
sudo modprobe fuse || true
# Verify setup
echo "FUSE setup verification:"
ls -la /dev/fuse
groups $USER | grep fuse || echo "User not in fuse group yet (will be effective after login)"
lsmod | grep fuse || echo "FUSE module not loaded"
- name: Install external test tools
if: ${{ github.event.inputs.enable_external_tests == 'true' }}
run: |
# Install nfstest for POSIX API verification
pip3 install --user nfstest
# Install FIO for performance testing
sudo apt-get install -y fio
# Install additional test utilities
sudo apt-get install -y stress-ng
- name: Build SeaweedFS
run: |
make
# Verify binary exists and is executable (it's installed to GOPATH/bin)
weed version
# Make weed binary available in PATH (it's already there from go install)
sudo cp $(which weed) /usr/local/bin/weed
which weed
weed version
- name: Set up SeaweedFS cluster
run: |
start_and_wait() {
local name=$1
local url=$2
local pidfile=$3
shift 3
local cmd="$@"
echo "Starting $name..."
$cmd > "/tmp/seaweedfs/$name.log" 2>&1 &
local pid=$!
echo $pid > "$pidfile"
echo "Waiting for $name to start..."
for i in {1..30}; do
if curl -sf "$url" > /dev/null 2>&1; then
echo "$name is ready"
return 0
fi
if [ $i -eq 30 ]; then
echo "$name failed to start"
cat "/tmp/seaweedfs/$name.log"
exit 1
fi
sleep 2
done
}
mkdir -p /tmp/seaweedfs/{master,volume,filer,mount}
start_and_wait "master" "http://127.0.0.1:9333/cluster/status" "/tmp/seaweedfs/master.pid" \
"weed master -ip=127.0.0.1 -port=9333 -mdir=/tmp/seaweedfs/master -raftBootstrap=true"
start_and_wait "volume" "http://127.0.0.1:8080/status" "/tmp/seaweedfs/volume.pid" \
"weed volume -mserver=127.0.0.1:9333 -ip=127.0.0.1 -port=8080 -dir=/tmp/seaweedfs/volume -max=100"
start_and_wait "filer" "http://127.0.0.1:8888/" "/tmp/seaweedfs/filer.pid" \
"weed filer -master=127.0.0.1:9333 -ip=127.0.0.1 -port=8888"
echo "SeaweedFS cluster status:"
curl -s http://127.0.0.1:9333/cluster/status || true
- name: Set up FUSE mount
run: |
# Create mount point
MOUNT_POINT="/tmp/seaweedfs/mount"
mkdir -p $MOUNT_POINT
echo "Mounting SeaweedFS FUSE filesystem..."
# Mount SeaweedFS FUSE filesystem in background
weed mount \
-filer=127.0.0.1:8888 \
-dir=$MOUNT_POINT \
-filer.path=/ \
-dirAutoCreate=true \
-allowOthers=true \
-nonempty=true \
> /tmp/seaweedfs/mount.log 2>&1 &
MOUNT_PID=$!
echo $MOUNT_PID > /tmp/seaweedfs/mount.pid
# Wait for mount to be ready. The mount point directory already
# exists, so only a real mount counts — a directory listing would
# pass on the local disk even if weed mount died.
echo "Waiting for FUSE mount to be ready..."
for i in {1..30}; do
if mountpoint -q $MOUNT_POINT 2>/dev/null; then
echo "FUSE mount is ready"
break
fi
if ! kill -0 $MOUNT_PID 2>/dev/null; then
echo "FUSE mount process exited"
cat /tmp/seaweedfs/mount.log
exit 1
fi
if [ $i -eq 30 ]; then
echo "FUSE mount failed to be ready"
cat /tmp/seaweedfs/mount.log
echo "Mount status:"
mount | grep fuse || echo "No FUSE mounts found"
ps aux | grep weed || echo "No weed processes found"
exit 1
fi
sleep 2
done
# Verify mount is working
echo "Testing FUSE mount functionality..."
echo "test" > $MOUNT_POINT/test_file.txt
if [ "$(cat $MOUNT_POINT/test_file.txt)" = "test" ]; then
echo "FUSE mount is working correctly"
rm $MOUNT_POINT/test_file.txt
else
echo "FUSE mount is not working properly"
exit 1
fi
# Export mount point for tests
echo "SEAWEEDFS_MOUNT_POINT=$MOUNT_POINT" >> $GITHUB_ENV
- name: Set up test environment
run: |
cd test/fuse_integration
# Initialize Go module for tests
if [ ! -f go.mod ]; then
go mod init seaweedfs-posix-tests
go mod tidy
fi
# Create reports directory
mkdir -p reports
# Set up external tools if requested
# Note: External tools setup is now optional and won't fail the build
if [ "${{ github.event.inputs.enable_external_tests }}" = "true" ]; then
echo "Setting up external POSIX testing tools..."
make -f posix_Makefile setup-external-tools || true
# setup-external-tools exits 0 even when a sub-target failed, so
# require the tools only when they are actually installed
if [ -d external_tools/pjdfstest ] && command -v nfstest_posix >/dev/null 2>&1; then
export POSIX_REQUIRE_EXTERNAL_TOOLS="true"
echo "POSIX_REQUIRE_EXTERNAL_TOOLS=true" >> $GITHUB_ENV
echo "✅ External tools setup completed successfully"
else
echo "⚠️ External tools setup incomplete - continuing with basic tests only"
echo "External tools will be skipped but tests will continue"
echo "This is expected behavior when build tools are not available"
fi
fi
# Verify SeaweedFS cluster is accessible
echo "Verifying SeaweedFS cluster accessibility..."
curl -sf http://127.0.0.1:9333/cluster/status
curl -sf http://127.0.0.1:8080/status
curl -sf http://127.0.0.1:8888/
# Verify mount point
echo "Mount point: $SEAWEEDFS_MOUNT_POINT"
ls -la $SEAWEEDFS_MOUNT_POINT
# Verify mount is actually working by testing file operations
echo "Verifying SeaweedFS mount functionality..."
TEST_FILE="$SEAWEEDFS_MOUNT_POINT/.mount_verification_test"
echo "SeaweedFS mount test" > "$TEST_FILE"
if [ "$(cat "$TEST_FILE")" = "SeaweedFS mount test" ]; then
echo "✅ Mount verification successful"
rm -f "$TEST_FILE"
else
echo "❌ Mount verification failed - mount may not be working properly"
exit 1
fi
# Check if this is actually a FUSE mount — the plain directory created
# above passes the file test, so without this the suite can run
# against the local disk and report a pass it never earned
if mount | grep -q "$SEAWEEDFS_MOUNT_POINT"; then
echo "✅ FUSE mount detected:"
mount | grep "$SEAWEEDFS_MOUNT_POINT"
else
echo "❌ Mount point not found in mount table - refusing to test a local directory"
exit 1
fi
- name: Run POSIX compliance tests
id: posix-tests
env:
SEAWEEDFS_MOUNT_POINT: ${{ env.SEAWEEDFS_MOUNT_POINT }}
POSIX_REQUIRE_EXTERNAL_TOOLS: ${{ env.POSIX_REQUIRE_EXTERNAL_TOOLS }}
run: |
cd test/fuse_integration
# Determine which tests to run
TEST_TYPE="${{ github.event.inputs.test_type }}"
if [ -z "$TEST_TYPE" ]; then
TEST_TYPE="full"
fi
echo "Running POSIX tests: $TEST_TYPE"
if [ "$TEST_TYPE" = "full" ]; then
echo "🚀 Running COMPREHENSIVE POSIX test suite (including external test suites)"
else
echo "Using mount point: $SEAWEEDFS_MOUNT_POINT"
fi
# Set test configuration to use our mounted filesystem
export TEST_MOUNT_POINT="$SEAWEEDFS_MOUNT_POINT"
export TEST_SKIP_CLUSTER_SETUP="true"
# Debug: Show environment variables
echo "🔍 Test Environment:"
echo " TEST_MOUNT_POINT=$TEST_MOUNT_POINT"
echo " TEST_SKIP_CLUSTER_SETUP=$TEST_SKIP_CLUSTER_SETUP"
echo " SEAWEEDFS_MOUNT_POINT=$SEAWEEDFS_MOUNT_POINT"
case "$TEST_TYPE" in
"critical")
echo "Running critical POSIX tests (basic compliance only)..."
make -f posix_Makefile test-posix-critical
;;
"basic")
echo "Running basic POSIX tests (core file operations)..."
make -f posix_Makefile test-posix-basic
;;
"extended")
echo "Running extended POSIX tests (including advanced features)..."
make -f posix_Makefile test-posix-extended
;;
"full")
echo "🚀 Running COMPREHENSIVE POSIX test suite (all tests including external)..."
echo " This includes: basic, extended, and external test suites"
echo " External suites: pjdfstest, nfstest_posix, and custom stress tests"
make -f posix_Makefile test-posix-full
;;
*)
echo "Unknown test type: $TEST_TYPE, defaulting to full suite"
echo "🚀 Running full POSIX test suite (all tests including external)..."
make -f posix_Makefile test-posix-full
;;
esac
- name: Run external test suites
if: ${{ github.event.inputs.enable_external_tests == 'true' }}
continue-on-error: true
run: |
cd test/fuse_integration
# Point the external suites at the same mount as the Go tests;
# without these the Makefile falls back to a local-disk path.
export TEST_MOUNT_POINT="$SEAWEEDFS_MOUNT_POINT"
export TEST_SKIP_CLUSTER_SETUP="true"
# Run external tests (may fail on some systems)
make -f posix_Makefile test-nfstest-posix || echo "nfstest failed or not available"
make -f posix_Makefile test-fio-posix || echo "FIO tests failed or not available"
- name: Generate compliance report
if: always()
run: |
cd test/fuse_integration
make -f posix_Makefile generate-report
- name: Collect system information for debugging
if: failure()
run: |
cd test/fuse_integration
mkdir -p reports/debug
echo "System Information" > reports/debug/system_info.txt
uname -a >> reports/debug/system_info.txt
echo "" >> reports/debug/system_info.txt
echo "Mount Information" >> reports/debug/system_info.txt
mount >> reports/debug/system_info.txt
echo "" >> reports/debug/system_info.txt
echo "Disk Space" >> reports/debug/system_info.txt
timeout 15 df -h >> reports/debug/system_info.txt || true
echo "" >> reports/debug/system_info.txt
echo "Memory Information" >> reports/debug/system_info.txt
free -h >> reports/debug/system_info.txt
echo "" >> reports/debug/system_info.txt
echo "FUSE Information" >> reports/debug/system_info.txt
ls -la /dev/fuse >> reports/debug/system_info.txt
lsmod | grep fuse >> reports/debug/system_info.txt
echo "" >> reports/debug/system_info.txt
echo "SeaweedFS Version" >> reports/debug/system_info.txt
weed version >> reports/debug/system_info.txt
echo "" >> reports/debug/system_info.txt
echo "Go Version" >> reports/debug/system_info.txt
go version >> reports/debug/system_info.txt
echo "" >> reports/debug/system_info.txt
echo "Running Processes" >> reports/debug/system_info.txt
ps aux | grep -E "(weed|fuse)" >> reports/debug/system_info.txt
echo "" >> reports/debug/system_info.txt
# Collect SeaweedFS service logs
echo "=== SeaweedFS Service Logs ===" >> reports/debug/system_info.txt
if [ -f /tmp/seaweedfs/master.log ]; then
echo "Master Log:" >> reports/debug/system_info.txt
tail -50 /tmp/seaweedfs/master.log >> reports/debug/system_info.txt
echo "" >> reports/debug/system_info.txt
fi
if [ -f /tmp/seaweedfs/volume.log ]; then
echo "Volume Log:" >> reports/debug/system_info.txt
tail -50 /tmp/seaweedfs/volume.log >> reports/debug/system_info.txt
echo "" >> reports/debug/system_info.txt
fi
if [ -f /tmp/seaweedfs/filer.log ]; then
echo "Filer Log:" >> reports/debug/system_info.txt
tail -50 /tmp/seaweedfs/filer.log >> reports/debug/system_info.txt
echo "" >> reports/debug/system_info.txt
fi
if [ -f /tmp/seaweedfs/mount.log ]; then
echo "Mount Log:" >> reports/debug/system_info.txt
tail -50 /tmp/seaweedfs/mount.log >> reports/debug/system_info.txt
echo "" >> reports/debug/system_info.txt
fi
# Copy full logs as separate files
if [ -f /tmp/seaweedfs/master.log ]; then
cp /tmp/seaweedfs/master.log reports/debug/master.log
fi
if [ -f /tmp/seaweedfs/volume.log ]; then
cp /tmp/seaweedfs/volume.log reports/debug/volume.log
fi
if [ -f /tmp/seaweedfs/filer.log ]; then
cp /tmp/seaweedfs/filer.log reports/debug/filer.log
fi
if [ -f /tmp/seaweedfs/mount.log ]; then
cp /tmp/seaweedfs/mount.log reports/debug/mount.log
fi
- name: Cleanup SeaweedFS cluster and FUSE mount
if: always()
run: |
stop_process() {
local name=$1
local pidfile=$2
if [ -f "$pidfile" ]; then
local pid=$(cat "$pidfile")
if kill -0 $pid 2>/dev/null; then
echo "Stopping $name process (PID: $pid)..."
kill -TERM $pid || true
sleep 2
kill -KILL $pid 2>/dev/null || true
fi
fi
}
echo "Cleaning up SeaweedFS cluster and FUSE mount..."
MOUNT_POINT="/tmp/seaweedfs/mount"
if mountpoint -q $MOUNT_POINT 2>/dev/null; then
echo "Unmounting FUSE filesystem..."
fusermount -u $MOUNT_POINT || umount $MOUNT_POINT || true
fi
stop_process "mount" "/tmp/seaweedfs/mount.pid"
stop_process "filer" "/tmp/seaweedfs/filer.pid"
stop_process "volume" "/tmp/seaweedfs/volume.pid"
stop_process "master" "/tmp/seaweedfs/master.pid"
pkill -f "weed " || true
fusermount -u $MOUNT_POINT 2>/dev/null || true
umount $MOUNT_POINT 2>/dev/null || true
rm -rf /tmp/seaweedfs || true
echo "Cleanup completed"
- name: Upload test results
uses: actions/upload-artifact@v7
if: always()
with:
name: posix-test-results-ubuntu-fuse${{ matrix.fuse-version }}
path: |
test/fuse_integration/reports/
test/fuse_integration/*.log
retention-days: 30
- name: Upload test coverage
uses: actions/upload-artifact@v7
if: always() && (github.event.inputs.test_type == 'full' || github.event.inputs.test_type == 'extended')
with:
name: posix-coverage-ubuntu
path: test/fuse_integration/reports/posix_coverage.html
retention-days: 14
- name: Comment PR with results
if: github.event_name == 'pull_request' && always()
uses: actions/github-script@v7
with:
script: |
const fs = require('fs');
const path = require('path');
const reportPath = 'test/fuse_integration/reports/posix_compliance_summary.txt';
if (fs.existsSync(reportPath)) {
const report = fs.readFileSync(reportPath, 'utf8');
const comment = `## POSIX Compliance Test Results
**Go Version:** ${{ env.GO_VERSION }}
**FUSE Version:** ${{ matrix.fuse-version }}
**Test Type:** ${{ github.event.inputs.test_type || 'full' }}
<details>
<summary>Test Summary</summary>
\`\`\`
${report}
\`\`\`
</details>
📊 Full results available in [test artifacts](https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }})
`;
try {
await github.rest.issues.createComment({
issue_number: context.issue.number,
owner: context.repo.owner,
repo: context.repo.repo,
body: comment
});
} catch (err) {
console.log('Could not post PR comment: ' + err.message);
}
}
posix-compliance-macos:
runs-on: macos-latest
timeout-minutes: 60
steps:
- name: Checkout code
uses: actions/checkout@v7
- name: Set up Go
uses: actions/setup-go@v7
with:
go-version: ${{ env.GO_VERSION }}
cache: ${{ runner.environment == 'github-hosted' }}
- name: Install macFUSE
run: |
# Install macFUSE
brew install macfuse
# Note: macFUSE may require system extension approval on macOS
# This step may need manual intervention in some cases
- name: Build SeaweedFS
run: |
make
# Verify binary exists and is executable (it's installed to GOPATH/bin)
weed version
- name: Run critical POSIX tests (macOS)
continue-on-error: true # macOS FUSE can be more restrictive
run: |
cd test/fuse_integration
if [ ! -f go.mod ]; then
go mod init seaweedfs-posix-tests
go mod tidy
fi
mkdir -p reports
# Run basic tests only on macOS due to FUSE limitations
make -f posix_Makefile test-posix-critical
- name: Upload macOS test results
uses: actions/upload-artifact@v7
if: always()
with:
name: posix-test-results-macos
path: |
test/fuse_integration/reports/
test/fuse_integration/*.log
retention-days: 30
security-analysis:
runs-on: ubuntu-latest
if: github.event_name == 'pull_request'
steps:
- name: Checkout code
uses: actions/checkout@v7
- name: Set up Go
uses: actions/setup-go@v7
with:
go-version: ${{ env.GO_VERSION }}
cache: ${{ runner.environment == 'github-hosted' }}
- name: Install security analysis tools
run: |
echo "Installing staticcheck for security analysis..."
go install honnef.co/go/tools/cmd/staticcheck@latest
- name: Run security analysis on FUSE code
run: |
# Analyze mount and FUSE-related code for security and quality issues
echo "Running staticcheck security analysis..."
# Run staticcheck on FUSE-related code
echo "Analyzing FUSE mount code..."
staticcheck ./weed/mount/... > staticcheck-report.txt 2>&1 || true
staticcheck ./weed/command/mount* ./weed/command/fuse* >> staticcheck-report.txt 2>&1 || true
# Convert to JSON format for consistency
echo "{" > gosec-report.json
echo " \"tool\": \"staticcheck\"," >> gosec-report.json
echo " \"timestamp\": \"$(date -Iseconds)\"," >> gosec-report.json
echo " \"issues\": [" >> gosec-report.json
if [ -s staticcheck-report.txt ]; then
echo " \"$(cat staticcheck-report.txt | head -20 | sed 's/"/\\"/g' | tr '\n' ' ')\"" >> gosec-report.json
fi
echo " ]," >> gosec-report.json
echo " \"stats\": {" >> gosec-report.json
echo " \"files_analyzed\": $(find ./weed/mount ./weed/command -name '*.go' | wc -l)," >> gosec-report.json
echo " \"issues_found\": $(wc -l < staticcheck-report.txt 2>/dev/null || echo 0)" >> gosec-report.json
echo " }" >> gosec-report.json
echo "}" >> gosec-report.json
echo "Security analysis completed"
echo "Issues found: $(wc -l < staticcheck-report.txt 2>/dev/null || echo 0)"
- name: Upload security analysis results
uses: actions/upload-artifact@v7
if: always()
with:
name: security-analysis-results
path: gosec-report.json
retention-days: 30
performance-baseline:
runs-on: ubuntu-latest
if: github.event.inputs.test_type == 'full' || github.event_name == 'schedule'
steps:
- name: Checkout code
uses: actions/checkout@v7
- name: Set up Go
uses: actions/setup-go@v7
with:
go-version: ${{ env.GO_VERSION }}
cache: ${{ runner.environment == 'github-hosted' }}
- name: Install dependencies
run: |
sudo apt-get update
sudo apt-get install -y fuse libfuse-dev fio
sudo usermod -a -G fuse $USER
sudo chmod 666 /dev/fuse
# The posix suites mount with -allowOthers; allow_other needs this
echo 'user_allow_other' | sudo tee -a /etc/fuse.conf
- name: Build SeaweedFS
run: make
- name: Run performance baseline tests
run: |
cd test/fuse_integration
if [ ! -f go.mod ]; then
go mod init seaweedfs-posix-tests
go mod tidy
fi
mkdir -p reports
# Run performance benchmarks
make -f posix_Makefile benchmark-posix
# This job has no FUSE mount, so the fio suite can only skip; keep the
# run green but honest about producing no mounted-FS results
make -f posix_Makefile test-fio-posix || echo "FIO skipped - no mounted filesystem in this job"
- name: Store performance baseline
uses: actions/upload-artifact@v7
with:
name: performance-baseline-results
path: |
test/fuse_integration/reports/posix_benchmark_results.log
test/fuse_integration/reports/fio_*_results.log
retention-days: 90
# Schedule regular compliance checks
scheduled-compliance-check:
runs-on: ubuntu-latest
if: github.event_name == 'schedule'
steps:
- name: Checkout code
uses: actions/checkout@v7
- name: Set up Go
uses: actions/setup-go@v7
with:
go-version: ${{ env.GO_VERSION }}
cache: ${{ runner.environment == 'github-hosted' }}
- name: Install dependencies
run: |
sudo apt-get update
sudo apt-get install -y fuse libfuse-dev
pip3 install --user nfstest
# The posix suites mount with -allowOthers; allow_other needs this
echo 'user_allow_other' | sudo tee -a /etc/fuse.conf
- name: Build SeaweedFS
run: make
- name: Run comprehensive compliance check
run: |
cd test/fuse_integration
make -f posix_Makefile test-posix-full
make -f posix_Makefile generate-report
- name: Create compliance issue if tests fail
if: failure()
uses: actions/github-script@v7
with:
script: |
const issue = await github.rest.issues.create({
owner: context.repo.owner,
repo: context.repo.repo,
title: 'POSIX Compliance Check Failed - ' + new Date().toISOString().split('T')[0],
body: `## POSIX Compliance Check Failure
The scheduled POSIX compliance check has failed. This may indicate:
- Regression in FUSE mount functionality
- Changes affecting POSIX compatibility
- Infrastructure issues with the test environment
**Action Required:** Review the test results and investigate any failures.
**Test Run:** [Workflow Run](https://github.com/${context.repo.owner}/${context.repo.repo}/actions/runs/${context.runId})
**Date:** ${new Date().toISOString()}
---
This issue was automatically created by the POSIX compliance workflow.
`,
labels: ['bug', 'posix-compliance', 'automated-issue']
});
console.log('Created issue:', issue.data.number);
# Schedule this workflow to run weekly
# Uncomment the following lines to enable scheduled runs:
# schedule:
# - cron: '0 2 * * 1' # Every Monday at 2 AM UTC
+2 -4
View File
@@ -23,8 +23,8 @@ on:
- '.github/workflows/postgres-tests.yml'
concurrency:
group: ${{ github.workflow }}-${{ github.event.pull_request.number || (github.event_name == 'push' && github.ref) || github.run_id }}
cancel-in-progress: ${{ github.event_name == 'pull_request' || (github.event_name == 'push' && !startsWith(github.ref, 'refs/tags/')) }}
group: ${{ github.head_ref }}/postgres-tests
cancel-in-progress: true
permissions:
contents: read
@@ -41,7 +41,6 @@ jobs:
- name: Set up Go 1.x
uses: actions/setup-go@v7
with:
cache: ${{ runner.environment == 'github-hosted' }}
go-version: ^1.26
id: go
@@ -57,7 +56,6 @@ jobs:
uses: docker/setup-buildx-action@v4
- name: Cache Docker layers
if: runner.environment == 'github-hosted'
uses: actions/cache@v6
with:
path: /tmp/.buildx-cache
+54 -82
View File
@@ -21,8 +21,8 @@ on:
- '.github/workflows/rust-volume-server-tests.yml'
concurrency:
group: ${{ github.workflow }}-${{ github.event.pull_request.number || (github.event_name == 'push' && github.ref) || github.run_id }}
cancel-in-progress: ${{ github.event_name == 'pull_request' || (github.event_name == 'push' && !startsWith(github.ref, 'refs/tags/')) }}
group: ${{ github.workflow }}-${{ github.head_ref || github.ref }}
cancel-in-progress: true
permissions:
contents: read
@@ -52,65 +52,6 @@ jobs:
- 'seaweed-volume/**'
- '.github/workflows/rust-volume-server-tests.yml'
build-test-binaries:
name: Build Test Binaries
runs-on: ubuntu-22.04
timeout-minutes: 30
steps:
- name: Checkout code
uses: actions/checkout@v7
- name: Set up Go
uses: actions/setup-go@v7
with:
go-version-file: 'go.mod'
cache: ${{ runner.environment == 'github-hosted' }}
- name: Install Rust toolchain
uses: dtolnay/rust-toolchain@stable
# cargo tracks its own inputs but not the runner's C toolchain, so a cached
# target/ can carry C objects built against a different glibc than we link against.
- name: Fingerprint build toolchain
id: toolchain
run: echo "fingerprint=$(getconf GNU_LIBC_VERSION | tr ' ' '-')-rustc-$(rustc -V | awk '{print $2}')" >> "$GITHUB_OUTPUT"
- name: Cache cargo registry and target
if: runner.environment == 'github-hosted'
uses: actions/cache@v6
with:
path: |
~/.cargo/registry
~/.cargo/git
seaweed-volume/target
key: rust-${{ steps.toolchain.outputs.fingerprint }}-${{ hashFiles('seaweed-volume/Cargo.lock', 'seaweed-common/Cargo.lock') }}
restore-keys: |
rust-${{ steps.toolchain.outputs.fingerprint }}-
- name: Build Go weed binary
run: |
cd weed
go build -tags 5BytesOffset -o weed .
chmod +x weed
./weed version
- name: Build Rust volume binary
run: cd seaweed-volume && cargo build --release
- name: Package binaries
run: tar cf rust-test-binaries.tar weed/weed seaweed-volume/target/release/weed-volume
- name: Upload test binaries
uses: actions/upload-artifact@v7
with:
# archive layout mirrors the workspace: weed/weed and
# seaweed-volume/target/release/weed-volume
name: rust-test-binaries
if-no-files-found: error
retention-days: 1
path: rust-test-binaries.tar
rust-unit-tests:
name: Rust Unit Tests
runs-on: ubuntu-22.04
@@ -130,7 +71,6 @@ jobs:
run: echo "fingerprint=$(getconf GNU_LIBC_VERSION | tr ' ' '-')-rustc-$(rustc -V | awk '{print $2}')" >> "$GITHUB_OUTPUT"
- name: Cache cargo registry and target
if: runner.environment == 'github-hosted'
uses: actions/cache@v6
with:
path: |
@@ -194,7 +134,6 @@ jobs:
run: echo "fingerprint=windows-rustc-$(rustc -V | awk '{print $2}')" >> "$GITHUB_OUTPUT"
- name: Cache cargo registry and target
if: runner.environment == 'github-hosted'
uses: actions/cache@v6
with:
path: |
@@ -215,7 +154,6 @@ jobs:
name: Rust Integration Tests
runs-on: ubuntu-22.04
timeout-minutes: 30
needs: [build-test-binaries]
steps:
- name: Checkout code
@@ -225,18 +163,36 @@ jobs:
uses: actions/setup-go@v7
with:
go-version-file: 'go.mod'
cache: ${{ runner.environment == 'github-hosted' }}
- name: Download test binaries
uses: actions/download-artifact@v7
- name: Install Rust toolchain
uses: dtolnay/rust-toolchain@stable
# cargo tracks its own inputs but not the runner's C toolchain, so a cached
# target/ can carry C objects built against a different glibc than we link against.
- name: Fingerprint build toolchain
id: toolchain
run: echo "fingerprint=$(getconf GNU_LIBC_VERSION | tr ' ' '-')-rustc-$(rustc -V | awk '{print $2}')" >> "$GITHUB_OUTPUT"
- name: Cache cargo registry and target
uses: actions/cache@v6
with:
name: rust-test-binaries
path: |
~/.cargo/registry
~/.cargo/git
seaweed-volume/target
key: rust-${{ steps.toolchain.outputs.fingerprint }}-${{ hashFiles('seaweed-volume/Cargo.lock', 'seaweed-common/Cargo.lock') }}
restore-keys: |
rust-${{ steps.toolchain.outputs.fingerprint }}-
- name: Restore binaries
- name: Build Go weed binary
run: |
tar xf rust-test-binaries.tar
chmod +x weed/weed seaweed-volume/target/release/weed-volume
./weed/weed version
cd weed
go build -tags 5BytesOffset -o weed .
chmod +x weed
./weed version
- name: Build Rust volume binary
run: cd seaweed-volume && cargo build --release
- name: Run integration tests
env:
@@ -272,7 +228,6 @@ jobs:
name: Go Tests with Rust Volume (${{ matrix.test-type }} - Shard ${{ matrix.shard }})
runs-on: ubuntu-22.04
timeout-minutes: 45
needs: [build-test-binaries]
env:
# Keep in step with the length of matrix.shard below.
SHARD_COUNT: 3
@@ -290,19 +245,36 @@ jobs:
uses: actions/setup-go@v7
with:
go-version-file: 'go.mod'
cache: ${{ runner.environment == 'github-hosted' }}
- name: Download test binaries
uses: actions/download-artifact@v7
- name: Install Rust toolchain
uses: dtolnay/rust-toolchain@stable
# cargo tracks its own inputs but not the runner's C toolchain, so a cached
# target/ can carry C objects built against a different glibc than we link against.
- name: Fingerprint build toolchain
id: toolchain
run: echo "fingerprint=$(getconf GNU_LIBC_VERSION | tr ' ' '-')-rustc-$(rustc -V | awk '{print $2}')" >> "$GITHUB_OUTPUT"
- name: Cache cargo registry and target
uses: actions/cache@v6
with:
name: rust-test-binaries
path: |
~/.cargo/registry
~/.cargo/git
seaweed-volume/target
key: rust-${{ steps.toolchain.outputs.fingerprint }}-${{ hashFiles('seaweed-volume/Cargo.lock', 'seaweed-common/Cargo.lock') }}
restore-keys: |
rust-${{ steps.toolchain.outputs.fingerprint }}-
- name: Restore binaries
- name: Build Go weed binary
run: |
tar xf rust-test-binaries.tar
chmod +x weed/weed seaweed-volume/target/release/weed-volume
./weed/weed version
./seaweed-volume/target/release/weed-volume --help >/dev/null || true
cd weed
go build -tags 5BytesOffset -o weed .
chmod +x weed
./weed version
- name: Build Rust volume binary
run: cd seaweed-volume && cargo build --release
# Dealing the listed tests out one by one keeps the shards even. Bucketing
# them by first letter did not: names cluster, so ^Test[I-S] drew 50 of
+2 -3
View File
@@ -17,8 +17,8 @@ on:
- '.github/workflows/rust-worker-tests.yml'
concurrency:
group: ${{ github.workflow }}-${{ github.event.pull_request.number || (github.event_name == 'push' && github.ref) || github.run_id }}
cancel-in-progress: ${{ github.event_name == 'pull_request' || (github.event_name == 'push' && !startsWith(github.ref, 'refs/tags/')) }}
group: ${{ github.workflow }}-${{ github.head_ref || github.ref }}
cancel-in-progress: true
permissions:
contents: read
@@ -45,7 +45,6 @@ jobs:
run: echo "fingerprint=$(getconf GNU_LIBC_VERSION | tr ' ' '-')-rustc-$(rustc -V | awk '{print $2}')" >> "$GITHUB_OUTPUT"
- name: Cache cargo registry and target
if: runner.environment == 'github-hosted'
uses: actions/cache@v6
with:
path: |
-2
View File
@@ -46,7 +46,6 @@ jobs:
uses: dtolnay/rust-toolchain@stable
- name: Cache cargo registry and target
if: runner.environment == 'github-hosted'
uses: actions/cache@v6
with:
path: |
@@ -115,7 +114,6 @@ jobs:
targets: ${{ matrix.target }}
- name: Cache cargo registry and target
if: runner.environment == 'github-hosted'
uses: actions/cache@v6
with:
path: |
@@ -48,7 +48,6 @@ jobs:
echo "OPENSSL_LIB_DIR=/usr/lib/aarch64-linux-gnu" >> "$GITHUB_ENV"
- name: Cache cargo registry and target
if: runner.environment == 'github-hosted'
uses: actions/cache@v6
with:
path: |
@@ -154,7 +153,6 @@ jobs:
echo "CARGO_TARGET_AARCH64_UNKNOWN_LINUX_GNU_LINKER=aarch64-linux-gnu-gcc" >> "$GITHUB_ENV"
- name: Cache cargo registry and target
if: runner.environment == 'github-hosted'
uses: actions/cache@v6
with:
path: |
@@ -232,7 +230,6 @@ jobs:
targets: ${{ matrix.target }}
- name: Cache cargo registry and target
if: runner.environment == 'github-hosted'
uses: actions/cache@v6
with:
path: |
@@ -308,7 +305,6 @@ jobs:
uses: dtolnay/rust-toolchain@stable
- name: Cache cargo registry and target
if: runner.environment == 'github-hosted'
uses: actions/cache@v6
with:
path: |
+2 -3
View File
@@ -21,8 +21,8 @@ on:
- '.github/workflows/s3-etag-acl-tests.yml'
concurrency:
group: ${{ github.workflow }}-${{ github.event.pull_request.number || (github.event_name == 'push' && github.ref) || github.run_id }}
cancel-in-progress: ${{ github.event_name == 'pull_request' || (github.event_name == 'push' && !startsWith(github.ref, 'refs/tags/')) }}
group: ${{ github.head_ref || github.ref }}/s3-etag-acl-tests
cancel-in-progress: true
permissions:
contents: read
@@ -40,7 +40,6 @@ jobs:
uses: actions/setup-go@v7
with:
go-version-file: 'go.mod'
cache: ${{ runner.environment == 'github-hosted' }}
- name: Install SeaweedFS
run: |
@@ -13,8 +13,8 @@ on:
- '.github/workflows/s3-example-integration-tests.yml'
concurrency:
group: ${{ github.workflow }}-${{ github.event.pull_request.number || (github.event_name == 'push' && github.ref) || github.run_id }}
cancel-in-progress: ${{ github.event_name == 'pull_request' || (github.event_name == 'push' && !startsWith(github.ref, 'refs/tags/')) }}
group: ${{ github.head_ref }}/s3-integration-tests
cancel-in-progress: true
permissions:
contents: read
@@ -33,7 +33,6 @@ jobs:
uses: actions/setup-go@v7
with:
go-version-file: 'go.mod'
cache: ${{ runner.environment == 'github-hosted' }}
id: go
- name: Build SeaweedFS
+3 -7
View File
@@ -12,8 +12,8 @@ on:
- '.github/workflows/s3-filer-group-tests.yml'
concurrency:
group: ${{ github.workflow }}-${{ github.event.pull_request.number || (github.event_name == 'push' && github.ref) || github.run_id }}
cancel-in-progress: ${{ github.event_name == 'pull_request' || (github.event_name == 'push' && !startsWith(github.ref, 'refs/tags/')) }}
group: ${{ github.head_ref }}/s3-filer-group-tests
cancel-in-progress: true
permissions:
contents: read
@@ -32,14 +32,10 @@ jobs:
- name: Check out code
uses: actions/checkout@v7
- name: Clean up stale FUSE mounts
uses: ./.github/actions/cleanup-stale-fuse-mounts
- name: Set up Go
uses: actions/setup-go@v7
with:
go-version-file: 'go.mod'
cache: ${{ runner.environment == 'github-hosted' }}
id: go
- name: Install SeaweedFS
@@ -54,7 +50,7 @@ jobs:
echo "=== System Information ==="
uname -a
free -h
timeout 15 df -h || true
df -h
echo "=== Starting Filer Group Tests ==="
# Run tests with automatic server management
+6 -31
View File
@@ -4,7 +4,6 @@ on:
pull_request:
paths:
- 'weed/s3api/**'
- 'weed/remote_storage/**'
- 'weed/filer/**'
- 'weed/server/**'
- 'test/s3/**'
@@ -13,8 +12,8 @@ on:
- '.github/workflows/s3-go-tests.yml'
concurrency:
group: ${{ github.workflow }}-${{ github.event.pull_request.number || (github.event_name == 'push' && github.ref) || github.run_id }}
cancel-in-progress: ${{ github.event_name == 'pull_request' || (github.event_name == 'push' && !startsWith(github.ref, 'refs/tags/')) }}
group: ${{ github.head_ref }}/s3-go-tests
cancel-in-progress: true
permissions:
contents: read
@@ -36,14 +35,10 @@ jobs:
- name: Check out code
uses: actions/checkout@v7
- name: Clean up stale FUSE mounts
uses: ./.github/actions/cleanup-stale-fuse-mounts
- name: Set up Go
uses: actions/setup-go@v7
with:
go-version-file: 'go.mod'
cache: ${{ runner.environment == 'github-hosted' }}
id: go
- name: Install SeaweedFS
@@ -58,7 +53,7 @@ jobs:
echo "=== System Information ==="
uname -a
free -h
timeout 15 df -h || true
df -h
echo "=== Starting Tests ==="
# Run tests with automatic server management
@@ -108,7 +103,6 @@ jobs:
uses: actions/setup-go@v7
with:
go-version-file: 'go.mod'
cache: ${{ runner.environment == 'github-hosted' }}
id: go
- name: Install SeaweedFS
@@ -157,7 +151,6 @@ jobs:
uses: actions/setup-go@v7
with:
go-version-file: 'go.mod'
cache: ${{ runner.environment == 'github-hosted' }}
id: go
- name: Install SeaweedFS
@@ -205,14 +198,10 @@ jobs:
- name: Check out code
uses: actions/checkout@v7
- name: Clean up stale FUSE mounts
uses: ./.github/actions/cleanup-stale-fuse-mounts
- name: Set up Go
uses: actions/setup-go@v7
with:
go-version-file: 'go.mod'
cache: ${{ runner.environment == 'github-hosted' }}
id: go
- name: Install SeaweedFS
@@ -227,7 +216,7 @@ jobs:
echo "=== System Information ==="
uname -a
free -h
timeout 15 df -h || true
df -h
echo "=== Starting Tests ==="
# Run tests with automatic server management
@@ -294,7 +283,6 @@ jobs:
- TestLifecycleSuspendedVersioningExpiration
- TestLifecycleTagFilter
- TestLifecycleVersionedBucketCreatesDeleteMarker
- TestLifecycleVersionedBucketExpiresOnSecondRun
steps:
- name: Check out code
@@ -304,7 +292,6 @@ jobs:
uses: actions/setup-go@v7
with:
go-version-file: 'go.mod'
cache: ${{ runner.environment == 'github-hosted' }}
id: go
- name: Install SeaweedFS
@@ -346,14 +333,10 @@ jobs:
- name: Check out code
uses: actions/checkout@v7
- name: Clean up stale FUSE mounts
uses: ./.github/actions/cleanup-stale-fuse-mounts
- name: Set up Go
uses: actions/setup-go@v7
with:
go-version-file: 'go.mod'
cache: ${{ runner.environment == 'github-hosted' }}
id: go
- name: Install SeaweedFS
@@ -368,7 +351,7 @@ jobs:
echo "=== System Information ==="
uname -a
free -h
timeout 15 df -h || true
df -h
echo "=== Starting Tests ==="
make test-with-server
@@ -408,14 +391,10 @@ jobs:
- name: Check out code
uses: actions/checkout@v7
- name: Clean up stale FUSE mounts
uses: ./.github/actions/cleanup-stale-fuse-mounts
- name: Set up Go
uses: actions/setup-go@v7
with:
go-version-file: 'go.mod'
cache: ${{ runner.environment == 'github-hosted' }}
id: go
- name: Install SeaweedFS
@@ -430,7 +409,7 @@ jobs:
echo "=== System Information ==="
uname -a
free -h
timeout 15 df -h || true
df -h
echo "=== Starting Tests ==="
# Run tests with automatic server management
@@ -480,7 +459,6 @@ jobs:
uses: actions/setup-go@v7
with:
go-version-file: 'go.mod'
cache: ${{ runner.environment == 'github-hosted' }}
id: go
- name: Install SeaweedFS
@@ -530,7 +508,6 @@ jobs:
uses: actions/setup-go@v7
with:
go-version-file: 'go.mod'
cache: ${{ runner.environment == 'github-hosted' }}
id: go
- name: Install SeaweedFS
@@ -599,7 +576,6 @@ jobs:
uses: actions/setup-go@v7
with:
go-version-file: 'go.mod'
cache: ${{ runner.environment == 'github-hosted' }}
id: go
- name: Install SeaweedFS
@@ -655,7 +631,6 @@ jobs:
uses: actions/setup-go@v7
with:
go-version-file: 'go.mod'
cache: ${{ runner.environment == 'github-hosted' }}
id: go
- name: Install SeaweedFS
+3 -10
View File
@@ -20,8 +20,8 @@ on:
- '.github/workflows/s3-iam-tests.yml'
concurrency:
group: ${{ github.workflow }}-${{ github.event.pull_request.number || (github.event_name == 'push' && github.ref) || github.run_id }}
cancel-in-progress: ${{ github.event_name == 'pull_request' || (github.event_name == 'push' && !startsWith(github.ref, 'refs/tags/')) }}
group: ${{ github.head_ref }}/s3-iam-tests
cancel-in-progress: true
permissions:
contents: read
@@ -45,7 +45,6 @@ jobs:
uses: actions/setup-go@v7
with:
go-version-file: 'go.mod'
cache: ${{ runner.environment == 'github-hosted' }}
id: go
- name: Get dependencies
@@ -91,14 +90,10 @@ jobs:
- name: Check out code
uses: actions/checkout@v7
- name: Clean up stale FUSE mounts
uses: ./.github/actions/cleanup-stale-fuse-mounts
- name: Set up Go
uses: actions/setup-go@v7
with:
go-version-file: 'go.mod'
cache: ${{ runner.environment == 'github-hosted' }}
id: go
- name: Install SeaweedFS
@@ -114,7 +109,7 @@ jobs:
echo "=== System Information ==="
uname -a
free -h
timeout 15 df -h || true
df -h
echo "=== Starting S3 IAM Integration Tests (${{ matrix.test-type }}) ==="
# Set WEED_BINARY to use the installed version
@@ -213,7 +208,6 @@ jobs:
uses: actions/setup-go@v7
with:
go-version-file: 'go.mod'
cache: ${{ runner.environment == 'github-hosted' }}
id: go
- name: Install SeaweedFS
@@ -274,7 +268,6 @@ jobs:
uses: actions/setup-go@v7
with:
go-version-file: 'go.mod'
cache: ${{ runner.environment == 'github-hosted' }}
id: go
- name: Install SeaweedFS
+3 -7
View File
@@ -16,8 +16,8 @@ on:
- '.github/workflows/s3-keycloak-tests.yml'
concurrency:
group: ${{ github.workflow }}-${{ github.event.pull_request.number || (github.event_name == 'push' && github.ref) || github.run_id }}
cancel-in-progress: ${{ github.event_name == 'pull_request' || (github.event_name == 'push' && !startsWith(github.ref, 'refs/tags/')) }}
group: ${{ github.head_ref }}/s3-keycloak-tests
cancel-in-progress: true
permissions:
contents: read
@@ -37,14 +37,10 @@ jobs:
- name: Check out code
uses: actions/checkout@v7
- name: Clean up stale FUSE mounts
uses: ./.github/actions/cleanup-stale-fuse-mounts
- name: Set up Go
uses: actions/setup-go@v7
with:
go-version-file: 'go.mod'
cache: ${{ runner.environment == 'github-hosted' }}
id: go
- name: Install SeaweedFS
@@ -60,7 +56,7 @@ jobs:
echo "=== System Information ==="
uname -a
free -h
timeout 15 df -h || true
df -h
echo "=== Starting S3 Keycloak Integration Tests ==="
# Set WEED_BINARY to use the installed version
@@ -12,8 +12,8 @@ on:
- '.github/workflows/s3-mutation-regression-tests.yml'
concurrency:
group: ${{ github.workflow }}-${{ github.event.pull_request.number || (github.event_name == 'push' && github.ref) || github.run_id }}
cancel-in-progress: ${{ github.event_name == 'pull_request' || (github.event_name == 'push' && !startsWith(github.ref, 'refs/tags/')) }}
group: ${{ github.head_ref }}/s3-mutation-regression-tests
cancel-in-progress: true
permissions:
contents: read
@@ -32,7 +32,6 @@ jobs:
uses: actions/setup-go@v7
with:
go-version-file: 'go.mod'
cache: ${{ runner.environment == 'github-hosted' }}
- name: Run S3 versioning regression tests
timeout-minutes: 20
@@ -86,7 +85,6 @@ jobs:
uses: actions/setup-go@v7
with:
go-version-file: 'go.mod'
cache: ${{ runner.environment == 'github-hosted' }}
- name: Run S3 delete regression tests
timeout-minutes: 15
@@ -125,7 +123,6 @@ jobs:
uses: actions/setup-go@v7
with:
go-version-file: 'go.mod'
cache: ${{ runner.environment == 'github-hosted' }}
- name: Build SeaweedFS
run: |
+3 -7
View File
@@ -23,10 +23,6 @@ env:
S3_ENDPOINT_URL: http://localhost:8333
BUCKET_NAME: test-parquet-bucket
concurrency:
group: ${{ github.workflow }}-${{ github.event.pull_request.number || (github.event_name == 'push' && github.ref) || github.run_id }}
cancel-in-progress: ${{ github.event_name == 'pull_request' || (github.event_name == 'push' && !startsWith(github.ref, 'refs/tags/')) }}
jobs:
parquet-integration-tests:
name: PyArrow Parquet Tests (Python ${{ matrix.python-version }})
@@ -46,13 +42,13 @@ jobs:
uses: actions/setup-go@v7
with:
go-version: ^1.26
cache: ${{ runner.environment == 'github-hosted' }}
cache: true
- name: Set up Python ${{ matrix.python-version }}
uses: actions/setup-python@v7
with:
python-version: ${{ matrix.python-version }}
cache: ${{ runner.environment == 'github-hosted' && 'pip' || '' }}
cache: 'pip'
cache-dependency-path: 'test/s3/parquet/requirements.txt'
- name: Install system dependencies
@@ -154,7 +150,7 @@ jobs:
uses: actions/setup-go@v7
with:
go-version: ^1.26
cache: ${{ runner.environment == 'github-hosted' }}
cache: true
- name: Run Go unit tests
run: |
+4 -15
View File
@@ -24,8 +24,8 @@ on:
- '.github/workflows/s3-policy-tests.yml'
concurrency:
group: ${{ github.workflow }}-${{ github.event.pull_request.number || (github.event_name == 'push' && github.ref) || github.run_id }}
cancel-in-progress: ${{ github.event_name == 'pull_request' || (github.event_name == 'push' && !startsWith(github.ref, 'refs/tags/')) }}
group: ${{ github.head_ref }}/s3-policy-tests
cancel-in-progress: true
permissions:
contents: read
@@ -49,7 +49,6 @@ jobs:
uses: actions/setup-go@v7
with:
go-version-file: 'go.mod'
cache: ${{ runner.environment == 'github-hosted' }}
id: go
- name: Get dependencies
@@ -89,14 +88,10 @@ jobs:
- name: Check out code
uses: actions/checkout@v7
- name: Clean up stale FUSE mounts
uses: ./.github/actions/cleanup-stale-fuse-mounts
- name: Set up Go
uses: actions/setup-go@v7
with:
go-version-file: 'go.mod'
cache: ${{ runner.environment == 'github-hosted' }}
id: go
- name: Install SeaweedFS
@@ -111,7 +106,7 @@ jobs:
echo "=== System Information ==="
uname -a
free -h
timeout 15 df -h || true
df -h
echo "=== Starting S3 Policy Variables Integration Tests ==="
@@ -202,14 +197,10 @@ jobs:
- name: Check out code
uses: actions/checkout@v7
- name: Clean up stale FUSE mounts
uses: ./.github/actions/cleanup-stale-fuse-mounts
- name: Set up Go
uses: actions/setup-go@v7
with:
go-version-file: 'go.mod'
cache: ${{ runner.environment == 'github-hosted' }}
id: go
- name: Install SeaweedFS
@@ -224,7 +215,7 @@ jobs:
echo "=== System Information ==="
uname -a
free -h
timeout 15 df -h || true
df -h
echo "=== Starting S3 Policy Enforcement Tests (${{ matrix.test-case }}) ==="
@@ -328,7 +319,6 @@ jobs:
uses: actions/setup-go@v7
with:
go-version-file: 'go.mod'
cache: ${{ runner.environment == 'github-hosted' }}
id: go
- name: Install SeaweedFS
@@ -416,7 +406,6 @@ jobs:
uses: actions/setup-go@v7
with:
go-version-file: 'go.mod'
cache: ${{ runner.environment == 'github-hosted' }}
id: go
- name: Install SeaweedFS
@@ -21,8 +21,8 @@ on:
- '.github/workflows/s3-proxy-signature-tests.yml'
concurrency:
group: ${{ github.workflow }}-${{ github.event.pull_request.number || (github.event_name == 'push' && github.ref) || github.run_id }}
cancel-in-progress: ${{ github.event_name == 'pull_request' || (github.event_name == 'push' && !startsWith(github.ref, 'refs/tags/')) }}
group: ${{ github.head_ref || github.ref }}/s3-proxy-signature-tests
cancel-in-progress: true
permissions:
contents: read
@@ -40,7 +40,6 @@ jobs:
uses: actions/setup-go@v7
with:
go-version-file: 'go.mod'
cache: ${{ runner.environment == 'github-hosted' }}
id: go
- name: Configure Docker Hub mirror
@@ -15,8 +15,8 @@ on:
- '.github/workflows/s3-sdk-v2-routing-tests.yml'
concurrency:
group: ${{ github.workflow }}-${{ github.event.pull_request.number || (github.event_name == 'push' && github.ref) || github.run_id }}
cancel-in-progress: ${{ github.event_name == 'pull_request' || (github.event_name == 'push' && !startsWith(github.ref, 'refs/tags/')) }}
group: ${{ github.head_ref || github.ref }}/s3-sdk-v2-routing-tests
cancel-in-progress: true
permissions:
contents: read
@@ -34,7 +34,6 @@ jobs:
uses: actions/setup-go@v7
with:
go-version-file: 'go.mod'
cache: ${{ runner.environment == 'github-hosted' }}
- name: Install SeaweedFS
run: |
+1 -2
View File
@@ -60,7 +60,6 @@ jobs:
uses: actions/setup-go@v7
with:
go-version-file: 'go.mod'
cache: ${{ runner.environment == 'github-hosted' }}
id: go
- name: Set up Java
@@ -68,7 +67,7 @@ jobs:
with:
java-version: '17'
distribution: 'temurin'
cache: ${{ runner.environment == 'github-hosted' && 'maven' || '' }}
cache: 'maven'
- name: Install SeaweedFS
run: |
+3 -7
View File
@@ -11,8 +11,8 @@ on:
workflow_dispatch:
concurrency:
group: ${{ github.workflow }}-${{ github.event.pull_request.number || (github.event_name == 'push' && github.ref) || github.run_id }}
cancel-in-progress: ${{ github.event_name == 'pull_request' || (github.event_name == 'push' && !startsWith(github.ref, 'refs/tags/')) }}
group: ${{ github.head_ref }}/s3-spark-tests
cancel-in-progress: true
permissions:
contents: read
@@ -27,14 +27,10 @@ jobs:
- name: Check out code
uses: actions/checkout@v7
- name: Clean up stale FUSE mounts
uses: ./.github/actions/cleanup-stale-fuse-mounts
- name: Set up Go
uses: actions/setup-go@v7
with:
go-version-file: 'go.mod'
cache: ${{ runner.environment == 'github-hosted' }}
id: go
- name: Configure Docker Hub mirror
@@ -60,7 +56,7 @@ jobs:
echo "=== System Information ==="
uname -a
free -h
timeout 15 df -h || true
df -h
echo "=== Starting S3 Spark Integration Tests ==="
go test -v -timeout 30m . 2>&1 | tee test-output.log || {
+3 -13
View File
@@ -23,8 +23,8 @@ on:
- 'test/s3/sse/**'
concurrency:
group: ${{ github.workflow }}-${{ github.event.pull_request.number || (github.event_name == 'push' && github.ref) || github.run_id }}
cancel-in-progress: ${{ github.event_name == 'pull_request' || (github.event_name == 'push' && !startsWith(github.ref, 'refs/tags/')) }}
group: ${{ github.head_ref }}/s3-sse-tests
cancel-in-progress: true
permissions:
contents: read
@@ -46,14 +46,10 @@ jobs:
- name: Check out code
uses: actions/checkout@v7
- name: Clean up stale FUSE mounts
uses: ./.github/actions/cleanup-stale-fuse-mounts
- name: Set up Go
uses: actions/setup-go@v7
with:
go-version-file: 'go.mod'
cache: ${{ runner.environment == 'github-hosted' }}
id: go
- name: Install SeaweedFS
@@ -68,7 +64,7 @@ jobs:
echo "=== System Information ==="
uname -a
free -h
timeout 15 df -h || true
df -h
echo "=== Starting SSE Tests ==="
# Run tests with automatic server management
@@ -122,7 +118,6 @@ jobs:
uses: actions/setup-go@v7
with:
go-version-file: 'go.mod'
cache: ${{ runner.environment == 'github-hosted' }}
id: go
- name: Install SeaweedFS
@@ -171,7 +166,6 @@ jobs:
uses: actions/setup-go@v7
with:
go-version-file: 'go.mod'
cache: ${{ runner.environment == 'github-hosted' }}
id: go
- name: Install SeaweedFS
@@ -221,7 +215,6 @@ jobs:
uses: actions/setup-go@v7
with:
go-version-file: 'go.mod'
cache: ${{ runner.environment == 'github-hosted' }}
id: go
- name: Install SeaweedFS
@@ -271,7 +264,6 @@ jobs:
uses: actions/setup-go@v7
with:
go-version-file: 'go.mod'
cache: ${{ runner.environment == 'github-hosted' }}
id: go
- name: Install SeaweedFS
@@ -323,7 +315,6 @@ jobs:
uses: actions/setup-go@v7
with:
go-version-file: 'go.mod'
cache: ${{ runner.environment == 'github-hosted' }}
id: go
- name: Install SeaweedFS
@@ -372,7 +363,6 @@ jobs:
uses: actions/setup-go@v7
with:
go-version-file: 'go.mod'
cache: ${{ runner.environment == 'github-hosted' }}
id: go
- name: Install SeaweedFS
+16 -90
View File
@@ -12,10 +12,6 @@ on:
- 'go.sum'
- '.github/workflows/s3-tables-tests.yml'
concurrency:
group: ${{ github.workflow }}-${{ github.event.pull_request.number || (github.event_name == 'push' && github.ref) || github.run_id }}
cancel-in-progress: ${{ github.event_name == 'pull_request' || (github.event_name == 'push' && !startsWith(github.ref, 'refs/tags/')) }}
permissions:
contents: read
@@ -30,14 +26,10 @@ jobs:
- name: Check out code
uses: actions/checkout@v7
- name: Clean up stale FUSE mounts
uses: ./.github/actions/cleanup-stale-fuse-mounts
- name: Set up Go
uses: actions/setup-go@v7
with:
go-version-file: 'go.mod'
cache: ${{ runner.environment == 'github-hosted' }}
id: go
- name: Run go mod tidy
@@ -56,8 +48,7 @@ jobs:
echo "=== System Information ==="
uname -a
free -h
# A stale FUSE mount can hang or fail df; keep diagnostics best effort.
timeout 15 df -h || true
df -h
echo "=== Starting S3 Tables Tests ==="
# Run S3 Tables integration tests
@@ -95,14 +86,10 @@ jobs:
- name: Check out code
uses: actions/checkout@v7
- name: Clean up stale FUSE mounts
uses: ./.github/actions/cleanup-stale-fuse-mounts
- name: Set up Go
uses: actions/setup-go@v7
with:
go-version-file: 'go.mod'
cache: ${{ runner.environment == 'github-hosted' }}
id: go
- name: Configure Docker Hub mirror
@@ -122,7 +109,7 @@ jobs:
echo "=== System Information ==="
uname -a
free -h
timeout 15 df -h || true
df -h
echo "=== Starting Iceberg Catalog Tests ==="
# Run Iceberg catalog integration tests using Makefile (handles build)
@@ -160,14 +147,10 @@ jobs:
- name: Check out code
uses: actions/checkout@v7
- name: Clean up stale FUSE mounts
uses: ./.github/actions/cleanup-stale-fuse-mounts
- name: Set up Go
uses: actions/setup-go@v7
with:
go-version-file: 'go.mod'
cache: ${{ runner.environment == 'github-hosted' }}
id: go
- name: Configure Docker Hub mirror
@@ -196,7 +179,7 @@ jobs:
echo "=== System Information ==="
uname -a
free -h
timeout 15 df -h || true
df -h
echo "=== Starting Trino Iceberg Catalog Tests ==="
# Run Trino + Iceberg catalog integration tests
@@ -234,14 +217,10 @@ jobs:
- name: Check out code
uses: actions/checkout@v7
- name: Clean up stale FUSE mounts
uses: ./.github/actions/cleanup-stale-fuse-mounts
- name: Set up Go
uses: actions/setup-go@v7
with:
go-version-file: 'go.mod'
cache: ${{ runner.environment == 'github-hosted' }}
id: go
- name: Configure Docker Hub mirror
@@ -271,7 +250,7 @@ jobs:
echo "=== System Information ==="
uname -a
free -h
timeout 15 df -h || true
df -h
docker info
echo "=== Starting Dremio Iceberg Catalog Tests ==="
@@ -311,14 +290,10 @@ jobs:
- name: Check out code
uses: actions/checkout@v7
- name: Clean up stale FUSE mounts
uses: ./.github/actions/cleanup-stale-fuse-mounts
- name: Set up Go
uses: actions/setup-go@v7
with:
go-version-file: 'go.mod'
cache: ${{ runner.environment == 'github-hosted' }}
id: go
- name: Configure Docker Hub mirror
@@ -348,7 +323,7 @@ jobs:
echo "=== System Information ==="
uname -a
free -h
timeout 15 df -h || true
df -h
docker info
echo "=== Starting Doris Iceberg Catalog Tests ==="
@@ -398,14 +373,10 @@ jobs:
- name: Check out code
uses: actions/checkout@v7
- name: Clean up stale FUSE mounts
uses: ./.github/actions/cleanup-stale-fuse-mounts
- name: Set up Go
uses: actions/setup-go@v7
with:
go-version-file: 'go.mod'
cache: ${{ runner.environment == 'github-hosted' }}
id: go
- name: Configure Docker Hub mirror
@@ -437,7 +408,7 @@ jobs:
echo "=== System Information ==="
uname -a
free -h
timeout 15 df -h || true
df -h
docker info
echo "=== Starting ClickHouse Iceberg Catalog Tests ==="
@@ -489,14 +460,10 @@ jobs:
- name: Check out code
uses: actions/checkout@v7
- name: Clean up stale FUSE mounts
uses: ./.github/actions/cleanup-stale-fuse-mounts
- name: Set up Go
uses: actions/setup-go@v7
with:
go-version-file: 'go.mod'
cache: ${{ runner.environment == 'github-hosted' }}
id: go
- name: Configure Docker Hub mirror
@@ -529,7 +496,7 @@ jobs:
echo "=== System Information ==="
uname -a
free -h
timeout 15 df -h || true
df -h
docker info
echo "=== Starting OLake Iceberg Catalog Tests ==="
@@ -592,14 +559,10 @@ jobs:
- name: Check out code
uses: actions/checkout@v7
- name: Clean up stale FUSE mounts
uses: ./.github/actions/cleanup-stale-fuse-mounts
- name: Set up Go
uses: actions/setup-go@v7
with:
go-version-file: 'go.mod'
cache: ${{ runner.environment == 'github-hosted' }}
id: go
- name: Run go mod tidy
@@ -627,7 +590,7 @@ jobs:
echo "=== System Information ==="
uname -a
free -h
timeout 15 df -h || true
df -h
echo "=== Starting Polaris Tests ==="
go test -v -timeout 20m ./test/s3tables/polaris 2>&1 | tee test/s3tables/polaris/test-output.log || {
@@ -664,14 +627,10 @@ jobs:
- name: Check out code
uses: actions/checkout@v7
- name: Clean up stale FUSE mounts
uses: ./.github/actions/cleanup-stale-fuse-mounts
- name: Set up Go
uses: actions/setup-go@v7
with:
go-version-file: 'go.mod'
cache: ${{ runner.environment == 'github-hosted' }}
id: go
- name: Configure Docker Hub mirror
@@ -700,7 +659,7 @@ jobs:
echo "=== System Information ==="
uname -a
free -h
timeout 15 df -h || true
df -h
echo "=== Starting Spark Iceberg Catalog Tests ==="
# Run Spark + Iceberg catalog integration tests
@@ -738,14 +697,10 @@ jobs:
- name: Check out code
uses: actions/checkout@v7
- name: Clean up stale FUSE mounts
uses: ./.github/actions/cleanup-stale-fuse-mounts
- name: Set up Go
uses: actions/setup-go@v7
with:
go-version-file: 'go.mod'
cache: ${{ runner.environment == 'github-hosted' }}
id: go
- name: Configure Docker Hub mirror
@@ -775,7 +730,7 @@ jobs:
echo "=== System Information ==="
uname -a
free -h
timeout 15 df -h || true
df -h
echo "=== Starting RisingWave Catalog Tests ==="
# Run RisingWave catalog integration tests
@@ -813,14 +768,10 @@ jobs:
- name: Check out code
uses: actions/checkout@v7
- name: Clean up stale FUSE mounts
uses: ./.github/actions/cleanup-stale-fuse-mounts
- name: Set up Go
uses: actions/setup-go@v7
with:
go-version-file: 'go.mod'
cache: ${{ runner.environment == 'github-hosted' }}
id: go
- name: Configure Docker Hub mirror
@@ -834,7 +785,6 @@ jobs:
- name: Restore python:3 image cache
id: python-image
if: runner.environment == 'github-hosted'
uses: actions/cache@v6
with:
path: /tmp/python3-image.tar
@@ -874,7 +824,7 @@ jobs:
echo "=== System Information ==="
uname -a
free -h
timeout 15 df -h || true
df -h
echo "=== Starting STS Integration Tests ==="
# Run STS integration tests
@@ -912,14 +862,10 @@ jobs:
- name: Check out code
uses: actions/checkout@v7
- name: Clean up stale FUSE mounts
uses: ./.github/actions/cleanup-stale-fuse-mounts
- name: Set up Go
uses: actions/setup-go@v7
with:
go-version-file: 'go.mod'
cache: ${{ runner.environment == 'github-hosted' }}
id: go
- name: Run go mod tidy
@@ -938,7 +884,7 @@ jobs:
echo "=== System Information ==="
uname -a
free -h
timeout 15 df -h || true
df -h
echo "=== Starting Lakekeeper Integration Tests ==="
# Run Lakekeeper integration tests
@@ -976,14 +922,10 @@ jobs:
- name: Check out code
uses: actions/checkout@v7
- name: Clean up stale FUSE mounts
uses: ./.github/actions/cleanup-stale-fuse-mounts
- name: Set up Go
uses: actions/setup-go@v7
with:
go-version-file: 'go.mod'
cache: ${{ runner.environment == 'github-hosted' }}
id: go
- name: Configure Docker Hub mirror
@@ -1013,7 +955,7 @@ jobs:
echo "=== System Information ==="
uname -a
free -h
timeout 15 df -h || true
df -h
docker info
echo "=== Starting Unity Catalog Tests ==="
@@ -1053,14 +995,10 @@ jobs:
- name: Check out code
uses: actions/checkout@v7
- name: Clean up stale FUSE mounts
uses: ./.github/actions/cleanup-stale-fuse-mounts
- name: Set up Go
uses: actions/setup-go@v7
with:
go-version-file: 'go.mod'
cache: ${{ runner.environment == 'github-hosted' }}
id: go
- name: Configure Docker Hub mirror
@@ -1089,7 +1027,7 @@ jobs:
echo "=== System Information ==="
uname -a
free -h
timeout 15 df -h || true
df -h
docker info
echo "=== Starting LanceDB Namespace Tests ==="
@@ -1131,7 +1069,6 @@ jobs:
uses: actions/setup-go@v7
with:
go-version-file: 'go.mod'
cache: ${{ runner.environment == 'github-hosted' }}
id: go
- name: Configure Docker Hub mirror
@@ -1200,14 +1137,10 @@ jobs:
# so do not leave a token in the checkout for it to pick up.
persist-credentials: false
- name: Clean up stale FUSE mounts
uses: ./.github/actions/cleanup-stale-fuse-mounts
- name: Set up Go
uses: actions/setup-go@v7
with:
go-version-file: 'go.mod'
cache: ${{ runner.environment == 'github-hosted' }}
id: go
- name: Configure Docker Hub mirror
@@ -1237,7 +1170,7 @@ jobs:
echo "=== System Information ==="
uname -a
free -h
timeout 15 df -h || true
df -h
docker info
echo "=== Starting DuckDB Lance Tests ==="
@@ -1279,14 +1212,10 @@ jobs:
# so do not leave a token in the checkout for it to pick up.
persist-credentials: false
- name: Clean up stale FUSE mounts
uses: ./.github/actions/cleanup-stale-fuse-mounts
- name: Set up Go
uses: actions/setup-go@v7
with:
go-version-file: 'go.mod'
cache: ${{ runner.environment == 'github-hosted' }}
id: go
- name: Configure Docker Hub mirror
@@ -1315,7 +1244,7 @@ jobs:
echo "=== System Information ==="
uname -a
free -h
timeout 15 df -h || true
df -h
docker info
echo "=== Starting Spark Lance Namespace Tests ==="
@@ -1357,7 +1286,6 @@ jobs:
uses: actions/setup-go@v7
with:
go-version-file: 'go.mod'
cache: ${{ runner.environment == 'github-hosted' }}
id: go
- name: Verify S3 Tables Package Builds
@@ -1423,7 +1351,6 @@ jobs:
uses: actions/setup-go@v7
with:
go-version-file: 'go.mod'
cache: ${{ runner.environment == 'github-hosted' }}
id: go
- name: Check Go Format
@@ -1475,7 +1402,6 @@ jobs:
uses: actions/setup-go@v7
with:
go-version-file: 'go.mod'
cache: ${{ runner.environment == 'github-hosted' }}
id: go
- name: Run Go Vet
+14 -19
View File
@@ -25,8 +25,8 @@ on:
- '.github/workflows/s3tests.yml'
concurrency:
group: ${{ github.workflow }}-${{ github.event.pull_request.number || (github.event_name == 'push' && github.ref) || github.run_id }}
cancel-in-progress: ${{ github.event_name == 'pull_request' || (github.event_name == 'push' && !startsWith(github.ref, 'refs/tags/')) }}
group: ${{ github.head_ref }}/s3tests
cancel-in-progress: true
permissions:
contents: read
@@ -44,7 +44,6 @@ jobs:
uses: actions/setup-go@v7
with:
go-version-file: 'go.mod'
cache: ${{ runner.environment == 'github-hosted' }}
id: go
- name: Set up Python
@@ -58,9 +57,9 @@ jobs:
cd s3-tests
sudo apt-get update -qq
sudo apt-get install -y -qq libxml2-dev libxslt1-dev zlib1g-dev
python3 -m pip install -r requirements.txt
python3 -m pip install tox
python3 -m pip install -e .
pip install -r requirements.txt
pip install tox
pip install -e .
- name: Fix S3 tests bucket creation conflicts
run: |
@@ -397,7 +396,6 @@ jobs:
uses: actions/setup-go@v7
with:
go-version-file: 'go.mod'
cache: ${{ runner.environment == 'github-hosted' }}
id: go
- name: Set up Python
@@ -411,9 +409,9 @@ jobs:
cd s3-tests
sudo apt-get update -qq
sudo apt-get install -y -qq libxml2-dev libxslt1-dev zlib1g-dev
python3 -m pip install -r requirements.txt
python3 -m pip install tox
python3 -m pip install -e .
pip install -r requirements.txt
pip install tox
pip install -e .
- name: Fix S3 tests bucket creation conflicts
run: |
@@ -568,7 +566,6 @@ jobs:
uses: actions/setup-go@v7
with:
go-version-file: 'go.mod'
cache: ${{ runner.environment == 'github-hosted' }}
id: go
- name: Set up Python
@@ -582,9 +579,9 @@ jobs:
cd s3-tests
sudo apt-get update -qq
sudo apt-get install -y -qq libxml2-dev libxslt1-dev zlib1g-dev
python3 -m pip install -r requirements.txt
python3 -m pip install tox
python3 -m pip install -e .
pip install -r requirements.txt
pip install tox
pip install -e .
- name: Run S3 CORS tests
timeout-minutes: 10
@@ -694,7 +691,6 @@ jobs:
uses: actions/setup-go@v7
with:
go-version-file: 'go.mod'
cache: ${{ runner.environment == 'github-hosted' }}
id: go
- name: Run SeaweedFS Custom S3 Copy tests
@@ -797,7 +793,6 @@ jobs:
uses: actions/setup-go@v7
with:
go-version-file: 'go.mod'
cache: ${{ runner.environment == 'github-hosted' }}
id: go
- name: Set up Python
@@ -811,9 +806,9 @@ jobs:
cd s3-tests
sudo apt-get update -qq
sudo apt-get install -y -qq libxml2-dev libxslt1-dev zlib1g-dev
python3 -m pip install -r requirements.txt
python3 -m pip install tox
python3 -m pip install -e .
pip install -r requirements.txt
pip install tox
pip install -e .
- name: Run Ceph S3 tests with SQL store
timeout-minutes: 15
+1 -9
View File
@@ -40,23 +40,15 @@ jobs:
uses: actions/setup-go@v7
with:
go-version-file: 'go.mod'
cache: ${{ runner.environment == 'github-hosted' }}
- name: Start local Docker registry
run: |
if ! docker start registry 2>/dev/null; then
docker rm -f registry >/dev/null 2>&1 || true
docker run -d --restart=always -p 5000:5000 --name registry registry:2
fi
run: docker run -d --restart=always -p 5000:5000 --name registry registry:2
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v4
with:
driver-opts: network=host
- name: Set up Docker Compose
uses: docker/setup-compose-action@v2
- name: Build weed race binary
run: |
cd docker
+2 -3
View File
@@ -17,8 +17,8 @@ on:
- '.github/workflows/sftp-tests.yml'
concurrency:
group: ${{ github.workflow }}-${{ github.event.pull_request.number || (github.event_name == 'push' && github.ref) || github.run_id }}
cancel-in-progress: ${{ github.event_name == 'pull_request' || (github.event_name == 'push' && !startsWith(github.ref, 'refs/tags/')) }}
group: ${{ github.head_ref }}/sftp-tests
cancel-in-progress: true
permissions:
contents: read
@@ -40,7 +40,6 @@ jobs:
uses: actions/setup-go@v7
with:
go-version-file: 'go.mod'
cache: ${{ runner.environment == 'github-hosted' }}
- name: Install dependencies
run: |
@@ -15,10 +15,6 @@ on:
- '.github/workflows/spark-integration-tests.yml'
workflow_dispatch:
concurrency:
group: ${{ github.workflow }}-${{ github.event.pull_request.number || (github.event_name == 'push' && github.ref) || github.run_id }}
cancel-in-progress: ${{ github.event_name == 'pull_request' || (github.event_name == 'push' && !startsWith(github.ref, 'refs/tags/')) }}
permissions:
contents: read
checks: write
@@ -42,13 +38,12 @@ jobs:
with:
java-version: '11'
distribution: 'temurin'
cache: ${{ runner.environment == 'github-hosted' && 'maven' || '' }}
cache: maven
- name: Set up Go
uses: actions/setup-go@v7
with:
go-version-file: 'go.mod'
cache: ${{ runner.environment == 'github-hosted' }}
- name: Build SeaweedFS binary
run: |
+6 -5
View File
@@ -10,11 +10,12 @@ permissions:
contents: write
concurrency:
# Only one chart regeneration per branch at a time so overlapping runs never
# conflict on note/star_history.svg during rebase. Scoped by ref so a manual
# run on another branch can't cancel the daily master update.
# Only one chart regeneration per branch at a time; a newer run on the same
# branch cancels an in-flight one so overlapping runs never conflict on
# note/star_history.svg during rebase. Scoped by ref so a manual run on
# another branch can't cancel the daily master update.
group: star-history-${{ github.ref }}
cancel-in-progress: ${{ github.event_name == 'pull_request' || (github.event_name == 'push' && !startsWith(github.ref, 'refs/tags/')) }}
cancel-in-progress: true
jobs:
render:
@@ -32,7 +33,7 @@ jobs:
uses: actions/setup-python@v7
with:
python-version: "3.x"
cache: ${{ runner.environment == 'github-hosted' && 'pip' || '' }}
cache: pip
cache-dependency-path: .github/scripts/star_history.py
- name: Install matplotlib
@@ -14,10 +14,6 @@ on:
- 'weed/telemetry/**'
- '.github/workflows/telemetry-integration.yml'
concurrency:
group: ${{ github.workflow }}-${{ github.event.pull_request.number || (github.event_name == 'push' && github.ref) || github.run_id }}
cancel-in-progress: ${{ github.event_name == 'pull_request' || (github.event_name == 'push' && !startsWith(github.ref, 'refs/tags/')) }}
permissions:
contents: read
@@ -34,7 +30,6 @@ jobs:
uses: actions/setup-go@v7
with:
go-version-file: 'go.mod'
cache: ${{ runner.environment == 'github-hosted' }}
- name: Build telemetry server
run: cd telemetry/server && go build -o telemetry-server .
-5
View File
@@ -8,10 +8,6 @@ on:
branches: [ master ]
paths: ['terraform/**', '.github/workflows/terraform_ci.yml']
concurrency:
group: ${{ github.workflow }}-${{ github.event.pull_request.number || (github.event_name == 'push' && github.ref) || github.run_id }}
cancel-in-progress: ${{ github.event_name == 'pull_request' || (github.event_name == 'push' && !startsWith(github.ref, 'refs/tags/')) }}
permissions:
contents: read
@@ -66,7 +62,6 @@ jobs:
uses: actions/setup-go@v7
with:
go-version-file: go.mod
cache: ${{ runner.environment == 'github-hosted' }}
- name: Build weed
run: go build -o "$RUNNER_TEMP/weed" ./weed
@@ -27,10 +27,6 @@ defaults:
run:
working-directory: weed
concurrency:
group: ${{ github.workflow }}-${{ github.event.pull_request.number || (github.event_name == 'push' && github.ref) || github.run_id }}
cancel-in-progress: ${{ github.event_name == 'pull_request' || (github.event_name == 'push' && !startsWith(github.ref, 'refs/tags/')) }}
jobs:
awscli-tests:
runs-on: ubuntu-latest
@@ -41,7 +37,6 @@ jobs:
- uses: actions/setup-go@v7
with:
go-version: ^1.26
cache: ${{ runner.environment == 'github-hosted' }}
- name: Build SeaweedFS
run: |
-5
View File
@@ -18,10 +18,6 @@ on:
- 'go.sum'
- '.github/workflows/tls-rotation-tests.yml'
concurrency:
group: ${{ github.workflow }}-${{ github.event.pull_request.number || (github.event_name == 'push' && github.ref) || github.run_id }}
cancel-in-progress: ${{ github.event_name == 'pull_request' || (github.event_name == 'push' && !startsWith(github.ref, 'refs/tags/')) }}
permissions:
contents: read
@@ -34,7 +30,6 @@ jobs:
- name: Set up Go 1.x
uses: actions/setup-go@v7
with:
cache: ${{ runner.environment == 'github-hosted' }}
go-version: ^1.26
id: go
+3 -7
View File
@@ -11,8 +11,8 @@ on:
- '.github/workflows/tus-tests.yml'
concurrency:
group: ${{ github.workflow }}-${{ github.event.pull_request.number || (github.event_name == 'push' && github.ref) || github.run_id }}
cancel-in-progress: ${{ github.event_name == 'pull_request' || (github.event_name == 'push' && !startsWith(github.ref, 'refs/tags/')) }}
group: ${{ github.head_ref || github.ref }}/tus-tests
cancel-in-progress: true
permissions:
contents: read
@@ -31,14 +31,10 @@ jobs:
- name: Check out code
uses: actions/checkout@v7
- name: Clean up stale FUSE mounts
uses: ./.github/actions/cleanup-stale-fuse-mounts
- name: Set up Go
uses: actions/setup-go@v7
with:
go-version-file: 'go.mod'
cache: ${{ runner.environment == 'github-hosted' }}
id: go
- name: Install SeaweedFS
@@ -53,7 +49,7 @@ jobs:
echo "=== System Information ==="
uname -a
free -h
timeout 15 df -h || true
df -h
echo "=== Starting TUS Tests ==="
# Run tests with automatic server management
@@ -18,10 +18,6 @@ on:
- 'go.sum'
- '.github/workflows/vacuum-integration-tests.yml'
concurrency:
group: ${{ github.workflow }}-${{ github.event.pull_request.number || (github.event_name == 'push' && github.ref) || github.run_id }}
cancel-in-progress: ${{ github.event_name == 'pull_request' || (github.event_name == 'push' && !startsWith(github.ref, 'refs/tags/')) }}
permissions:
contents: read
@@ -34,7 +30,6 @@ jobs:
- name: Set up Go 1.x
uses: actions/setup-go@v7
with:
cache: ${{ runner.environment == 'github-hosted' }}
go-version: ^1.26
id: go
@@ -9,7 +9,6 @@ on:
- 'weed/storage/**'
- 'weed/pb/volume_server.proto'
- 'weed/pb/volume_server_pb/**'
- 'rust/volume_server/**'
- '.github/workflows/volume-server-integration-tests.yml'
push:
branches: [ master, main ]
@@ -19,12 +18,11 @@ on:
- 'weed/storage/**'
- 'weed/pb/volume_server.proto'
- 'weed/pb/volume_server_pb/**'
- 'rust/volume_server/**'
- '.github/workflows/volume-server-integration-tests.yml'
concurrency:
group: ${{ github.workflow }}-${{ github.event.pull_request.number || (github.event_name == 'push' && github.ref) || github.run_id }}
cancel-in-progress: ${{ github.event_name == 'pull_request' || (github.event_name == 'push' && !startsWith(github.ref, 'refs/tags/')) }}
group: ${{ github.head_ref || github.ref }}/volume-server-integration-tests
cancel-in-progress: true
permissions:
contents: read
@@ -35,44 +33,10 @@ env:
SHARD_COUNT: 3
jobs:
build-weed:
name: Build SeaweedFS Binary
runs-on: ubuntu-22.04
timeout-minutes: 15
steps:
- name: Checkout code
uses: actions/checkout@v7
- name: Set up Go
uses: actions/setup-go@v7
with:
go-version-file: 'go.mod'
cache: ${{ runner.environment == 'github-hosted' }}
- name: Build SeaweedFS binary
run: |
cd weed
go build -o weed .
chmod +x weed
./weed version
- name: Package binary
run: tar cf weed-binary.tar weed/weed
- name: Upload binary
uses: actions/upload-artifact@v7
with:
name: weed-binary
if-no-files-found: error
retention-days: 1
path: weed-binary.tar
volume-server-integration-tests:
name: Volume Server Integration Tests (${{ matrix.test-type }} - Shard ${{ matrix.shard }})
runs-on: ubuntu-22.04
timeout-minutes: 45
needs: [build-weed]
strategy:
fail-fast: false
matrix:
@@ -87,18 +51,13 @@ jobs:
uses: actions/setup-go@v7
with:
go-version-file: 'go.mod'
cache: ${{ runner.environment == 'github-hosted' }}
- name: Download binary
uses: actions/download-artifact@v7
with:
name: weed-binary
- name: Restore binary
- name: Build SeaweedFS binary
run: |
tar xf weed-binary.tar
chmod +x weed/weed
./weed/weed version
cd weed
go build -o weed .
chmod +x weed
./weed version
# Dealing the listed tests out one by one keeps the shards even. Bucketing
# them by first letter did not: names cluster, so ^Test[I-S] drew 50 of
@@ -143,77 +102,3 @@ jobs:
echo "## Volume Server Integration Test Summary (${{ matrix.test-type }} - Shard ${{ matrix.shard }})" >> "$GITHUB_STEP_SUMMARY"
echo "- Suite: test/volume_server/${{ matrix.test-type }} (shard ${{ matrix.shard }} of ${SHARD_COUNT}, see 'Select this shard's tests' for the split)" >> "$GITHUB_STEP_SUMMARY"
echo "- Command: go test -v -count=1 -timeout=${{ env.TEST_TIMEOUT }} ./test/volume_server/${{ matrix.test-type }}/... -run \"\${TEST_PATTERN}\"" >> "$GITHUB_STEP_SUMMARY"
volume-server-rust-smoke:
name: Volume Server Rust Smoke (${{ matrix.mode }})
runs-on: ubuntu-22.04
timeout-minutes: 40
strategy:
fail-fast: false
matrix:
mode: [exec, proxy, native]
steps:
- name: Checkout code
uses: actions/checkout@v6
- name: Set up Go
uses: actions/setup-go@v6
with:
go-version-file: 'go.mod'
- name: Set up Rust toolchain
uses: dtolnay/rust-toolchain@stable
- name: Build SeaweedFS binary
run: |
cd weed
go build -o weed .
chmod +x weed
./weed version
- name: Build Rust volume launcher
run: |
cd rust/volume_server
cargo build --release
- name: Run Rust-mode HTTP smoke tests
env:
WEED_BINARY: ${{ github.workspace }}/weed/weed
VOLUME_SERVER_IMPL: rust
RUST_VOLUME_BINARY: ${{ github.workspace }}/rust/volume_server/target/release/weed-volume-rs
VOLUME_SERVER_RUST_MODE: ${{ matrix.mode }}
run: |
go test -v -count=1 -timeout=25m ./test/volume_server/http/... -run "TestAdminStatusAndHealthz|TestUploadReadRangeHeadDeleteRoundTrip"
- name: Run Rust-mode gRPC smoke tests
env:
WEED_BINARY: ${{ github.workspace }}/weed/weed
VOLUME_SERVER_IMPL: rust
RUST_VOLUME_BINARY: ${{ github.workspace }}/rust/volume_server/target/release/weed-volume-rs
VOLUME_SERVER_RUST_MODE: ${{ matrix.mode }}
run: |
go test -v -count=1 -timeout=25m ./test/volume_server/grpc/... -run "TestStateAndStatusRPCs|TestVolumeSyncStatusAndReadVolumeFileStatus"
- name: Collect logs on failure
if: failure()
run: |
mkdir -p /tmp/volume-server-rust-smoke-logs
find /tmp -maxdepth 1 -type d -name "seaweedfs_volume_server_it_*" -print -exec cp -r {} /tmp/volume-server-rust-smoke-logs/ \; || true
- name: Archive logs on failure
if: failure()
uses: actions/upload-artifact@v7
with:
name: volume-server-rust-smoke-logs-${{ matrix.mode }}
path: /tmp/volume-server-rust-smoke-logs/
if-no-files-found: warn
retention-days: 7
- name: Rust smoke summary
if: always()
run: |
echo "## Volume Server Rust Smoke Summary (${{ matrix.mode }})" >> "$GITHUB_STEP_SUMMARY"
echo "- Mode: Go master + Rust volume server launcher (VOLUME_SERVER_IMPL=rust, VOLUME_SERVER_RUST_MODE=${{ matrix.mode }})" >> "$GITHUB_STEP_SUMMARY"
echo "- HTTP Command: go test -v -count=1 -timeout=25m ./test/volume_server/http/... -run \"TestAdminStatusAndHealthz|TestUploadReadRangeHeadDeleteRoundTrip\"" >> "$GITHUB_STEP_SUMMARY"
echo "- gRPC Command: go test -v -count=1 -timeout=25m ./test/volume_server/grpc/... -run \"TestStateAndStatusRPCs|TestVolumeSyncStatusAndReadVolumeFileStatus\"" >> "$GITHUB_STEP_SUMMARY"
-1
View File
@@ -144,4 +144,3 @@ weed_bin
telemetry/server/telemetry-server
.aider*
/seaweed-volume/docs
__pycache__/
+1 -1
View File
@@ -3,7 +3,7 @@ FROM golang:1.26 AS builder
RUN apt-get update
RUN apt-get install -y build-essential libsnappy-dev zlib1g-dev libbz2-dev libgflags-dev liblz4-dev libzstd-dev
ARG ROCKSDB_VERSION=v11.1.2
ARG ROCKSDB_VERSION=v10.10.1
ENV ROCKSDB_VERSION=${ROCKSDB_VERSION}
# build RocksDB
+1 -1
View File
@@ -3,7 +3,7 @@ FROM golang:1.26 AS builder
RUN apt-get update
RUN apt-get install -y build-essential libsnappy-dev zlib1g-dev libbz2-dev libgflags-dev liblz4-dev libzstd-dev
ARG ROCKSDB_VERSION=v11.1.2
ARG ROCKSDB_VERSION=v10.10.1
ENV ROCKSDB_VERSION=${ROCKSDB_VERSION}
# build RocksDB
+1 -1
View File
@@ -362,7 +362,7 @@ require (
github.com/flynn/noise v1.1.0 // indirect
github.com/gabriel-vasile/mimetype v1.4.13 // indirect
github.com/geoffgarside/ber v1.2.0 // indirect
github.com/go-chi/chi/v5 v5.4.0 // indirect
github.com/go-chi/chi/v5 v5.3.1 // indirect
github.com/go-darwin/apfs v0.0.0-20211011131704-f84b94dbf348 // indirect
github.com/go-jose/go-jose/v4 v4.1.5 // indirect
github.com/go-logr/logr v1.4.4 // indirect
+2 -2
View File
@@ -483,8 +483,8 @@ github.com/gin-gonic/gin v1.11.0 h1:OW/6PLjyusp2PPXtyxKHU0RbX6I/l28FTdDlae5ueWk=
github.com/gin-gonic/gin v1.11.0/go.mod h1:+iq/FyxlGzII0KHiBGjuNn4UNENUlKbGlNmc+W50Dls=
github.com/go-asn1-ber/asn1-ber v1.5.8 h1:H9AZkK22UOmfX8J84ubyaZxKJZ3FMHVwn8swoMML7iQ=
github.com/go-asn1-ber/asn1-ber v1.5.8/go.mod h1:hEBeB/ic+5LoWskz+yKT7vGhhPYkProFKoKdwZRWMe0=
github.com/go-chi/chi/v5 v5.4.0 h1:+ufzvFTGKyin26Or5DLrlIDq4ygIDBVcj7oVNeeaYFE=
github.com/go-chi/chi/v5 v5.4.0/go.mod h1:d1IG7GtKPSjPyBP365PH8sAGVHk3qfB/IhjFNRH1wWM=
github.com/go-chi/chi/v5 v5.3.1 h1:3j4HZLGZQ3JpMCrPJF/Jl3mYJfWLKBfNJ6quurUGCf8=
github.com/go-chi/chi/v5 v5.3.1/go.mod h1:R+tYY2hNuVUUjxoPtqUdgBqevM9s9njzkTLutVsOCto=
github.com/go-darwin/apfs v0.0.0-20211011131704-f84b94dbf348 h1:JnrjqG5iR07/8k7NqrLNilRsl3s1EPRQEGvbPyOce68=
github.com/go-darwin/apfs v0.0.0-20211011131704-f84b94dbf348/go.mod h1:Czxo/d1g948LtrALAZdL04TL/HnkopquAjxYUuI02bo=
github.com/go-errors/errors v1.5.1 h1:ZwEMSLRCapFLflTpT7NKaAc7ukJ8ZPEjzlxt8rPN8bk=
+2 -2
View File
@@ -1,6 +1,6 @@
apiVersion: v1
description: SeaweedFS
name: seaweedfs
appVersion: "4.50"
appVersion: "4.48"
# Dev note: Trigger a helm chart release by `git tag -a helm-<version>`
version: 4.50.0
version: 4.48.2
@@ -170,6 +170,9 @@ spec:
{{- if .Values.filer.metricsIp }}
-metricsIp={{ .Values.filer.metricsIp }} \
{{- end }}
{{- if .Values.filer.redirectOnRead }}
-redirectOnRead \
{{- end }}
{{- if .Values.filer.disableHttp }}
-disableHttp \
{{- end }}
@@ -172,8 +172,7 @@ spec:
{{- $bucketName := .name }}
{{- $bucketLock := or .lock .objectLock .withLock }}
bucket_list=$(/bin/echo 's3.bucket.list' | /usr/bin/weed shell) || { echo "Error listing s3 buckets"; exit 1; }
{{- /* One awk: grep -q exits early, awk takes SIGPIPE, pipefail fails the test. */}}
if printf '%s\n' "$bucket_list" | awk -v bucket="{{ $bucketName }}" '$1 "" == bucket { found = 1 } END { exit !found }'; then
if echo "$bucket_list" | awk '{print $1}' | grep -Fxq "{{ $bucketName }}"; then
echo "Bucket '{{ $bucketName }}' already exists, skipping creation."
else
echo "Creating bucket '{{ $bucketName }}'..."
+3
View File
@@ -740,6 +740,9 @@ filer:
# encrypt data on volume servers
encryptVolumeData: false
# Whether proxy or redirect to volume server during file GET request
redirectOnRead: false
# Limit sub dir listing size (default 100000)
dirListLimit: 100000
+1099 -1034
View File
File diff suppressed because it is too large. Load diff

Before

Width:  |  Height:  |  Size: 53 KiB

After

Width:  |  Height:  |  Size: 55 KiB

+1 -2
View File
@@ -771,8 +771,7 @@ message FilerConf {
string replication = 3;
string ttl = 4;
string disk_type = 5;
// unset inherits from the enclosing path rule, set overrides it
optional bool fsync = 6;
bool fsync = 6;
uint32 volume_growth_count = 7;
bool read_only = 8;
string data_center = 9;
-7
View File
@@ -1,7 +0,0 @@
# This file is automatically @generated by Cargo.
# It is not intended for manual editing.
version = 4
[[package]]
name = "weed-volume-rs"
version = "0.1.0"
-6
View File
@@ -1,6 +0,0 @@
[package]
name = "weed-volume-rs"
version = "0.1.0"
edition = "2021"
[dependencies]
-245
View File
@@ -1,245 +0,0 @@
# Rust Volume Server Parity Implementation Plan
## Objective
Implement a native Rust volume server that replicates Go volume-server behavior for HTTP and gRPC APIs, so it can become a drop-in replacement validated by existing integration suites.
## Current Focus (2026-02-16)
- Program focus is now Rust implementation parity, not broad test expansion.
- `test/volume_server` is treated as the parity gate.
- Existing Rust launcher modes (`exec`, `proxy`) are transition tools; they are not the final target.
## Current Status
- Rust crate and launcher are in place.
- Integration harness can run:
- Go master + Go volume (default)
- Go master + Rust launcher (`VOLUME_SERVER_IMPL=rust`)
- Rust launcher `proxy` mode has full-suite integration pass while delegating backend handlers to Go.
- Rust launcher `native` mode is wired as the default Rust entrypoint and currently bootstraps via supervised Go backend delegation.
- Native Rust HTTP handlers now serve control/surface paths in `native` mode:
- `/status`, `/healthz`
- `OPTIONS` admin/public behavior (method allow-list + CORS preflight headers)
- `/ui/index.html` with config-driven JWT/access-ui gating
- `/favicon.ico` and `/seaweedfsstatic/*`
- public non-read methods (`POST`/`PUT`/`DELETE`/unsupported verbs) as `200` no-op passthrough parity
- admin unsupported verbs as `400` parity
- Native Rust control-surface parity now also includes:
- `/healthz` status mirroring from backend state transitions (e.g. leave/stopping => `503`)
- absolute-form HTTP request-target normalization before native route matching
- Native Rust HTTP data-path prevalidation now includes:
- early malformed vid/fid rejection (`400`) for GET/HEAD/POST/PUT fid-route shapes before delegation
- slash-form fid-route parsing for `/{vid}/{fid}` and `/{vid}/{fid}/{filename}` with reserved-path exclusions (`/status`, `/healthz`, `/ui/index.html`, `/stats/*`, static assets)
- write-path error prevalidation for fid routes:
- malformed multipart form-data requests without a boundary => `400`
- requests containing `Content-MD5` header => `400` mismatch parity branch
- request bodies exceeding configured `-fileSizeLimitMB` (by `Content-Length`) => `400`
- Native Rust API/storage handlers are not implemented yet.
## Parity Exit Criteria
1. Native mode passes:
- `env VOLUME_SERVER_IMPL=rust VOLUME_SERVER_RUST_MODE=native go test -count=1 ./test/volume_server/http`
- `env VOLUME_SERVER_IMPL=rust VOLUME_SERVER_RUST_MODE=native go test -count=1 ./test/volume_server/grpc`
2. CI runs native Rust mode integration coverage (at least smoke, then expanded shards).
3. Rust mode defaults to native behavior for integration harness.
4. Go-backend delegation is removed (or retained only as explicit fallback mode).
## Architecture Workstreams
### A. Runtime and Configuration Parity
- [x] Add `native` runtime mode in `weed-volume-rs` (bootstrap delegation path).
- [ ] Parse and honor volume-server CLI/config flags used by integration harness:
- [ ] network/bind ports (`-ip`, `-port`, `-port.grpc`, `-port.public`)
- [ ] master target/config dir/read mode/throttling/JWT-related config
- [ ] size/timeout controls and maintenance state defaults
- [ ] Implement graceful lifecycle behavior (signals, shutdown, readiness).
### B. Native HTTP Surface
- [ ] Admin/control endpoints:
- [x] `GET /status` (native Rust in `native` mode)
- [x] `GET /healthz` (native Rust in `native` mode)
- [x] `OPTIONS` admin/public method+CORS control behavior (native Rust in `native` mode)
- [x] static/UI endpoints currently exercised
- [ ] Data read path parity:
- [ ] fid parsing/path variants
- [ ] conditional headers (`If-Modified-Since`, `If-None-Match`)
- [ ] range handling (single/multi/invalid)
- [ ] deleted reads, auth checks, read-mode branches
- [ ] chunk-manifest and compression/image transformation branches
- [ ] Data write/delete parity:
- [ ] write success/unchanged/error paths
- [ ] replication and file-size-limit paths
- [ ] delete and chunk-manifest delete branches
- [ ] Method/CORS/public-port parity for split admin/public behavior.
### C. Native gRPC Surface
- [ ] Control-plane RPCs:
- [ ] `GetState`, `SetState`, `VolumeServerStatus`, `Ping`, `VolumeServerLeave`
- [ ] admin lifecycle: allocate/mount/unmount/delete/configure/readonly/writable
- [ ] Data RPCs:
- [ ] `ReadNeedleBlob`, `ReadNeedleMeta`, `WriteNeedleBlob`
- [ ] `BatchDelete`, `ReadAllNeedles`
- [ ] sync/copy/receive and status endpoints
- [ ] Stream RPCs:
- [ ] tail sender/receiver
- [ ] vacuum streams
- [ ] query streams
- [ ] Advanced families:
- [ ] erasure coding RPC set
- [ ] tiering/remote fetch
- [ ] scrub/query mode matrix
### D. Storage Compatibility Layer
- [ ] Implement volume data/index handling compatible with Go on-disk format.
- [ ] Preserve cookie/checksum/timestamp semantics used by tests.
- [ ] Match read/write/delete consistency and error mapping behavior.
- [ ] Ensure EC metadata/data-path compatibility with existing files.
### E. Operational Hardening
- [ ] Deterministic startup/readiness and shutdown semantics.
- [ ] Log/error parity sufficient for debugging and CI triage.
- [ ] Concurrency/timeout behavior alignment for throttling and streams.
- [ ] Performance baseline checks vs Go for key flows.
## Milestone Plan
### M0 (Completed): Harness + Launcher Transition
- [x] Rust launcher integrated into harness.
- [x] Proxy mode full-suite validation with Go backend delegation.
### M1: Native Skeleton (Control Plane First)
- [ ] `native` mode boots and serves:
- [x] `/status`, `/healthz`
- [ ] `GetState`, `SetState`, `VolumeServerStatus`, `Ping`, `VolumeServerLeave`
- Gate:
- [x] targeted HTTP/grpc control tests pass in `native` mode (delegated backend path).
### M2: Native Core Data Paths
- [ ] Native HTTP read/write/delete baseline parity.
- [ ] Native gRPC data baseline parity (`Read/WriteNeedle*`, `BatchDelete`, `ReadAllNeedles`).
- Gate:
- core HTTP and gRPC data suites pass in `native` mode.
### M3: Native Stream + Copy/Sync
- [ ] Tail/copy/receive/sync paths in native mode.
- Gate:
- stream/copy families pass in `native` mode.
### M4: Native Advanced Feature Families
- [ ] EC, tiering, scrub/query advanced branches.
- Gate:
- full `/test/volume_server/http` and `/test/volume_server/grpc` pass in `native` mode.
### M5: CI/Cutover
- [x] Add/expand native-mode CI jobs (smoke matrix includes `native`).
- [x] Make native mode default for Rust integration runs.
- [ ] Keep `exec`/`proxy` only as explicit fallback modes during rollout.
## Immediate Next Steps
1. Implement minimal native gRPC state/ping RPC handlers (`GetState`, `SetState`, `VolumeServerStatus`, `Ping`, `VolumeServerLeave`).
2. Expand native HTTP parity from control/static surface into data-path handlers (read/write/delete variants) while preserving current native control behavior.
3. Keep rerunning native-mode integration suites as each delegated branch is replaced.
4. Add mismatch triage notes for each API moved from delegation to native implementation.
## Risk Register
- On-disk format mismatch risk:
- Mitigation: implement format-level compatibility tests early (idx/dat/needle encoding).
- Behavioral drift in edge branches:
- Mitigation: use integration suite failures as primary truth; only add tests for newly discovered untracked branches.
- Stream/concurrency semantic mismatch:
- Mitigation: stabilize with focused interruption/timeout parity tests.
## Progress Log
- Date: 2026-02-15
- Change: Added Rust launcher integration (`exec`) and harness wiring.
- Validation: Rust launcher mode passed smoke and full integration suites while delegating to Go backend.
- Commits: `7beab85c2`, `880c2e1da`, `63d08e8a9`, `d402573ea`, `3bd20e6a1`, `6ce4d7ede`
- Date: 2026-02-15
- Change: Added Rust proxy supervisor mode and validated full integration suite.
- Validation:
- `env VOLUME_SERVER_IMPL=rust VOLUME_SERVER_RUST_MODE=proxy go test -count=1 ./test/volume_server/http`
- `env VOLUME_SERVER_IMPL=rust VOLUME_SERVER_RUST_MODE=proxy go test -count=1 ./test/volume_server/grpc`
- Commits: `a7f50d23b`, `548b3d9a3`
- Date: 2026-02-16
- Change: Re-focused plan from test expansion to native Rust implementation parity.
- Validation basis: latest Rust proxy full-suite pass keeps regression baseline stable while native implementation starts.
- Commits: `14c863dbf`
- Date: 2026-02-16
- Change: Added native Rust launcher mode bootstrap, set Rust launcher default mode to `native`, and expanded CI Rust smoke matrix to include `native`.
- Validation:
- `env VOLUME_SERVER_IMPL=rust VOLUME_SERVER_RUST_MODE=native go test -count=1 ./test/volume_server/http`
- `env VOLUME_SERVER_IMPL=rust VOLUME_SERVER_RUST_MODE=native go test -count=1 ./test/volume_server/grpc`
- `env VOLUME_SERVER_IMPL=rust go test -count=1 ./test/volume_server/http -run '^TestAdminStatusAndHealthz$'`
- `env VOLUME_SERVER_IMPL=rust go test -count=1 ./test/volume_server/grpc -run '^TestStateAndStatusRPCs$'`
- Commits: `70ddbee37`, `61befd10f`, `2e65966c0`
- Date: 2026-02-16
- Change: Implemented first native Rust HTTP handlers in `native` mode for `/status` and `/healthz` on the admin listener while preserving proxy delegation for other APIs.
- Validation:
- `env VOLUME_SERVER_IMPL=rust VOLUME_SERVER_RUST_MODE=native go test -count=1 ./test/volume_server/http -run '^TestAdminStatusAndHealthz$'`
- `env VOLUME_SERVER_IMPL=rust VOLUME_SERVER_RUST_MODE=native go test -count=1 ./test/volume_server/http`
- `env VOLUME_SERVER_IMPL=rust VOLUME_SERVER_RUST_MODE=native go test -count=1 ./test/volume_server/grpc`
- Commits: `7e6e0261a`
- Date: 2026-02-16
- Change: Implemented native Rust `OPTIONS` handling for admin and public listeners in `native` mode, including method allow-list and origin-driven CORS response parity used by integration tests.
- Validation:
- `env VOLUME_SERVER_IMPL=rust VOLUME_SERVER_RUST_MODE=native go test -count=1 ./test/volume_server/http -run '^TestOptionsMethodsByPort$|^TestOptionsWithOriginIncludesCorsHeaders$'`
- `env VOLUME_SERVER_IMPL=rust VOLUME_SERVER_RUST_MODE=native go test -count=1 ./test/volume_server/http`
- `env VOLUME_SERVER_IMPL=rust VOLUME_SERVER_RUST_MODE=native go test -count=1 ./test/volume_server/grpc`
- Commits: `fbff2cb39`
- Date: 2026-02-16
- Change: Added native Rust UI/static/public-surface handlers in `native` mode:
- `/ui/index.html` with config-driven JWT/access-ui gating parity
- `/favicon.ico` and `/seaweedfsstatic/*` static endpoint parity
- public non-read methods as `200` no-op passthrough parity
- admin unsupported methods as `400` parity
- Validation:
- `env VOLUME_SERVER_IMPL=rust VOLUME_SERVER_RUST_MODE=native go test -count=1 ./test/volume_server/http -run '^TestAdminStatusAndHealthz$|^TestUiIndexNotExposedWhenJwtSigningEnabled$|^TestUiIndexExposedWhenJwtSigningEnabledAndAccessUITrue$|^TestStaticAssetEndpoints$|^TestStaticAssetEndpointsOnPublicPort$'`
- `env VOLUME_SERVER_IMPL=rust VOLUME_SERVER_RUST_MODE=native go test -count=1 ./test/volume_server/http -run '^TestOptionsMethodsByPort$|^TestOptionsWithOriginIncludesCorsHeaders$|^TestPublicPortReadOnlyMethodBehavior$|^TestCorsAndUnsupportedMethodBehavior$|^TestUnsupportedMethodTraceParity$|^TestUnsupportedMethodPropfindParity$|^TestUnsupportedMethodConnectParity$|^TestUnsupportedMethodMkcolParity$'`
- `env VOLUME_SERVER_IMPL=rust VOLUME_SERVER_RUST_MODE=native go test -count=1 ./test/volume_server/http`
- `env VOLUME_SERVER_IMPL=rust VOLUME_SERVER_RUST_MODE=native go test -count=1 ./test/volume_server/grpc`
- Commits: `23e4497b2`
- Date: 2026-02-16
- Change: Hardened native Rust HTTP control-path parity:
- `/healthz` now mirrors backend health status (`200`/`503`) in native mode
- absolute-form request-target parsing (e.g. `GET http://host/path HTTP/1.1`) now normalizes to route path parity
- Validation:
- `env VOLUME_SERVER_IMPL=rust VOLUME_SERVER_RUST_MODE=native go test -count=1 ./test/volume_server/http -run '^TestAdminStatusAndHealthz$'`
- `env VOLUME_SERVER_IMPL=rust VOLUME_SERVER_RUST_MODE=native go test -count=1 ./test/volume_server/http/...`
- `env VOLUME_SERVER_IMPL=rust VOLUME_SERVER_RUST_MODE=native go test -count=1 ./test/volume_server/grpc/...`
- Commits: `d6ff6ed6d`
- Date: 2026-02-16
- Change: Added native malformed fid-route validation in `native` mode:
- GET/HEAD/POST/PUT requests matching fid URL shapes now return Rust-native `400 Bad Request` for invalid vid/fid tokens
- valid fid-route requests still delegate to backend handlers for storage/data execution paths
- Validation:
- `env VOLUME_SERVER_IMPL=rust VOLUME_SERVER_RUST_MODE=native go test -count=1 ./test/volume_server/http -run '^TestInvalidReadPathReturnsBadRequest$|^TestWriteInvalidVidAndFidReturnBadRequest$|^TestMalformedVidFidPathReturnsBadRequest$|^TestDownloadLimitInvalidVidWhileOverLimitReturnsBadRequest$'`
- `env VOLUME_SERVER_IMPL=rust VOLUME_SERVER_RUST_MODE=native go test -count=1 ./test/volume_server/http/...`
- `env VOLUME_SERVER_IMPL=rust VOLUME_SERVER_RUST_MODE=native go test -count=1 ./test/volume_server/grpc/...`
- Commits: `1ce0174b2`
- Date: 2026-02-16
- Change: Added native Rust write-path prevalidation branches for fid routes in `native` mode:
- multipart form uploads without boundary now return native `400`
- `Content-MD5` requests now return native `400` mismatch branch
- `Content-Length` over configured `-fileSizeLimitMB` now returns native `400` limit branch
- Validation:
- `env VOLUME_SERVER_IMPL=rust VOLUME_SERVER_RUST_MODE=native go test -count=1 ./test/volume_server/http -run '^TestUploadReadRangeHeadDeleteRoundTrip$|^TestWriteMalformedMultipartAndMD5Mismatch$|^TestWriteRejectsPayloadOverFileSizeLimit$'`
- `env VOLUME_SERVER_IMPL=rust VOLUME_SERVER_RUST_MODE=native go test -count=1 ./test/volume_server/http/...`
- `env VOLUME_SERVER_IMPL=rust VOLUME_SERVER_RUST_MODE=native go test -count=1 ./test/volume_server/grpc/...`
- Commits: `94cefd6f4`
- Date: 2026-02-16
- Change: Broadened native fid-route parser to cover slash-form URLs:
- `/{vid}/{fid}` and `/{vid}/{fid}/{filename}` now flow through native malformed vid/fid validation
- added explicit non-fid exclusions for control/stats/static paths to preserve route parity
- Validation:
- `env VOLUME_SERVER_IMPL=rust VOLUME_SERVER_RUST_MODE=native go test -count=1 ./test/volume_server/http -run '^TestAdminStatusAndHealthz$|^TestReadPathShapesAndIfModifiedSince$|^TestMalformedVidFidPathReturnsBadRequest$'`
- `env VOLUME_SERVER_IMPL=rust VOLUME_SERVER_RUST_MODE=native go test -count=1 ./test/volume_server/http/...`
- `env VOLUME_SERVER_IMPL=rust VOLUME_SERVER_RUST_MODE=native go test -count=1 ./test/volume_server/grpc/...`
- Commits: `e66983e9b`
File diff suppressed because it is too large. Load diff
+14 -39
View File
@@ -86,7 +86,7 @@ version = "1.1.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "40c48f72fd53cd289104fc64099abca73db4166ad86ea0b4341abe65af83dadc"
dependencies = [
"windows-sys 0.60.2",
"windows-sys 0.61.2",
]
[[package]]
@@ -97,7 +97,7 @@ checksum = "291e6a250ff86cd4a820112fb8898808a366d8f9f58ce16d1f538353ad55747d"
dependencies = [
"anstyle",
"once_cell_polyfill",
"windows-sys 0.60.2",
"windows-sys 0.61.2",
]
[[package]]
@@ -510,12 +510,10 @@ dependencies = [
[[package]]
name = "aws-smithy-json"
version = "0.62.7"
version = "0.62.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "701a947f4797e52a911e114a898667c746c39feea467bbd1abd7b3721f702ffa"
checksum = "9648b0bb82a2eedd844052c6ad2a1a822d1f8e3adee5fbf668366717e428856a"
dependencies = [
"aws-smithy-runtime-api",
"aws-smithy-schema",
"aws-smithy-types",
]
@@ -565,12 +563,11 @@ dependencies = [
[[package]]
name = "aws-smithy-runtime-api"
version = "1.12.3"
version = "1.11.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9db177daa6ba8afb9ee1aefcf548c907abcf52065e394ee11a92780057fe0e8c"
checksum = "876ab3c9c29791ba4ba02b780a3049e21ec63dabda09268b175272c3733a79e6"
dependencies = [
"aws-smithy-async",
"aws-smithy-runtime-api-macros",
"aws-smithy-types",
"bytes",
"http 0.2.12",
@@ -581,33 +578,11 @@ dependencies = [
"zeroize",
]
[[package]]
name = "aws-smithy-runtime-api-macros"
version = "1.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8d7396fd9500589e62e460e987ecb671bad374934e55ec3b5f498cc7a8a8a7b7"
dependencies = [
"proc-macro2",
"quote",
"syn",
]
[[package]]
name = "aws-smithy-schema"
version = "0.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7442cb268338f0eb8278140a107c046756aa01093d8ef5e99628d34ae09c94f5"
dependencies = [
"aws-smithy-runtime-api",
"aws-smithy-types",
"http 1.4.0",
]
[[package]]
name = "aws-smithy-types"
version = "1.5.0"
version = "1.4.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "32b42fcf341259d85ca10fac9a2f6448a8ec691c6955a18e45bc3b71a85fab85"
checksum = "9d73dbfbaa8e4bc57b9045137680b958d274823509a360abfd8e1d514d40c95c"
dependencies = [
"base64-simd",
"bytes",
@@ -1350,7 +1325,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb"
dependencies = [
"libc",
"windows-sys 0.52.0",
"windows-sys 0.61.2",
]
[[package]]
@@ -2404,7 +2379,7 @@ version = "0.50.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7957b9740744892f114936ab4a57b3f487491bbeafaf8083688b16841a4240e5"
dependencies = [
"windows-sys 0.59.0",
"windows-sys 0.61.2",
]
[[package]]
@@ -3416,7 +3391,7 @@ dependencies = [
"errno 0.3.14",
"libc",
"linux-raw-sys 0.12.1",
"windows-sys 0.52.0",
"windows-sys 0.61.2",
]
[[package]]
@@ -3701,7 +3676,7 @@ version = "1.4.8"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c4db69cba1110affc0e9f7bcd48bbf87b3f4fc7c61fc9155afd4c469eb3d6c1b"
dependencies = [
"errno 0.2.8",
"errno 0.3.14",
"libc",
]
@@ -3768,7 +3743,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3a766e1110788c36f4fa1c2b71b387a7815aa65f88ce0229841826633d93723e"
dependencies = [
"libc",
"windows-sys 0.60.2",
"windows-sys 0.61.2",
]
[[package]]
@@ -3902,7 +3877,7 @@ dependencies = [
"getrandom 0.4.2",
"once_cell",
"rustix 1.1.4",
"windows-sys 0.52.0",
"windows-sys 0.61.2",
]
[[package]]
-2
View File
@@ -157,8 +157,6 @@ windows-sys = { version = "0.61", features = ["Win32_Storage_FileSystem"] }
[dev-dependencies]
tempfile = "3"
# Paused-time tests drive the write-queue batch window deterministically.
tokio = { version = "1", features = ["full", "test-util"] }
# Already a transitive dependency of aws-sdk-s3 at a single locked version;
# needed directly only for the canned HttpClient in remote_storage::s3 tests.
aws-smithy-runtime-api = "1"
-30
View File
@@ -233,25 +233,6 @@ pub struct Cli {
#[arg(long = "volume.allowUntrustedRemoteEndpoints", default_value_t = false)]
pub allow_untrusted_remote_endpoints: bool,
/// Minimum bytes an fsync batch gathers before a drained queue flushes
/// without waiting.
#[arg(long = "fsyncMinBatchBytes", default_value_t = 1024 * 1024)]
pub fsync_min_batch_bytes: u64,
/// Minimum requests an fsync batch gathers before a drained queue flushes
/// without waiting.
#[arg(long = "fsyncMinBatchRequests", default_value_t = 128)]
pub fsync_min_batch_requests: usize,
/// How long a below-minimum fsync batch waits for more requests once the
/// queue drains; 0 flushes immediately.
#[arg(
long = "fsyncBatchWait",
default_value = "500us",
value_parser = parse_go_duration
)]
pub fsync_batch_wait: std::time::Duration,
/// A file of command line options, each line in optionName=optionValue format.
#[arg(long = "options", default_value = "")]
pub options: String,
@@ -327,14 +308,6 @@ pub struct VolumeServerConfig {
/// If true, FetchAndWriteNeedle skips remote S3 endpoint validation
/// (allows loopback / link-local / metadata hosts).
pub allow_untrusted_remote_endpoints: bool,
/// Minimum bytes an fsync batch gathers before a drained queue flushes
/// without waiting.
pub fsync_min_batch_bytes: u64,
/// Minimum requests an fsync batch gathers before a drained queue flushes
/// without waiting.
pub fsync_min_batch_requests: usize,
/// How long a below-minimum fsync batch waits for stragglers.
pub fsync_batch_wait: std::time::Duration,
}
pub use crate::storage::needle_map::NeedleMapKind;
@@ -965,9 +938,6 @@ fn resolve_config_with_env(cli: Cli, env: EnvLookup<'_>) -> VolumeServerConfig {
concurrent_download_limit: cli.concurrent_download_limit_mb as i64 * 1024 * 1024,
inflight_upload_data_timeout: cli.inflight_upload_data_timeout,
inflight_download_data_timeout: cli.inflight_download_data_timeout,
fsync_min_batch_bytes: cli.fsync_min_batch_bytes,
fsync_min_batch_requests: cli.fsync_min_batch_requests,
fsync_batch_wait: cli.fsync_batch_wait,
has_slow_read: cli.has_slow_read,
read_buffer_size_mb: cli.read_buffer_size_mb,
ldb_timeout: cli.ldb_timeout,
+2 -12
View File
@@ -23,7 +23,7 @@ use seaweed_volume::server::request_id::GrpcRequestIdLayer;
use seaweed_volume::server::volume_server::{
RuntimeMetricsConfig, VolumeServerState, build_metrics_router,
};
use seaweed_volume::server::write_queue::{self, WriteQueue};
use seaweed_volume::server::write_queue::WriteQueue;
use seaweed_volume::storage::store::Store;
use seaweed_volume::storage::types::DiskType;
@@ -436,17 +436,7 @@ async fn run(
// The write queue is always on: durable (fsync) uploads go through it so
// a batch shares one .dat/.idx sync, the way Go's per-volume async writer
// group-commits. Non-fsync writes stay on the synchronous path.
let min_bytes = usize::try_from(config.fsync_min_batch_bytes)
.map_err(|_| "fsyncMinBatchBytes exceeds this platform's maximum usize value")?;
let wq = WriteQueue::new(
state.clone(),
128,
write_queue::SyncBatchOption {
min_bytes,
min_requests: config.fsync_min_batch_requests,
wait: config.fsync_batch_wait,
},
);
let wq = WriteQueue::new(state.clone(), 128);
let _ = state.write_queue.set(wq);
// Set initial metric gauges for concurrent limits and max volumes
-3
View File
@@ -393,9 +393,6 @@ mod tests {
fix_jpg_orientation: false,
read_mode: ReadMode::Local,
allow_untrusted_remote_endpoints: false,
fsync_min_batch_bytes: 1024 * 1024,
fsync_min_batch_requests: 128,
fsync_batch_wait: std::time::Duration::from_micros(500),
cpu_profile: String::new(),
mem_profile: String::new(),
compaction_byte_per_second: 0,
+67 -276
View File
@@ -207,15 +207,9 @@ fn read_needle_for_get(
if request.try_meta_first(plan.size()) {
plan.read_meta(&mut n)?;
if n.cookie != cookie {
// The meta read answered the request.
plan.note_meta_only_success();
return Ok(None);
}
if request.served_from_source(&n) {
if request.is_head {
// HEAD ends here; no payload read will record the success.
plan.note_meta_only_success();
}
let info = plan.into_stream_info(&n);
return Ok(Some((n, Some(info))));
}
@@ -243,9 +237,6 @@ struct StreamingBody {
hold_read_lock_for_stream: bool,
_held_read_lease: Option<crate::storage::volume::DataFileReadLease>,
io_unavailable: Arc<crate::storage::volume::IoUnavailable>,
/// Chunk read failures feed the volume's io-error streak, and finishing
/// the stream clears it — Go readNeedleDataInto parity.
io_errors: Arc<crate::storage::io_error::IoErrorTracker>,
/// Pending chunk read; it hands `buf` back with the result.
pending: Option<tokio::task::JoinHandle<(bytes::BytesMut, std::io::Result<bytes::Bytes>)>>,
/// Chunk buffer, reclaimed once the previous frame has been written out.
@@ -307,18 +298,12 @@ impl http_body::Body for StreamingBody {
),
)));
}
// Hyper may not poll again once Content-Length is
// satisfied, so the read's success is recorded here.
self.io_errors.check_read_write_error(None);
}
return std::task::Poll::Ready(Some(Ok(http_body::Frame::data(
chunk,
))));
}
Ok(Err(e)) => {
self.io_errors.check_read_write_error(Some(&e));
return std::task::Poll::Ready(Some(Err(e)));
}
Ok(Err(e)) => return std::task::Poll::Ready(Some(Err(e))),
Err(e) => {
return std::task::Poll::Ready(Some(Err(std::io::Error::other(e))));
}
@@ -329,7 +314,6 @@ impl http_body::Body for StreamingBody {
let total = self.data_size as usize;
if self.pos >= total {
self.io_errors.check_read_write_error(None);
return std::task::Poll::Ready(None);
}
@@ -1121,10 +1105,6 @@ async fn get_or_head_handler_inner(
let (etag, last_modified_str) = etag_and_last_modified(&n);
if let Some(resp) = not_modified_response(&n, &headers, &etag, &last_modified_str) {
// A request that ends here was answered by the meta read alone.
if let Some(io_errors) = strategy.io_errors() {
io_errors.check_read_write_error(None);
}
return resp;
}
@@ -1218,19 +1198,6 @@ struct ReadPlan {
strategy: ReadStrategy,
}
impl ReadStrategy {
/// The tracker a terminal meta read reports success to, when the reply
/// is served from the data file.
fn io_errors(&self) -> Option<&Arc<crate::storage::io_error::IoErrorTracker>> {
match self {
ReadStrategy::Stream(info)
| ReadStrategy::HeadFromMeta(info)
| ReadStrategy::RangeFromSource(info) => Some(&info.io_errors),
ReadStrategy::Buffered => None,
}
}
}
/// The 401 reply for a bad read JWT; Go checks it before NewVolumeId, so a
/// bad path with JWT enabled is a 401, not a 400.
fn reject_read_jwt(
@@ -1519,78 +1486,41 @@ fn not_modified_response(
etag: &str,
last_modified_str: &Option<String>,
) -> Option<Response> {
let not_modified = || {
let mut resp = StatusCode::NOT_MODIFIED.into_response();
resp.headers_mut()
.insert(header::ETAG, etag.parse().unwrap());
if let Some(lm) = last_modified_str {
resp.headers_mut()
.insert(header::LAST_MODIFIED, lm.parse().unwrap());
}
resp
};
// If-Modified-Since applies only when there is no If-None-Match (RFC 9110 13.2.2).
if headers.contains_key(header::IF_NONE_MATCH) {
let inm = headers
.get_all(header::IF_NONE_MATCH)
.iter()
.map(|v| v.to_str().unwrap_or(""))
.collect::<Vec<_>>()
.join(",");
return etag_list_matches(&inm, etag).then(not_modified);
}
// If-Modified-Since first (Go order), then If-None-Match.
if n.last_modified > 0
&& let Some(ims_str) = headers
.get(header::IF_MODIFIED_SINCE)
.and_then(|v| v.to_str().ok())
&& let Some(ims_header) = headers.get(header::IF_MODIFIED_SINCE)
&& let Ok(ims_str) = ims_header.to_str()
{
// Parse HTTP date format: "Mon, 02 Jan 2006 15:04:05 GMT"
if let Ok(ims_time) =
chrono::NaiveDateTime::parse_from_str(ims_str, "%a, %d %b %Y %H:%M:%S GMT")
&& (n.last_modified as i64) <= ims_time.and_utc().timestamp()
{
return Some(not_modified());
let mut resp = StatusCode::NOT_MODIFIED.into_response();
if let Some(lm) = last_modified_str {
resp.headers_mut()
.insert(header::LAST_MODIFIED, lm.parse().unwrap());
}
// Go sets ETag after the 304 paths, so a 304 has no ETag.
return Some(resp);
}
}
if let Some(if_none_match) = headers.get(header::IF_NONE_MATCH)
&& let Ok(inm) = if_none_match.to_str()
&& inm == etag
{
let mut resp = StatusCode::NOT_MODIFIED.into_response();
if let Some(lm) = last_modified_str {
resp.headers_mut()
.insert(header::LAST_MODIFIED, lm.parse().unwrap());
}
// Go sets ETag after the 304 paths, so a 304 has no ETag.
return Some(resp);
}
None
}
/// Weak comparison against "*" or a comma-separated entity-tag list (RFC 9110 13.1.2).
fn etag_list_matches(header: &str, etag: &str) -> bool {
if header.trim() == "*" {
return true;
}
let etag = etag.trim_matches('"');
split_etag_list(header).into_iter().any(|tag| {
let tag = tag.trim();
let tag = tag.strip_prefix("W/").unwrap_or(tag);
// Empty list elements are ignored, so padding like a trailing
// comma cannot match an empty opaque tag.
!tag.is_empty() && tag.trim_matches('"') == etag
})
}
/// Splits an entity-tag list on the commas between tags; a comma inside a
/// quoted opaque tag does not split.
fn split_etag_list(header: &str) -> Vec<&str> {
let mut tags = Vec::new();
let (mut start, mut quoted) = (0, false);
for (i, b) in header.bytes().enumerate() {
match b {
b'"' => quoted = !quoted,
b',' if !quoted => {
tags.push(&header[start..i]);
start = i + 1;
}
_ => {}
}
}
tags.push(&header[start..]);
tags
}
/// The 200 reply headers, and the extension, which falls back to the stored name's.
fn read_response_headers(
n: &Needle,
@@ -1792,7 +1722,6 @@ fn stream_response(
data_file_access_control: info.data_file_access_control,
hold_read_lock_for_stream: !state.has_slow_read,
io_unavailable: info.io_unavailable,
io_errors: info.io_errors,
pending: None,
buf: bytes::BytesMut::new(),
state: tracking_state,
@@ -2192,21 +2121,9 @@ fn handle_range_request_from_source(
return Ok(Vec::new());
}
let mut buf = vec![0u8; length as usize];
match info
.source
.read_exact_at(&mut buf, info.data_file_offset + start as u64)
{
// A finished range read clears the streak a failed meta read
// counted, like readNeedleDataInto's terminal report.
Ok(()) => {
info.io_errors.check_read_write_error(None);
Ok(buf)
}
Err(e) => {
info.io_errors.check_read_write_error(Some(&e));
Err(e)
}
}
info.source
.read_exact_at(&mut buf, info.data_file_offset + start as u64)?;
Ok(buf)
};
if ranges.len() == 1 {
@@ -3039,62 +2956,50 @@ pub async fn post_handler(
let needle_etag = n.etag();
let needle_has_name = n.has_name();
// Replicate to remote volume servers if this volume has replicas.
// Matches Go's GetWritableRemoteReplications: skip if copy_count == 1.
let needs_replication = !is_replicate && {
let store = state.store.read().unwrap();
store
.find_volume(vid)
.is_some_and(|(_, v)| v.super_block().replica_placement.get_copy_count() > 1)
};
// The fan-out runs concurrently with the local needle write, like Go's
// ReplicatedWrite: it reads the already-buffered request body, so the two
// sides share no mutable state. The ack still waits for both.
let replication = if needs_replication {
let state_clone = state.clone();
let path_clone = path.clone();
let query_clone = query.clone();
let headers_clone = headers.clone();
let body_clone = body.clone();
Some(tokio::spawn(async move {
do_replicated_request(
&state_clone,
vid.0,
Method::POST,
&path_clone,
&query_clone,
&headers_clone,
Some(body_clone),
)
.await
}))
} else {
None
};
let write_result = write_upload_needle(&state, vid, n, fsync).await;
// Replicate to remote volume servers if this volume has replicas.
// Matches Go's GetWritableRemoteReplications: skip if copy_count == 1.
// No leader-known gate: when the lookup cannot run, the write must fail
// rather than be acknowledged on fewer copies than required. The local
// write result takes precedence, matching Go's error ordering.
let replication_result = match replication {
Some(task) => Some(
task.await
// rather than be acknowledged on fewer copies than required.
if !is_replicate && write_result.is_ok() {
let needs_replication = {
let store = state.store.read().unwrap();
store
.find_volume(vid)
.is_some_and(|(_, v)| v.super_block().replica_placement.get_copy_count() > 1)
};
if needs_replication {
let state_clone = state.clone();
let path_clone = path.clone();
let query_clone = query.clone();
let headers_clone = headers.clone();
let body_clone = body.clone();
let replication = tokio::spawn(async move {
do_replicated_request(
&state_clone,
vid.0,
Method::POST,
&path_clone,
&query_clone,
&headers_clone,
Some(body_clone),
)
.await
});
let replication_result = replication
.await
.map_err(|e| format!("replication task failed: {}", e))
.flatten(),
),
None => None,
};
if write_result.is_ok()
&& let Some(Err(e)) = replication_result
{
tracing::error!("replicated write failed: {}", e);
return json_error_with_query(
StatusCode::INTERNAL_SERVER_ERROR,
format!("replication failed: {}", e),
Some(&query),
);
.flatten();
if let Err(e) = replication_result {
tracing::error!("replicated write failed: {}", e);
return json_error_with_query(
StatusCode::INTERNAL_SERVER_ERROR,
format!("replication failed: {}", e),
Some(&query),
);
}
}
}
let resp = match write_result {
@@ -5320,7 +5225,6 @@ mod tests {
hold_read_lock_for_stream: true,
_held_read_lease: None,
io_unavailable: Arc::default(),
io_errors: Arc::default(),
pending: None,
buf: bytes::BytesMut::new(),
state: None,
@@ -5693,80 +5597,6 @@ mod tests {
delete_handler(State(state.clone()), request).await.status()
}
#[test]
fn test_not_modified_checks_if_none_match_first() {
let n = Needle {
last_modified: 1700000000,
..Needle::default()
};
let (etag, last_modified_str) = etag_and_last_modified(&n);
let bare = n.etag();
let before = "Tue, 14 Nov 2023 22:12:20 GMT".to_string();
let after = "Tue, 14 Nov 2023 22:14:20 GMT".to_string();
let cases: Vec<(Vec<(axum::http::HeaderName, String)>, bool)> = vec![
(vec![(header::IF_NONE_MATCH, format!("\"{bare}\""))], true),
(vec![(header::IF_NONE_MATCH, format!("W/\"{bare}\""))], true),
(
vec![(header::IF_NONE_MATCH, format!("\"x\", \"{bare}\""))],
true,
),
(vec![(header::IF_NONE_MATCH, "*".into())], true),
(vec![(header::IF_NONE_MATCH, "\"x\"".into())], false),
(
vec![
(header::IF_NONE_MATCH, "\"x\"".into()),
(header::IF_NONE_MATCH, format!("\"{bare}\"")),
],
true,
),
(
vec![(header::IF_NONE_MATCH, format!("\"{bare},old\""))],
false,
),
(
vec![(header::IF_NONE_MATCH, format!("\"x\",,\"{bare}\""))],
true,
),
(
vec![
(header::IF_NONE_MATCH, "\"x\"".into()),
(header::IF_MODIFIED_SINCE, after.clone()),
],
false,
),
(
vec![
(header::IF_NONE_MATCH, "".into()),
(header::IF_MODIFIED_SINCE, after.clone()),
],
false,
),
(vec![(header::IF_MODIFIED_SINCE, after)], true),
(vec![(header::IF_MODIFIED_SINCE, before)], false),
];
for (pairs, want) in cases {
let mut headers = HeaderMap::new();
for (name, value) in &pairs {
headers.append(name, value.parse().unwrap());
}
assert_eq!(
not_modified_response(&n, &headers, &etag, &last_modified_str).is_some(),
want,
"{pairs:?}"
);
}
let mut headers = HeaderMap::new();
headers.insert(
header::IF_NONE_MATCH,
format!("\"{bare}\"").parse().unwrap(),
);
let resp = not_modified_response(&n, &headers, &etag, &last_modified_str).unwrap();
assert!(resp.headers().contains_key(header::ETAG));
assert!(resp.headers().contains_key(header::LAST_MODIFIED));
}
/// A GET whose needle read is parked must not hold the store lock: a
/// writer, here a real append to the same volume, must get through.
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
@@ -5941,44 +5771,6 @@ mod tests {
assert_eq!(status, StatusCode::NOT_FOUND);
}
/// A ranged read served from the data file reports its outcome to the
/// io-error streak, like readNeedleDataInto: a finished range clears the
/// count a failed meta read left.
#[test]
fn test_range_from_source_reports_to_io_error_streak() {
use std::io::Write;
let tmp = tempfile::TempDir::new().unwrap();
let file = std::fs::OpenOptions::new()
.read(true)
.write(true)
.create(true)
.open(tmp.path().join("1.dat"))
.unwrap();
let data: Vec<u8> = (0..256u32).map(|i| i as u8).collect();
(&file).write_all(&data).unwrap();
let io_errors = Arc::new(crate::storage::io_error::IoErrorTracker::default());
let eio = crate::storage::volume::media_error_for_test();
io_errors.check_read_write_error(Some(&eio));
io_errors.check_read_write_error(Some(&eio));
assert_eq!(io_errors.get_io_error_state().1, 2);
let info = crate::storage::volume::NeedleStreamInfo {
source: crate::storage::volume::NeedleStreamSource::Local(file),
data_file_offset: 0,
data_size: data.len() as u32,
data_file_access_control: Arc::default(),
io_unavailable: Arc::default(),
io_errors: io_errors.clone(),
needle_id: NeedleId(1),
checksum: 0,
};
let resp = handle_range_request_from_source("bytes=10-19", info, HeaderMap::new(), None);
assert_eq!(resp.status(), StatusCode::PARTIAL_CONTENT);
assert_eq!(io_errors.get_io_error_state().1, 0);
}
/// Range is read as Go reads it: a byte its parseRange cannot parse is a
/// 416, Unicode whitespace around a range is trimmed, an empty value is no
/// range, and HEAD ignores it.
@@ -7431,7 +7223,6 @@ mod tests {
.set(crate::server::write_queue::WriteQueue::new(
state.clone(),
128,
crate::server::write_queue::SyncBatchOption::default(),
))
.is_ok()
);
-3
View File
@@ -132,9 +132,6 @@ mod tests {
fix_jpg_orientation: false,
read_mode: ReadMode::Local,
allow_untrusted_remote_endpoints: false,
fsync_min_batch_bytes: 1024 * 1024,
fsync_min_batch_requests: 128,
fsync_batch_wait: std::time::Duration::from_micros(500),
cpu_profile: String::new(),
mem_profile: String::new(),
compaction_byte_per_second: 0,
+12 -164
View File
@@ -56,27 +56,6 @@ const MAX_CONCURRENT_VOLUME_WRITES: usize = 8;
/// retirement is the volume leaving the store.
const IDLE_WORKER_POLL: std::time::Duration = std::time::Duration::from_secs(60);
/// Tunes the per-volume durable-write worker. Once the queue drains with the
/// batch still below min_bytes and min_requests, the batch stays open up to
/// wait for stragglers instead of paying an fsync per request. Mirrors Go's
/// storage.SyncBatchOption.
#[derive(Clone, Copy)]
pub struct SyncBatchOption {
pub min_bytes: usize,
pub min_requests: usize,
pub wait: std::time::Duration,
}
impl Default for SyncBatchOption {
fn default() -> Self {
SyncBatchOption {
min_bytes: 1024 * 1024,
min_requests: MAX_BATCH_SIZE,
wait: std::time::Duration::from_micros(500),
}
}
}
/// Handle for submitting write requests to the per-volume workers.
#[derive(Clone)]
pub struct WriteQueue {
@@ -88,35 +67,23 @@ pub struct WriteQueue {
/// alone would multiply the old single queue's capacity by volume count.
queue_permits: Arc<Semaphore>,
workers: Arc<Mutex<HashMap<VolumeId, mpsc::Sender<WriteRequest>>>>,
sync_batch: SyncBatchOption,
/// Submissions accepted, for tests asserting which writes route here.
#[cfg(test)]
submits: Arc<std::sync::atomic::AtomicU64>,
/// Times a worker armed the straggler window, for tests that must land a
/// write inside an already-open batch.
#[cfg(test)]
windows_armed: Arc<std::sync::atomic::AtomicU64>,
}
impl WriteQueue {
/// Create a new write queue. Per-volume workers are spawned lazily on the
/// first submission for that volume.
pub fn new(
state: Arc<VolumeServerState>,
capacity: usize,
sync_batch: SyncBatchOption,
) -> Self {
pub fn new(state: Arc<VolumeServerState>, capacity: usize) -> Self {
WriteQueue {
state,
capacity,
io_permits: Arc::new(Semaphore::new(MAX_CONCURRENT_VOLUME_WRITES)),
queue_permits: Arc::new(Semaphore::new(capacity)),
workers: Arc::new(Mutex::new(HashMap::new())),
sync_batch,
#[cfg(test)]
submits: Arc::new(std::sync::atomic::AtomicU64::new(0)),
#[cfg(test)]
windows_armed: Arc::new(std::sync::atomic::AtomicU64::new(0)),
}
}
@@ -126,12 +93,6 @@ impl WriteQueue {
self.submits.load(std::sync::atomic::Ordering::Relaxed)
}
/// How many batch windows workers have armed so far.
#[cfg(test)]
fn windows_armed_for_test(&self) -> u64 {
self.windows_armed.load(std::sync::atomic::Ordering::Relaxed)
}
/// Sender for the volume's worker, spawning one on first use.
fn worker_for(&self, volume_id: VolumeId) -> mpsc::Sender<WriteRequest> {
let mut workers = self.workers.lock().unwrap();
@@ -146,9 +107,6 @@ impl WriteQueue {
state: self.state.clone(),
io_permits: self.io_permits.clone(),
workers: self.workers.clone(),
sync_batch: self.sync_batch,
#[cfg(test)]
windows_armed: self.windows_armed.clone(),
}
.run(),
);
@@ -233,9 +191,6 @@ struct VolumeWriteWorker {
state: Arc<VolumeServerState>,
io_permits: Arc<Semaphore>,
workers: Arc<Mutex<HashMap<VolumeId, mpsc::Sender<WriteRequest>>>>,
sync_batch: SyncBatchOption,
#[cfg(test)]
windows_armed: Arc<std::sync::atomic::AtomicU64>,
}
impl VolumeWriteWorker {
@@ -283,40 +238,14 @@ impl VolumeWriteWorker {
let mut batch = Vec::with_capacity(MAX_BATCH_SIZE);
let mut batch_bytes: usize = first.needle.data.len();
batch.push(first);
// Armed once the queue first drains below the minimum batch; holds the
// batch open for stragglers so the wait bound is per batch, not per
// lull (Go's batchWait).
let mut window_deadline = None;
while batch.len() < MAX_BATCH_SIZE && batch_bytes < MAX_BATCH_BYTES {
if window_deadline.is_some_and(|d| tokio::time::Instant::now() >= d) {
break;
}
let req = match self.rx.try_recv() {
Ok(req) => req,
Err(mpsc::error::TryRecvError::Empty) => {
let sync = self.sync_batch;
if sync.wait.is_zero()
|| batch_bytes >= sync.min_bytes
|| batch.len() >= sync.min_requests
{
// a drained queue flushes right away once it holds enough
break;
}
let deadline = *window_deadline.get_or_insert_with(|| {
#[cfg(test)]
self.windows_armed
.fetch_add(1, std::sync::atomic::Ordering::Relaxed);
tokio::time::Instant::now() + sync.wait
});
match tokio::time::timeout_at(deadline, self.rx.recv()).await {
Ok(Some(req)) => req,
Ok(None) | Err(_) => break,
}
match self.rx.try_recv() {
Ok(req) => {
batch_bytes += req.needle.data.len();
batch.push(req);
}
Err(mpsc::error::TryRecvError::Disconnected) => break,
};
batch_bytes += req.needle.data.len();
batch.push(req);
Err(_) => break,
}
}
debug!("processing write batch of {} requests", batch.len());
@@ -451,7 +380,7 @@ mod tests {
async fn test_write_queue_submit_no_volume() {
// Submit a write to a non-existent volume -- should return VolumeError::NotFound.
let state = make_test_state();
let queue = WriteQueue::new(state, MAX_BATCH_SIZE, SyncBatchOption::default());
let queue = WriteQueue::new(state, MAX_BATCH_SIZE);
let needle = Needle {
id: 1.into(),
@@ -473,7 +402,7 @@ mod tests {
async fn test_write_queue_concurrent_submissions() {
// Submit multiple concurrent writes -- all should complete (with errors since no volume).
let state = make_test_state();
let queue = WriteQueue::new(state, MAX_BATCH_SIZE, SyncBatchOption::default());
let queue = WriteQueue::new(state, MAX_BATCH_SIZE);
let mut handles = Vec::new();
for i in 0..10u64 {
@@ -501,7 +430,7 @@ mod tests {
async fn test_write_queue_batching() {
// Verify that many concurrent writes get processed (testing the batching path).
let state = make_test_state();
let queue = WriteQueue::new(state, MAX_BATCH_SIZE, SyncBatchOption::default());
let queue = WriteQueue::new(state, MAX_BATCH_SIZE);
// Submit MAX_BATCH_SIZE requests concurrently
let mut handles = Vec::new();
@@ -593,7 +522,7 @@ mod tests {
async fn test_write_queue_dropped_sender() {
// When the queue is dropped, subsequent submits should fail gracefully.
let state = make_test_state();
let queue = WriteQueue::new(state, 1, SyncBatchOption::default());
let queue = WriteQueue::new(state, 1);
// Clone then drop the original -- the worker keeps running via its rx handle.
let queue2 = queue.clone();
@@ -736,7 +665,7 @@ mod tests {
.unwrap();
}
}
let queue = WriteQueue::new(state, MAX_BATCH_SIZE, SyncBatchOption::default());
let queue = WriteQueue::new(state, MAX_BATCH_SIZE);
let in_append = Arc::new(Barrier::new(2));
let release = Arc::new(Barrier::new(2));
@@ -805,85 +734,4 @@ mod tests {
assert!(a1.await.unwrap().is_ok());
assert!(a2.await.unwrap().is_ok());
}
/// A durable write below the minimum batch rides out its window so a
/// straggler joins the same sync instead of each paying an fsync alone —
/// Go's batchWait. Paused time makes the hold observable: a lone write can
/// only complete once the runtime auto-advances to the window's deadline.
#[tokio::test(start_paused = true)]
async fn test_batch_window_defers_flush_for_stragglers() {
use crate::config::MinFreeSpace;
use crate::storage::types::{DiskType, NeedleId};
use crate::storage::volume::VolumeSpec;
use std::time::Duration;
let tmp = tempfile::TempDir::new().unwrap();
let dir = tmp.path().to_str().unwrap();
let state = make_test_state();
{
let mut store = state.store.write().unwrap();
store
.add_location(
dir,
dir,
10,
DiskType::HardDrive,
MinFreeSpace::Percent(1.0),
Vec::new(),
)
.unwrap();
store
.add_volume(VolumeId(1), DiskType::HardDrive, &VolumeSpec::default())
.unwrap();
}
let queue = WriteQueue::new(
state,
MAX_BATCH_SIZE,
SyncBatchOption {
min_bytes: usize::MAX,
min_requests: MAX_BATCH_SIZE,
wait: Duration::from_secs(30),
},
);
let needle = |id: u64| Needle {
id: NeedleId(id),
cookie: 0x1111.into(),
data: vec![id as u8; 8],
data_size: 8,
..Needle::default()
};
let start = tokio::time::Instant::now();
queue.submit(VolumeId(1), needle(1), true).await.unwrap();
assert!(start.elapsed() >= Duration::from_secs(30));
// Wait for the worker to take the second write and open its batch
// window, so the straggler provably lands inside an open batch
// instead of arriving during the initial drain.
let armed = queue.windows_armed_for_test();
let second = tokio::spawn({
let queue = queue.clone();
async move { queue.submit(VolumeId(1), needle(2), true).await }
});
for _ in 0..10_000 {
if queue.windows_armed_for_test() > armed {
break;
}
tokio::task::yield_now().await;
}
assert!(queue.windows_armed_for_test() > armed);
let third = tokio::spawn({
let queue = queue.clone();
async move { queue.submit(VolumeId(1), needle(3), true).await }
});
let (second, third) = tokio::join!(second, third);
second.unwrap().unwrap();
third.unwrap().unwrap();
// One shared window ends both writes ~30s after their batch armed;
// back-to-back windows would keep the third out to ~90s.
let elapsed = start.elapsed();
assert!(elapsed >= Duration::from_secs(60) && elapsed < Duration::from_secs(90));
}
}
+1 -5
View File
@@ -740,11 +740,7 @@ impl DiskLocation {
if vol.is_read_only() {
continue;
}
// A preallocated volume already holds its space on disk.
let dat_size = vol
.dat_file_size()
.unwrap_or(0)
.max(vol.dat_allocated_size());
let dat_size = vol.dat_file_size().unwrap_or(0);
let idx_size = vol.idx_file_size();
let used = dat_size + idx_size;
if volume_size_limit > used {
+43 -142
View File
@@ -18,7 +18,7 @@ use crate::storage::erasure_coding::ec_volume::{
};
use crate::storage::needle::needle::{Needle, get_actual_size};
use crate::storage::needle_map::NeedleMapKind;
use crate::storage::super_block::{ReplicaPlacement, SUPER_BLOCK_SIZE, SuperBlock};
use crate::storage::super_block::{ReplicaPlacement, SUPER_BLOCK_SIZE};
use crate::storage::types::*;
use crate::storage::volume::{CompactionJob, VifVolumeInfo, Volume, VolumeError, VolumeSpec};
@@ -762,8 +762,8 @@ impl Store {
}
/// Configure a volume's replica placement on disk.
/// The volume must already be unmounted. Rewrites the .vif and the .dat
/// superblock so both carry the new placement.
/// The volume must already be unmounted. This opens the .dat file directly,
/// modifies the replica_placement byte (offset 1), and writes it back.
pub fn configure_volume(&self, vid: VolumeId, rp: ReplicaPlacement) -> Result<(), VolumeError> {
let (_, base_path, _) = self.find_volume_file_base(vid).ok_or_else(|| {
VolumeError::Io(io::Error::new(
@@ -775,7 +775,7 @@ impl Store {
let mut vif = load_vif_volume_info(&vif_path)?;
vif.replication = rp.to_string();
save_vif_volume_info(&vif_path, &vif)?;
update_dat_replica_placement(&format!("{}.dat", base_path), vid, rp)
Ok(())
}
// ---- Read / Write / Delete ----
@@ -943,7 +943,11 @@ impl Store {
let current = loc.max_volume_count.load(Ordering::Relaxed);
let (_, free) = super::disk_location::get_disk_stats(&loc.directory);
let unused_space = loc.unused_space(volume_size_limit);
let unused_space = if self.get_preallocate() {
0
} else {
loc.unused_space(volume_size_limit)
};
let unclaimed = (free as i64) - (unused_space as i64);
let vol_count = loc.volumes_len() as i32;
@@ -1740,61 +1744,6 @@ fn strip_volume_suffix(filename: &str) -> Option<&str> {
.or_else(|| filename.strip_suffix(".idx"))
}
/// Rewrites the replica placement in a .dat header for a volume that is not
/// mounted; a missing .dat means a remote-only volume, where the .vif carries
/// the placement.
fn update_dat_replica_placement(
dat_path: &str,
vid: VolumeId,
rp: ReplicaPlacement,
) -> Result<(), VolumeError> {
use std::io::{Read, Seek, SeekFrom, Write};
let mut file = match std::fs::OpenOptions::new()
.read(true)
.write(true)
.open(dat_path)
{
Ok(f) => f,
Err(e) if e.kind() == io::ErrorKind::NotFound => return Ok(()),
Err(e) => {
return Err(VolumeError::Io(io::Error::new(
e.kind(),
format!("volume {} failed to open {}: {}", vid, dat_path, e),
)))
}
};
let mut header = [0u8; SUPER_BLOCK_SIZE];
file.read_exact(&mut header).map_err(|e| {
VolumeError::Io(io::Error::new(
e.kind(),
format!("volume {} failed to parse superblock: {}", vid, e),
))
})?;
let extra_size = u16::from_be_bytes([header[6], header[7]]) as usize;
let mut block = header.to_vec();
block.resize(SUPER_BLOCK_SIZE + extra_size, 0);
if extra_size > 0 {
file.read_exact(&mut block[SUPER_BLOCK_SIZE..]).map_err(|e| {
VolumeError::Io(io::Error::new(
e.kind(),
format!("volume {} failed to parse superblock: {}", vid, e),
))
})?;
}
let mut super_block = SuperBlock::from_bytes(&block).map_err(|e| {
VolumeError::Io(io::Error::new(
io::ErrorKind::InvalidData,
format!("volume {} failed to parse superblock: {}", vid, e),
))
})?;
super_block.replica_placement = rp;
file.seek(SeekFrom::Start(0))?;
file.write_all(&super_block.to_bytes())?;
file.sync_all()?;
Ok(())
}
fn load_vif_volume_info(path: &str) -> Result<VifVolumeInfo, VolumeError> {
let content = match std::fs::read_to_string(path) {
Ok(c) => c,
@@ -2032,33 +1981,6 @@ mod tests {
assert_eq!(store.total_volume_count(), 1);
}
// Mirrors Go's TestConfigureVolumeUpdatesSuperblock: configure must leave
// the .dat header and the .vif both on the new placement.
#[test]
fn test_configure_volume_updates_superblock() {
let tmp = TempDir::new().unwrap();
let dir = tmp.path().to_str().unwrap();
let mut store = make_test_store(&[dir]);
store
.add_volume(VolumeId(1), DiskType::HardDrive, &VolumeSpec::default())
.unwrap();
assert!(store.unmount_volume(VolumeId(1)).unwrap());
let rp = ReplicaPlacement::from_string("010").unwrap();
store.configure_volume(VolumeId(1), rp).unwrap();
let base = volume_file_name(dir, "", VolumeId(1));
let vif = load_vif_volume_info(&format!("{}.vif", base)).unwrap();
assert_eq!(vif.replication, "010");
let dat = std::fs::read(format!("{}.dat", base)).unwrap();
let super_block = SuperBlock::from_bytes(&dat).unwrap();
assert_eq!(super_block.replica_placement, rp);
assert!(store.configure_volume(VolumeId(2), rp).is_err());
}
#[test]
fn test_mount_volume_by_id_hint_skips_lone_sidecar() {
// A lone .vif/.idx sidecar (e.g. an EC sidecar whose .ecx is on a
@@ -2557,7 +2479,7 @@ mod tests {
}
#[test]
fn test_maybe_adjust_volume_max_reserves_plain_volumes_while_preallocating() {
fn test_maybe_adjust_volume_max_honors_preallocate_flag() {
let tmp = TempDir::new().unwrap();
let dir = tmp.path().to_str().unwrap();
@@ -2572,6 +2494,13 @@ mod tests {
Vec::new(),
)
.unwrap();
// Use a large volume_size_limit so the unused-space difference between
// preallocate=true (0) and preallocate=false (~2 × limit) is big enough
// that integer-division rounding and disk-free fluctuations between the
// two get_disk_stats calls cannot make the quotients equal.
store
.volume_size_limit
.store(100 * 1024 * 1024, Ordering::Relaxed);
store
.add_volume(
VolumeId(61),
@@ -2582,67 +2511,39 @@ mod tests {
},
)
.unwrap();
// Room for 2.5 volumes: the plain volume claims one, leaving one more slot.
let (_, free) = crate::storage::disk_location::get_disk_stats(dir);
let volume_size_limit = (free as f64 / 2.5) as u64;
if volume_size_limit == 0 {
return; // cannot determine free disk space
}
store
.volume_size_limit
.store(volume_size_limit, Ordering::Relaxed);
store.locations[0].original_max_volume_count = 0;
store.set_preallocate(true);
store.maybe_adjust_volume_max();
assert_eq!(
store.locations[0].max_volume_count.load(Ordering::Relaxed),
2
);
}
#[cfg(target_os = "linux")]
#[test]
fn test_unused_space_counts_preallocated_volumes_as_full() {
const LIMIT: u64 = 1 << 20;
let tmp = TempDir::new().unwrap();
let dir = tmp.path().to_str().unwrap();
let mut store = Store::new(NeedleMapKind::InMemory);
store
.add_location(
dir,
dir,
2,
.add_volume(
VolumeId(62),
DiskType::HardDrive,
MinFreeSpace::Percent(1.0),
Vec::new(),
&VolumeSpec {
collection: "preallocate_case",
..Default::default()
},
)
.unwrap();
for (vid, preallocate) in [(VolumeId(71), LIMIT), (VolumeId(72), 0)] {
store
.add_volume(
vid,
DiskType::HardDrive,
&VolumeSpec {
collection: "preallocate_case",
preallocate,
..Default::default()
},
)
for vid in [VolumeId(61), VolumeId(62)] {
let dat_path = store.find_volume(vid).unwrap().1.dat_path();
std::fs::OpenOptions::new()
.write(true)
.open(dat_path)
.unwrap()
.set_len((crate::storage::super_block::SUPER_BLOCK_SIZE + 1) as u64)
.unwrap();
}
let preallocated = store.find_volume(VolumeId(71)).unwrap().1;
if preallocated.dat_allocated_size() < LIMIT {
return; // the filesystem does not support fallocate
}
store.locations[0].original_max_volume_count = 0;
store.locations[0]
.max_volume_count
.store(0, Ordering::Relaxed);
let plain = store.find_volume(VolumeId(72)).unwrap().1;
let plain_dat = plain.dat_file_size().unwrap();
// The filesystem may round a small file up to a larger allocation unit.
let plain_used = plain_dat.max(plain.dat_allocated_size()) + plain.idx_file_size();
assert_eq!(store.locations[0].unused_space(LIMIT), LIMIT - plain_used);
store.set_preallocate(false);
assert!(store.maybe_adjust_volume_max());
let without_preallocate = store.locations[0].max_volume_count.load(Ordering::Relaxed);
store.set_preallocate(true);
assert!(store.maybe_adjust_volume_max());
let with_preallocate = store.locations[0].max_volume_count.load(Ordering::Relaxed);
assert!(with_preallocate > without_preallocate);
}
#[test]
+9 -180
View File
@@ -836,7 +836,7 @@ impl NeedleReadPlan {
pub(crate) fn read_meta(&mut self, n: &mut Needle) -> Result<(), VolumeError> {
let (source, size, version) = (&self.source, self.size, self.version);
let read_at = |buf: &mut [u8], at: u64| Ok(source.read_exact_at(buf, at)?);
let result = with_offset_retry(&mut self.offset, |off| {
with_offset_retry(&mut self.offset, |off| {
if version == VERSION_1 {
// A V1 body is all data, so its "meta" is the whole record.
let mut buf = vec![0u8; get_actual_size(size, version) as usize];
@@ -847,26 +847,13 @@ impl NeedleReadPlan {
} else {
Volume::read_needle_meta_blob_and_parse(read_at, n, off, size, version)
}
});
// Report failures only: a successful meta read precedes the payload
// read, so clearing the streak here would reset the count before
// every failing download and the volume could never reach quarantine.
if let Err(VolumeError::Io(e)) = &result {
self.io_errors.check_read_write_error(Some(e));
}
result?;
})?;
if needle_expired(n) {
return Err(VolumeError::NotFound);
}
Ok(())
}
/// A successful meta read ended the request (e.g. HEAD), so it may reset
/// the streak — unlike a meta read that precedes a payload read.
pub(crate) fn note_meta_only_success(&self) {
self.io_errors.check_read_write_error(None);
}
/// For a reply served from the data file, after `read_meta` filled `n`.
pub(crate) fn into_stream_info(self, n: &Needle) -> NeedleStreamInfo {
// V1 data starts right after the header, V2/V3 after the DataSize field.
@@ -881,7 +868,6 @@ impl NeedleReadPlan {
data_size: n.data_size,
data_file_access_control: self.data_file_access_control,
io_unavailable: self.io_unavailable,
io_errors: self.io_errors,
needle_id: self.needle_id,
checksum: n.checksum.0,
}
@@ -1158,9 +1144,6 @@ pub struct NeedleStreamInfo {
pub data_file_access_control: Arc<DataFileAccessControl>,
/// Checked before each chunk: the volume can become unavailable mid-stream.
pub(crate) io_unavailable: Arc<IoUnavailable>,
/// Streamed chunks report read failures here and clear the streak when
/// the whole body has been emitted, mirroring readNeedleDataInto.
pub(crate) io_errors: Arc<IoErrorTracker>,
pub needle_id: NeedleId,
/// Checksum stored in the needle tail, verified once the last chunk has
/// been read — before that frame is emitted.
@@ -2935,52 +2918,18 @@ impl Volume {
}
// Cookie validation for existing needle (matches Go: check whenever nm.Get returns ok)
let indexed = match io.nm.as_ref() {
Some(nm) => nm.get(n.id)?,
None => None,
};
if let Some(nv) = indexed
&& !nv.offset.is_zero()
if let Some(nm) = &io.nm
&& let Some(nv) = nm.get(n.id)?
{
let mut existing = Needle::default();
// Read only the header to check cookie
self.read_needle_header_unlocked(io, &mut existing, nv.offset.to_actual_offset())?;
if existing.id != n.id && existing.size != nv.size {
// a 4-byte index entry can index past the 32 GiB wrap: the
// low offset may hold another needle while the real record
// sits at the wrapped offset, where reads already retry
#[cfg(not(feature = "5bytes"))]
if nv.offset.to_actual_offset() < MAX_POSSIBLE_VOLUME_SIZE as i64 {
let mut wrapped = Needle::default();
match self.read_needle_header_unlocked(
io,
&mut wrapped,
nv.offset.to_actual_offset() + MAX_POSSIBLE_VOLUME_SIZE as i64,
) {
Ok(()) => existing = wrapped,
Err(VolumeError::Io(e)) if e.kind() == io::ErrorKind::UnexpectedEof => {}
Err(e) => return Err(e),
}
}
if n.cookie.0 == 0 && !check_cookie {
n.cookie = existing.cookie;
}
if existing.id != n.id && existing.size != nv.size {
// the entry resolves to a record that is neither the indexed
// key nor the indexed size, so it is stale rather than a
// duplicate — skip the cookie check and let publish_write
// overwrite it once the append commits
warn!(
"volume {} index entry for needle {} resolves to needle {}; overwriting stale index entry",
self.id.0, n.id.0, existing.id.0
);
} else {
if n.cookie.0 == 0 && !check_cookie {
n.cookie = existing.cookie;
}
if existing.cookie != n.cookie {
return Err(VolumeError::CookieMismatch(n.cookie.0));
}
if existing.cookie != n.cookie {
return Err(VolumeError::CookieMismatch(n.cookie.0));
}
}
Ok(false)
@@ -5036,25 +4985,6 @@ impl Volume {
self.current_dat_file_size()
}
/// Disk space held by the local .dat file, or 0 when it is unknown. It
/// exceeds the file size when the file was preallocated with
/// `FALLOC_FL_KEEP_SIZE`.
pub fn dat_allocated_size(&self) -> u64 {
#[cfg(unix)]
{
use std::os::unix::fs::MetadataExt;
let io = self.io.read().unwrap();
io.dat_file
.as_ref()
.and_then(|f| f.metadata().ok())
.map_or(0, |m| m.blocks() * 512)
}
#[cfg(not(unix))]
{
0
}
}
/// Get the modification time of the .dat file as Unix seconds.
pub fn dat_file_mod_time(&self) -> u64 {
let io = self.io.read().unwrap();
@@ -6118,7 +6048,7 @@ fn preallocate_file(file: &File, size: u64) {
/// An OS error the platform reports for failing storage media, which is
/// what counts toward the I/O error streak.
#[cfg(test)]
pub(crate) fn media_error_for_test() -> io::Error {
fn media_error_for_test() -> io::Error {
#[cfg(unix)]
{
io::Error::from_raw_os_error(libc::EIO)
@@ -9095,107 +9025,6 @@ mod tests {
assert!(matches!(err, VolumeError::CookieMismatch(_)));
}
/// A damaged index entry resolving to a different needle must miss on
/// reads and be dropped by the next write for the key, not rejected as a
/// cookie conflict.
#[test]
fn test_stale_index_entry_heals_on_write() {
let tmp = TempDir::new().unwrap();
let dir = tmp.path().to_str().unwrap();
let v = make_test_volume(dir);
let mut first = Needle {
id: NeedleId(2),
cookie: Cookie(0x2222),
data: b"second".to_vec(),
data_size: 6,
..Needle::default()
};
v.write_needle(&mut first, true, false).unwrap();
// point key 9 at needle 2's record with a size that does not match
// the record, as a corrupt index entry would look
{
let mut io = v.io.write().unwrap();
let nm = io.nm.as_mut().unwrap();
let nv = nm.get(NeedleId(2)).unwrap().unwrap();
nm.put(NeedleId(9), nv.offset, Size(nv.size.0 + 1)).unwrap();
}
let mut probe = Needle {
id: NeedleId(9),
..Needle::default()
};
assert!(v.read_needle(&mut probe).is_err());
let mut n = Needle {
id: NeedleId(9),
cookie: Cookie(0xabcd),
data: b"ninth".to_vec(),
data_size: 5,
..Needle::default()
};
v.write_needle(&mut n, true, false).unwrap();
let mut back = Needle {
id: NeedleId(9),
..Needle::default()
};
v.read_needle(&mut back).unwrap();
assert_eq!(back.data, b"ninth");
assert_eq!(back.cookie, Cookie(0xabcd));
}
/// A stale index entry is only replaced once the new record commits: a
/// durable write rolled back by a failed fsync leaves the index exactly
/// as it found it.
#[test]
fn test_stale_index_entry_survives_failed_write() {
let tmp = TempDir::new().unwrap();
let dir = tmp.path().to_str().unwrap();
let v = make_test_volume(dir);
let mut existing = Needle {
id: NeedleId(2),
cookie: Cookie(0x2222),
data: b"second".to_vec(),
data_size: 6,
..Needle::default()
};
v.write_needle(&mut existing, true, false).unwrap();
let stale_nv = {
let mut io = v.io.write().unwrap();
let nm = io.nm.as_mut().unwrap();
let nv = nm.get(NeedleId(2)).unwrap().unwrap();
nm.put(NeedleId(9), nv.offset, Size(nv.size.0 + 1)).unwrap();
nm.get(NeedleId(9)).unwrap().unwrap()
};
v.fail_next_fsync_for_test(true);
let mut replacement = Needle {
id: NeedleId(9),
cookie: Cookie(0xabcd),
data: b"ninth".to_vec(),
data_size: 5,
..Needle::default()
};
assert!(v.write_needle(&mut replacement, true, true).is_err());
v.fail_next_fsync_for_test(false);
let now = {
let io = v.io.read().unwrap();
io.nm
.as_ref()
.unwrap()
.get(NeedleId(9))
.unwrap()
.expect("a failed write must not drop the index entry it meant to replace")
};
assert_eq!(now.offset, stale_nv.offset);
assert_eq!(now.size, stale_nv.size);
}
// A size disagreeing with the blob's own header indexes the needle at the wrong
// length and, on v3, stamps the append timestamp into the middle of the needle.
#[test]
+2 -2
View File
@@ -1340,11 +1340,11 @@ async fn non_ascii_fid_and_ttl_are_rejected_not_panicked() {
#[tokio::test]
async fn write_queue_upload_returns_same_etag_as_direct_write() {
use seaweed_volume::server::write_queue::{SyncBatchOption, WriteQueue};
use seaweed_volume::server::write_queue::WriteQueue;
let (direct_state, _direct_tmp) = test_state();
let (queued_state, _queued_tmp) = test_state();
let wq = WriteQueue::new(queued_state.clone(), 128, SyncBatchOption::default());
let wq = WriteQueue::new(queued_state.clone(), 128);
let _ = queued_state.write_queue.set(wq);
let compressible = "seaweedfs ".repeat(200).into_bytes();
+23 -68
View File
@@ -3,12 +3,9 @@ package erasure_coding
import (
"bytes"
"context"
"encoding/json"
"fmt"
"io"
"math"
"net"
"net/http"
"os"
"os/exec"
"path/filepath"
@@ -16,7 +13,6 @@ import (
"testing"
"time"
"github.com/seaweedfs/seaweedfs/test/testutil"
"github.com/seaweedfs/seaweedfs/weed/operation"
"github.com/seaweedfs/seaweedfs/weed/pb"
"github.com/seaweedfs/seaweedfs/weed/pb/volume_server_pb"
@@ -51,15 +47,17 @@ func TestECEncodingVolumeLocationTimingBug(t *testing.T) {
defer cluster.Stop()
// Wait for servers to be ready
require.NoError(t, waitForServer(cluster.masterAddr, 30*time.Second))
for _, addr := range cluster.volumeAddrs {
require.NoError(t, waitForServer(addr, 30*time.Second))
}
require.NoError(t, waitForMasterLeader(cluster.masterAddr, 30*time.Second))
require.NoError(t, waitForServer("127.0.0.1:9333", 30*time.Second))
require.NoError(t, waitForServer("127.0.0.1:8080", 30*time.Second))
require.NoError(t, waitForServer("127.0.0.1:8081", 30*time.Second))
require.NoError(t, waitForServer("127.0.0.1:8082", 30*time.Second))
require.NoError(t, waitForServer("127.0.0.1:8083", 30*time.Second))
require.NoError(t, waitForServer("127.0.0.1:8084", 30*time.Second))
require.NoError(t, waitForServer("127.0.0.1:8085", 30*time.Second))
// Create command environment
options := &shell.ShellOptions{
Masters: stringPtr(cluster.masterAddr),
Masters: stringPtr("127.0.0.1:9333"),
GrpcDialOption: grpc.WithInsecure(),
FilerGroup: stringPtr("default"),
}
@@ -73,7 +71,7 @@ func TestECEncodingVolumeLocationTimingBug(t *testing.T) {
// Upload some test data to create volumes
testData := []byte("This is test data for EC encoding integration test")
volumeId, err := uploadTestData(testData, cluster.masterAddr)
volumeId, err := uploadTestData(testData, "127.0.0.1:9333")
require.NoError(t, err)
t.Logf("Created volume %d with test data", volumeId)
@@ -259,13 +257,12 @@ func TestECEncodingMasterTimingRaceCondition(t *testing.T) {
defer cluster.Stop()
// Wait for servers to be ready
require.NoError(t, waitForServer(cluster.masterAddr, 30*time.Second))
require.NoError(t, waitForServer(cluster.volumeAddrs[0], 30*time.Second))
require.NoError(t, waitForMasterLeader(cluster.masterAddr, 30*time.Second))
require.NoError(t, waitForServer("127.0.0.1:9333", 30*time.Second))
require.NoError(t, waitForServer("127.0.0.1:8080", 30*time.Second))
// Create command environment
options := &shell.ShellOptions{
Masters: stringPtr(cluster.masterAddr),
Masters: stringPtr("127.0.0.1:9333"),
GrpcDialOption: grpc.WithInsecure(),
FilerGroup: stringPtr("default"),
}
@@ -279,7 +276,7 @@ func TestECEncodingMasterTimingRaceCondition(t *testing.T) {
// Upload test data
testData := []byte("Race condition test data")
volumeId, err := uploadTestData(testData, cluster.masterAddr)
volumeId, err := uploadTestData(testData, "127.0.0.1:9333")
require.NoError(t, err)
t.Logf("Created volume %d for race condition test", volumeId)
@@ -369,8 +366,6 @@ func TestECEncodingMasterTimingRaceCondition(t *testing.T) {
type TestCluster struct {
masterCmd *exec.Cmd
volumeServers []*exec.Cmd
masterAddr string
volumeAddrs []string
}
func (c *TestCluster) Stop() {
@@ -407,23 +402,9 @@ func startSeaweedFSCluster(ctx context.Context, dataDir string) (*TestCluster, e
securityToml := filepath.Join(dataDir, "security.toml")
os.WriteFile(securityToml, []byte("# Empty security config for testing\n"), 0644)
// Allocate ports instead of using the defaults: on a shared or reused
// runner a stale weed process can still hold 9333/808x, which kills the
// freshly started master at bind time and leaves the test talking to a
// raft follower that answers every assign with "not current leader".
// Mini-safe ports guarantee each HTTP port's derived gRPC port
// (port + 10000) is free as well.
ports, err := testutil.AllocateMiniPorts(7)
if err != nil {
return nil, fmt.Errorf("failed to allocate ports: %v", err)
}
masterPort := ports[0]
masterAddr := fmt.Sprintf("127.0.0.1:%d", masterPort)
cluster.masterAddr = masterAddr
// Start master server
masterCmd := exec.CommandContext(ctx, weedBinary, "master",
"-port", fmt.Sprintf("%d", masterPort),
"-port", "9333",
"-mdir", masterDir,
"-volumeSizeLimitMB", "10", // Small volumes for testing
"-ip", "127.0.0.1",
@@ -451,14 +432,13 @@ func startSeaweedFSCluster(ctx context.Context, dataDir string) (*TestCluster, e
volumeDir := filepath.Join(dataDir, fmt.Sprintf("volume%d", i))
os.MkdirAll(volumeDir, 0755)
port := fmt.Sprintf("%d", ports[1+i])
cluster.volumeAddrs = append(cluster.volumeAddrs, fmt.Sprintf("127.0.0.1:%d", ports[1+i]))
port := fmt.Sprintf("808%d", i)
rack := fmt.Sprintf("rack%d", i)
volumeCmd := exec.CommandContext(ctx, weedBinary, "volume",
"-port", port,
"-dir", volumeDir,
"-max", "10",
"-master", masterAddr,
"-master", "127.0.0.1:9333",
"-volume.allowUntrustedRemoteEndpoints",
"-ip", "127.0.0.1",
"-dataCenter", "dc1",
@@ -518,8 +498,7 @@ func findWeedBinary() string {
func waitForServer(address string, timeout time.Duration) error {
start := time.Now()
for time.Since(start) < timeout {
conn, err := net.DialTimeout("tcp", address, time.Second)
if err == nil {
if conn, err := grpc.NewClient(address, grpc.WithInsecure()); err == nil {
conn.Close()
return nil
}
@@ -528,29 +507,6 @@ func waitForServer(address string, timeout time.Duration) error {
return fmt.Errorf("timeout waiting for server %s", address)
}
// waitForMasterLeader polls the master's /cluster/status endpoint until it
// reports itself as the raft leader. A goraft master can take up to the
// configured election timeout (10s by default) before it is ready to serve
// raft-gated requests like volume assign, so a bare port check is not enough.
func waitForMasterLeader(masterAddr string, timeout time.Duration) error {
start := time.Now()
for time.Since(start) < timeout {
resp, err := http.Get(fmt.Sprintf("http://%s/cluster/status", masterAddr))
if err == nil {
var status struct {
IsLeader bool `json:"IsLeader"`
}
err = json.NewDecoder(resp.Body).Decode(&status)
resp.Body.Close()
if err == nil && status.IsLeader {
return nil
}
}
time.Sleep(500 * time.Millisecond)
}
return fmt.Errorf("timeout waiting for master %s to become raft leader", masterAddr)
}
func uploadTestData(data []byte, masterAddress string) (needle.VolumeId, error) {
// Upload data to get a file ID
assignResult, err := operation.Assign(context.Background(), func(ctx context.Context) pb.ServerAddress {
@@ -2253,15 +2209,14 @@ func TestECEncodeReplicatedVolumeSync(t *testing.T) {
defer cluster.Stop()
// Wait for servers to be ready
require.NoError(t, waitForServer(cluster.masterAddr, 30*time.Second))
for _, addr := range cluster.volumeAddrs {
require.NoError(t, waitForServer(addr, 30*time.Second))
require.NoError(t, waitForServer("127.0.0.1:9333", 30*time.Second))
for i := 0; i < 6; i++ {
require.NoError(t, waitForServer(fmt.Sprintf("127.0.0.1:%d", 8080+i), 30*time.Second))
}
require.NoError(t, waitForMasterLeader(cluster.masterAddr, 30*time.Second))
// Create command environment
options := &shell.ShellOptions{
Masters: stringPtr(cluster.masterAddr),
Masters: stringPtr("127.0.0.1:9333"),
GrpcDialOption: grpc.WithInsecure(),
FilerGroup: stringPtr("default"),
}
@@ -2285,7 +2240,7 @@ func TestECEncodeReplicatedVolumeSync(t *testing.T) {
var volumeId needle.VolumeId
var uploadErr error
for retry := 0; retry < 5; retry++ {
volumeId, uploadErr = uploadTestDataWithReplication(t, cluster.masterAddr, testCollection, testReplication)
volumeId, uploadErr = uploadTestDataWithReplication(t, "127.0.0.1:9333", testCollection, testReplication)
if uploadErr == nil {
break
}
@@ -2335,7 +2290,7 @@ func TestECEncodeReplicatedVolumeSync(t *testing.T) {
var volumeId needle.VolumeId
var uploadErr error
for retry := 0; retry < 5; retry++ {
volumeId, uploadErr = uploadTestDataWithReplication(t, cluster.masterAddr, testCollection, testReplication)
volumeId, uploadErr = uploadTestDataWithReplication(t, "127.0.0.1:9333", testCollection, testReplication)
if uploadErr == nil {
break
}
-15
View File
@@ -110,21 +110,6 @@ func startFailoverCluster(t testing.TB, numVolumes, numMounts int) *failoverClus
"mount %d not ready\n%s", i, c.tailLog(fmt.Sprintf("mount%d", i)))
}
require.NoError(t, c.WaitForVolumeServers(60*time.Second))
// A mounted filesystem and registered volume servers do not yet guarantee
// that the lock ring has settled or the first replicated volume is ready.
// Exercise the write/read path during setup so the append latency checks
// measure failover on a running cluster rather than its cold start.
probeName := ".failover-ready"
probeData := []byte("ready\n")
probePath := filepath.Join(c.MountDir(0), probeName)
require.NoError(t, os.WriteFile(probePath, probeData, 0644),
"readiness write failed\n%s", c.tailLog("mount0"))
for i := 0; i < numMounts; i++ {
_, ready := waitForContent(filepath.Join(c.MountDir(i), probeName), probeData, convergeTimeout)
require.True(t, ready, "mount %d did not observe readiness write\n%s", i, c.tailLog(fmt.Sprintf("mount%d", i)))
}
require.NoError(t, os.Remove(probePath))
return c
}
Loaded 100 of 513 files, more files were not shown because too many files have changed in this diff. Show more