Compare commits

..
Author SHA1 Message Date
Devin 9d84ecca2f upload -dir: finish every file before reporting a failed upload
A failed batch stopped all remaining batches, and the file-list path
returned before printing results, hiding ids of files that did upload.
Now every file is still attempted, results print for each batch, and
the command fails at the end if any upload failed.
2026-10-21 15:20:40 -07:00
Devin a1d2ebbb9b operation: warm lazy log fds in the file-part leak test 2026-10-21 15:15:01 -07:00
Devin 371d320854 operation: close open file parts when a batch fails to build
Batched -dir uploads now open up to 32 files at once; a mid-batch error
left every earlier handle open until process exit. Close the parts
already built, and close the handle when Stat fails after Open.
2026-10-21 15:14:12 -07:00
Chris LuandDevin 84cc179b80 upload -dir: fetch file ids in batches instead of per file
Directory upload submitted every file through its own SubmitFiles
call, so each file paid a separate serial assign RPC. Files are now
collected during the walk and submitted in batches, sharing one batch
assign and the parallel uploads.

Generated with [Devin](https://devin.ai)

Co-Authored-By: Devin <158243242+devin-ai-integration[bot]@users.noreply.github.com>
2026-10-21 15:10:30 -07:00
Chris LuandDevin c8002d63bb upload: submit batched files concurrently
SubmitFiles fetched all file ids in one assign RPC but then uploaded
the files one after another, so a multi-file upload spent its time in
serial per-file round trips. Uploads now run through a small worker
pool while results keep their input order.

Generated with [Devin](https://devin.ai)

Co-Authored-By: Devin <158243242+devin-ai-integration[bot]@users.noreply.github.com>
2026-10-21 15:10:30 -07:00
484 changed files with 3069 additions and 30540 deletions

No files matched your search

@@ -1,11 +0,0 @@
name: Clean up stale SeaweedFS test mounts
description: Detach disconnected FUSE test mounts left by earlier jobs on a shared runner.
runs:
using: composite
steps:
- name: Unmount disconnected SeaweedFS test mounts
if: runner.os == 'Linux' && runner.environment == 'self-hosted'
shell: bash
env:
CLEANUP_SCRIPT: ${{ github.action_path }}/cleanup.py
run: python3 "$CLEANUP_SCRIPT"
@@ -1,89 +0,0 @@
#!/usr/bin/env python3
"""Detach disconnected SeaweedFS integration-test mounts without touching live ones."""
import errno
import os
from pathlib import Path
import re
import shutil
import subprocess
import sys
import tempfile
def test_mounts(mountinfo):
"""Read the kernel table directly: traversing a stale mount may block."""
roots = {tempfile.gettempdir(), "/tmp", "/var/tmp"}
for line in mountinfo.splitlines():
fields, filesystem = line.split(" - ", 1)
if filesystem.split()[0].split(".")[0] != "fuse":
continue
fields = fields.split()
# mountinfo escapes whitespace and backslashes with octal sequences.
target = re.sub(r"\\([0-7]{3})", lambda m: chr(int(m[1], 8)), fields[4])
path = Path(target)
# Test mounts are a mount* leaf under a seaweedfs_fuse_* scratch dir
# (seaweedfs_fuse_test_*/mount, seaweedfs_fuse_failover_*/mount0, ...)
# or the posix-compliance workflow's fixed /tmp/seaweedfs/mount.
if (not path.name.startswith("mount")
or str(path.parent.parent) not in roots
or not (path.parent.name.startswith("seaweedfs_fuse")
or path.parent.name == "seaweedfs")):
continue
yield fields[0], target
def mounted(mount_id, target):
return (mount_id, target) in test_mounts(Path("/proc/self/mountinfo").read_text())
def cleanup():
failed = False
for mount_id, target in test_mounts(Path("/proc/self/mountinfo").read_text()):
try:
# A separate, bounded probe prevents a wedged FUSE daemon from
# hanging cleanup. Only ENOTCONN proves the daemon disconnected;
# a timeout or another error is not enough to detach a live mount.
probe = subprocess.run(
[sys.executable, __file__, "--probe", target], timeout=5,
capture_output=True, check=False,
)
except subprocess.TimeoutExpired:
print(f"::warning::FUSE probe timed out; leaving {target} mounted")
continue
if probe.returncode != errno.ENOTCONN or not mounted(mount_id, target):
continue
print(f"Unmounting disconnected FUSE test mount: {target}", flush=True)
# Lazy unmount detaches a dead mount even when an old process still
# holds it open. Prefer the unprivileged FUSE helper; sudo is fallback.
for command in (
["fusermount3", "-uz", "--", target],
["fusermount", "-uz", "--", target],
["sudo", "-n", "umount", "-l", "--", target],
):
try:
subprocess.run(command, timeout=10, check=False)
except (OSError, subprocess.TimeoutExpired) as error:
print(f"{command[0]}: {error}", file=sys.stderr)
if not mounted(mount_id, target):
break
else:
print(f"::error::Could not unmount disconnected FUSE test mount: {target}")
failed = True
continue
# The detached mount's parent still holds the crashed run's data and
# logs; test_mounts only yields targets under a SeaweedFS test dir in
# a temp root, so the parent is safe to reclaim here.
shutil.rmtree(Path(target).parent, ignore_errors=True)
return int(failed)
if __name__ == "__main__":
if len(sys.argv) == 3 and sys.argv[1] == "--probe":
try:
os.stat(sys.argv[2])
except OSError as error:
sys.exit(error.errno)
sys.exit(0)
sys.exit(cleanup())
@@ -1,137 +0,0 @@
import errno
import importlib.util
from pathlib import Path
import subprocess
import unittest
from unittest.mock import patch
spec = importlib.util.spec_from_file_location("cleanup", Path(__file__).with_name("cleanup.py"))
cleanup = importlib.util.module_from_spec(spec)
spec.loader.exec_module(cleanup)
TARGET = "/tmp/seaweedfs_fuse_test_123/mount"
MOUNTINFO = f"42 1 0:9 / {TARGET} rw - fuse.seaweedfs seaweedfs rw\n"
class CleanupTests(unittest.TestCase):
def run_cleanup(self, results, mounted=True):
with patch.object(cleanup.Path, "read_text", return_value=MOUNTINFO), \
patch.object(cleanup, "mounted", side_effect=mounted if isinstance(mounted, list) else None,
return_value=mounted) as check, \
patch.object(cleanup.shutil, "rmtree") as rmtree, \
patch.object(cleanup.subprocess, "run", side_effect=results) as run:
status = cleanup.cleanup()
return status, run, check, rmtree
def result(self, status):
return subprocess.CompletedProcess([], status)
def test_only_fuse_integration_test_mounts_are_candidates(self):
rows = MOUNTINFO
rows += MOUNTINFO.replace("fuse.seaweedfs", "ext4")
rows += MOUNTINFO.replace(TARGET, "/mnt/production")
rows += MOUNTINFO.replace(TARGET, "/tmp/seaweedfs_fuse_test_123/data")
rows += MOUNTINFO.replace(TARGET, "/home/user/seaweedfs_fuse_test_123/mount")
rows += MOUNTINFO.replace(TARGET, "/tmp/seaweedfs/data")
self.assertEqual(list(cleanup.test_mounts(rows)), [("42", TARGET)])
def test_all_test_mount_layouts_are_candidates(self):
targets = (
"/tmp/seaweedfs_fuse_test_123/mount",
"/tmp/seaweedfs_fuse_failover_456/mount0",
"/tmp/seaweedfs_fuse_failover_456/mount1",
"/var/tmp/seaweedfs_fuse_p2p_test_789/mount2",
"/tmp/seaweedfs/mount",
)
rows = "".join(
MOUNTINFO.replace("42", str(40 + i)).replace(TARGET, target)
for i, target in enumerate(targets)
)
self.assertEqual(
list(cleanup.test_mounts(rows)),
[(str(40 + i), target) for i, target in enumerate(targets)],
)
def test_mountinfo_paths_are_unescaped_once(self):
target = "/tmp/seaweedfs_fuse_test_space and\\slash/mount"
encoded = target.replace("\\", r"\134").replace(" ", r"\040")
rows = MOUNTINFO.replace(TARGET, encoded)
self.assertEqual(list(cleanup.test_mounts(rows)), [("42", target)])
def test_no_mounts_is_a_noop(self):
with patch.object(cleanup.Path, "read_text", return_value=""), \
patch.object(cleanup.subprocess, "run") as run:
self.assertEqual(cleanup.cleanup(), 0)
run.assert_not_called()
def test_healthy_mount_is_untouched(self):
status, run, _, _ = self.run_cleanup([self.result(0)])
self.assertEqual(status, 0)
self.assertEqual(run.call_count, 1)
def test_other_probe_errors_are_untouched(self):
for error in (errno.EACCES, errno.ENOENT, errno.EIO):
with self.subTest(error=error):
status, run, _, _ = self.run_cleanup([self.result(error)])
self.assertEqual(status, 0)
self.assertEqual(run.call_count, 1)
def test_probe_timeout_does_not_detach_a_possibly_live_mount(self):
status, run, _, _ = self.run_cleanup([subprocess.TimeoutExpired("probe", 5)])
self.assertEqual(status, 0)
self.assertEqual(run.call_count, 1)
def test_disconnected_mount_is_lazily_unmounted(self):
status, run, _, _ = self.run_cleanup([self.result(errno.ENOTCONN), self.result(0)], [True, False])
self.assertEqual(status, 0)
self.assertEqual(run.call_args_list[1].args[0], ["fusermount3", "-uz", "--", TARGET])
self.assertEqual(run.call_args_list[0].kwargs["timeout"], 5)
self.assertEqual(run.call_args_list[1].kwargs["timeout"], 10)
def test_replaced_or_removed_mount_is_untouched(self):
status, run, _, _ = self.run_cleanup([self.result(errno.ENOTCONN)], False)
self.assertEqual(status, 0)
self.assertEqual(run.call_count, 1)
def test_mount_identity_is_checked_without_stat(self):
with patch.object(cleanup.Path, "read_text", return_value=MOUNTINFO):
self.assertTrue(cleanup.mounted("42", TARGET))
self.assertFalse(cleanup.mounted("43", TARGET))
def test_missing_or_timed_out_helper_uses_fallback(self):
for error in (FileNotFoundError(), subprocess.TimeoutExpired("fusermount3", 10)):
with self.subTest(error=error):
status, run, _, _ = self.run_cleanup(
[self.result(errno.ENOTCONN), error, self.result(0)], [True, True, False])
self.assertEqual(status, 0)
self.assertEqual(run.call_args.args[0], ["fusermount", "-uz", "--", TARGET])
def test_failed_helpers_fall_back_to_sudo(self):
status, run, _, _ = self.run_cleanup(
[self.result(errno.ENOTCONN), self.result(1), self.result(1), self.result(0)],
[True, True, True, False])
self.assertEqual(status, 0)
self.assertEqual(run.call_args.args[0], ["sudo", "-n", "umount", "-l", "--", TARGET])
def test_cleanup_fails_when_stale_mount_remains(self):
status, run, _, rmtree = self.run_cleanup(
[self.result(errno.ENOTCONN), self.result(1), self.result(1), self.result(1)])
self.assertEqual(status, 1)
self.assertEqual(run.call_count, 4)
rmtree.assert_not_called()
def test_unmounted_test_dir_is_reclaimed(self):
status, _, _, rmtree = self.run_cleanup(
[self.result(errno.ENOTCONN), self.result(0)], [True, False])
self.assertEqual(status, 0)
rmtree.assert_called_once_with(Path(TARGET).parent, ignore_errors=True)
def test_probe_process_reports_errno(self):
for target, expected in ((str(Path(__file__)), 0), (str(Path(__file__) / "absent"), errno.ENOTDIR)):
with self.subTest(target=target):
result = subprocess.run([cleanup.sys.executable, cleanup.__file__, "--probe", target], check=False)
self.assertEqual(result.returncode, expected)
if __name__ == "__main__":
unittest.main()
@@ -23,14 +23,8 @@ runs:
bin=$(command -v fusermount3 || true)
if [ -z "$bin" ]; then
# Self-hosted runners are not guaranteed to carry fuse3.
sudo apt-get update
sudo apt-get install -y fuse3
bin=$(command -v fusermount3 || true)
if [ -z "$bin" ]; then
echo "no fusermount3 in PATH after installing fuse3" >&2
exit 1
fi
echo "no fusermount3 in PATH" >&2
exit 1
fi
if can_mount "$bin"; then
ls -l "$bin"
-30
View File
@@ -1,30 +0,0 @@
name: "actions: unit tests"
on:
push:
branches: [ master ]
paths:
- '.github/actions/**'
- '.github/workflows/action-tests.yml'
pull_request:
branches: [ master ]
paths:
- '.github/actions/**'
- '.github/workflows/action-tests.yml'
concurrency:
group: ${{ github.workflow }}-${{ github.event.pull_request.number || (github.event_name == 'push' && github.ref) || github.run_id }}
cancel-in-progress: ${{ github.event_name == 'pull_request' || (github.event_name == 'push' && !startsWith(github.ref, 'refs/tags/')) }}
permissions:
contents: read
jobs:
cleanup-stale-fuse-mounts:
runs-on: ubuntu-latest
steps:
- name: Check out code
uses: actions/checkout@v7
- name: Run cleanup action unit tests
run: python3 .github/actions/cleanup-stale-fuse-mounts/test_cleanup.py
+2 -8
View File
@@ -9,8 +9,8 @@ on:
- '.github/workflows/codeql.yml'
concurrency:
group: ${{ github.workflow }}-${{ github.event.pull_request.number || (github.event_name == 'push' && github.ref) || github.run_id }}
cancel-in-progress: ${{ github.event_name == 'pull_request' || (github.event_name == 'push' && !startsWith(github.ref, 'refs/tags/')) }}
group: ${{ github.head_ref }}/codeql
cancel-in-progress: true
jobs:
CodeQL-Build:
@@ -25,12 +25,6 @@ jobs:
- name: Checkout repository
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1
- name: Set up Go
uses: actions/setup-go@v7
with:
go-version-file: 'go.mod'
cache: ${{ runner.environment == 'github-hosted' }}
# Initializes the CodeQL tools for scanning.
- name: Initialize CodeQL
uses: github/codeql-action/init@v4.39.0
-4
View File
@@ -8,10 +8,6 @@ on:
pull_request:
branches: [master]
concurrency:
group: ${{ github.workflow }}-${{ github.event.pull_request.number || (github.event_name == 'push' && github.ref) || github.run_id }}
cancel-in-progress: ${{ github.event_name == 'pull_request' || (github.event_name == 'push' && !startsWith(github.ref, 'refs/tags/')) }}
permissions:
contents: read
-11
View File
@@ -13,10 +13,6 @@ on:
- 'go.sum'
- '.github/workflows/container_dev.yml'
concurrency:
group: ${{ github.workflow }}-${{ github.event.pull_request.number || (github.event_name == 'push' && github.ref) || github.run_id }}
cancel-in-progress: ${{ github.event_name == 'pull_request' || (github.event_name == 'push' && !startsWith(github.ref, 'refs/tags/')) }}
permissions:
contents: read
@@ -55,7 +51,6 @@ jobs:
echo "CFLAGS_aarch64_unknown_linux_musl=-U_FORTIFY_SOURCE" >> "$GITHUB_ENV"
- name: Cache cargo registry and target
if: runner.environment == 'github-hosted'
uses: actions/cache@v6
with:
path: |
@@ -198,12 +193,6 @@ jobs:
platforms: linux/amd64, linux/arm64
tags: ${{ steps.docker_meta.outputs.tags }}
labels: ${{ steps.docker_meta.outputs.labels }}
cache-from: type=gha
cache-to: type=gha,mode=max
# Key the checkout layer on the pushed sha so cached layers can't
# serve stale code; the Dockerfile fetch fallback picks it up.
build-args: |
BRANCH=${{ github.sha }}
- name: Sign
if: github.event_name != 'pull_request'
@@ -24,10 +24,6 @@ on:
required: false
default: ''
concurrency:
group: ${{ github.workflow }}-${{ github.event.pull_request.number || (github.event_name == 'push' && github.ref) || github.event.inputs.image_tag || format('foundationdb_{0}_seaweedfs_{1}', github.event.inputs.fdb_version || '7.4.5', github.event.inputs.seaweedfs_ref || github.ref_name) }}
cancel-in-progress: ${{ github.event_name == 'pull_request' || (github.event_name == 'push' && !startsWith(github.ref, 'refs/tags/')) }}
permissions:
contents: read
@@ -68,7 +64,6 @@ jobs:
uses: actions/setup-go@v7
with:
go-version-file: go.mod
cache: ${{ runner.environment == 'github-hosted' }}
- name: Run FoundationDB tagged tests
env:
+2 -11
View File
@@ -30,11 +30,6 @@ on:
type: boolean
default: false
concurrency:
group: ${{ github.workflow }}-${{ github.event.pull_request.number || (github.event_name == 'push' && github.ref) || github.event.inputs.image_tag || github.run_id }}
cancel-in-progress: false
queue: max
permissions:
contents: read
security-events: write
@@ -99,7 +94,6 @@ jobs:
echo "CFLAGS_aarch64_unknown_linux_musl=-U_FORTIFY_SOURCE" >> "$GITHUB_ENV"
- name: Cache cargo registry and target
if: runner.environment == 'github-hosted'
uses: actions/cache@v6
with:
path: |
@@ -170,13 +164,10 @@ jobs:
uses: actions/checkout@v7
with:
ref: ${{ github.event_name == 'workflow_dispatch' && github.event.inputs.source_ref || github.ref }}
- name: Clean up stale FUSE mounts
uses: ./.github/actions/cleanup-stale-fuse-mounts
- name: Free Disk Space
run: |
echo "Available disk space before cleanup:"
timeout 15 df -h || true
df -h
# Remove pre-installed tools
sudo rm -rf /usr/share/dotnet /usr/local/lib/android /opt/ghc /opt/hostedtoolcache/CodeQL
# Clean package managers
@@ -188,7 +179,7 @@ jobs:
[ -d ~/.cache/go-build ] && rm -rf ~/.cache/go-build || true
[ -d /go/pkg ] && rm -rf /go/pkg || true
echo "Available disk space after cleanup:"
timeout 15 df -h || true
df -h
- name: Configure variant
id: config
@@ -11,11 +11,6 @@ on:
required: true
default: ''
concurrency:
group: ${{ github.workflow }}-${{ github.event.pull_request.number || (github.event_name == 'push' && github.ref) || format('refs/tags/{0}', github.event.inputs.release_tag) || github.run_id }}
cancel-in-progress: false
queue: max
permissions:
contents: read
@@ -25,7 +25,7 @@ on:
rocksdb_version:
description: 'RocksDB git tag to use when variant=rocksdb'
required: false
default: 'v11.1.2'
default: 'v10.10.1'
permissions:
contents: read
@@ -78,7 +78,6 @@ jobs:
echo "CFLAGS_aarch64_unknown_linux_musl=-U_FORTIFY_SOURCE" >> "$GITHUB_ENV"
- name: Cache cargo registry and target
if: runner.environment == 'github-hosted'
uses: actions/cache@v6
with:
path: |
@@ -180,10 +179,6 @@ jobs:
if: github.event_name != 'workflow_dispatch' || github.event.inputs.variant == 'all' || github.event.inputs.variant == matrix.variant
uses: actions/checkout@v7
- name: Clean up stale FUSE mounts
if: github.event_name != 'workflow_dispatch' || github.event.inputs.variant == 'all' || github.event.inputs.variant == matrix.variant
uses: ./.github/actions/cleanup-stale-fuse-mounts
- name: Download pre-built Rust binaries
if: github.event_name != 'workflow_dispatch' || github.event.inputs.variant == 'all' || github.event.inputs.variant == matrix.variant
uses: actions/download-artifact@v8
@@ -223,7 +218,7 @@ jobs:
sudo docker system prune -af --volumes
[ -d ~/.cache/go-build ] && rm -rf ~/.cache/go-build || true
[ -d /go/pkg ] && rm -rf /go/pkg || true
timeout 15 df -h || true
df -h
- name: Docker meta
if: github.event_name != 'workflow_dispatch' || github.event.inputs.variant == 'all' || github.event.inputs.variant == matrix.variant
@@ -275,11 +270,8 @@ jobs:
file: ${{ matrix.dockerfile }}
platforms: ${{ matrix.platform }}
labels: ${{ steps.docker_meta.outputs.labels }}
# mode=max keeps resolvedDependencies complete and records the
# builder identity; attestations ride inside each pushed index and
# survive the imagetools merge into the multi-arch manifest.
provenance: mode=max
sbom: true
# Flat single-platform manifest so imagetools create assembles cleanly.
provenance: false
outputs: type=image,name=${{ env.IMAGE }},push-by-digest=true,name-canonical=true,push=true
cache-from: type=gha,scope=${{ matrix.variant }}-${{ matrix.arch }}
# max only for rocksdb: its RocksDB compile is sha-independent and worth
@@ -289,7 +281,7 @@ jobs:
${{ matrix.build_args }}
BUILDKIT_INLINE_CACHE=1
BRANCH=${{ github.sha }}
${{ matrix.variant == 'rocksdb' && format('ROCKSDB_VERSION={0}', github.event.inputs.rocksdb_version || 'v11.1.2') || '' }}
${{ matrix.variant == 'rocksdb' && format('ROCKSDB_VERSION={0}', github.event.inputs.rocksdb_version || 'v10.10.1') || '' }}
- name: Export digest
if: github.event_name != 'workflow_dispatch' || github.event.inputs.variant == 'all' || github.event.inputs.variant == matrix.variant
@@ -315,8 +307,6 @@ jobs:
permissions:
contents: read
id-token: write
attestations: write
packages: write
strategy:
fail-fast: false
matrix:
@@ -424,45 +414,6 @@ jobs:
${{ env.IMAGE }}@${{ steps.manifest.outputs.digest }}
chrislusf/seaweedfs@${{ steps.manifest.outputs.digest }}
# Attest the merged multi-arch digest itself, so digest-pinning consumers
# can verify provenance against the exact index they pull. Both steps
# reuse the GHCR login above and push as OCI referrers.
- name: Attest build provenance for ${{ matrix.variant }}
if: github.event_name != 'workflow_dispatch' || github.event.inputs.variant == 'all' || github.event.inputs.variant == matrix.variant
uses: actions/attest-build-provenance@v4
with:
subject-name: ${{ env.IMAGE }}
subject-digest: ${{ steps.manifest.outputs.digest }}
push-to-registry: true
# SBOMs are generated and attested per platform manifest: an index-wide
# scan only describes syft's default platform, while the manifests it
# merges differ (some ship the real binary, others placeholders), so the
# SBOM has to be bound to the platform digest a consumer actually pulls.
- name: Install syft
if: github.event_name != 'workflow_dispatch' || github.event.inputs.variant == 'all' || github.event.inputs.variant == matrix.variant
uses: anchore/sbom-action/download-syft@v0
- name: Attest per-platform SBOMs for ${{ matrix.variant }}
if: github.event_name != 'workflow_dispatch' || github.event.inputs.variant == 'all' || github.event.inputs.variant == matrix.variant
working-directory: /tmp/digests
run: |
set -euo pipefail
for f in *; do
# Each saved digest is an OCI index (image + attestation manifests);
# scan and attest the image manifest a same-platform pull resolves.
ref="${{ env.IMAGE }}@sha256:${f}"
image=$(crane manifest "$ref" | jq -r '
[.manifests[]? | select((.annotations["vnd.docker.reference.type"] // "") != "attestation-manifest"
and (.platform.os // "unknown") != "unknown") | .digest][0] // empty')
ref="${{ env.IMAGE }}@${image:-sha256:${f}}"
syft "registry:${ref}" -o "spdx-json=/tmp/sbom-${f}.spdx.json"
cosign attest --yes \
--predicate "/tmp/sbom-${f}.spdx.json" \
--type spdxjson \
"$ref"
done
# Report-only trivy scan: uploads fixable HIGH/CRITICAL findings to GitHub
# Security for visibility, but never blocks the release. Releases (including
# `latest`) ship regardless — vulnerabilities are tracked, not gated, since
@@ -501,9 +452,7 @@ jobs:
scanners: vuln
vuln-type: os,library
severity: HIGH,CRITICAL
# Keep unfixed findings in the report so digest-pinning consumers see
# the complete picture, not just what a patch could resolve.
ignore-unfixed: false
ignore-unfixed: true
limit-severities-for-sarif: true
format: sarif
output: trivy-results.sarif
@@ -4,9 +4,9 @@ on:
workflow_dispatch:
inputs:
rocksdb_version:
description: 'RocksDB git tag or branch to build (e.g. v11.1.2)'
description: 'RocksDB git tag or branch to build (e.g. v10.10.1)'
required: true
default: 'v11.1.2'
default: 'v10.10.1'
seaweedfs_ref:
description: 'SeaweedFS git tag, branch, or commit to build'
required: true
@@ -16,11 +16,6 @@ on:
required: false
default: ''
concurrency:
group: ${{ github.workflow }}-${{ github.event.inputs.image_tag || format('rocksdb_{0}_seaweedfs_{1}', github.event.inputs.rocksdb_version, github.event.inputs.seaweedfs_ref) }}
cancel-in-progress: false
queue: max
permissions:
contents: read
-1
View File
@@ -27,7 +27,6 @@ jobs:
uses: actions/setup-go@v7
with:
go-version-file: 'telemetry/server/go.mod'
cache: ${{ runner.environment == 'github-hosted' }}
- name: Build Telemetry Server
if: github.event_name == 'workflow_dispatch' && inputs.deploy
-4
View File
@@ -1,10 +1,6 @@
name: 'Dependency Review'
on: [pull_request]
concurrency:
group: ${{ github.workflow }}-${{ github.event.pull_request.number || (github.event_name == 'push' && github.ref) || github.run_id }}
cancel-in-progress: ${{ github.event_name == 'pull_request' || (github.event_name == 'push' && !startsWith(github.ref, 'refs/tags/')) }}
permissions:
contents: read
+2 -4
View File
@@ -19,8 +19,8 @@ on:
- '.github/workflows/e2e.yml'
concurrency:
group: ${{ github.workflow }}-${{ github.event.pull_request.number || (github.event_name == 'push' && github.ref) || github.run_id }}
cancel-in-progress: ${{ github.event_name == 'pull_request' || (github.event_name == 'push' && !startsWith(github.ref, 'refs/tags/')) }}
group: ${{ github.head_ref }}/e2e
cancel-in-progress: true
permissions:
contents: read
@@ -42,7 +42,6 @@ jobs:
uses: actions/setup-go@v7
with:
go-version-file: 'go.mod'
cache: ${{ runner.environment == 'github-hosted' }}
- name: Configure Docker Hub mirror
run: |
@@ -53,7 +52,6 @@ jobs:
uses: docker/setup-buildx-action@v4
- name: Cache Docker layers
if: runner.environment == 'github-hosted'
uses: actions/cache@v6
with:
path: /tmp/.buildx-cache
@@ -18,10 +18,6 @@ on:
- 'go.sum'
- '.github/workflows/ec-integration-tests.yml'
concurrency:
group: ${{ github.workflow }}-${{ github.event.pull_request.number || (github.event_name == 'push' && github.ref) || github.run_id }}
cancel-in-progress: ${{ github.event_name == 'pull_request' || (github.event_name == 'push' && !startsWith(github.ref, 'refs/tags/')) }}
permissions:
contents: read
@@ -34,7 +30,6 @@ jobs:
- name: Set up Go 1.x
uses: actions/setup-go@v7
with:
cache: ${{ runner.environment == 'github-hosted' }}
go-version: ^1.26
id: go
+2 -3
View File
@@ -19,8 +19,8 @@ on:
- '.github/actions/fix-fusermount-setuid/**'
concurrency:
group: ${{ github.workflow }}-${{ github.event.pull_request.number || (github.event_name == 'push' && github.ref) || github.run_id }}
cancel-in-progress: ${{ github.event_name == 'pull_request' || (github.event_name == 'push' && !startsWith(github.ref, 'refs/tags/')) }}
group: ${{ github.head_ref || github.ref }}/fuse-dlm-integration
cancel-in-progress: true
permissions:
contents: read
@@ -39,7 +39,6 @@ jobs:
uses: actions/setup-go@v7
with:
go-version-file: 'go.mod'
cache: ${{ runner.environment == 'github-hosted' }}
- name: Configure FUSE
run: |
+2 -6
View File
@@ -24,8 +24,8 @@ on:
- '.github/actions/fix-fusermount-setuid/**'
concurrency:
group: ${{ github.workflow }}-${{ github.event.pull_request.number || (github.event_name == 'push' && github.ref) || github.run_id }}
cancel-in-progress: ${{ github.event_name == 'pull_request' || (github.event_name == 'push' && !startsWith(github.ref, 'refs/tags/')) }}
group: ${{ github.head_ref || github.ref }}/fuse-failover
cancel-in-progress: true
permissions:
contents: read
@@ -42,14 +42,10 @@ jobs:
with:
persist-credentials: false
- name: Clean up stale FUSE mounts
uses: ./.github/actions/cleanup-stale-fuse-mounts
- name: Set up Go
uses: actions/setup-go@v7
with:
go-version-file: 'go.mod'
cache: ${{ runner.environment == 'github-hosted' }}
- name: Configure FUSE
run: |
+2 -3
View File
@@ -17,8 +17,8 @@ on:
- '.github/actions/fix-fusermount-setuid/**'
concurrency:
group: ${{ github.workflow }}-${{ github.event.pull_request.number || (github.event_name == 'push' && github.ref) || github.run_id }}
cancel-in-progress: ${{ github.event_name == 'pull_request' || (github.event_name == 'push' && !startsWith(github.ref, 'refs/tags/')) }}
group: ${{ github.head_ref }}/fuse-integration
cancel-in-progress: true
permissions:
contents: read
@@ -37,7 +37,6 @@ jobs:
uses: actions/setup-go@v7
with:
go-version-file: 'go.mod'
cache: ${{ runner.environment == 'github-hosted' }}
- name: Configure FUSE
run: |
+2 -6
View File
@@ -25,8 +25,8 @@ on:
- '.github/actions/fix-fusermount-setuid/**'
concurrency:
group: ${{ github.workflow }}-${{ github.event.pull_request.number || (github.event_name == 'push' && github.ref) || github.run_id }}
cancel-in-progress: ${{ github.event_name == 'pull_request' || (github.event_name == 'push' && !startsWith(github.ref, 'refs/tags/')) }}
group: ${{ github.head_ref || github.ref }}/fuse-p2p-integration
cancel-in-progress: true
permissions:
contents: read
@@ -41,14 +41,10 @@ jobs:
- name: Check out code
uses: actions/checkout@v7
- name: Clean up stale FUSE mounts
uses: ./.github/actions/cleanup-stale-fuse-mounts
- name: Set up Go
uses: actions/setup-go@v7
with:
go-version-file: 'go.mod'
cache: ${{ runner.environment == 'github-hosted' }}
- name: Configure FUSE
run: |
+2 -9
View File
@@ -17,8 +17,8 @@ on:
- '.github/workflows/go.yml'
concurrency:
group: ${{ github.workflow }}-${{ github.event.pull_request.number || (github.event_name == 'push' && github.ref) || github.run_id }}
cancel-in-progress: ${{ github.event_name == 'pull_request' || (github.event_name == 'push' && !startsWith(github.ref, 'refs/tags/')) }}
group: ${{ github.head_ref }}/go
cancel-in-progress: true
permissions:
contents: read
@@ -35,7 +35,6 @@ jobs:
uses: actions/setup-go@v7
with:
go-version-file: 'go.mod'
cache: ${{ runner.environment == 'github-hosted' }}
- name: Get dependencies
run: |
cd weed; go get -v -t -d ./...
@@ -58,7 +57,6 @@ jobs:
uses: actions/setup-go@v7
with:
go-version-file: 'go.mod'
cache: ${{ runner.environment == 'github-hosted' }}
- name: Go Vet linux/386 (type-checks code and tests for 32-bit int overflows)
run: |
GOOS=linux GOARCH=386 go vet ./... 2>&1 | grep -v "MessageState contains sync.Mutex" | grep -v "IdentityAccessManagement contains sync.RWMutex" | tee vet-32bit-output.txt
@@ -74,7 +72,6 @@ jobs:
uses: actions/setup-go@v7
with:
go-version-file: 'go.mod'
cache: ${{ runner.environment == 'github-hosted' }}
- name: Build
run: cd weed; go build -tags "elastic gocdk sqlite ydb tarantool tikv rclone" -v .
@@ -97,7 +94,6 @@ jobs:
uses: actions/setup-go@v7
with:
go-version-file: 'go.mod'
cache: ${{ runner.environment == 'github-hosted' }}
# Nothing else on a PR compiles these, so per-OS syscall constants creep
# back into shared files unnoticed and only a release build catches it.
- name: Cross-compile
@@ -132,7 +128,6 @@ jobs:
uses: actions/setup-go@v7
with:
go-version-file: 'go.mod'
cache: ${{ runner.environment == 'github-hosted' }}
- name: Test
env:
RUN_REDIS_TESTS: "1"
@@ -148,7 +143,6 @@ jobs:
uses: actions/setup-go@v7
with:
go-version-file: 'go.mod'
cache: ${{ runner.environment == 'github-hosted' }}
# 386 test binaries run natively on the amd64 runner. This catches what vet
# can't: unaligned 64-bit atomics and arithmetic that wraps at runtime.
# -short skips the e2e suites already covered on amd64.
@@ -165,7 +159,6 @@ jobs:
uses: actions/setup-go@v7
with:
go-version-file: 'go.mod'
cache: ${{ runner.environment == 'github-hosted' }}
# weed/kms/azure is excluded from every default build, so without this
# nothing compiles it and tag-gated code silently rots.
- name: Build and test with azurekms
-4
View File
@@ -8,10 +8,6 @@ on:
branches: [ master ]
paths: ['k8s/**', '.github/workflows/helm_ci.yml', '.github/scripts/helm_jwt_keys.py']
concurrency:
group: ${{ github.workflow }}-${{ github.event.pull_request.number || (github.event_name == 'push' && github.ref) || github.run_id }}
cancel-in-progress: ${{ github.event_name == 'pull_request' || (github.event_name == 'push' && !startsWith(github.ref, 'refs/tags/')) }}
permissions:
contents: read
+1 -7
View File
@@ -14,10 +14,6 @@ on:
- 'weed/**'
- '.github/workflows/java_integration_tests.yml'
concurrency:
group: ${{ github.workflow }}-${{ github.event.pull_request.number || (github.event_name == 'push' && github.ref) || github.run_id }}
cancel-in-progress: ${{ github.event_name == 'pull_request' || (github.event_name == 'push' && !startsWith(github.ref, 'refs/tags/')) }}
jobs:
test:
name: Java Integration Tests
@@ -35,7 +31,6 @@ jobs:
uses: actions/setup-go@v7
with:
go-version-file: 'go.mod'
cache: ${{ runner.environment == 'github-hosted' }}
id: go
- name: Set up Java
@@ -43,7 +38,7 @@ jobs:
with:
java-version: ${{ matrix.java }}
distribution: 'temurin'
cache: ${{ runner.environment == 'github-hosted' && 'maven' || '' }}
cache: 'maven'
- name: Build SeaweedFS
run: |
@@ -60,7 +55,6 @@ jobs:
# Start SeaweedFS with optimized settings for CI
# Include S3 API for s3copier integration tests
weed server -dir="$WEED_DATA_DIR" \
-ip=127.0.0.1 -ip.bind=127.0.0.1 \
-master.raftHashicorp \
-master.electionTimeout=1s \
-master.volumeSizeLimitMB=100 \
+1 -1
View File
@@ -77,7 +77,7 @@ jobs:
with:
java-version: '17'
distribution: 'temurin'
cache: ${{ runner.environment == 'github-hosted' && 'maven' || '' }}
cache: 'maven'
server-id: central
server-username: MAVEN_CENTRAL_USERNAME
server-password: MAVEN_CENTRAL_PASSWORD
+1 -5
View File
@@ -12,10 +12,6 @@ on:
- 'other/java/**'
- '.github/workflows/java_unit_tests.yml'
concurrency:
group: ${{ github.workflow }}-${{ github.event.pull_request.number || (github.event_name == 'push' && github.ref) || github.run_id }}
cancel-in-progress: ${{ github.event_name == 'pull_request' || (github.event_name == 'push' && !startsWith(github.ref, 'refs/tags/')) }}
jobs:
test:
name: Java Unit Tests
@@ -34,7 +30,7 @@ jobs:
with:
java-version: ${{ matrix.java }}
distribution: 'temurin'
cache: ${{ runner.environment == 'github-hosted' && 'maven' || '' }}
cache: 'maven'
- name: Build and Install SeaweedFS Client
working-directory: other/java/client
+3 -3
View File
@@ -24,8 +24,8 @@ on:
workflow_dispatch: # Allow manual trigger
concurrency:
group: ${{ github.workflow }}-${{ github.event.pull_request.number || (github.event_name == 'push' && github.ref) || github.run_id }}
cancel-in-progress: ${{ github.event_name == 'pull_request' || (github.event_name == 'push' && !startsWith(github.ref, 'refs/tags/')) }}
group: ${{ github.head_ref }}/kafka-quicktest
cancel-in-progress: true
permissions:
contents: read
@@ -43,7 +43,7 @@ jobs:
uses: actions/setup-go@v7
with:
go-version: ^1.26
cache: ${{ runner.environment == 'github-hosted' }}
cache: true
cache-dependency-path: |
**/go.sum
id: go
+6 -9
View File
@@ -23,8 +23,8 @@ on:
- '.github/workflows/kafka-tests.yml'
concurrency:
group: ${{ github.workflow }}-${{ github.event.pull_request.number || (github.event_name == 'push' && github.ref) || github.run_id }}
cancel-in-progress: ${{ github.event_name == 'pull_request' || (github.event_name == 'push' && !startsWith(github.ref, 'refs/tags/')) }}
group: ${{ github.head_ref }}/kafka-tests
cancel-in-progress: true
# Force different runners for better isolation
env:
@@ -53,7 +53,6 @@ jobs:
- name: Set up Go 1.x
uses: actions/setup-go@v7
with:
cache: ${{ runner.environment == 'github-hosted' }}
go-version: ^1.26
id: go
@@ -99,7 +98,6 @@ jobs:
- name: Set up Go 1.x
uses: actions/setup-go@v7
with:
cache: ${{ runner.environment == 'github-hosted' }}
go-version: ^1.26
id: go
@@ -151,7 +149,7 @@ jobs:
uses: actions/setup-go@v7
with:
go-version: ^1.26
cache: ${{ runner.environment == 'github-hosted' }}
cache: true
cache-dependency-path: |
**/go.sum
id: go
@@ -330,7 +328,7 @@ jobs:
uses: actions/setup-go@v7
with:
go-version: ^1.26
cache: ${{ runner.environment == 'github-hosted' }}
cache: true
cache-dependency-path: |
**/go.sum
id: go
@@ -492,7 +490,7 @@ jobs:
uses: actions/setup-go@v7
with:
go-version: ^1.26
cache: ${{ runner.environment == 'github-hosted' }}
cache: true
cache-dependency-path: |
**/go.sum
id: go
@@ -650,7 +648,7 @@ jobs:
uses: actions/setup-go@v7
with:
go-version: ^1.26
cache: ${{ runner.environment == 'github-hosted' }}
cache: true
cache-dependency-path: |
**/go.sum
id: go
@@ -807,7 +805,6 @@ jobs:
- name: Set up Go 1.x
uses: actions/setup-go@v7
with:
cache: ${{ runner.environment == 'github-hosted' }}
go-version: ^1.26
id: go
+2 -4
View File
@@ -19,8 +19,8 @@ on:
- 'test/kms/**'
concurrency:
group: ${{ github.workflow }}-${{ github.event.pull_request.number || (github.event_name == 'push' && github.ref) || github.run_id }}
cancel-in-progress: ${{ github.event_name == 'pull_request' || (github.event_name == 'push' && !startsWith(github.ref, 'refs/tags/')) }}
group: ${{ github.head_ref || github.ref }}-kms-tests
cancel-in-progress: true
permissions:
contents: read
@@ -43,7 +43,6 @@ jobs:
uses: actions/setup-go@v7
with:
go-version-file: 'go.mod'
cache: ${{ runner.environment == 'github-hosted' }}
id: go
- name: Install SeaweedFS
@@ -89,7 +88,6 @@ jobs:
uses: actions/setup-go@v7
with:
go-version-file: 'go.mod'
cache: ${{ runner.environment == 'github-hosted' }}
id: go
- name: Install SeaweedFS
@@ -21,8 +21,8 @@ on:
- '.github/workflows/master-cold-start-tests.yml'
concurrency:
group: ${{ github.workflow }}-${{ github.event.pull_request.number || (github.event_name == 'push' && github.ref) || github.run_id }}
cancel-in-progress: ${{ github.event_name == 'pull_request' || (github.event_name == 'push' && !startsWith(github.ref, 'refs/tags/')) }}
group: ${{ github.head_ref || github.ref }}/master-cold-start-tests
cancel-in-progress: true
permissions:
contents: read
@@ -42,7 +42,6 @@ jobs:
uses: actions/setup-go@v7
with:
go-version-file: 'go.mod'
cache: ${{ runner.environment == 'github-hosted' }}
- name: Build weed binary
run: |
@@ -25,8 +25,8 @@ on:
- '.github/workflows/metadata-subscribe-tests.yml'
concurrency:
group: ${{ github.workflow }}-${{ github.event.pull_request.number || (github.event_name == 'push' && github.ref) || github.run_id }}
cancel-in-progress: ${{ github.event_name == 'pull_request' || (github.event_name == 'push' && !startsWith(github.ref, 'refs/tags/')) }}
group: ${{ github.head_ref }}/metadata-subscribe-tests
cancel-in-progress: true
permissions:
contents: read
@@ -48,7 +48,6 @@ jobs:
uses: actions/setup-go@v7
with:
go-version-file: 'go.mod'
cache: ${{ runner.environment == 'github-hosted' }}
- name: Build SeaweedFS
run: |
+2 -4
View File
@@ -10,8 +10,8 @@ on:
branches: [ 'winfsp-bench**' ]
concurrency:
group: ${{ github.workflow }}-${{ github.event.pull_request.number || (github.event_name == 'push' && github.ref) || github.run_id }}
cancel-in-progress: ${{ github.event_name == 'pull_request' || (github.event_name == 'push' && !startsWith(github.ref, 'refs/tags/')) }}
group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }}
cancel-in-progress: true
permissions:
contents: read
@@ -32,7 +32,6 @@ jobs:
- uses: actions/setup-go@v7
with:
go-version-file: 'go.mod'
cache: ${{ runner.environment == 'github-hosted' }}
- name: Install WinFsp and rclone
run: choco install winfsp rclone -y --no-progress
@@ -151,7 +150,6 @@ jobs:
- uses: actions/setup-go@v7
with:
go-version-file: 'go.mod'
cache: ${{ runner.environment == 'github-hosted' }}
- name: Repair the fusermount3 setuid bit
uses: ./.github/actions/fix-fusermount-setuid
@@ -24,8 +24,8 @@ on:
- '.github/workflows/mount-windows-conformance.yml'
concurrency:
group: ${{ github.workflow }}-${{ github.event.pull_request.number || (github.event_name == 'push' && github.ref) || github.run_id }}
cancel-in-progress: ${{ github.event_name == 'pull_request' || (github.event_name == 'push' && !startsWith(github.ref, 'refs/tags/')) }}
group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }}
cancel-in-progress: true
permissions:
contents: read
@@ -49,7 +49,6 @@ jobs:
- uses: actions/setup-go@v7
with:
go-version-file: 'go.mod'
cache: ${{ runner.environment == 'github-hosted' }}
# cgofuse loads winfsp-x64.dll at run time, so WinFsp is needed here but
# not to build.
+2 -3
View File
@@ -24,8 +24,8 @@ on:
- '.github/workflows/mount-windows.yml'
concurrency:
group: ${{ github.workflow }}-${{ github.event.pull_request.number || (github.event_name == 'push' && github.ref) || github.run_id }}
cancel-in-progress: ${{ github.event_name == 'pull_request' || (github.event_name == 'push' && !startsWith(github.ref, 'refs/tags/')) }}
group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }}
cancel-in-progress: true
permissions:
contents: read
@@ -49,7 +49,6 @@ jobs:
- uses: actions/setup-go@v7
with:
go-version-file: 'go.mod'
cache: ${{ runner.environment == 'github-hosted' }}
# cgofuse loads winfsp-x64.dll at run time, so WinFsp is needed here but
# not to build.
+2 -3
View File
@@ -21,8 +21,8 @@ on:
- '.github/workflows/multi-master-tests.yml'
concurrency:
group: ${{ github.workflow }}-${{ github.event.pull_request.number || (github.event_name == 'push' && github.ref) || github.run_id }}
cancel-in-progress: ${{ github.event_name == 'pull_request' || (github.event_name == 'push' && !startsWith(github.ref, 'refs/tags/')) }}
group: ${{ github.head_ref || github.ref }}/multi-master-tests
cancel-in-progress: true
permissions:
contents: read
@@ -40,7 +40,6 @@ jobs:
uses: actions/setup-go@v7
with:
go-version-file: 'go.mod'
cache: ${{ runner.environment == 'github-hosted' }}
- name: Install SeaweedFS
run: |
+4 -7
View File
@@ -45,8 +45,8 @@ on:
type: string
concurrency:
group: ${{ github.workflow }}-${{ github.event.pull_request.number || (github.event_name == 'push' && github.ref) || github.run_id }}
cancel-in-progress: ${{ github.event_name == 'pull_request' || (github.event_name == 'push' && !startsWith(github.ref, 'refs/tags/')) }}
group: ${{ github.head_ref || github.ref }}/performance
cancel-in-progress: true
permissions:
contents: read
@@ -67,7 +67,6 @@ jobs:
uses: actions/setup-go@v7
with:
go-version-file: 'go.mod'
cache: ${{ runner.environment == 'github-hosted' }}
- name: Build weed
run: go build -o weed_bin ./weed
@@ -176,14 +175,13 @@ jobs:
uses: actions/setup-go@v7
with:
go-version-file: 'go.mod'
cache: ${{ runner.environment == 'github-hosted' }}
- name: Install Rust toolchain
if: matrix.impl == 'rust'
uses: dtolnay/rust-toolchain@stable
- name: Cache cargo registry and target
if: matrix.impl == 'rust' && runner.environment == 'github-hosted'
if: matrix.impl == 'rust'
uses: actions/cache@v6
with:
path: |
@@ -329,14 +327,13 @@ jobs:
uses: actions/setup-go@v7
with:
go-version-file: 'go.mod'
cache: ${{ runner.environment == 'github-hosted' }}
- name: Install Rust toolchain
if: matrix.impl == 'rust'
uses: dtolnay/rust-toolchain@stable
- name: Cache cargo registry and target
if: matrix.impl == 'rust' && runner.environment == 'github-hosted'
if: matrix.impl == 'rust'
uses: actions/cache@v6
with:
path: |
+1 -9
View File
@@ -38,23 +38,15 @@ jobs:
uses: actions/setup-go@v7
with:
go-version-file: 'go.mod'
cache: ${{ runner.environment == 'github-hosted' }}
- name: Start local Docker registry
run: |
if ! docker start registry 2>/dev/null; then
docker rm -f registry >/dev/null 2>&1 || true
docker run -d --restart=always -p 5000:5000 --name registry registry:2
fi
run: docker run -d --restart=always -p 5000:5000 --name registry registry:2
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v4
with:
driver-opts: network=host
- name: Set up Docker Compose
uses: docker/setup-compose-action@v2
- name: Build weed race binary
run: |
cd docker
-5
View File
@@ -18,10 +18,6 @@ on:
- 'go.sum'
- '.github/workflows/plugin-workers.yml'
concurrency:
group: ${{ github.workflow }}-${{ github.event.pull_request.number || (github.event_name == 'push' && github.ref) || github.run_id }}
cancel-in-progress: ${{ github.event_name == 'pull_request' || (github.event_name == 'push' && !startsWith(github.ref, 'refs/tags/')) }}
permissions:
contents: read
@@ -45,7 +41,6 @@ jobs:
- name: Set up Go 1.x
uses: actions/setup-go@v7
with:
cache: ${{ runner.environment == 'github-hosted' }}
go-version: ^1.26
id: go
-791
View File
@@ -1,791 +0,0 @@
name: POSIX Compliance Tests
on:
push:
branches: [ main, master, develop ]
paths:
- 'weed/mount/**'
- 'weed/command/mount*'
- 'weed/command/fuse*'
- 'test/fuse_integration/**'
pull_request:
branches: [ main, master, develop ]
paths:
- 'weed/mount/**'
- 'weed/command/mount*'
- 'weed/command/fuse*'
- 'test/fuse_integration/**'
workflow_dispatch:
inputs:
test_type:
description: 'Type of POSIX tests to run (default: full for comprehensive testing)'
required: true
default: 'full'
type: choice
options:
- critical
- basic
- extended
- full
enable_external_tests:
description: 'Run external test suites (slower)'
required: false
default: false
type: boolean
permissions:
contents: read
pull-requests: write
issues: write
env:
GO_VERSION: '1.26'
TIMEOUT: '45m'
jobs:
posix-compliance-ubuntu:
runs-on: ubuntu-latest
timeout-minutes: 60
strategy:
matrix:
fuse-version: ['2.9', '3.0']
fail-fast: false
steps:
- name: Checkout code
uses: actions/checkout@v7
with:
fetch-depth: 1
- name: Clean up stale FUSE mounts
uses: ./.github/actions/cleanup-stale-fuse-mounts
- name: Set up Go ${{ env.GO_VERSION }}
uses: actions/setup-go@v7
with:
go-version: ${{ env.GO_VERSION }}
cache: ${{ runner.environment == 'github-hosted' }}
cache-dependency-path: |
go.sum
test/fuse_integration/go.sum
- name: Install system dependencies
run: |
sudo apt-get update
sudo apt-get install -y \
attr \
acl \
build-essential \
git \
python3-pip
# Install FUSE version specific packages
if [ "${{ matrix.fuse-version }}" = "2.9" ]; then
echo "Installing FUSE 2.9..."
sudo apt-get install -y fuse libfuse-dev
elif [ "${{ matrix.fuse-version }}" = "3.0" ]; then
echo "Installing FUSE 3.0..."
sudo apt-get install -y fuse3 libfuse3-dev
# Also install fuse2 for compatibility if needed
sudo apt-get install -y fuse libfuse-dev
fi
- name: Set up user permissions for FUSE
run: |
# Create fuse group if it doesn't exist
sudo groupadd -f fuse
# Add user to fuse group
sudo usermod -a -G fuse $USER
# Set permissions on FUSE device
sudo chmod 666 /dev/fuse
# Allow non-root mounts to pass -o allow_other (weed -allowOthers)
echo 'user_allow_other' | sudo tee -a /etc/fuse.conf
sudo chmod 644 /etc/fuse.conf
# Ensure fuse module is loaded
sudo modprobe fuse || true
# Verify setup
echo "FUSE setup verification:"
ls -la /dev/fuse
groups $USER | grep fuse || echo "User not in fuse group yet (will be effective after login)"
lsmod | grep fuse || echo "FUSE module not loaded"
- name: Install external test tools
if: ${{ github.event.inputs.enable_external_tests == 'true' }}
run: |
# Install nfstest for POSIX API verification
pip3 install --user nfstest
# Install FIO for performance testing
sudo apt-get install -y fio
# Install additional test utilities
sudo apt-get install -y stress-ng
- name: Build SeaweedFS
run: |
make
# Verify binary exists and is executable (it's installed to GOPATH/bin)
weed version
# Make weed binary available in PATH (it's already there from go install)
sudo cp $(which weed) /usr/local/bin/weed
which weed
weed version
- name: Set up SeaweedFS cluster
run: |
start_and_wait() {
local name=$1
local url=$2
local pidfile=$3
shift 3
local cmd="$@"
echo "Starting $name..."
$cmd > "/tmp/seaweedfs/$name.log" 2>&1 &
local pid=$!
echo $pid > "$pidfile"
echo "Waiting for $name to start..."
for i in {1..30}; do
if curl -sf "$url" > /dev/null 2>&1; then
echo "$name is ready"
return 0
fi
if [ $i -eq 30 ]; then
echo "$name failed to start"
cat "/tmp/seaweedfs/$name.log"
exit 1
fi
sleep 2
done
}
mkdir -p /tmp/seaweedfs/{master,volume,filer,mount}
start_and_wait "master" "http://127.0.0.1:9333/cluster/status" "/tmp/seaweedfs/master.pid" \
"weed master -ip=127.0.0.1 -port=9333 -mdir=/tmp/seaweedfs/master -raftBootstrap=true"
start_and_wait "volume" "http://127.0.0.1:8080/status" "/tmp/seaweedfs/volume.pid" \
"weed volume -mserver=127.0.0.1:9333 -ip=127.0.0.1 -port=8080 -dir=/tmp/seaweedfs/volume -max=100"
start_and_wait "filer" "http://127.0.0.1:8888/" "/tmp/seaweedfs/filer.pid" \
"weed filer -master=127.0.0.1:9333 -ip=127.0.0.1 -port=8888"
echo "SeaweedFS cluster status:"
curl -s http://127.0.0.1:9333/cluster/status || true
- name: Set up FUSE mount
run: |
# Create mount point
MOUNT_POINT="/tmp/seaweedfs/mount"
mkdir -p $MOUNT_POINT
echo "Mounting SeaweedFS FUSE filesystem..."
# Mount SeaweedFS FUSE filesystem in background
weed mount \
-filer=127.0.0.1:8888 \
-dir=$MOUNT_POINT \
-filer.path=/ \
-dirAutoCreate=true \
-allowOthers=true \
-nonempty=true \
> /tmp/seaweedfs/mount.log 2>&1 &
MOUNT_PID=$!
echo $MOUNT_PID > /tmp/seaweedfs/mount.pid
# Wait for mount to be ready. The mount point directory already
# exists, so only a real mount counts — a directory listing would
# pass on the local disk even if weed mount died.
echo "Waiting for FUSE mount to be ready..."
for i in {1..30}; do
if mountpoint -q $MOUNT_POINT 2>/dev/null; then
echo "FUSE mount is ready"
break
fi
if ! kill -0 $MOUNT_PID 2>/dev/null; then
echo "FUSE mount process exited"
cat /tmp/seaweedfs/mount.log
exit 1
fi
if [ $i -eq 30 ]; then
echo "FUSE mount failed to be ready"
cat /tmp/seaweedfs/mount.log
echo "Mount status:"
mount | grep fuse || echo "No FUSE mounts found"
ps aux | grep weed || echo "No weed processes found"
exit 1
fi
sleep 2
done
# Verify mount is working
echo "Testing FUSE mount functionality..."
echo "test" > $MOUNT_POINT/test_file.txt
if [ "$(cat $MOUNT_POINT/test_file.txt)" = "test" ]; then
echo "FUSE mount is working correctly"
rm $MOUNT_POINT/test_file.txt
else
echo "FUSE mount is not working properly"
exit 1
fi
# Export mount point for tests
echo "SEAWEEDFS_MOUNT_POINT=$MOUNT_POINT" >> $GITHUB_ENV
- name: Set up test environment
run: |
cd test/fuse_integration
# Initialize Go module for tests
if [ ! -f go.mod ]; then
go mod init seaweedfs-posix-tests
go mod tidy
fi
# Create reports directory
mkdir -p reports
# Set up external tools if requested
# Note: External tools setup is now optional and won't fail the build
if [ "${{ github.event.inputs.enable_external_tests }}" = "true" ]; then
echo "Setting up external POSIX testing tools..."
make -f posix_Makefile setup-external-tools || true
# setup-external-tools exits 0 even when a sub-target failed, so
# require the tools only when they are actually installed
if [ -d external_tools/pjdfstest ] && command -v nfstest_posix >/dev/null 2>&1; then
export POSIX_REQUIRE_EXTERNAL_TOOLS="true"
echo "POSIX_REQUIRE_EXTERNAL_TOOLS=true" >> $GITHUB_ENV
echo "✅ External tools setup completed successfully"
else
echo "⚠️ External tools setup incomplete - continuing with basic tests only"
echo "External tools will be skipped but tests will continue"
echo "This is expected behavior when build tools are not available"
fi
fi
# Verify SeaweedFS cluster is accessible
echo "Verifying SeaweedFS cluster accessibility..."
curl -sf http://127.0.0.1:9333/cluster/status
curl -sf http://127.0.0.1:8080/status
curl -sf http://127.0.0.1:8888/
# Verify mount point
echo "Mount point: $SEAWEEDFS_MOUNT_POINT"
ls -la $SEAWEEDFS_MOUNT_POINT
# Verify mount is actually working by testing file operations
echo "Verifying SeaweedFS mount functionality..."
TEST_FILE="$SEAWEEDFS_MOUNT_POINT/.mount_verification_test"
echo "SeaweedFS mount test" > "$TEST_FILE"
if [ "$(cat "$TEST_FILE")" = "SeaweedFS mount test" ]; then
echo "✅ Mount verification successful"
rm -f "$TEST_FILE"
else
echo "❌ Mount verification failed - mount may not be working properly"
exit 1
fi
# Check if this is actually a FUSE mount — the plain directory created
# above passes the file test, so without this the suite can run
# against the local disk and report a pass it never earned
if mount | grep -q "$SEAWEEDFS_MOUNT_POINT"; then
echo "✅ FUSE mount detected:"
mount | grep "$SEAWEEDFS_MOUNT_POINT"
else
echo "❌ Mount point not found in mount table - refusing to test a local directory"
exit 1
fi
- name: Run POSIX compliance tests
id: posix-tests
env:
SEAWEEDFS_MOUNT_POINT: ${{ env.SEAWEEDFS_MOUNT_POINT }}
POSIX_REQUIRE_EXTERNAL_TOOLS: ${{ env.POSIX_REQUIRE_EXTERNAL_TOOLS }}
run: |
cd test/fuse_integration
# Determine which tests to run
TEST_TYPE="${{ github.event.inputs.test_type }}"
if [ -z "$TEST_TYPE" ]; then
TEST_TYPE="full"
fi
echo "Running POSIX tests: $TEST_TYPE"
if [ "$TEST_TYPE" = "full" ]; then
echo "🚀 Running COMPREHENSIVE POSIX test suite (including external test suites)"
else
echo "Using mount point: $SEAWEEDFS_MOUNT_POINT"
fi
# Set test configuration to use our mounted filesystem
export TEST_MOUNT_POINT="$SEAWEEDFS_MOUNT_POINT"
export TEST_SKIP_CLUSTER_SETUP="true"
# Debug: Show environment variables
echo "🔍 Test Environment:"
echo " TEST_MOUNT_POINT=$TEST_MOUNT_POINT"
echo " TEST_SKIP_CLUSTER_SETUP=$TEST_SKIP_CLUSTER_SETUP"
echo " SEAWEEDFS_MOUNT_POINT=$SEAWEEDFS_MOUNT_POINT"
case "$TEST_TYPE" in
"critical")
echo "Running critical POSIX tests (basic compliance only)..."
make -f posix_Makefile test-posix-critical
;;
"basic")
echo "Running basic POSIX tests (core file operations)..."
make -f posix_Makefile test-posix-basic
;;
"extended")
echo "Running extended POSIX tests (including advanced features)..."
make -f posix_Makefile test-posix-extended
;;
"full")
echo "🚀 Running COMPREHENSIVE POSIX test suite (all tests including external)..."
echo " This includes: basic, extended, and external test suites"
echo " External suites: pjdfstest, nfstest_posix, and custom stress tests"
make -f posix_Makefile test-posix-full
;;
*)
echo "Unknown test type: $TEST_TYPE, defaulting to full suite"
echo "🚀 Running full POSIX test suite (all tests including external)..."
make -f posix_Makefile test-posix-full
;;
esac
- name: Run external test suites
if: ${{ github.event.inputs.enable_external_tests == 'true' }}
continue-on-error: true
run: |
cd test/fuse_integration
# Point the external suites at the same mount as the Go tests;
# without these the Makefile falls back to a local-disk path.
export TEST_MOUNT_POINT="$SEAWEEDFS_MOUNT_POINT"
export TEST_SKIP_CLUSTER_SETUP="true"
# Run external tests (may fail on some systems)
make -f posix_Makefile test-nfstest-posix || echo "nfstest failed or not available"
make -f posix_Makefile test-fio-posix || echo "FIO tests failed or not available"
- name: Generate compliance report
if: always()
run: |
cd test/fuse_integration
make -f posix_Makefile generate-report
- name: Collect system information for debugging
if: failure()
run: |
cd test/fuse_integration
mkdir -p reports/debug
echo "System Information" > reports/debug/system_info.txt
uname -a >> reports/debug/system_info.txt
echo "" >> reports/debug/system_info.txt
echo "Mount Information" >> reports/debug/system_info.txt
mount >> reports/debug/system_info.txt
echo "" >> reports/debug/system_info.txt
echo "Disk Space" >> reports/debug/system_info.txt
timeout 15 df -h >> reports/debug/system_info.txt || true
echo "" >> reports/debug/system_info.txt
echo "Memory Information" >> reports/debug/system_info.txt
free -h >> reports/debug/system_info.txt
echo "" >> reports/debug/system_info.txt
echo "FUSE Information" >> reports/debug/system_info.txt
ls -la /dev/fuse >> reports/debug/system_info.txt
lsmod | grep fuse >> reports/debug/system_info.txt
echo "" >> reports/debug/system_info.txt
echo "SeaweedFS Version" >> reports/debug/system_info.txt
weed version >> reports/debug/system_info.txt
echo "" >> reports/debug/system_info.txt
echo "Go Version" >> reports/debug/system_info.txt
go version >> reports/debug/system_info.txt
echo "" >> reports/debug/system_info.txt
echo "Running Processes" >> reports/debug/system_info.txt
ps aux | grep -E "(weed|fuse)" >> reports/debug/system_info.txt
echo "" >> reports/debug/system_info.txt
# Collect SeaweedFS service logs
echo "=== SeaweedFS Service Logs ===" >> reports/debug/system_info.txt
if [ -f /tmp/seaweedfs/master.log ]; then
echo "Master Log:" >> reports/debug/system_info.txt
tail -50 /tmp/seaweedfs/master.log >> reports/debug/system_info.txt
echo "" >> reports/debug/system_info.txt
fi
if [ -f /tmp/seaweedfs/volume.log ]; then
echo "Volume Log:" >> reports/debug/system_info.txt
tail -50 /tmp/seaweedfs/volume.log >> reports/debug/system_info.txt
echo "" >> reports/debug/system_info.txt
fi
if [ -f /tmp/seaweedfs/filer.log ]; then
echo "Filer Log:" >> reports/debug/system_info.txt
tail -50 /tmp/seaweedfs/filer.log >> reports/debug/system_info.txt
echo "" >> reports/debug/system_info.txt
fi
if [ -f /tmp/seaweedfs/mount.log ]; then
echo "Mount Log:" >> reports/debug/system_info.txt
tail -50 /tmp/seaweedfs/mount.log >> reports/debug/system_info.txt
echo "" >> reports/debug/system_info.txt
fi
# Copy full logs as separate files
if [ -f /tmp/seaweedfs/master.log ]; then
cp /tmp/seaweedfs/master.log reports/debug/master.log
fi
if [ -f /tmp/seaweedfs/volume.log ]; then
cp /tmp/seaweedfs/volume.log reports/debug/volume.log
fi
if [ -f /tmp/seaweedfs/filer.log ]; then
cp /tmp/seaweedfs/filer.log reports/debug/filer.log
fi
if [ -f /tmp/seaweedfs/mount.log ]; then
cp /tmp/seaweedfs/mount.log reports/debug/mount.log
fi
- name: Cleanup SeaweedFS cluster and FUSE mount
if: always()
run: |
stop_process() {
local name=$1
local pidfile=$2
if [ -f "$pidfile" ]; then
local pid=$(cat "$pidfile")
if kill -0 $pid 2>/dev/null; then
echo "Stopping $name process (PID: $pid)..."
kill -TERM $pid || true
sleep 2
kill -KILL $pid 2>/dev/null || true
fi
fi
}
echo "Cleaning up SeaweedFS cluster and FUSE mount..."
MOUNT_POINT="/tmp/seaweedfs/mount"
if mountpoint -q $MOUNT_POINT 2>/dev/null; then
echo "Unmounting FUSE filesystem..."
fusermount -u $MOUNT_POINT || umount $MOUNT_POINT || true
fi
stop_process "mount" "/tmp/seaweedfs/mount.pid"
stop_process "filer" "/tmp/seaweedfs/filer.pid"
stop_process "volume" "/tmp/seaweedfs/volume.pid"
stop_process "master" "/tmp/seaweedfs/master.pid"
pkill -f "weed " || true
fusermount -u $MOUNT_POINT 2>/dev/null || true
umount $MOUNT_POINT 2>/dev/null || true
rm -rf /tmp/seaweedfs || true
echo "Cleanup completed"
- name: Upload test results
uses: actions/upload-artifact@v7
if: always()
with:
name: posix-test-results-ubuntu-fuse${{ matrix.fuse-version }}
path: |
test/fuse_integration/reports/
test/fuse_integration/*.log
retention-days: 30
- name: Upload test coverage
uses: actions/upload-artifact@v7
if: always() && (github.event.inputs.test_type == 'full' || github.event.inputs.test_type == 'extended')
with:
name: posix-coverage-ubuntu
path: test/fuse_integration/reports/posix_coverage.html
retention-days: 14
- name: Comment PR with results
if: github.event_name == 'pull_request' && always()
uses: actions/github-script@v7
with:
script: |
const fs = require('fs');
const path = require('path');
const reportPath = 'test/fuse_integration/reports/posix_compliance_summary.txt';
if (fs.existsSync(reportPath)) {
const report = fs.readFileSync(reportPath, 'utf8');
const comment = `## POSIX Compliance Test Results
**Go Version:** ${{ env.GO_VERSION }}
**FUSE Version:** ${{ matrix.fuse-version }}
**Test Type:** ${{ github.event.inputs.test_type || 'full' }}
<details>
<summary>Test Summary</summary>
\`\`\`
${report}
\`\`\`
</details>
📊 Full results available in [test artifacts](https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }})
`;
try {
await github.rest.issues.createComment({
issue_number: context.issue.number,
owner: context.repo.owner,
repo: context.repo.repo,
body: comment
});
} catch (err) {
console.log('Could not post PR comment: ' + err.message);
}
}
posix-compliance-macos:
runs-on: macos-latest
timeout-minutes: 60
steps:
- name: Checkout code
uses: actions/checkout@v7
- name: Set up Go
uses: actions/setup-go@v7
with:
go-version: ${{ env.GO_VERSION }}
cache: ${{ runner.environment == 'github-hosted' }}
- name: Install macFUSE
run: |
# Install macFUSE
brew install macfuse
# Note: macFUSE may require system extension approval on macOS
# This step may need manual intervention in some cases
- name: Build SeaweedFS
run: |
make
# Verify binary exists and is executable (it's installed to GOPATH/bin)
weed version
- name: Run critical POSIX tests (macOS)
continue-on-error: true # macOS FUSE can be more restrictive
run: |
cd test/fuse_integration
if [ ! -f go.mod ]; then
go mod init seaweedfs-posix-tests
go mod tidy
fi
mkdir -p reports
# Run basic tests only on macOS due to FUSE limitations
make -f posix_Makefile test-posix-critical
- name: Upload macOS test results
uses: actions/upload-artifact@v7
if: always()
with:
name: posix-test-results-macos
path: |
test/fuse_integration/reports/
test/fuse_integration/*.log
retention-days: 30
security-analysis:
runs-on: ubuntu-latest
if: github.event_name == 'pull_request'
steps:
- name: Checkout code
uses: actions/checkout@v7
- name: Set up Go
uses: actions/setup-go@v7
with:
go-version: ${{ env.GO_VERSION }}
cache: ${{ runner.environment == 'github-hosted' }}
- name: Install security analysis tools
run: |
echo "Installing staticcheck for security analysis..."
go install honnef.co/go/tools/cmd/staticcheck@latest
- name: Run security analysis on FUSE code
run: |
# Analyze mount and FUSE-related code for security and quality issues
echo "Running staticcheck security analysis..."
# Run staticcheck on FUSE-related code
echo "Analyzing FUSE mount code..."
staticcheck ./weed/mount/... > staticcheck-report.txt 2>&1 || true
staticcheck ./weed/command/mount* ./weed/command/fuse* >> staticcheck-report.txt 2>&1 || true
# Convert to JSON format for consistency
echo "{" > gosec-report.json
echo " \"tool\": \"staticcheck\"," >> gosec-report.json
echo " \"timestamp\": \"$(date -Iseconds)\"," >> gosec-report.json
echo " \"issues\": [" >> gosec-report.json
if [ -s staticcheck-report.txt ]; then
echo " \"$(cat staticcheck-report.txt | head -20 | sed 's/"/\\"/g' | tr '\n' ' ')\"" >> gosec-report.json
fi
echo " ]," >> gosec-report.json
echo " \"stats\": {" >> gosec-report.json
echo " \"files_analyzed\": $(find ./weed/mount ./weed/command -name '*.go' | wc -l)," >> gosec-report.json
echo " \"issues_found\": $(wc -l < staticcheck-report.txt 2>/dev/null || echo 0)" >> gosec-report.json
echo " }" >> gosec-report.json
echo "}" >> gosec-report.json
echo "Security analysis completed"
echo "Issues found: $(wc -l < staticcheck-report.txt 2>/dev/null || echo 0)"
- name: Upload security analysis results
uses: actions/upload-artifact@v7
if: always()
with:
name: security-analysis-results
path: gosec-report.json
retention-days: 30
performance-baseline:
runs-on: ubuntu-latest
if: github.event.inputs.test_type == 'full' || github.event_name == 'schedule'
steps:
- name: Checkout code
uses: actions/checkout@v7
- name: Set up Go
uses: actions/setup-go@v7
with:
go-version: ${{ env.GO_VERSION }}
cache: ${{ runner.environment == 'github-hosted' }}
- name: Install dependencies
run: |
sudo apt-get update
sudo apt-get install -y fuse libfuse-dev fio
sudo usermod -a -G fuse $USER
sudo chmod 666 /dev/fuse
# The posix suites mount with -allowOthers; allow_other needs this
echo 'user_allow_other' | sudo tee -a /etc/fuse.conf
- name: Build SeaweedFS
run: make
- name: Run performance baseline tests
run: |
cd test/fuse_integration
if [ ! -f go.mod ]; then
go mod init seaweedfs-posix-tests
go mod tidy
fi
mkdir -p reports
# Run performance benchmarks
make -f posix_Makefile benchmark-posix
# This job has no FUSE mount, so the fio suite can only skip; keep the
# run green but honest about producing no mounted-FS results
make -f posix_Makefile test-fio-posix || echo "FIO skipped - no mounted filesystem in this job"
- name: Store performance baseline
uses: actions/upload-artifact@v7
with:
name: performance-baseline-results
path: |
test/fuse_integration/reports/posix_benchmark_results.log
test/fuse_integration/reports/fio_*_results.log
retention-days: 90
# Schedule regular compliance checks
scheduled-compliance-check:
runs-on: ubuntu-latest
if: github.event_name == 'schedule'
steps:
- name: Checkout code
uses: actions/checkout@v7
- name: Set up Go
uses: actions/setup-go@v7
with:
go-version: ${{ env.GO_VERSION }}
cache: ${{ runner.environment == 'github-hosted' }}
- name: Install dependencies
run: |
sudo apt-get update
sudo apt-get install -y fuse libfuse-dev
pip3 install --user nfstest
# The posix suites mount with -allowOthers; allow_other needs this
echo 'user_allow_other' | sudo tee -a /etc/fuse.conf
- name: Build SeaweedFS
run: make
- name: Run comprehensive compliance check
run: |
cd test/fuse_integration
make -f posix_Makefile test-posix-full
make -f posix_Makefile generate-report
- name: Create compliance issue if tests fail
if: failure()
uses: actions/github-script@v7
with:
script: |
const issue = await github.rest.issues.create({
owner: context.repo.owner,
repo: context.repo.repo,
title: 'POSIX Compliance Check Failed - ' + new Date().toISOString().split('T')[0],
body: `## POSIX Compliance Check Failure
The scheduled POSIX compliance check has failed. This may indicate:
- Regression in FUSE mount functionality
- Changes affecting POSIX compatibility
- Infrastructure issues with the test environment
**Action Required:** Review the test results and investigate any failures.
**Test Run:** [Workflow Run](https://github.com/${context.repo.owner}/${context.repo.repo}/actions/runs/${context.runId})
**Date:** ${new Date().toISOString()}
---
This issue was automatically created by the POSIX compliance workflow.
`,
labels: ['bug', 'posix-compliance', 'automated-issue']
});
console.log('Created issue:', issue.data.number);
# Schedule this workflow to run weekly
# Uncomment the following lines to enable scheduled runs:
# schedule:
# - cron: '0 2 * * 1' # Every Monday at 2 AM UTC
+2 -4
View File
@@ -23,8 +23,8 @@ on:
- '.github/workflows/postgres-tests.yml'
concurrency:
group: ${{ github.workflow }}-${{ github.event.pull_request.number || (github.event_name == 'push' && github.ref) || github.run_id }}
cancel-in-progress: ${{ github.event_name == 'pull_request' || (github.event_name == 'push' && !startsWith(github.ref, 'refs/tags/')) }}
group: ${{ github.head_ref }}/postgres-tests
cancel-in-progress: true
permissions:
contents: read
@@ -41,7 +41,6 @@ jobs:
- name: Set up Go 1.x
uses: actions/setup-go@v7
with:
cache: ${{ runner.environment == 'github-hosted' }}
go-version: ^1.26
id: go
@@ -57,7 +56,6 @@ jobs:
uses: docker/setup-buildx-action@v4
- name: Cache Docker layers
if: runner.environment == 'github-hosted'
uses: actions/cache@v6
with:
path: /tmp/.buildx-cache
+54 -82
View File
@@ -21,8 +21,8 @@ on:
- '.github/workflows/rust-volume-server-tests.yml'
concurrency:
group: ${{ github.workflow }}-${{ github.event.pull_request.number || (github.event_name == 'push' && github.ref) || github.run_id }}
cancel-in-progress: ${{ github.event_name == 'pull_request' || (github.event_name == 'push' && !startsWith(github.ref, 'refs/tags/')) }}
group: ${{ github.workflow }}-${{ github.head_ref || github.ref }}
cancel-in-progress: true
permissions:
contents: read
@@ -52,65 +52,6 @@ jobs:
- 'seaweed-volume/**'
- '.github/workflows/rust-volume-server-tests.yml'
build-test-binaries:
name: Build Test Binaries
runs-on: ubuntu-22.04
timeout-minutes: 30
steps:
- name: Checkout code
uses: actions/checkout@v7
- name: Set up Go
uses: actions/setup-go@v7
with:
go-version-file: 'go.mod'
cache: ${{ runner.environment == 'github-hosted' }}
- name: Install Rust toolchain
uses: dtolnay/rust-toolchain@stable
# cargo tracks its own inputs but not the runner's C toolchain, so a cached
# target/ can carry C objects built against a different glibc than we link against.
- name: Fingerprint build toolchain
id: toolchain
run: echo "fingerprint=$(getconf GNU_LIBC_VERSION | tr ' ' '-')-rustc-$(rustc -V | awk '{print $2}')" >> "$GITHUB_OUTPUT"
- name: Cache cargo registry and target
if: runner.environment == 'github-hosted'
uses: actions/cache@v6
with:
path: |
~/.cargo/registry
~/.cargo/git
seaweed-volume/target
key: rust-${{ steps.toolchain.outputs.fingerprint }}-${{ hashFiles('seaweed-volume/Cargo.lock', 'seaweed-common/Cargo.lock') }}
restore-keys: |
rust-${{ steps.toolchain.outputs.fingerprint }}-
- name: Build Go weed binary
run: |
cd weed
go build -tags 5BytesOffset -o weed .
chmod +x weed
./weed version
- name: Build Rust volume binary
run: cd seaweed-volume && cargo build --release
- name: Package binaries
run: tar cf rust-test-binaries.tar weed/weed seaweed-volume/target/release/weed-volume
- name: Upload test binaries
uses: actions/upload-artifact@v7
with:
# archive layout mirrors the workspace: weed/weed and
# seaweed-volume/target/release/weed-volume
name: rust-test-binaries
if-no-files-found: error
retention-days: 1
path: rust-test-binaries.tar
rust-unit-tests:
name: Rust Unit Tests
runs-on: ubuntu-22.04
@@ -130,7 +71,6 @@ jobs:
run: echo "fingerprint=$(getconf GNU_LIBC_VERSION | tr ' ' '-')-rustc-$(rustc -V | awk '{print $2}')" >> "$GITHUB_OUTPUT"
- name: Cache cargo registry and target
if: runner.environment == 'github-hosted'
uses: actions/cache@v6
with:
path: |
@@ -194,7 +134,6 @@ jobs:
run: echo "fingerprint=windows-rustc-$(rustc -V | awk '{print $2}')" >> "$GITHUB_OUTPUT"
- name: Cache cargo registry and target
if: runner.environment == 'github-hosted'
uses: actions/cache@v6
with:
path: |
@@ -215,7 +154,6 @@ jobs:
name: Rust Integration Tests
runs-on: ubuntu-22.04
timeout-minutes: 30
needs: [build-test-binaries]
steps:
- name: Checkout code
@@ -225,18 +163,36 @@ jobs:
uses: actions/setup-go@v7
with:
go-version-file: 'go.mod'
cache: ${{ runner.environment == 'github-hosted' }}
- name: Download test binaries
uses: actions/download-artifact@v7
- name: Install Rust toolchain
uses: dtolnay/rust-toolchain@stable
# cargo tracks its own inputs but not the runner's C toolchain, so a cached
# target/ can carry C objects built against a different glibc than we link against.
- name: Fingerprint build toolchain
id: toolchain
run: echo "fingerprint=$(getconf GNU_LIBC_VERSION | tr ' ' '-')-rustc-$(rustc -V | awk '{print $2}')" >> "$GITHUB_OUTPUT"
- name: Cache cargo registry and target
uses: actions/cache@v6
with:
name: rust-test-binaries
path: |
~/.cargo/registry
~/.cargo/git
seaweed-volume/target
key: rust-${{ steps.toolchain.outputs.fingerprint }}-${{ hashFiles('seaweed-volume/Cargo.lock', 'seaweed-common/Cargo.lock') }}
restore-keys: |
rust-${{ steps.toolchain.outputs.fingerprint }}-
- name: Restore binaries
- name: Build Go weed binary
run: |
tar xf rust-test-binaries.tar
chmod +x weed/weed seaweed-volume/target/release/weed-volume
./weed/weed version
cd weed
go build -tags 5BytesOffset -o weed .
chmod +x weed
./weed version
- name: Build Rust volume binary
run: cd seaweed-volume && cargo build --release
- name: Run integration tests
env:
@@ -272,7 +228,6 @@ jobs:
name: Go Tests with Rust Volume (${{ matrix.test-type }} - Shard ${{ matrix.shard }})
runs-on: ubuntu-22.04
timeout-minutes: 45
needs: [build-test-binaries]
env:
# Keep in step with the length of matrix.shard below.
SHARD_COUNT: 3
@@ -290,19 +245,36 @@ jobs:
uses: actions/setup-go@v7
with:
go-version-file: 'go.mod'
cache: ${{ runner.environment == 'github-hosted' }}
- name: Download test binaries
uses: actions/download-artifact@v7
- name: Install Rust toolchain
uses: dtolnay/rust-toolchain@stable
# cargo tracks its own inputs but not the runner's C toolchain, so a cached
# target/ can carry C objects built against a different glibc than we link against.
- name: Fingerprint build toolchain
id: toolchain
run: echo "fingerprint=$(getconf GNU_LIBC_VERSION | tr ' ' '-')-rustc-$(rustc -V | awk '{print $2}')" >> "$GITHUB_OUTPUT"
- name: Cache cargo registry and target
uses: actions/cache@v6
with:
name: rust-test-binaries
path: |
~/.cargo/registry
~/.cargo/git
seaweed-volume/target
key: rust-${{ steps.toolchain.outputs.fingerprint }}-${{ hashFiles('seaweed-volume/Cargo.lock', 'seaweed-common/Cargo.lock') }}
restore-keys: |
rust-${{ steps.toolchain.outputs.fingerprint }}-
- name: Restore binaries
- name: Build Go weed binary
run: |
tar xf rust-test-binaries.tar
chmod +x weed/weed seaweed-volume/target/release/weed-volume
./weed/weed version
./seaweed-volume/target/release/weed-volume --help >/dev/null || true
cd weed
go build -tags 5BytesOffset -o weed .
chmod +x weed
./weed version
- name: Build Rust volume binary
run: cd seaweed-volume && cargo build --release
# Dealing the listed tests out one by one keeps the shards even. Bucketing
# them by first letter did not: names cluster, so ^Test[I-S] drew 50 of
+2 -3
View File
@@ -17,8 +17,8 @@ on:
- '.github/workflows/rust-worker-tests.yml'
concurrency:
group: ${{ github.workflow }}-${{ github.event.pull_request.number || (github.event_name == 'push' && github.ref) || github.run_id }}
cancel-in-progress: ${{ github.event_name == 'pull_request' || (github.event_name == 'push' && !startsWith(github.ref, 'refs/tags/')) }}
group: ${{ github.workflow }}-${{ github.head_ref || github.ref }}
cancel-in-progress: true
permissions:
contents: read
@@ -45,7 +45,6 @@ jobs:
run: echo "fingerprint=$(getconf GNU_LIBC_VERSION | tr ' ' '-')-rustc-$(rustc -V | awk '{print $2}')" >> "$GITHUB_OUTPUT"
- name: Cache cargo registry and target
if: runner.environment == 'github-hosted'
uses: actions/cache@v6
with:
path: |
-2
View File
@@ -46,7 +46,6 @@ jobs:
uses: dtolnay/rust-toolchain@stable
- name: Cache cargo registry and target
if: runner.environment == 'github-hosted'
uses: actions/cache@v6
with:
path: |
@@ -115,7 +114,6 @@ jobs:
targets: ${{ matrix.target }}
- name: Cache cargo registry and target
if: runner.environment == 'github-hosted'
uses: actions/cache@v6
with:
path: |
@@ -48,7 +48,6 @@ jobs:
echo "OPENSSL_LIB_DIR=/usr/lib/aarch64-linux-gnu" >> "$GITHUB_ENV"
- name: Cache cargo registry and target
if: runner.environment == 'github-hosted'
uses: actions/cache@v6
with:
path: |
@@ -154,7 +153,6 @@ jobs:
echo "CARGO_TARGET_AARCH64_UNKNOWN_LINUX_GNU_LINKER=aarch64-linux-gnu-gcc" >> "$GITHUB_ENV"
- name: Cache cargo registry and target
if: runner.environment == 'github-hosted'
uses: actions/cache@v6
with:
path: |
@@ -232,7 +230,6 @@ jobs:
targets: ${{ matrix.target }}
- name: Cache cargo registry and target
if: runner.environment == 'github-hosted'
uses: actions/cache@v6
with:
path: |
@@ -308,7 +305,6 @@ jobs:
uses: dtolnay/rust-toolchain@stable
- name: Cache cargo registry and target
if: runner.environment == 'github-hosted'
uses: actions/cache@v6
with:
path: |
+2 -3
View File
@@ -21,8 +21,8 @@ on:
- '.github/workflows/s3-etag-acl-tests.yml'
concurrency:
group: ${{ github.workflow }}-${{ github.event.pull_request.number || (github.event_name == 'push' && github.ref) || github.run_id }}
cancel-in-progress: ${{ github.event_name == 'pull_request' || (github.event_name == 'push' && !startsWith(github.ref, 'refs/tags/')) }}
group: ${{ github.head_ref || github.ref }}/s3-etag-acl-tests
cancel-in-progress: true
permissions:
contents: read
@@ -40,7 +40,6 @@ jobs:
uses: actions/setup-go@v7
with:
go-version-file: 'go.mod'
cache: ${{ runner.environment == 'github-hosted' }}
- name: Install SeaweedFS
run: |
@@ -13,8 +13,8 @@ on:
- '.github/workflows/s3-example-integration-tests.yml'
concurrency:
group: ${{ github.workflow }}-${{ github.event.pull_request.number || (github.event_name == 'push' && github.ref) || github.run_id }}
cancel-in-progress: ${{ github.event_name == 'pull_request' || (github.event_name == 'push' && !startsWith(github.ref, 'refs/tags/')) }}
group: ${{ github.head_ref }}/s3-integration-tests
cancel-in-progress: true
permissions:
contents: read
@@ -33,7 +33,6 @@ jobs:
uses: actions/setup-go@v7
with:
go-version-file: 'go.mod'
cache: ${{ runner.environment == 'github-hosted' }}
id: go
- name: Build SeaweedFS
+3 -7
View File
@@ -12,8 +12,8 @@ on:
- '.github/workflows/s3-filer-group-tests.yml'
concurrency:
group: ${{ github.workflow }}-${{ github.event.pull_request.number || (github.event_name == 'push' && github.ref) || github.run_id }}
cancel-in-progress: ${{ github.event_name == 'pull_request' || (github.event_name == 'push' && !startsWith(github.ref, 'refs/tags/')) }}
group: ${{ github.head_ref }}/s3-filer-group-tests
cancel-in-progress: true
permissions:
contents: read
@@ -32,14 +32,10 @@ jobs:
- name: Check out code
uses: actions/checkout@v7
- name: Clean up stale FUSE mounts
uses: ./.github/actions/cleanup-stale-fuse-mounts
- name: Set up Go
uses: actions/setup-go@v7
with:
go-version-file: 'go.mod'
cache: ${{ runner.environment == 'github-hosted' }}
id: go
- name: Install SeaweedFS
@@ -54,7 +50,7 @@ jobs:
echo "=== System Information ==="
uname -a
free -h
timeout 15 df -h || true
df -h
echo "=== Starting Filer Group Tests ==="
# Run tests with automatic server management
+6 -31
View File
@@ -4,7 +4,6 @@ on:
pull_request:
paths:
- 'weed/s3api/**'
- 'weed/remote_storage/**'
- 'weed/filer/**'
- 'weed/server/**'
- 'test/s3/**'
@@ -13,8 +12,8 @@ on:
- '.github/workflows/s3-go-tests.yml'
concurrency:
group: ${{ github.workflow }}-${{ github.event.pull_request.number || (github.event_name == 'push' && github.ref) || github.run_id }}
cancel-in-progress: ${{ github.event_name == 'pull_request' || (github.event_name == 'push' && !startsWith(github.ref, 'refs/tags/')) }}
group: ${{ github.head_ref }}/s3-go-tests
cancel-in-progress: true
permissions:
contents: read
@@ -36,14 +35,10 @@ jobs:
- name: Check out code
uses: actions/checkout@v7
- name: Clean up stale FUSE mounts
uses: ./.github/actions/cleanup-stale-fuse-mounts
- name: Set up Go
uses: actions/setup-go@v7
with:
go-version-file: 'go.mod'
cache: ${{ runner.environment == 'github-hosted' }}
id: go
- name: Install SeaweedFS
@@ -58,7 +53,7 @@ jobs:
echo "=== System Information ==="
uname -a
free -h
timeout 15 df -h || true
df -h
echo "=== Starting Tests ==="
# Run tests with automatic server management
@@ -108,7 +103,6 @@ jobs:
uses: actions/setup-go@v7
with:
go-version-file: 'go.mod'
cache: ${{ runner.environment == 'github-hosted' }}
id: go
- name: Install SeaweedFS
@@ -157,7 +151,6 @@ jobs:
uses: actions/setup-go@v7
with:
go-version-file: 'go.mod'
cache: ${{ runner.environment == 'github-hosted' }}
id: go
- name: Install SeaweedFS
@@ -205,14 +198,10 @@ jobs:
- name: Check out code
uses: actions/checkout@v7
- name: Clean up stale FUSE mounts
uses: ./.github/actions/cleanup-stale-fuse-mounts
- name: Set up Go
uses: actions/setup-go@v7
with:
go-version-file: 'go.mod'
cache: ${{ runner.environment == 'github-hosted' }}
id: go
- name: Install SeaweedFS
@@ -227,7 +216,7 @@ jobs:
echo "=== System Information ==="
uname -a
free -h
timeout 15 df -h || true
df -h
echo "=== Starting Tests ==="
# Run tests with automatic server management
@@ -294,7 +283,6 @@ jobs:
- TestLifecycleSuspendedVersioningExpiration
- TestLifecycleTagFilter
- TestLifecycleVersionedBucketCreatesDeleteMarker
- TestLifecycleVersionedBucketExpiresOnSecondRun
steps:
- name: Check out code
@@ -304,7 +292,6 @@ jobs:
uses: actions/setup-go@v7
with:
go-version-file: 'go.mod'
cache: ${{ runner.environment == 'github-hosted' }}
id: go
- name: Install SeaweedFS
@@ -346,14 +333,10 @@ jobs:
- name: Check out code
uses: actions/checkout@v7
- name: Clean up stale FUSE mounts
uses: ./.github/actions/cleanup-stale-fuse-mounts
- name: Set up Go
uses: actions/setup-go@v7
with:
go-version-file: 'go.mod'
cache: ${{ runner.environment == 'github-hosted' }}
id: go
- name: Install SeaweedFS
@@ -368,7 +351,7 @@ jobs:
echo "=== System Information ==="
uname -a
free -h
timeout 15 df -h || true
df -h
echo "=== Starting Tests ==="
make test-with-server
@@ -408,14 +391,10 @@ jobs:
- name: Check out code
uses: actions/checkout@v7
- name: Clean up stale FUSE mounts
uses: ./.github/actions/cleanup-stale-fuse-mounts
- name: Set up Go
uses: actions/setup-go@v7
with:
go-version-file: 'go.mod'
cache: ${{ runner.environment == 'github-hosted' }}
id: go
- name: Install SeaweedFS
@@ -430,7 +409,7 @@ jobs:
echo "=== System Information ==="
uname -a
free -h
timeout 15 df -h || true
df -h
echo "=== Starting Tests ==="
# Run tests with automatic server management
@@ -480,7 +459,6 @@ jobs:
uses: actions/setup-go@v7
with:
go-version-file: 'go.mod'
cache: ${{ runner.environment == 'github-hosted' }}
id: go
- name: Install SeaweedFS
@@ -530,7 +508,6 @@ jobs:
uses: actions/setup-go@v7
with:
go-version-file: 'go.mod'
cache: ${{ runner.environment == 'github-hosted' }}
id: go
- name: Install SeaweedFS
@@ -599,7 +576,6 @@ jobs:
uses: actions/setup-go@v7
with:
go-version-file: 'go.mod'
cache: ${{ runner.environment == 'github-hosted' }}
id: go
- name: Install SeaweedFS
@@ -655,7 +631,6 @@ jobs:
uses: actions/setup-go@v7
with:
go-version-file: 'go.mod'
cache: ${{ runner.environment == 'github-hosted' }}
id: go
- name: Install SeaweedFS
+3 -10
View File
@@ -20,8 +20,8 @@ on:
- '.github/workflows/s3-iam-tests.yml'
concurrency:
group: ${{ github.workflow }}-${{ github.event.pull_request.number || (github.event_name == 'push' && github.ref) || github.run_id }}
cancel-in-progress: ${{ github.event_name == 'pull_request' || (github.event_name == 'push' && !startsWith(github.ref, 'refs/tags/')) }}
group: ${{ github.head_ref }}/s3-iam-tests
cancel-in-progress: true
permissions:
contents: read
@@ -45,7 +45,6 @@ jobs:
uses: actions/setup-go@v7
with:
go-version-file: 'go.mod'
cache: ${{ runner.environment == 'github-hosted' }}
id: go
- name: Get dependencies
@@ -91,14 +90,10 @@ jobs:
- name: Check out code
uses: actions/checkout@v7
- name: Clean up stale FUSE mounts
uses: ./.github/actions/cleanup-stale-fuse-mounts
- name: Set up Go
uses: actions/setup-go@v7
with:
go-version-file: 'go.mod'
cache: ${{ runner.environment == 'github-hosted' }}
id: go
- name: Install SeaweedFS
@@ -114,7 +109,7 @@ jobs:
echo "=== System Information ==="
uname -a
free -h
timeout 15 df -h || true
df -h
echo "=== Starting S3 IAM Integration Tests (${{ matrix.test-type }}) ==="
# Set WEED_BINARY to use the installed version
@@ -213,7 +208,6 @@ jobs:
uses: actions/setup-go@v7
with:
go-version-file: 'go.mod'
cache: ${{ runner.environment == 'github-hosted' }}
id: go
- name: Install SeaweedFS
@@ -274,7 +268,6 @@ jobs:
uses: actions/setup-go@v7
with:
go-version-file: 'go.mod'
cache: ${{ runner.environment == 'github-hosted' }}
id: go
- name: Install SeaweedFS
+3 -7
View File
@@ -16,8 +16,8 @@ on:
- '.github/workflows/s3-keycloak-tests.yml'
concurrency:
group: ${{ github.workflow }}-${{ github.event.pull_request.number || (github.event_name == 'push' && github.ref) || github.run_id }}
cancel-in-progress: ${{ github.event_name == 'pull_request' || (github.event_name == 'push' && !startsWith(github.ref, 'refs/tags/')) }}
group: ${{ github.head_ref }}/s3-keycloak-tests
cancel-in-progress: true
permissions:
contents: read
@@ -37,14 +37,10 @@ jobs:
- name: Check out code
uses: actions/checkout@v7
- name: Clean up stale FUSE mounts
uses: ./.github/actions/cleanup-stale-fuse-mounts
- name: Set up Go
uses: actions/setup-go@v7
with:
go-version-file: 'go.mod'
cache: ${{ runner.environment == 'github-hosted' }}
id: go
- name: Install SeaweedFS
@@ -60,7 +56,7 @@ jobs:
echo "=== System Information ==="
uname -a
free -h
timeout 15 df -h || true
df -h
echo "=== Starting S3 Keycloak Integration Tests ==="
# Set WEED_BINARY to use the installed version
@@ -12,8 +12,8 @@ on:
- '.github/workflows/s3-mutation-regression-tests.yml'
concurrency:
group: ${{ github.workflow }}-${{ github.event.pull_request.number || (github.event_name == 'push' && github.ref) || github.run_id }}
cancel-in-progress: ${{ github.event_name == 'pull_request' || (github.event_name == 'push' && !startsWith(github.ref, 'refs/tags/')) }}
group: ${{ github.head_ref }}/s3-mutation-regression-tests
cancel-in-progress: true
permissions:
contents: read
@@ -32,7 +32,6 @@ jobs:
uses: actions/setup-go@v7
with:
go-version-file: 'go.mod'
cache: ${{ runner.environment == 'github-hosted' }}
- name: Run S3 versioning regression tests
timeout-minutes: 20
@@ -86,7 +85,6 @@ jobs:
uses: actions/setup-go@v7
with:
go-version-file: 'go.mod'
cache: ${{ runner.environment == 'github-hosted' }}
- name: Run S3 delete regression tests
timeout-minutes: 15
@@ -125,7 +123,6 @@ jobs:
uses: actions/setup-go@v7
with:
go-version-file: 'go.mod'
cache: ${{ runner.environment == 'github-hosted' }}
- name: Build SeaweedFS
run: |
+3 -7
View File
@@ -23,10 +23,6 @@ env:
S3_ENDPOINT_URL: http://localhost:8333
BUCKET_NAME: test-parquet-bucket
concurrency:
group: ${{ github.workflow }}-${{ github.event.pull_request.number || (github.event_name == 'push' && github.ref) || github.run_id }}
cancel-in-progress: ${{ github.event_name == 'pull_request' || (github.event_name == 'push' && !startsWith(github.ref, 'refs/tags/')) }}
jobs:
parquet-integration-tests:
name: PyArrow Parquet Tests (Python ${{ matrix.python-version }})
@@ -46,13 +42,13 @@ jobs:
uses: actions/setup-go@v7
with:
go-version: ^1.26
cache: ${{ runner.environment == 'github-hosted' }}
cache: true
- name: Set up Python ${{ matrix.python-version }}
uses: actions/setup-python@v7
with:
python-version: ${{ matrix.python-version }}
cache: ${{ runner.environment == 'github-hosted' && 'pip' || '' }}
cache: 'pip'
cache-dependency-path: 'test/s3/parquet/requirements.txt'
- name: Install system dependencies
@@ -154,7 +150,7 @@ jobs:
uses: actions/setup-go@v7
with:
go-version: ^1.26
cache: ${{ runner.environment == 'github-hosted' }}
cache: true
- name: Run Go unit tests
run: |
+4 -15
View File
@@ -24,8 +24,8 @@ on:
- '.github/workflows/s3-policy-tests.yml'
concurrency:
group: ${{ github.workflow }}-${{ github.event.pull_request.number || (github.event_name == 'push' && github.ref) || github.run_id }}
cancel-in-progress: ${{ github.event_name == 'pull_request' || (github.event_name == 'push' && !startsWith(github.ref, 'refs/tags/')) }}
group: ${{ github.head_ref }}/s3-policy-tests
cancel-in-progress: true
permissions:
contents: read
@@ -49,7 +49,6 @@ jobs:
uses: actions/setup-go@v7
with:
go-version-file: 'go.mod'
cache: ${{ runner.environment == 'github-hosted' }}
id: go
- name: Get dependencies
@@ -89,14 +88,10 @@ jobs:
- name: Check out code
uses: actions/checkout@v7
- name: Clean up stale FUSE mounts
uses: ./.github/actions/cleanup-stale-fuse-mounts
- name: Set up Go
uses: actions/setup-go@v7
with:
go-version-file: 'go.mod'
cache: ${{ runner.environment == 'github-hosted' }}
id: go
- name: Install SeaweedFS
@@ -111,7 +106,7 @@ jobs:
echo "=== System Information ==="
uname -a
free -h
timeout 15 df -h || true
df -h
echo "=== Starting S3 Policy Variables Integration Tests ==="
@@ -202,14 +197,10 @@ jobs:
- name: Check out code
uses: actions/checkout@v7
- name: Clean up stale FUSE mounts
uses: ./.github/actions/cleanup-stale-fuse-mounts
- name: Set up Go
uses: actions/setup-go@v7
with:
go-version-file: 'go.mod'
cache: ${{ runner.environment == 'github-hosted' }}
id: go
- name: Install SeaweedFS
@@ -224,7 +215,7 @@ jobs:
echo "=== System Information ==="
uname -a
free -h
timeout 15 df -h || true
df -h
echo "=== Starting S3 Policy Enforcement Tests (${{ matrix.test-case }}) ==="
@@ -328,7 +319,6 @@ jobs:
uses: actions/setup-go@v7
with:
go-version-file: 'go.mod'
cache: ${{ runner.environment == 'github-hosted' }}
id: go
- name: Install SeaweedFS
@@ -416,7 +406,6 @@ jobs:
uses: actions/setup-go@v7
with:
go-version-file: 'go.mod'
cache: ${{ runner.environment == 'github-hosted' }}
id: go
- name: Install SeaweedFS
@@ -21,8 +21,8 @@ on:
- '.github/workflows/s3-proxy-signature-tests.yml'
concurrency:
group: ${{ github.workflow }}-${{ github.event.pull_request.number || (github.event_name == 'push' && github.ref) || github.run_id }}
cancel-in-progress: ${{ github.event_name == 'pull_request' || (github.event_name == 'push' && !startsWith(github.ref, 'refs/tags/')) }}
group: ${{ github.head_ref || github.ref }}/s3-proxy-signature-tests
cancel-in-progress: true
permissions:
contents: read
@@ -40,7 +40,6 @@ jobs:
uses: actions/setup-go@v7
with:
go-version-file: 'go.mod'
cache: ${{ runner.environment == 'github-hosted' }}
id: go
- name: Configure Docker Hub mirror
@@ -15,8 +15,8 @@ on:
- '.github/workflows/s3-sdk-v2-routing-tests.yml'
concurrency:
group: ${{ github.workflow }}-${{ github.event.pull_request.number || (github.event_name == 'push' && github.ref) || github.run_id }}
cancel-in-progress: ${{ github.event_name == 'pull_request' || (github.event_name == 'push' && !startsWith(github.ref, 'refs/tags/')) }}
group: ${{ github.head_ref || github.ref }}/s3-sdk-v2-routing-tests
cancel-in-progress: true
permissions:
contents: read
@@ -34,7 +34,6 @@ jobs:
uses: actions/setup-go@v7
with:
go-version-file: 'go.mod'
cache: ${{ runner.environment == 'github-hosted' }}
- name: Install SeaweedFS
run: |
+1 -2
View File
@@ -60,7 +60,6 @@ jobs:
uses: actions/setup-go@v7
with:
go-version-file: 'go.mod'
cache: ${{ runner.environment == 'github-hosted' }}
id: go
- name: Set up Java
@@ -68,7 +67,7 @@ jobs:
with:
java-version: '17'
distribution: 'temurin'
cache: ${{ runner.environment == 'github-hosted' && 'maven' || '' }}
cache: 'maven'
- name: Install SeaweedFS
run: |
+3 -7
View File
@@ -11,8 +11,8 @@ on:
workflow_dispatch:
concurrency:
group: ${{ github.workflow }}-${{ github.event.pull_request.number || (github.event_name == 'push' && github.ref) || github.run_id }}
cancel-in-progress: ${{ github.event_name == 'pull_request' || (github.event_name == 'push' && !startsWith(github.ref, 'refs/tags/')) }}
group: ${{ github.head_ref }}/s3-spark-tests
cancel-in-progress: true
permissions:
contents: read
@@ -27,14 +27,10 @@ jobs:
- name: Check out code
uses: actions/checkout@v7
- name: Clean up stale FUSE mounts
uses: ./.github/actions/cleanup-stale-fuse-mounts
- name: Set up Go
uses: actions/setup-go@v7
with:
go-version-file: 'go.mod'
cache: ${{ runner.environment == 'github-hosted' }}
id: go
- name: Configure Docker Hub mirror
@@ -60,7 +56,7 @@ jobs:
echo "=== System Information ==="
uname -a
free -h
timeout 15 df -h || true
df -h
echo "=== Starting S3 Spark Integration Tests ==="
go test -v -timeout 30m . 2>&1 | tee test-output.log || {
+3 -13
View File
@@ -23,8 +23,8 @@ on:
- 'test/s3/sse/**'
concurrency:
group: ${{ github.workflow }}-${{ github.event.pull_request.number || (github.event_name == 'push' && github.ref) || github.run_id }}
cancel-in-progress: ${{ github.event_name == 'pull_request' || (github.event_name == 'push' && !startsWith(github.ref, 'refs/tags/')) }}
group: ${{ github.head_ref }}/s3-sse-tests
cancel-in-progress: true
permissions:
contents: read
@@ -46,14 +46,10 @@ jobs:
- name: Check out code
uses: actions/checkout@v7
- name: Clean up stale FUSE mounts
uses: ./.github/actions/cleanup-stale-fuse-mounts
- name: Set up Go
uses: actions/setup-go@v7
with:
go-version-file: 'go.mod'
cache: ${{ runner.environment == 'github-hosted' }}
id: go
- name: Install SeaweedFS
@@ -68,7 +64,7 @@ jobs:
echo "=== System Information ==="
uname -a
free -h
timeout 15 df -h || true
df -h
echo "=== Starting SSE Tests ==="
# Run tests with automatic server management
@@ -122,7 +118,6 @@ jobs:
uses: actions/setup-go@v7
with:
go-version-file: 'go.mod'
cache: ${{ runner.environment == 'github-hosted' }}
id: go
- name: Install SeaweedFS
@@ -171,7 +166,6 @@ jobs:
uses: actions/setup-go@v7
with:
go-version-file: 'go.mod'
cache: ${{ runner.environment == 'github-hosted' }}
id: go
- name: Install SeaweedFS
@@ -221,7 +215,6 @@ jobs:
uses: actions/setup-go@v7
with:
go-version-file: 'go.mod'
cache: ${{ runner.environment == 'github-hosted' }}
id: go
- name: Install SeaweedFS
@@ -271,7 +264,6 @@ jobs:
uses: actions/setup-go@v7
with:
go-version-file: 'go.mod'
cache: ${{ runner.environment == 'github-hosted' }}
id: go
- name: Install SeaweedFS
@@ -323,7 +315,6 @@ jobs:
uses: actions/setup-go@v7
with:
go-version-file: 'go.mod'
cache: ${{ runner.environment == 'github-hosted' }}
id: go
- name: Install SeaweedFS
@@ -372,7 +363,6 @@ jobs:
uses: actions/setup-go@v7
with:
go-version-file: 'go.mod'
cache: ${{ runner.environment == 'github-hosted' }}
id: go
- name: Install SeaweedFS
+16 -90
View File
@@ -12,10 +12,6 @@ on:
- 'go.sum'
- '.github/workflows/s3-tables-tests.yml'
concurrency:
group: ${{ github.workflow }}-${{ github.event.pull_request.number || (github.event_name == 'push' && github.ref) || github.run_id }}
cancel-in-progress: ${{ github.event_name == 'pull_request' || (github.event_name == 'push' && !startsWith(github.ref, 'refs/tags/')) }}
permissions:
contents: read
@@ -30,14 +26,10 @@ jobs:
- name: Check out code
uses: actions/checkout@v7
- name: Clean up stale FUSE mounts
uses: ./.github/actions/cleanup-stale-fuse-mounts
- name: Set up Go
uses: actions/setup-go@v7
with:
go-version-file: 'go.mod'
cache: ${{ runner.environment == 'github-hosted' }}
id: go
- name: Run go mod tidy
@@ -56,8 +48,7 @@ jobs:
echo "=== System Information ==="
uname -a
free -h
# A stale FUSE mount can hang or fail df; keep diagnostics best effort.
timeout 15 df -h || true
df -h
echo "=== Starting S3 Tables Tests ==="
# Run S3 Tables integration tests
@@ -95,14 +86,10 @@ jobs:
- name: Check out code
uses: actions/checkout@v7
- name: Clean up stale FUSE mounts
uses: ./.github/actions/cleanup-stale-fuse-mounts
- name: Set up Go
uses: actions/setup-go@v7
with:
go-version-file: 'go.mod'
cache: ${{ runner.environment == 'github-hosted' }}
id: go
- name: Configure Docker Hub mirror
@@ -122,7 +109,7 @@ jobs:
echo "=== System Information ==="
uname -a
free -h
timeout 15 df -h || true
df -h
echo "=== Starting Iceberg Catalog Tests ==="
# Run Iceberg catalog integration tests using Makefile (handles build)
@@ -160,14 +147,10 @@ jobs:
- name: Check out code
uses: actions/checkout@v7
- name: Clean up stale FUSE mounts
uses: ./.github/actions/cleanup-stale-fuse-mounts
- name: Set up Go
uses: actions/setup-go@v7
with:
go-version-file: 'go.mod'
cache: ${{ runner.environment == 'github-hosted' }}
id: go
- name: Configure Docker Hub mirror
@@ -196,7 +179,7 @@ jobs:
echo "=== System Information ==="
uname -a
free -h
timeout 15 df -h || true
df -h
echo "=== Starting Trino Iceberg Catalog Tests ==="
# Run Trino + Iceberg catalog integration tests
@@ -234,14 +217,10 @@ jobs:
- name: Check out code
uses: actions/checkout@v7
- name: Clean up stale FUSE mounts
uses: ./.github/actions/cleanup-stale-fuse-mounts
- name: Set up Go
uses: actions/setup-go@v7
with:
go-version-file: 'go.mod'
cache: ${{ runner.environment == 'github-hosted' }}
id: go
- name: Configure Docker Hub mirror
@@ -271,7 +250,7 @@ jobs:
echo "=== System Information ==="
uname -a
free -h
timeout 15 df -h || true
df -h
docker info
echo "=== Starting Dremio Iceberg Catalog Tests ==="
@@ -311,14 +290,10 @@ jobs:
- name: Check out code
uses: actions/checkout@v7
- name: Clean up stale FUSE mounts
uses: ./.github/actions/cleanup-stale-fuse-mounts
- name: Set up Go
uses: actions/setup-go@v7
with:
go-version-file: 'go.mod'
cache: ${{ runner.environment == 'github-hosted' }}
id: go
- name: Configure Docker Hub mirror
@@ -348,7 +323,7 @@ jobs:
echo "=== System Information ==="
uname -a
free -h
timeout 15 df -h || true
df -h
docker info
echo "=== Starting Doris Iceberg Catalog Tests ==="
@@ -398,14 +373,10 @@ jobs:
- name: Check out code
uses: actions/checkout@v7
- name: Clean up stale FUSE mounts
uses: ./.github/actions/cleanup-stale-fuse-mounts
- name: Set up Go
uses: actions/setup-go@v7
with:
go-version-file: 'go.mod'
cache: ${{ runner.environment == 'github-hosted' }}
id: go
- name: Configure Docker Hub mirror
@@ -437,7 +408,7 @@ jobs:
echo "=== System Information ==="
uname -a
free -h
timeout 15 df -h || true
df -h
docker info
echo "=== Starting ClickHouse Iceberg Catalog Tests ==="
@@ -489,14 +460,10 @@ jobs:
- name: Check out code
uses: actions/checkout@v7
- name: Clean up stale FUSE mounts
uses: ./.github/actions/cleanup-stale-fuse-mounts
- name: Set up Go
uses: actions/setup-go@v7
with:
go-version-file: 'go.mod'
cache: ${{ runner.environment == 'github-hosted' }}
id: go
- name: Configure Docker Hub mirror
@@ -529,7 +496,7 @@ jobs:
echo "=== System Information ==="
uname -a
free -h
timeout 15 df -h || true
df -h
docker info
echo "=== Starting OLake Iceberg Catalog Tests ==="
@@ -592,14 +559,10 @@ jobs:
- name: Check out code
uses: actions/checkout@v7
- name: Clean up stale FUSE mounts
uses: ./.github/actions/cleanup-stale-fuse-mounts
- name: Set up Go
uses: actions/setup-go@v7
with:
go-version-file: 'go.mod'
cache: ${{ runner.environment == 'github-hosted' }}
id: go
- name: Run go mod tidy
@@ -627,7 +590,7 @@ jobs:
echo "=== System Information ==="
uname -a
free -h
timeout 15 df -h || true
df -h
echo "=== Starting Polaris Tests ==="
go test -v -timeout 20m ./test/s3tables/polaris 2>&1 | tee test/s3tables/polaris/test-output.log || {
@@ -664,14 +627,10 @@ jobs:
- name: Check out code
uses: actions/checkout@v7
- name: Clean up stale FUSE mounts
uses: ./.github/actions/cleanup-stale-fuse-mounts
- name: Set up Go
uses: actions/setup-go@v7
with:
go-version-file: 'go.mod'
cache: ${{ runner.environment == 'github-hosted' }}
id: go
- name: Configure Docker Hub mirror
@@ -700,7 +659,7 @@ jobs:
echo "=== System Information ==="
uname -a
free -h
timeout 15 df -h || true
df -h
echo "=== Starting Spark Iceberg Catalog Tests ==="
# Run Spark + Iceberg catalog integration tests
@@ -738,14 +697,10 @@ jobs:
- name: Check out code
uses: actions/checkout@v7
- name: Clean up stale FUSE mounts
uses: ./.github/actions/cleanup-stale-fuse-mounts
- name: Set up Go
uses: actions/setup-go@v7
with:
go-version-file: 'go.mod'
cache: ${{ runner.environment == 'github-hosted' }}
id: go
- name: Configure Docker Hub mirror
@@ -775,7 +730,7 @@ jobs:
echo "=== System Information ==="
uname -a
free -h
timeout 15 df -h || true
df -h
echo "=== Starting RisingWave Catalog Tests ==="
# Run RisingWave catalog integration tests
@@ -813,14 +768,10 @@ jobs:
- name: Check out code
uses: actions/checkout@v7
- name: Clean up stale FUSE mounts
uses: ./.github/actions/cleanup-stale-fuse-mounts
- name: Set up Go
uses: actions/setup-go@v7
with:
go-version-file: 'go.mod'
cache: ${{ runner.environment == 'github-hosted' }}
id: go
- name: Configure Docker Hub mirror
@@ -834,7 +785,6 @@ jobs:
- name: Restore python:3 image cache
id: python-image
if: runner.environment == 'github-hosted'
uses: actions/cache@v6
with:
path: /tmp/python3-image.tar
@@ -874,7 +824,7 @@ jobs:
echo "=== System Information ==="
uname -a
free -h
timeout 15 df -h || true
df -h
echo "=== Starting STS Integration Tests ==="
# Run STS integration tests
@@ -912,14 +862,10 @@ jobs:
- name: Check out code
uses: actions/checkout@v7
- name: Clean up stale FUSE mounts
uses: ./.github/actions/cleanup-stale-fuse-mounts
- name: Set up Go
uses: actions/setup-go@v7
with:
go-version-file: 'go.mod'
cache: ${{ runner.environment == 'github-hosted' }}
id: go
- name: Run go mod tidy
@@ -938,7 +884,7 @@ jobs:
echo "=== System Information ==="
uname -a
free -h
timeout 15 df -h || true
df -h
echo "=== Starting Lakekeeper Integration Tests ==="
# Run Lakekeeper integration tests
@@ -976,14 +922,10 @@ jobs:
- name: Check out code
uses: actions/checkout@v7
- name: Clean up stale FUSE mounts
uses: ./.github/actions/cleanup-stale-fuse-mounts
- name: Set up Go
uses: actions/setup-go@v7
with:
go-version-file: 'go.mod'
cache: ${{ runner.environment == 'github-hosted' }}
id: go
- name: Configure Docker Hub mirror
@@ -1013,7 +955,7 @@ jobs:
echo "=== System Information ==="
uname -a
free -h
timeout 15 df -h || true
df -h
docker info
echo "=== Starting Unity Catalog Tests ==="
@@ -1053,14 +995,10 @@ jobs:
- name: Check out code
uses: actions/checkout@v7
- name: Clean up stale FUSE mounts
uses: ./.github/actions/cleanup-stale-fuse-mounts
- name: Set up Go
uses: actions/setup-go@v7
with:
go-version-file: 'go.mod'
cache: ${{ runner.environment == 'github-hosted' }}
id: go
- name: Configure Docker Hub mirror
@@ -1089,7 +1027,7 @@ jobs:
echo "=== System Information ==="
uname -a
free -h
timeout 15 df -h || true
df -h
docker info
echo "=== Starting LanceDB Namespace Tests ==="
@@ -1131,7 +1069,6 @@ jobs:
uses: actions/setup-go@v7
with:
go-version-file: 'go.mod'
cache: ${{ runner.environment == 'github-hosted' }}
id: go
- name: Configure Docker Hub mirror
@@ -1200,14 +1137,10 @@ jobs:
# so do not leave a token in the checkout for it to pick up.
persist-credentials: false
- name: Clean up stale FUSE mounts
uses: ./.github/actions/cleanup-stale-fuse-mounts
- name: Set up Go
uses: actions/setup-go@v7
with:
go-version-file: 'go.mod'
cache: ${{ runner.environment == 'github-hosted' }}
id: go
- name: Configure Docker Hub mirror
@@ -1237,7 +1170,7 @@ jobs:
echo "=== System Information ==="
uname -a
free -h
timeout 15 df -h || true
df -h
docker info
echo "=== Starting DuckDB Lance Tests ==="
@@ -1279,14 +1212,10 @@ jobs:
# so do not leave a token in the checkout for it to pick up.
persist-credentials: false
- name: Clean up stale FUSE mounts
uses: ./.github/actions/cleanup-stale-fuse-mounts
- name: Set up Go
uses: actions/setup-go@v7
with:
go-version-file: 'go.mod'
cache: ${{ runner.environment == 'github-hosted' }}
id: go
- name: Configure Docker Hub mirror
@@ -1315,7 +1244,7 @@ jobs:
echo "=== System Information ==="
uname -a
free -h
timeout 15 df -h || true
df -h
docker info
echo "=== Starting Spark Lance Namespace Tests ==="
@@ -1357,7 +1286,6 @@ jobs:
uses: actions/setup-go@v7
with:
go-version-file: 'go.mod'
cache: ${{ runner.environment == 'github-hosted' }}
id: go
- name: Verify S3 Tables Package Builds
@@ -1423,7 +1351,6 @@ jobs:
uses: actions/setup-go@v7
with:
go-version-file: 'go.mod'
cache: ${{ runner.environment == 'github-hosted' }}
id: go
- name: Check Go Format
@@ -1475,7 +1402,6 @@ jobs:
uses: actions/setup-go@v7
with:
go-version-file: 'go.mod'
cache: ${{ runner.environment == 'github-hosted' }}
id: go
- name: Run Go Vet
+14 -19
View File
@@ -25,8 +25,8 @@ on:
- '.github/workflows/s3tests.yml'
concurrency:
group: ${{ github.workflow }}-${{ github.event.pull_request.number || (github.event_name == 'push' && github.ref) || github.run_id }}
cancel-in-progress: ${{ github.event_name == 'pull_request' || (github.event_name == 'push' && !startsWith(github.ref, 'refs/tags/')) }}
group: ${{ github.head_ref }}/s3tests
cancel-in-progress: true
permissions:
contents: read
@@ -44,7 +44,6 @@ jobs:
uses: actions/setup-go@v7
with:
go-version-file: 'go.mod'
cache: ${{ runner.environment == 'github-hosted' }}
id: go
- name: Set up Python
@@ -58,9 +57,9 @@ jobs:
cd s3-tests
sudo apt-get update -qq
sudo apt-get install -y -qq libxml2-dev libxslt1-dev zlib1g-dev
python3 -m pip install -r requirements.txt
python3 -m pip install tox
python3 -m pip install -e .
pip install -r requirements.txt
pip install tox
pip install -e .
- name: Fix S3 tests bucket creation conflicts
run: |
@@ -397,7 +396,6 @@ jobs:
uses: actions/setup-go@v7
with:
go-version-file: 'go.mod'
cache: ${{ runner.environment == 'github-hosted' }}
id: go
- name: Set up Python
@@ -411,9 +409,9 @@ jobs:
cd s3-tests
sudo apt-get update -qq
sudo apt-get install -y -qq libxml2-dev libxslt1-dev zlib1g-dev
python3 -m pip install -r requirements.txt
python3 -m pip install tox
python3 -m pip install -e .
pip install -r requirements.txt
pip install tox
pip install -e .
- name: Fix S3 tests bucket creation conflicts
run: |
@@ -568,7 +566,6 @@ jobs:
uses: actions/setup-go@v7
with:
go-version-file: 'go.mod'
cache: ${{ runner.environment == 'github-hosted' }}
id: go
- name: Set up Python
@@ -582,9 +579,9 @@ jobs:
cd s3-tests
sudo apt-get update -qq
sudo apt-get install -y -qq libxml2-dev libxslt1-dev zlib1g-dev
python3 -m pip install -r requirements.txt
python3 -m pip install tox
python3 -m pip install -e .
pip install -r requirements.txt
pip install tox
pip install -e .
- name: Run S3 CORS tests
timeout-minutes: 10
@@ -694,7 +691,6 @@ jobs:
uses: actions/setup-go@v7
with:
go-version-file: 'go.mod'
cache: ${{ runner.environment == 'github-hosted' }}
id: go
- name: Run SeaweedFS Custom S3 Copy tests
@@ -797,7 +793,6 @@ jobs:
uses: actions/setup-go@v7
with:
go-version-file: 'go.mod'
cache: ${{ runner.environment == 'github-hosted' }}
id: go
- name: Set up Python
@@ -811,9 +806,9 @@ jobs:
cd s3-tests
sudo apt-get update -qq
sudo apt-get install -y -qq libxml2-dev libxslt1-dev zlib1g-dev
python3 -m pip install -r requirements.txt
python3 -m pip install tox
python3 -m pip install -e .
pip install -r requirements.txt
pip install tox
pip install -e .
- name: Run Ceph S3 tests with SQL store
timeout-minutes: 15
+1 -9
View File
@@ -40,23 +40,15 @@ jobs:
uses: actions/setup-go@v7
with:
go-version-file: 'go.mod'
cache: ${{ runner.environment == 'github-hosted' }}
- name: Start local Docker registry
run: |
if ! docker start registry 2>/dev/null; then
docker rm -f registry >/dev/null 2>&1 || true
docker run -d --restart=always -p 5000:5000 --name registry registry:2
fi
run: docker run -d --restart=always -p 5000:5000 --name registry registry:2
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v4
with:
driver-opts: network=host
- name: Set up Docker Compose
uses: docker/setup-compose-action@v2
- name: Build weed race binary
run: |
cd docker
+2 -3
View File
@@ -17,8 +17,8 @@ on:
- '.github/workflows/sftp-tests.yml'
concurrency:
group: ${{ github.workflow }}-${{ github.event.pull_request.number || (github.event_name == 'push' && github.ref) || github.run_id }}
cancel-in-progress: ${{ github.event_name == 'pull_request' || (github.event_name == 'push' && !startsWith(github.ref, 'refs/tags/')) }}
group: ${{ github.head_ref }}/sftp-tests
cancel-in-progress: true
permissions:
contents: read
@@ -40,7 +40,6 @@ jobs:
uses: actions/setup-go@v7
with:
go-version-file: 'go.mod'
cache: ${{ runner.environment == 'github-hosted' }}
- name: Install dependencies
run: |
@@ -15,10 +15,6 @@ on:
- '.github/workflows/spark-integration-tests.yml'
workflow_dispatch:
concurrency:
group: ${{ github.workflow }}-${{ github.event.pull_request.number || (github.event_name == 'push' && github.ref) || github.run_id }}
cancel-in-progress: ${{ github.event_name == 'pull_request' || (github.event_name == 'push' && !startsWith(github.ref, 'refs/tags/')) }}
permissions:
contents: read
checks: write
@@ -42,13 +38,12 @@ jobs:
with:
java-version: '11'
distribution: 'temurin'
cache: ${{ runner.environment == 'github-hosted' && 'maven' || '' }}
cache: maven
- name: Set up Go
uses: actions/setup-go@v7
with:
go-version-file: 'go.mod'
cache: ${{ runner.environment == 'github-hosted' }}
- name: Build SeaweedFS binary
run: |
+6 -5
View File
@@ -10,11 +10,12 @@ permissions:
contents: write
concurrency:
# Only one chart regeneration per branch at a time so overlapping runs never
# conflict on note/star_history.svg during rebase. Scoped by ref so a manual
# run on another branch can't cancel the daily master update.
# Only one chart regeneration per branch at a time; a newer run on the same
# branch cancels an in-flight one so overlapping runs never conflict on
# note/star_history.svg during rebase. Scoped by ref so a manual run on
# another branch can't cancel the daily master update.
group: star-history-${{ github.ref }}
cancel-in-progress: ${{ github.event_name == 'pull_request' || (github.event_name == 'push' && !startsWith(github.ref, 'refs/tags/')) }}
cancel-in-progress: true
jobs:
render:
@@ -32,7 +33,7 @@ jobs:
uses: actions/setup-python@v7
with:
python-version: "3.x"
cache: ${{ runner.environment == 'github-hosted' && 'pip' || '' }}
cache: pip
cache-dependency-path: .github/scripts/star_history.py
- name: Install matplotlib
@@ -14,10 +14,6 @@ on:
- 'weed/telemetry/**'
- '.github/workflows/telemetry-integration.yml'
concurrency:
group: ${{ github.workflow }}-${{ github.event.pull_request.number || (github.event_name == 'push' && github.ref) || github.run_id }}
cancel-in-progress: ${{ github.event_name == 'pull_request' || (github.event_name == 'push' && !startsWith(github.ref, 'refs/tags/')) }}
permissions:
contents: read
@@ -34,7 +30,6 @@ jobs:
uses: actions/setup-go@v7
with:
go-version-file: 'go.mod'
cache: ${{ runner.environment == 'github-hosted' }}
- name: Build telemetry server
run: cd telemetry/server && go build -o telemetry-server .
-5
View File
@@ -8,10 +8,6 @@ on:
branches: [ master ]
paths: ['terraform/**', '.github/workflows/terraform_ci.yml']
concurrency:
group: ${{ github.workflow }}-${{ github.event.pull_request.number || (github.event_name == 'push' && github.ref) || github.run_id }}
cancel-in-progress: ${{ github.event_name == 'pull_request' || (github.event_name == 'push' && !startsWith(github.ref, 'refs/tags/')) }}
permissions:
contents: read
@@ -66,7 +62,6 @@ jobs:
uses: actions/setup-go@v7
with:
go-version-file: go.mod
cache: ${{ runner.environment == 'github-hosted' }}
- name: Build weed
run: go build -o "$RUNNER_TEMP/weed" ./weed
@@ -27,10 +27,6 @@ defaults:
run:
working-directory: weed
concurrency:
group: ${{ github.workflow }}-${{ github.event.pull_request.number || (github.event_name == 'push' && github.ref) || github.run_id }}
cancel-in-progress: ${{ github.event_name == 'pull_request' || (github.event_name == 'push' && !startsWith(github.ref, 'refs/tags/')) }}
jobs:
awscli-tests:
runs-on: ubuntu-latest
@@ -41,7 +37,6 @@ jobs:
- uses: actions/setup-go@v7
with:
go-version: ^1.26
cache: ${{ runner.environment == 'github-hosted' }}
- name: Build SeaweedFS
run: |
-5
View File
@@ -18,10 +18,6 @@ on:
- 'go.sum'
- '.github/workflows/tls-rotation-tests.yml'
concurrency:
group: ${{ github.workflow }}-${{ github.event.pull_request.number || (github.event_name == 'push' && github.ref) || github.run_id }}
cancel-in-progress: ${{ github.event_name == 'pull_request' || (github.event_name == 'push' && !startsWith(github.ref, 'refs/tags/')) }}
permissions:
contents: read
@@ -34,7 +30,6 @@ jobs:
- name: Set up Go 1.x
uses: actions/setup-go@v7
with:
cache: ${{ runner.environment == 'github-hosted' }}
go-version: ^1.26
id: go
+3 -7
View File
@@ -11,8 +11,8 @@ on:
- '.github/workflows/tus-tests.yml'
concurrency:
group: ${{ github.workflow }}-${{ github.event.pull_request.number || (github.event_name == 'push' && github.ref) || github.run_id }}
cancel-in-progress: ${{ github.event_name == 'pull_request' || (github.event_name == 'push' && !startsWith(github.ref, 'refs/tags/')) }}
group: ${{ github.head_ref || github.ref }}/tus-tests
cancel-in-progress: true
permissions:
contents: read
@@ -31,14 +31,10 @@ jobs:
- name: Check out code
uses: actions/checkout@v7
- name: Clean up stale FUSE mounts
uses: ./.github/actions/cleanup-stale-fuse-mounts
- name: Set up Go
uses: actions/setup-go@v7
with:
go-version-file: 'go.mod'
cache: ${{ runner.environment == 'github-hosted' }}
id: go
- name: Install SeaweedFS
@@ -53,7 +49,7 @@ jobs:
echo "=== System Information ==="
uname -a
free -h
timeout 15 df -h || true
df -h
echo "=== Starting TUS Tests ==="
# Run tests with automatic server management
@@ -18,10 +18,6 @@ on:
- 'go.sum'
- '.github/workflows/vacuum-integration-tests.yml'
concurrency:
group: ${{ github.workflow }}-${{ github.event.pull_request.number || (github.event_name == 'push' && github.ref) || github.run_id }}
cancel-in-progress: ${{ github.event_name == 'pull_request' || (github.event_name == 'push' && !startsWith(github.ref, 'refs/tags/')) }}
permissions:
contents: read
@@ -34,7 +30,6 @@ jobs:
- name: Set up Go 1.x
uses: actions/setup-go@v7
with:
cache: ${{ runner.environment == 'github-hosted' }}
go-version: ^1.26
id: go
@@ -21,8 +21,8 @@ on:
- '.github/workflows/volume-server-integration-tests.yml'
concurrency:
group: ${{ github.workflow }}-${{ github.event.pull_request.number || (github.event_name == 'push' && github.ref) || github.run_id }}
cancel-in-progress: ${{ github.event_name == 'pull_request' || (github.event_name == 'push' && !startsWith(github.ref, 'refs/tags/')) }}
group: ${{ github.head_ref || github.ref }}/volume-server-integration-tests
cancel-in-progress: true
permissions:
contents: read
@@ -33,44 +33,10 @@ env:
SHARD_COUNT: 3
jobs:
build-weed:
name: Build SeaweedFS Binary
runs-on: ubuntu-22.04
timeout-minutes: 15
steps:
- name: Checkout code
uses: actions/checkout@v7
- name: Set up Go
uses: actions/setup-go@v7
with:
go-version-file: 'go.mod'
cache: ${{ runner.environment == 'github-hosted' }}
- name: Build SeaweedFS binary
run: |
cd weed
go build -o weed .
chmod +x weed
./weed version
- name: Package binary
run: tar cf weed-binary.tar weed/weed
- name: Upload binary
uses: actions/upload-artifact@v7
with:
name: weed-binary
if-no-files-found: error
retention-days: 1
path: weed-binary.tar
volume-server-integration-tests:
name: Volume Server Integration Tests (${{ matrix.test-type }} - Shard ${{ matrix.shard }})
runs-on: ubuntu-22.04
timeout-minutes: 45
needs: [build-weed]
strategy:
fail-fast: false
matrix:
@@ -85,18 +51,13 @@ jobs:
uses: actions/setup-go@v7
with:
go-version-file: 'go.mod'
cache: ${{ runner.environment == 'github-hosted' }}
- name: Download binary
uses: actions/download-artifact@v7
with:
name: weed-binary
- name: Restore binary
- name: Build SeaweedFS binary
run: |
tar xf weed-binary.tar
chmod +x weed/weed
./weed/weed version
cd weed
go build -o weed .
chmod +x weed
./weed version
# Dealing the listed tests out one by one keeps the shards even. Bucketing
# them by first letter did not: names cluster, so ^Test[I-S] drew 50 of
-1
View File
@@ -144,4 +144,3 @@ weed_bin
telemetry/server/telemetry-server
.aider*
/seaweed-volume/docs
__pycache__/
+1 -1
View File
@@ -3,7 +3,7 @@ FROM golang:1.26 AS builder
RUN apt-get update
RUN apt-get install -y build-essential libsnappy-dev zlib1g-dev libbz2-dev libgflags-dev liblz4-dev libzstd-dev
ARG ROCKSDB_VERSION=v11.1.2
ARG ROCKSDB_VERSION=v10.10.1
ENV ROCKSDB_VERSION=${ROCKSDB_VERSION}
# build RocksDB
+1 -1
View File
@@ -3,7 +3,7 @@ FROM golang:1.26 AS builder
RUN apt-get update
RUN apt-get install -y build-essential libsnappy-dev zlib1g-dev libbz2-dev libgflags-dev liblz4-dev libzstd-dev
ARG ROCKSDB_VERSION=v11.1.2
ARG ROCKSDB_VERSION=v10.10.1
ENV ROCKSDB_VERSION=${ROCKSDB_VERSION}
# build RocksDB
+1 -1
View File
@@ -362,7 +362,7 @@ require (
github.com/flynn/noise v1.1.0 // indirect
github.com/gabriel-vasile/mimetype v1.4.13 // indirect
github.com/geoffgarside/ber v1.2.0 // indirect
github.com/go-chi/chi/v5 v5.4.0 // indirect
github.com/go-chi/chi/v5 v5.3.1 // indirect
github.com/go-darwin/apfs v0.0.0-20211011131704-f84b94dbf348 // indirect
github.com/go-jose/go-jose/v4 v4.1.5 // indirect
github.com/go-logr/logr v1.4.4 // indirect
+2 -2
View File
@@ -483,8 +483,8 @@ github.com/gin-gonic/gin v1.11.0 h1:OW/6PLjyusp2PPXtyxKHU0RbX6I/l28FTdDlae5ueWk=
github.com/gin-gonic/gin v1.11.0/go.mod h1:+iq/FyxlGzII0KHiBGjuNn4UNENUlKbGlNmc+W50Dls=
github.com/go-asn1-ber/asn1-ber v1.5.8 h1:H9AZkK22UOmfX8J84ubyaZxKJZ3FMHVwn8swoMML7iQ=
github.com/go-asn1-ber/asn1-ber v1.5.8/go.mod h1:hEBeB/ic+5LoWskz+yKT7vGhhPYkProFKoKdwZRWMe0=
github.com/go-chi/chi/v5 v5.4.0 h1:+ufzvFTGKyin26Or5DLrlIDq4ygIDBVcj7oVNeeaYFE=
github.com/go-chi/chi/v5 v5.4.0/go.mod h1:d1IG7GtKPSjPyBP365PH8sAGVHk3qfB/IhjFNRH1wWM=
github.com/go-chi/chi/v5 v5.3.1 h1:3j4HZLGZQ3JpMCrPJF/Jl3mYJfWLKBfNJ6quurUGCf8=
github.com/go-chi/chi/v5 v5.3.1/go.mod h1:R+tYY2hNuVUUjxoPtqUdgBqevM9s9njzkTLutVsOCto=
github.com/go-darwin/apfs v0.0.0-20211011131704-f84b94dbf348 h1:JnrjqG5iR07/8k7NqrLNilRsl3s1EPRQEGvbPyOce68=
github.com/go-darwin/apfs v0.0.0-20211011131704-f84b94dbf348/go.mod h1:Czxo/d1g948LtrALAZdL04TL/HnkopquAjxYUuI02bo=
github.com/go-errors/errors v1.5.1 h1:ZwEMSLRCapFLflTpT7NKaAc7ukJ8ZPEjzlxt8rPN8bk=
+2 -2
View File
@@ -1,6 +1,6 @@
apiVersion: v1
description: SeaweedFS
name: seaweedfs
appVersion: "4.49"
appVersion: "4.48"
# Dev note: Trigger a helm chart release by `git tag -a helm-<version>`
version: 4.49.0
version: 4.48.2
@@ -172,8 +172,7 @@ spec:
{{- $bucketName := .name }}
{{- $bucketLock := or .lock .objectLock .withLock }}
bucket_list=$(/bin/echo 's3.bucket.list' | /usr/bin/weed shell) || { echo "Error listing s3 buckets"; exit 1; }
{{- /* One awk: grep -q exits early, awk takes SIGPIPE, pipefail fails the test. */}}
if printf '%s\n' "$bucket_list" | awk -v bucket="{{ $bucketName }}" '$1 "" == bucket { found = 1 } END { exit !found }'; then
if echo "$bucket_list" | awk '{print $1}' | grep -Fxq "{{ $bucketName }}"; then
echo "Bucket '{{ $bucketName }}' already exists, skipping creation."
else
echo "Creating bucket '{{ $bucketName }}'..."
+1099 -1080
View File
File diff suppressed because it is too large. Load diff

Before

Width:  |  Height:  |  Size: 55 KiB

After

Width:  |  Height:  |  Size: 55 KiB

+1 -2
View File
@@ -771,8 +771,7 @@ message FilerConf {
string replication = 3;
string ttl = 4;
string disk_type = 5;
// unset inherits from the enclosing path rule, set overrides it
optional bool fsync = 6;
bool fsync = 6;
uint32 volume_growth_count = 7;
bool read_only = 8;
string data_center = 9;
+14 -39
View File
@@ -86,7 +86,7 @@ version = "1.1.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "40c48f72fd53cd289104fc64099abca73db4166ad86ea0b4341abe65af83dadc"
dependencies = [
"windows-sys 0.60.2",
"windows-sys 0.61.2",
]
[[package]]
@@ -97,7 +97,7 @@ checksum = "291e6a250ff86cd4a820112fb8898808a366d8f9f58ce16d1f538353ad55747d"
dependencies = [
"anstyle",
"once_cell_polyfill",
"windows-sys 0.60.2",
"windows-sys 0.61.2",
]
[[package]]
@@ -510,12 +510,10 @@ dependencies = [
[[package]]
name = "aws-smithy-json"
version = "0.62.7"
version = "0.62.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "701a947f4797e52a911e114a898667c746c39feea467bbd1abd7b3721f702ffa"
checksum = "9648b0bb82a2eedd844052c6ad2a1a822d1f8e3adee5fbf668366717e428856a"
dependencies = [
"aws-smithy-runtime-api",
"aws-smithy-schema",
"aws-smithy-types",
]
@@ -565,12 +563,11 @@ dependencies = [
[[package]]
name = "aws-smithy-runtime-api"
version = "1.12.3"
version = "1.11.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9db177daa6ba8afb9ee1aefcf548c907abcf52065e394ee11a92780057fe0e8c"
checksum = "876ab3c9c29791ba4ba02b780a3049e21ec63dabda09268b175272c3733a79e6"
dependencies = [
"aws-smithy-async",
"aws-smithy-runtime-api-macros",
"aws-smithy-types",
"bytes",
"http 0.2.12",
@@ -581,33 +578,11 @@ dependencies = [
"zeroize",
]
[[package]]
name = "aws-smithy-runtime-api-macros"
version = "1.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8d7396fd9500589e62e460e987ecb671bad374934e55ec3b5f498cc7a8a8a7b7"
dependencies = [
"proc-macro2",
"quote",
"syn",
]
[[package]]
name = "aws-smithy-schema"
version = "0.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7442cb268338f0eb8278140a107c046756aa01093d8ef5e99628d34ae09c94f5"
dependencies = [
"aws-smithy-runtime-api",
"aws-smithy-types",
"http 1.4.0",
]
[[package]]
name = "aws-smithy-types"
version = "1.5.0"
version = "1.4.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "32b42fcf341259d85ca10fac9a2f6448a8ec691c6955a18e45bc3b71a85fab85"
checksum = "9d73dbfbaa8e4bc57b9045137680b958d274823509a360abfd8e1d514d40c95c"
dependencies = [
"base64-simd",
"bytes",
@@ -1350,7 +1325,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb"
dependencies = [
"libc",
"windows-sys 0.52.0",
"windows-sys 0.61.2",
]
[[package]]
@@ -2404,7 +2379,7 @@ version = "0.50.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7957b9740744892f114936ab4a57b3f487491bbeafaf8083688b16841a4240e5"
dependencies = [
"windows-sys 0.59.0",
"windows-sys 0.61.2",
]
[[package]]
@@ -3416,7 +3391,7 @@ dependencies = [
"errno 0.3.14",
"libc",
"linux-raw-sys 0.12.1",
"windows-sys 0.52.0",
"windows-sys 0.61.2",
]
[[package]]
@@ -3701,7 +3676,7 @@ version = "1.4.8"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c4db69cba1110affc0e9f7bcd48bbf87b3f4fc7c61fc9155afd4c469eb3d6c1b"
dependencies = [
"errno 0.2.8",
"errno 0.3.14",
"libc",
]
@@ -3768,7 +3743,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3a766e1110788c36f4fa1c2b71b387a7815aa65f88ce0229841826633d93723e"
dependencies = [
"libc",
"windows-sys 0.60.2",
"windows-sys 0.61.2",
]
[[package]]
@@ -3902,7 +3877,7 @@ dependencies = [
"getrandom 0.4.2",
"once_cell",
"rustix 1.1.4",
"windows-sys 0.52.0",
"windows-sys 0.61.2",
]
[[package]]
-2
View File
@@ -157,8 +157,6 @@ windows-sys = { version = "0.61", features = ["Win32_Storage_FileSystem"] }
[dev-dependencies]
tempfile = "3"
# Paused-time tests drive the write-queue batch window deterministically.
tokio = { version = "1", features = ["full", "test-util"] }
# Already a transitive dependency of aws-sdk-s3 at a single locked version;
# needed directly only for the canned HttpClient in remote_storage::s3 tests.
aws-smithy-runtime-api = "1"
-30
View File
@@ -233,25 +233,6 @@ pub struct Cli {
#[arg(long = "volume.allowUntrustedRemoteEndpoints", default_value_t = false)]
pub allow_untrusted_remote_endpoints: bool,
/// Minimum bytes an fsync batch gathers before a drained queue flushes
/// without waiting.
#[arg(long = "fsyncMinBatchBytes", default_value_t = 1024 * 1024)]
pub fsync_min_batch_bytes: u64,
/// Minimum requests an fsync batch gathers before a drained queue flushes
/// without waiting.
#[arg(long = "fsyncMinBatchRequests", default_value_t = 128)]
pub fsync_min_batch_requests: usize,
/// How long a below-minimum fsync batch waits for more requests once the
/// queue drains; 0 flushes immediately.
#[arg(
long = "fsyncBatchWait",
default_value = "500us",
value_parser = parse_go_duration
)]
pub fsync_batch_wait: std::time::Duration,
/// A file of command line options, each line in optionName=optionValue format.
#[arg(long = "options", default_value = "")]
pub options: String,
@@ -327,14 +308,6 @@ pub struct VolumeServerConfig {
/// If true, FetchAndWriteNeedle skips remote S3 endpoint validation
/// (allows loopback / link-local / metadata hosts).
pub allow_untrusted_remote_endpoints: bool,
/// Minimum bytes an fsync batch gathers before a drained queue flushes
/// without waiting.
pub fsync_min_batch_bytes: u64,
/// Minimum requests an fsync batch gathers before a drained queue flushes
/// without waiting.
pub fsync_min_batch_requests: usize,
/// How long a below-minimum fsync batch waits for stragglers.
pub fsync_batch_wait: std::time::Duration,
}
pub use crate::storage::needle_map::NeedleMapKind;
@@ -965,9 +938,6 @@ fn resolve_config_with_env(cli: Cli, env: EnvLookup<'_>) -> VolumeServerConfig {
concurrent_download_limit: cli.concurrent_download_limit_mb as i64 * 1024 * 1024,
inflight_upload_data_timeout: cli.inflight_upload_data_timeout,
inflight_download_data_timeout: cli.inflight_download_data_timeout,
fsync_min_batch_bytes: cli.fsync_min_batch_bytes,
fsync_min_batch_requests: cli.fsync_min_batch_requests,
fsync_batch_wait: cli.fsync_batch_wait,
has_slow_read: cli.has_slow_read,
read_buffer_size_mb: cli.read_buffer_size_mb,
ldb_timeout: cli.ldb_timeout,
+2 -12
View File
@@ -23,7 +23,7 @@ use seaweed_volume::server::request_id::GrpcRequestIdLayer;
use seaweed_volume::server::volume_server::{
RuntimeMetricsConfig, VolumeServerState, build_metrics_router,
};
use seaweed_volume::server::write_queue::{self, WriteQueue};
use seaweed_volume::server::write_queue::WriteQueue;
use seaweed_volume::storage::store::Store;
use seaweed_volume::storage::types::DiskType;
@@ -436,17 +436,7 @@ async fn run(
// The write queue is always on: durable (fsync) uploads go through it so
// a batch shares one .dat/.idx sync, the way Go's per-volume async writer
// group-commits. Non-fsync writes stay on the synchronous path.
let min_bytes = usize::try_from(config.fsync_min_batch_bytes)
.map_err(|_| "fsyncMinBatchBytes exceeds this platform's maximum usize value")?;
let wq = WriteQueue::new(
state.clone(),
128,
write_queue::SyncBatchOption {
min_bytes,
min_requests: config.fsync_min_batch_requests,
wait: config.fsync_batch_wait,
},
);
let wq = WriteQueue::new(state.clone(), 128);
let _ = state.write_queue.set(wq);
// Set initial metric gauges for concurrent limits and max volumes
-3
View File
@@ -393,9 +393,6 @@ mod tests {
fix_jpg_orientation: false,
read_mode: ReadMode::Local,
allow_untrusted_remote_endpoints: false,
fsync_min_batch_bytes: 1024 * 1024,
fsync_min_batch_requests: 128,
fsync_batch_wait: std::time::Duration::from_micros(500),
cpu_profile: String::new(),
mem_profile: String::new(),
compaction_byte_per_second: 0,
+27 -224
View File
@@ -207,15 +207,9 @@ fn read_needle_for_get(
if request.try_meta_first(plan.size()) {
plan.read_meta(&mut n)?;
if n.cookie != cookie {
// The meta read answered the request.
plan.note_meta_only_success();
return Ok(None);
}
if request.served_from_source(&n) {
if request.is_head {
// HEAD ends here; no payload read will record the success.
plan.note_meta_only_success();
}
let info = plan.into_stream_info(&n);
return Ok(Some((n, Some(info))));
}
@@ -243,9 +237,6 @@ struct StreamingBody {
hold_read_lock_for_stream: bool,
_held_read_lease: Option<crate::storage::volume::DataFileReadLease>,
io_unavailable: Arc<crate::storage::volume::IoUnavailable>,
/// Chunk read failures feed the volume's io-error streak, and finishing
/// the stream clears it — Go readNeedleDataInto parity.
io_errors: Arc<crate::storage::io_error::IoErrorTracker>,
/// Pending chunk read; it hands `buf` back with the result.
pending: Option<tokio::task::JoinHandle<(bytes::BytesMut, std::io::Result<bytes::Bytes>)>>,
/// Chunk buffer, reclaimed once the previous frame has been written out.
@@ -307,18 +298,12 @@ impl http_body::Body for StreamingBody {
),
)));
}
// Hyper may not poll again once Content-Length is
// satisfied, so the read's success is recorded here.
self.io_errors.check_read_write_error(None);
}
return std::task::Poll::Ready(Some(Ok(http_body::Frame::data(
chunk,
))));
}
Ok(Err(e)) => {
self.io_errors.check_read_write_error(Some(&e));
return std::task::Poll::Ready(Some(Err(e)));
}
Ok(Err(e)) => return std::task::Poll::Ready(Some(Err(e))),
Err(e) => {
return std::task::Poll::Ready(Some(Err(std::io::Error::other(e))));
}
@@ -329,7 +314,6 @@ impl http_body::Body for StreamingBody {
let total = self.data_size as usize;
if self.pos >= total {
self.io_errors.check_read_write_error(None);
return std::task::Poll::Ready(None);
}
@@ -1121,10 +1105,6 @@ async fn get_or_head_handler_inner(
let (etag, last_modified_str) = etag_and_last_modified(&n);
if let Some(resp) = not_modified_response(&n, &headers, &etag, &last_modified_str) {
// A request that ends here was answered by the meta read alone.
if let Some(io_errors) = strategy.io_errors() {
io_errors.check_read_write_error(None);
}
return resp;
}
@@ -1218,19 +1198,6 @@ struct ReadPlan {
strategy: ReadStrategy,
}
impl ReadStrategy {
/// The tracker a terminal meta read reports success to, when the reply
/// is served from the data file.
fn io_errors(&self) -> Option<&Arc<crate::storage::io_error::IoErrorTracker>> {
match self {
ReadStrategy::Stream(info)
| ReadStrategy::HeadFromMeta(info)
| ReadStrategy::RangeFromSource(info) => Some(&info.io_errors),
ReadStrategy::Buffered => None,
}
}
}
/// The 401 reply for a bad read JWT; Go checks it before NewVolumeId, so a
/// bad path with JWT enabled is a 401, not a 400.
fn reject_read_jwt(
@@ -1519,78 +1486,41 @@ fn not_modified_response(
etag: &str,
last_modified_str: &Option<String>,
) -> Option<Response> {
let not_modified = || {
let mut resp = StatusCode::NOT_MODIFIED.into_response();
resp.headers_mut()
.insert(header::ETAG, etag.parse().unwrap());
if let Some(lm) = last_modified_str {
resp.headers_mut()
.insert(header::LAST_MODIFIED, lm.parse().unwrap());
}
resp
};
// If-Modified-Since applies only when there is no If-None-Match (RFC 9110 13.2.2).
if headers.contains_key(header::IF_NONE_MATCH) {
let inm = headers
.get_all(header::IF_NONE_MATCH)
.iter()
.map(|v| v.to_str().unwrap_or(""))
.collect::<Vec<_>>()
.join(",");
return etag_list_matches(&inm, etag).then(not_modified);
}
// If-Modified-Since first (Go order), then If-None-Match.
if n.last_modified > 0
&& let Some(ims_str) = headers
.get(header::IF_MODIFIED_SINCE)
.and_then(|v| v.to_str().ok())
&& let Some(ims_header) = headers.get(header::IF_MODIFIED_SINCE)
&& let Ok(ims_str) = ims_header.to_str()
{
// Parse HTTP date format: "Mon, 02 Jan 2006 15:04:05 GMT"
if let Ok(ims_time) =
chrono::NaiveDateTime::parse_from_str(ims_str, "%a, %d %b %Y %H:%M:%S GMT")
&& (n.last_modified as i64) <= ims_time.and_utc().timestamp()
{
return Some(not_modified());
let mut resp = StatusCode::NOT_MODIFIED.into_response();
if let Some(lm) = last_modified_str {
resp.headers_mut()
.insert(header::LAST_MODIFIED, lm.parse().unwrap());
}
// Go sets ETag after the 304 paths, so a 304 has no ETag.
return Some(resp);
}
}
if let Some(if_none_match) = headers.get(header::IF_NONE_MATCH)
&& let Ok(inm) = if_none_match.to_str()
&& inm == etag
{
let mut resp = StatusCode::NOT_MODIFIED.into_response();
if let Some(lm) = last_modified_str {
resp.headers_mut()
.insert(header::LAST_MODIFIED, lm.parse().unwrap());
}
// Go sets ETag after the 304 paths, so a 304 has no ETag.
return Some(resp);
}
None
}
/// Weak comparison against "*" or a comma-separated entity-tag list (RFC 9110 13.1.2).
fn etag_list_matches(header: &str, etag: &str) -> bool {
if header.trim() == "*" {
return true;
}
let etag = etag.trim_matches('"');
split_etag_list(header).into_iter().any(|tag| {
let tag = tag.trim();
let tag = tag.strip_prefix("W/").unwrap_or(tag);
// Empty list elements are ignored, so padding like a trailing
// comma cannot match an empty opaque tag.
!tag.is_empty() && tag.trim_matches('"') == etag
})
}
/// Splits an entity-tag list on the commas between tags; a comma inside a
/// quoted opaque tag does not split.
fn split_etag_list(header: &str) -> Vec<&str> {
let mut tags = Vec::new();
let (mut start, mut quoted) = (0, false);
for (i, b) in header.bytes().enumerate() {
match b {
b'"' => quoted = !quoted,
b',' if !quoted => {
tags.push(&header[start..i]);
start = i + 1;
}
_ => {}
}
}
tags.push(&header[start..]);
tags
}
/// The 200 reply headers, and the extension, which falls back to the stored name's.
fn read_response_headers(
n: &Needle,
@@ -1792,7 +1722,6 @@ fn stream_response(
data_file_access_control: info.data_file_access_control,
hold_read_lock_for_stream: !state.has_slow_read,
io_unavailable: info.io_unavailable,
io_errors: info.io_errors,
pending: None,
buf: bytes::BytesMut::new(),
state: tracking_state,
@@ -2192,21 +2121,9 @@ fn handle_range_request_from_source(
return Ok(Vec::new());
}
let mut buf = vec![0u8; length as usize];
match info
.source
.read_exact_at(&mut buf, info.data_file_offset + start as u64)
{
// A finished range read clears the streak a failed meta read
// counted, like readNeedleDataInto's terminal report.
Ok(()) => {
info.io_errors.check_read_write_error(None);
Ok(buf)
}
Err(e) => {
info.io_errors.check_read_write_error(Some(&e));
Err(e)
}
}
info.source
.read_exact_at(&mut buf, info.data_file_offset + start as u64)?;
Ok(buf)
};
if ranges.len() == 1 {
@@ -5308,7 +5225,6 @@ mod tests {
hold_read_lock_for_stream: true,
_held_read_lease: None,
io_unavailable: Arc::default(),
io_errors: Arc::default(),
pending: None,
buf: bytes::BytesMut::new(),
state: None,
@@ -5681,80 +5597,6 @@ mod tests {
delete_handler(State(state.clone()), request).await.status()
}
#[test]
fn test_not_modified_checks_if_none_match_first() {
let n = Needle {
last_modified: 1700000000,
..Needle::default()
};
let (etag, last_modified_str) = etag_and_last_modified(&n);
let bare = n.etag();
let before = "Tue, 14 Nov 2023 22:12:20 GMT".to_string();
let after = "Tue, 14 Nov 2023 22:14:20 GMT".to_string();
let cases: Vec<(Vec<(axum::http::HeaderName, String)>, bool)> = vec![
(vec![(header::IF_NONE_MATCH, format!("\"{bare}\""))], true),
(vec![(header::IF_NONE_MATCH, format!("W/\"{bare}\""))], true),
(
vec![(header::IF_NONE_MATCH, format!("\"x\", \"{bare}\""))],
true,
),
(vec![(header::IF_NONE_MATCH, "*".into())], true),
(vec![(header::IF_NONE_MATCH, "\"x\"".into())], false),
(
vec![
(header::IF_NONE_MATCH, "\"x\"".into()),
(header::IF_NONE_MATCH, format!("\"{bare}\"")),
],
true,
),
(
vec![(header::IF_NONE_MATCH, format!("\"{bare},old\""))],
false,
),
(
vec![(header::IF_NONE_MATCH, format!("\"x\",,\"{bare}\""))],
true,
),
(
vec![
(header::IF_NONE_MATCH, "\"x\"".into()),
(header::IF_MODIFIED_SINCE, after.clone()),
],
false,
),
(
vec![
(header::IF_NONE_MATCH, "".into()),
(header::IF_MODIFIED_SINCE, after.clone()),
],
false,
),
(vec![(header::IF_MODIFIED_SINCE, after)], true),
(vec![(header::IF_MODIFIED_SINCE, before)], false),
];
for (pairs, want) in cases {
let mut headers = HeaderMap::new();
for (name, value) in &pairs {
headers.append(name, value.parse().unwrap());
}
assert_eq!(
not_modified_response(&n, &headers, &etag, &last_modified_str).is_some(),
want,
"{pairs:?}"
);
}
let mut headers = HeaderMap::new();
headers.insert(
header::IF_NONE_MATCH,
format!("\"{bare}\"").parse().unwrap(),
);
let resp = not_modified_response(&n, &headers, &etag, &last_modified_str).unwrap();
assert!(resp.headers().contains_key(header::ETAG));
assert!(resp.headers().contains_key(header::LAST_MODIFIED));
}
/// A GET whose needle read is parked must not hold the store lock: a
/// writer, here a real append to the same volume, must get through.
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
@@ -5929,44 +5771,6 @@ mod tests {
assert_eq!(status, StatusCode::NOT_FOUND);
}
/// A ranged read served from the data file reports its outcome to the
/// io-error streak, like readNeedleDataInto: a finished range clears the
/// count a failed meta read left.
#[test]
fn test_range_from_source_reports_to_io_error_streak() {
use std::io::Write;
let tmp = tempfile::TempDir::new().unwrap();
let file = std::fs::OpenOptions::new()
.read(true)
.write(true)
.create(true)
.open(tmp.path().join("1.dat"))
.unwrap();
let data: Vec<u8> = (0..256u32).map(|i| i as u8).collect();
(&file).write_all(&data).unwrap();
let io_errors = Arc::new(crate::storage::io_error::IoErrorTracker::default());
let eio = crate::storage::volume::media_error_for_test();
io_errors.check_read_write_error(Some(&eio));
io_errors.check_read_write_error(Some(&eio));
assert_eq!(io_errors.get_io_error_state().1, 2);
let info = crate::storage::volume::NeedleStreamInfo {
source: crate::storage::volume::NeedleStreamSource::Local(file),
data_file_offset: 0,
data_size: data.len() as u32,
data_file_access_control: Arc::default(),
io_unavailable: Arc::default(),
io_errors: io_errors.clone(),
needle_id: NeedleId(1),
checksum: 0,
};
let resp = handle_range_request_from_source("bytes=10-19", info, HeaderMap::new(), None);
assert_eq!(resp.status(), StatusCode::PARTIAL_CONTENT);
assert_eq!(io_errors.get_io_error_state().1, 0);
}
/// Range is read as Go reads it: a byte its parseRange cannot parse is a
/// 416, Unicode whitespace around a range is trimmed, an empty value is no
/// range, and HEAD ignores it.
@@ -7419,7 +7223,6 @@ mod tests {
.set(crate::server::write_queue::WriteQueue::new(
state.clone(),
128,
crate::server::write_queue::SyncBatchOption::default(),
))
.is_ok()
);
-3
View File
@@ -132,9 +132,6 @@ mod tests {
fix_jpg_orientation: false,
read_mode: ReadMode::Local,
allow_untrusted_remote_endpoints: false,
fsync_min_batch_bytes: 1024 * 1024,
fsync_min_batch_requests: 128,
fsync_batch_wait: std::time::Duration::from_micros(500),
cpu_profile: String::new(),
mem_profile: String::new(),
compaction_byte_per_second: 0,
+12 -164
View File
@@ -56,27 +56,6 @@ const MAX_CONCURRENT_VOLUME_WRITES: usize = 8;
/// retirement is the volume leaving the store.
const IDLE_WORKER_POLL: std::time::Duration = std::time::Duration::from_secs(60);
/// Tunes the per-volume durable-write worker. Once the queue drains with the
/// batch still below min_bytes and min_requests, the batch stays open up to
/// wait for stragglers instead of paying an fsync per request. Mirrors Go's
/// storage.SyncBatchOption.
#[derive(Clone, Copy)]
pub struct SyncBatchOption {
pub min_bytes: usize,
pub min_requests: usize,
pub wait: std::time::Duration,
}
impl Default for SyncBatchOption {
fn default() -> Self {
SyncBatchOption {
min_bytes: 1024 * 1024,
min_requests: MAX_BATCH_SIZE,
wait: std::time::Duration::from_micros(500),
}
}
}
/// Handle for submitting write requests to the per-volume workers.
#[derive(Clone)]
pub struct WriteQueue {
@@ -88,35 +67,23 @@ pub struct WriteQueue {
/// alone would multiply the old single queue's capacity by volume count.
queue_permits: Arc<Semaphore>,
workers: Arc<Mutex<HashMap<VolumeId, mpsc::Sender<WriteRequest>>>>,
sync_batch: SyncBatchOption,
/// Submissions accepted, for tests asserting which writes route here.
#[cfg(test)]
submits: Arc<std::sync::atomic::AtomicU64>,
/// Times a worker armed the straggler window, for tests that must land a
/// write inside an already-open batch.
#[cfg(test)]
windows_armed: Arc<std::sync::atomic::AtomicU64>,
}
impl WriteQueue {
/// Create a new write queue. Per-volume workers are spawned lazily on the
/// first submission for that volume.
pub fn new(
state: Arc<VolumeServerState>,
capacity: usize,
sync_batch: SyncBatchOption,
) -> Self {
pub fn new(state: Arc<VolumeServerState>, capacity: usize) -> Self {
WriteQueue {
state,
capacity,
io_permits: Arc::new(Semaphore::new(MAX_CONCURRENT_VOLUME_WRITES)),
queue_permits: Arc::new(Semaphore::new(capacity)),
workers: Arc::new(Mutex::new(HashMap::new())),
sync_batch,
#[cfg(test)]
submits: Arc::new(std::sync::atomic::AtomicU64::new(0)),
#[cfg(test)]
windows_armed: Arc::new(std::sync::atomic::AtomicU64::new(0)),
}
}
@@ -126,12 +93,6 @@ impl WriteQueue {
self.submits.load(std::sync::atomic::Ordering::Relaxed)
}
/// How many batch windows workers have armed so far.
#[cfg(test)]
fn windows_armed_for_test(&self) -> u64 {
self.windows_armed.load(std::sync::atomic::Ordering::Relaxed)
}
/// Sender for the volume's worker, spawning one on first use.
fn worker_for(&self, volume_id: VolumeId) -> mpsc::Sender<WriteRequest> {
let mut workers = self.workers.lock().unwrap();
@@ -146,9 +107,6 @@ impl WriteQueue {
state: self.state.clone(),
io_permits: self.io_permits.clone(),
workers: self.workers.clone(),
sync_batch: self.sync_batch,
#[cfg(test)]
windows_armed: self.windows_armed.clone(),
}
.run(),
);
@@ -233,9 +191,6 @@ struct VolumeWriteWorker {
state: Arc<VolumeServerState>,
io_permits: Arc<Semaphore>,
workers: Arc<Mutex<HashMap<VolumeId, mpsc::Sender<WriteRequest>>>>,
sync_batch: SyncBatchOption,
#[cfg(test)]
windows_armed: Arc<std::sync::atomic::AtomicU64>,
}
impl VolumeWriteWorker {
@@ -283,40 +238,14 @@ impl VolumeWriteWorker {
let mut batch = Vec::with_capacity(MAX_BATCH_SIZE);
let mut batch_bytes: usize = first.needle.data.len();
batch.push(first);
// Armed once the queue first drains below the minimum batch; holds the
// batch open for stragglers so the wait bound is per batch, not per
// lull (Go's batchWait).
let mut window_deadline = None;
while batch.len() < MAX_BATCH_SIZE && batch_bytes < MAX_BATCH_BYTES {
if window_deadline.is_some_and(|d| tokio::time::Instant::now() >= d) {
break;
}
let req = match self.rx.try_recv() {
Ok(req) => req,
Err(mpsc::error::TryRecvError::Empty) => {
let sync = self.sync_batch;
if sync.wait.is_zero()
|| batch_bytes >= sync.min_bytes
|| batch.len() >= sync.min_requests
{
// a drained queue flushes right away once it holds enough
break;
}
let deadline = *window_deadline.get_or_insert_with(|| {
#[cfg(test)]
self.windows_armed
.fetch_add(1, std::sync::atomic::Ordering::Relaxed);
tokio::time::Instant::now() + sync.wait
});
match tokio::time::timeout_at(deadline, self.rx.recv()).await {
Ok(Some(req)) => req,
Ok(None) | Err(_) => break,
}
match self.rx.try_recv() {
Ok(req) => {
batch_bytes += req.needle.data.len();
batch.push(req);
}
Err(mpsc::error::TryRecvError::Disconnected) => break,
};
batch_bytes += req.needle.data.len();
batch.push(req);
Err(_) => break,
}
}
debug!("processing write batch of {} requests", batch.len());
@@ -451,7 +380,7 @@ mod tests {
async fn test_write_queue_submit_no_volume() {
// Submit a write to a non-existent volume -- should return VolumeError::NotFound.
let state = make_test_state();
let queue = WriteQueue::new(state, MAX_BATCH_SIZE, SyncBatchOption::default());
let queue = WriteQueue::new(state, MAX_BATCH_SIZE);
let needle = Needle {
id: 1.into(),
@@ -473,7 +402,7 @@ mod tests {
async fn test_write_queue_concurrent_submissions() {
// Submit multiple concurrent writes -- all should complete (with errors since no volume).
let state = make_test_state();
let queue = WriteQueue::new(state, MAX_BATCH_SIZE, SyncBatchOption::default());
let queue = WriteQueue::new(state, MAX_BATCH_SIZE);
let mut handles = Vec::new();
for i in 0..10u64 {
@@ -501,7 +430,7 @@ mod tests {
async fn test_write_queue_batching() {
// Verify that many concurrent writes get processed (testing the batching path).
let state = make_test_state();
let queue = WriteQueue::new(state, MAX_BATCH_SIZE, SyncBatchOption::default());
let queue = WriteQueue::new(state, MAX_BATCH_SIZE);
// Submit MAX_BATCH_SIZE requests concurrently
let mut handles = Vec::new();
@@ -593,7 +522,7 @@ mod tests {
async fn test_write_queue_dropped_sender() {
// When the queue is dropped, subsequent submits should fail gracefully.
let state = make_test_state();
let queue = WriteQueue::new(state, 1, SyncBatchOption::default());
let queue = WriteQueue::new(state, 1);
// Clone then drop the original -- the worker keeps running via its rx handle.
let queue2 = queue.clone();
@@ -736,7 +665,7 @@ mod tests {
.unwrap();
}
}
let queue = WriteQueue::new(state, MAX_BATCH_SIZE, SyncBatchOption::default());
let queue = WriteQueue::new(state, MAX_BATCH_SIZE);
let in_append = Arc::new(Barrier::new(2));
let release = Arc::new(Barrier::new(2));
@@ -805,85 +734,4 @@ mod tests {
assert!(a1.await.unwrap().is_ok());
assert!(a2.await.unwrap().is_ok());
}
/// A durable write below the minimum batch rides out its window so a
/// straggler joins the same sync instead of each paying an fsync alone —
/// Go's batchWait. Paused time makes the hold observable: a lone write can
/// only complete once the runtime auto-advances to the window's deadline.
#[tokio::test(start_paused = true)]
async fn test_batch_window_defers_flush_for_stragglers() {
use crate::config::MinFreeSpace;
use crate::storage::types::{DiskType, NeedleId};
use crate::storage::volume::VolumeSpec;
use std::time::Duration;
let tmp = tempfile::TempDir::new().unwrap();
let dir = tmp.path().to_str().unwrap();
let state = make_test_state();
{
let mut store = state.store.write().unwrap();
store
.add_location(
dir,
dir,
10,
DiskType::HardDrive,
MinFreeSpace::Percent(1.0),
Vec::new(),
)
.unwrap();
store
.add_volume(VolumeId(1), DiskType::HardDrive, &VolumeSpec::default())
.unwrap();
}
let queue = WriteQueue::new(
state,
MAX_BATCH_SIZE,
SyncBatchOption {
min_bytes: usize::MAX,
min_requests: MAX_BATCH_SIZE,
wait: Duration::from_secs(30),
},
);
let needle = |id: u64| Needle {
id: NeedleId(id),
cookie: 0x1111.into(),
data: vec![id as u8; 8],
data_size: 8,
..Needle::default()
};
let start = tokio::time::Instant::now();
queue.submit(VolumeId(1), needle(1), true).await.unwrap();
assert!(start.elapsed() >= Duration::from_secs(30));
// Wait for the worker to take the second write and open its batch
// window, so the straggler provably lands inside an open batch
// instead of arriving during the initial drain.
let armed = queue.windows_armed_for_test();
let second = tokio::spawn({
let queue = queue.clone();
async move { queue.submit(VolumeId(1), needle(2), true).await }
});
for _ in 0..10_000 {
if queue.windows_armed_for_test() > armed {
break;
}
tokio::task::yield_now().await;
}
assert!(queue.windows_armed_for_test() > armed);
let third = tokio::spawn({
let queue = queue.clone();
async move { queue.submit(VolumeId(1), needle(3), true).await }
});
let (second, third) = tokio::join!(second, third);
second.unwrap().unwrap();
third.unwrap().unwrap();
// One shared window ends both writes ~30s after their batch armed;
// back-to-back windows would keep the third out to ~90s.
let elapsed = start.elapsed();
assert!(elapsed >= Duration::from_secs(60) && elapsed < Duration::from_secs(90));
}
}
+1 -5
View File
@@ -740,11 +740,7 @@ impl DiskLocation {
if vol.is_read_only() {
continue;
}
// A preallocated volume already holds its space on disk.
let dat_size = vol
.dat_file_size()
.unwrap_or(0)
.max(vol.dat_allocated_size());
let dat_size = vol.dat_file_size().unwrap_or(0);
let idx_size = vol.idx_file_size();
let used = dat_size + idx_size;
if volume_size_limit > used {
+43 -142
View File
@@ -18,7 +18,7 @@ use crate::storage::erasure_coding::ec_volume::{
};
use crate::storage::needle::needle::{Needle, get_actual_size};
use crate::storage::needle_map::NeedleMapKind;
use crate::storage::super_block::{ReplicaPlacement, SUPER_BLOCK_SIZE, SuperBlock};
use crate::storage::super_block::{ReplicaPlacement, SUPER_BLOCK_SIZE};
use crate::storage::types::*;
use crate::storage::volume::{CompactionJob, VifVolumeInfo, Volume, VolumeError, VolumeSpec};
@@ -762,8 +762,8 @@ impl Store {
}
/// Configure a volume's replica placement on disk.
/// The volume must already be unmounted. Rewrites the .vif and the .dat
/// superblock so both carry the new placement.
/// The volume must already be unmounted. This opens the .dat file directly,
/// modifies the replica_placement byte (offset 1), and writes it back.
pub fn configure_volume(&self, vid: VolumeId, rp: ReplicaPlacement) -> Result<(), VolumeError> {
let (_, base_path, _) = self.find_volume_file_base(vid).ok_or_else(|| {
VolumeError::Io(io::Error::new(
@@ -775,7 +775,7 @@ impl Store {
let mut vif = load_vif_volume_info(&vif_path)?;
vif.replication = rp.to_string();
save_vif_volume_info(&vif_path, &vif)?;
update_dat_replica_placement(&format!("{}.dat", base_path), vid, rp)
Ok(())
}
// ---- Read / Write / Delete ----
@@ -943,7 +943,11 @@ impl Store {
let current = loc.max_volume_count.load(Ordering::Relaxed);
let (_, free) = super::disk_location::get_disk_stats(&loc.directory);
let unused_space = loc.unused_space(volume_size_limit);
let unused_space = if self.get_preallocate() {
0
} else {
loc.unused_space(volume_size_limit)
};
let unclaimed = (free as i64) - (unused_space as i64);
let vol_count = loc.volumes_len() as i32;
@@ -1740,61 +1744,6 @@ fn strip_volume_suffix(filename: &str) -> Option<&str> {
.or_else(|| filename.strip_suffix(".idx"))
}
/// Rewrites the replica placement in a .dat header for a volume that is not
/// mounted; a missing .dat means a remote-only volume, where the .vif carries
/// the placement.
fn update_dat_replica_placement(
dat_path: &str,
vid: VolumeId,
rp: ReplicaPlacement,
) -> Result<(), VolumeError> {
use std::io::{Read, Seek, SeekFrom, Write};
let mut file = match std::fs::OpenOptions::new()
.read(true)
.write(true)
.open(dat_path)
{
Ok(f) => f,
Err(e) if e.kind() == io::ErrorKind::NotFound => return Ok(()),
Err(e) => {
return Err(VolumeError::Io(io::Error::new(
e.kind(),
format!("volume {} failed to open {}: {}", vid, dat_path, e),
)))
}
};
let mut header = [0u8; SUPER_BLOCK_SIZE];
file.read_exact(&mut header).map_err(|e| {
VolumeError::Io(io::Error::new(
e.kind(),
format!("volume {} failed to parse superblock: {}", vid, e),
))
})?;
let extra_size = u16::from_be_bytes([header[6], header[7]]) as usize;
let mut block = header.to_vec();
block.resize(SUPER_BLOCK_SIZE + extra_size, 0);
if extra_size > 0 {
file.read_exact(&mut block[SUPER_BLOCK_SIZE..]).map_err(|e| {
VolumeError::Io(io::Error::new(
e.kind(),
format!("volume {} failed to parse superblock: {}", vid, e),
))
})?;
}
let mut super_block = SuperBlock::from_bytes(&block).map_err(|e| {
VolumeError::Io(io::Error::new(
io::ErrorKind::InvalidData,
format!("volume {} failed to parse superblock: {}", vid, e),
))
})?;
super_block.replica_placement = rp;
file.seek(SeekFrom::Start(0))?;
file.write_all(&super_block.to_bytes())?;
file.sync_all()?;
Ok(())
}
fn load_vif_volume_info(path: &str) -> Result<VifVolumeInfo, VolumeError> {
let content = match std::fs::read_to_string(path) {
Ok(c) => c,
@@ -2032,33 +1981,6 @@ mod tests {
assert_eq!(store.total_volume_count(), 1);
}
// Mirrors Go's TestConfigureVolumeUpdatesSuperblock: configure must leave
// the .dat header and the .vif both on the new placement.
#[test]
fn test_configure_volume_updates_superblock() {
let tmp = TempDir::new().unwrap();
let dir = tmp.path().to_str().unwrap();
let mut store = make_test_store(&[dir]);
store
.add_volume(VolumeId(1), DiskType::HardDrive, &VolumeSpec::default())
.unwrap();
assert!(store.unmount_volume(VolumeId(1)).unwrap());
let rp = ReplicaPlacement::from_string("010").unwrap();
store.configure_volume(VolumeId(1), rp).unwrap();
let base = volume_file_name(dir, "", VolumeId(1));
let vif = load_vif_volume_info(&format!("{}.vif", base)).unwrap();
assert_eq!(vif.replication, "010");
let dat = std::fs::read(format!("{}.dat", base)).unwrap();
let super_block = SuperBlock::from_bytes(&dat).unwrap();
assert_eq!(super_block.replica_placement, rp);
assert!(store.configure_volume(VolumeId(2), rp).is_err());
}
#[test]
fn test_mount_volume_by_id_hint_skips_lone_sidecar() {
// A lone .vif/.idx sidecar (e.g. an EC sidecar whose .ecx is on a
@@ -2557,7 +2479,7 @@ mod tests {
}
#[test]
fn test_maybe_adjust_volume_max_reserves_plain_volumes_while_preallocating() {
fn test_maybe_adjust_volume_max_honors_preallocate_flag() {
let tmp = TempDir::new().unwrap();
let dir = tmp.path().to_str().unwrap();
@@ -2572,6 +2494,13 @@ mod tests {
Vec::new(),
)
.unwrap();
// Use a large volume_size_limit so the unused-space difference between
// preallocate=true (0) and preallocate=false (~2 × limit) is big enough
// that integer-division rounding and disk-free fluctuations between the
// two get_disk_stats calls cannot make the quotients equal.
store
.volume_size_limit
.store(100 * 1024 * 1024, Ordering::Relaxed);
store
.add_volume(
VolumeId(61),
@@ -2582,67 +2511,39 @@ mod tests {
},
)
.unwrap();
// Room for 2.5 volumes: the plain volume claims one, leaving one more slot.
let (_, free) = crate::storage::disk_location::get_disk_stats(dir);
let volume_size_limit = (free as f64 / 2.5) as u64;
if volume_size_limit == 0 {
return; // cannot determine free disk space
}
store
.volume_size_limit
.store(volume_size_limit, Ordering::Relaxed);
store.locations[0].original_max_volume_count = 0;
store.set_preallocate(true);
store.maybe_adjust_volume_max();
assert_eq!(
store.locations[0].max_volume_count.load(Ordering::Relaxed),
2
);
}
#[cfg(target_os = "linux")]
#[test]
fn test_unused_space_counts_preallocated_volumes_as_full() {
const LIMIT: u64 = 1 << 20;
let tmp = TempDir::new().unwrap();
let dir = tmp.path().to_str().unwrap();
let mut store = Store::new(NeedleMapKind::InMemory);
store
.add_location(
dir,
dir,
2,
.add_volume(
VolumeId(62),
DiskType::HardDrive,
MinFreeSpace::Percent(1.0),
Vec::new(),
&VolumeSpec {
collection: "preallocate_case",
..Default::default()
},
)
.unwrap();
for (vid, preallocate) in [(VolumeId(71), LIMIT), (VolumeId(72), 0)] {
store
.add_volume(
vid,
DiskType::HardDrive,
&VolumeSpec {
collection: "preallocate_case",
preallocate,
..Default::default()
},
)
for vid in [VolumeId(61), VolumeId(62)] {
let dat_path = store.find_volume(vid).unwrap().1.dat_path();
std::fs::OpenOptions::new()
.write(true)
.open(dat_path)
.unwrap()
.set_len((crate::storage::super_block::SUPER_BLOCK_SIZE + 1) as u64)
.unwrap();
}
let preallocated = store.find_volume(VolumeId(71)).unwrap().1;
if preallocated.dat_allocated_size() < LIMIT {
return; // the filesystem does not support fallocate
}
store.locations[0].original_max_volume_count = 0;
store.locations[0]
.max_volume_count
.store(0, Ordering::Relaxed);
let plain = store.find_volume(VolumeId(72)).unwrap().1;
let plain_dat = plain.dat_file_size().unwrap();
// The filesystem may round a small file up to a larger allocation unit.
let plain_used = plain_dat.max(plain.dat_allocated_size()) + plain.idx_file_size();
assert_eq!(store.locations[0].unused_space(LIMIT), LIMIT - plain_used);
store.set_preallocate(false);
assert!(store.maybe_adjust_volume_max());
let without_preallocate = store.locations[0].max_volume_count.load(Ordering::Relaxed);
store.set_preallocate(true);
assert!(store.maybe_adjust_volume_max());
let with_preallocate = store.locations[0].max_volume_count.load(Ordering::Relaxed);
assert!(with_preallocate > without_preallocate);
}
#[test]
+9 -180
View File
@@ -836,7 +836,7 @@ impl NeedleReadPlan {
pub(crate) fn read_meta(&mut self, n: &mut Needle) -> Result<(), VolumeError> {
let (source, size, version) = (&self.source, self.size, self.version);
let read_at = |buf: &mut [u8], at: u64| Ok(source.read_exact_at(buf, at)?);
let result = with_offset_retry(&mut self.offset, |off| {
with_offset_retry(&mut self.offset, |off| {
if version == VERSION_1 {
// A V1 body is all data, so its "meta" is the whole record.
let mut buf = vec![0u8; get_actual_size(size, version) as usize];
@@ -847,26 +847,13 @@ impl NeedleReadPlan {
} else {
Volume::read_needle_meta_blob_and_parse(read_at, n, off, size, version)
}
});
// Report failures only: a successful meta read precedes the payload
// read, so clearing the streak here would reset the count before
// every failing download and the volume could never reach quarantine.
if let Err(VolumeError::Io(e)) = &result {
self.io_errors.check_read_write_error(Some(e));
}
result?;
})?;
if needle_expired(n) {
return Err(VolumeError::NotFound);
}
Ok(())
}
/// A successful meta read ended the request (e.g. HEAD), so it may reset
/// the streak — unlike a meta read that precedes a payload read.
pub(crate) fn note_meta_only_success(&self) {
self.io_errors.check_read_write_error(None);
}
/// For a reply served from the data file, after `read_meta` filled `n`.
pub(crate) fn into_stream_info(self, n: &Needle) -> NeedleStreamInfo {
// V1 data starts right after the header, V2/V3 after the DataSize field.
@@ -881,7 +868,6 @@ impl NeedleReadPlan {
data_size: n.data_size,
data_file_access_control: self.data_file_access_control,
io_unavailable: self.io_unavailable,
io_errors: self.io_errors,
needle_id: self.needle_id,
checksum: n.checksum.0,
}
@@ -1158,9 +1144,6 @@ pub struct NeedleStreamInfo {
pub data_file_access_control: Arc<DataFileAccessControl>,
/// Checked before each chunk: the volume can become unavailable mid-stream.
pub(crate) io_unavailable: Arc<IoUnavailable>,
/// Streamed chunks report read failures here and clear the streak when
/// the whole body has been emitted, mirroring readNeedleDataInto.
pub(crate) io_errors: Arc<IoErrorTracker>,
pub needle_id: NeedleId,
/// Checksum stored in the needle tail, verified once the last chunk has
/// been read — before that frame is emitted.
@@ -2935,52 +2918,18 @@ impl Volume {
}
// Cookie validation for existing needle (matches Go: check whenever nm.Get returns ok)
let indexed = match io.nm.as_ref() {
Some(nm) => nm.get(n.id)?,
None => None,
};
if let Some(nv) = indexed
&& !nv.offset.is_zero()
if let Some(nm) = &io.nm
&& let Some(nv) = nm.get(n.id)?
{
let mut existing = Needle::default();
// Read only the header to check cookie
self.read_needle_header_unlocked(io, &mut existing, nv.offset.to_actual_offset())?;
if existing.id != n.id && existing.size != nv.size {
// a 4-byte index entry can index past the 32 GiB wrap: the
// low offset may hold another needle while the real record
// sits at the wrapped offset, where reads already retry
#[cfg(not(feature = "5bytes"))]
if nv.offset.to_actual_offset() < MAX_POSSIBLE_VOLUME_SIZE as i64 {
let mut wrapped = Needle::default();
match self.read_needle_header_unlocked(
io,
&mut wrapped,
nv.offset.to_actual_offset() + MAX_POSSIBLE_VOLUME_SIZE as i64,
) {
Ok(()) => existing = wrapped,
Err(VolumeError::Io(e)) if e.kind() == io::ErrorKind::UnexpectedEof => {}
Err(e) => return Err(e),
}
}
if n.cookie.0 == 0 && !check_cookie {
n.cookie = existing.cookie;
}
if existing.id != n.id && existing.size != nv.size {
// the entry resolves to a record that is neither the indexed
// key nor the indexed size, so it is stale rather than a
// duplicate — skip the cookie check and let publish_write
// overwrite it once the append commits
warn!(
"volume {} index entry for needle {} resolves to needle {}; overwriting stale index entry",
self.id.0, n.id.0, existing.id.0
);
} else {
if n.cookie.0 == 0 && !check_cookie {
n.cookie = existing.cookie;
}
if existing.cookie != n.cookie {
return Err(VolumeError::CookieMismatch(n.cookie.0));
}
if existing.cookie != n.cookie {
return Err(VolumeError::CookieMismatch(n.cookie.0));
}
}
Ok(false)
@@ -5036,25 +4985,6 @@ impl Volume {
self.current_dat_file_size()
}
/// Disk space held by the local .dat file, or 0 when it is unknown. It
/// exceeds the file size when the file was preallocated with
/// `FALLOC_FL_KEEP_SIZE`.
pub fn dat_allocated_size(&self) -> u64 {
#[cfg(unix)]
{
use std::os::unix::fs::MetadataExt;
let io = self.io.read().unwrap();
io.dat_file
.as_ref()
.and_then(|f| f.metadata().ok())
.map_or(0, |m| m.blocks() * 512)
}
#[cfg(not(unix))]
{
0
}
}
/// Get the modification time of the .dat file as Unix seconds.
pub fn dat_file_mod_time(&self) -> u64 {
let io = self.io.read().unwrap();
@@ -6118,7 +6048,7 @@ fn preallocate_file(file: &File, size: u64) {
/// An OS error the platform reports for failing storage media, which is
/// what counts toward the I/O error streak.
#[cfg(test)]
pub(crate) fn media_error_for_test() -> io::Error {
fn media_error_for_test() -> io::Error {
#[cfg(unix)]
{
io::Error::from_raw_os_error(libc::EIO)
@@ -9095,107 +9025,6 @@ mod tests {
assert!(matches!(err, VolumeError::CookieMismatch(_)));
}
/// A damaged index entry resolving to a different needle must miss on
/// reads and be dropped by the next write for the key, not rejected as a
/// cookie conflict.
#[test]
fn test_stale_index_entry_heals_on_write() {
let tmp = TempDir::new().unwrap();
let dir = tmp.path().to_str().unwrap();
let v = make_test_volume(dir);
let mut first = Needle {
id: NeedleId(2),
cookie: Cookie(0x2222),
data: b"second".to_vec(),
data_size: 6,
..Needle::default()
};
v.write_needle(&mut first, true, false).unwrap();
// point key 9 at needle 2's record with a size that does not match
// the record, as a corrupt index entry would look
{
let mut io = v.io.write().unwrap();
let nm = io.nm.as_mut().unwrap();
let nv = nm.get(NeedleId(2)).unwrap().unwrap();
nm.put(NeedleId(9), nv.offset, Size(nv.size.0 + 1)).unwrap();
}
let mut probe = Needle {
id: NeedleId(9),
..Needle::default()
};
assert!(v.read_needle(&mut probe).is_err());
let mut n = Needle {
id: NeedleId(9),
cookie: Cookie(0xabcd),
data: b"ninth".to_vec(),
data_size: 5,
..Needle::default()
};
v.write_needle(&mut n, true, false).unwrap();
let mut back = Needle {
id: NeedleId(9),
..Needle::default()
};
v.read_needle(&mut back).unwrap();
assert_eq!(back.data, b"ninth");
assert_eq!(back.cookie, Cookie(0xabcd));
}
/// A stale index entry is only replaced once the new record commits: a
/// durable write rolled back by a failed fsync leaves the index exactly
/// as it found it.
#[test]
fn test_stale_index_entry_survives_failed_write() {
let tmp = TempDir::new().unwrap();
let dir = tmp.path().to_str().unwrap();
let v = make_test_volume(dir);
let mut existing = Needle {
id: NeedleId(2),
cookie: Cookie(0x2222),
data: b"second".to_vec(),
data_size: 6,
..Needle::default()
};
v.write_needle(&mut existing, true, false).unwrap();
let stale_nv = {
let mut io = v.io.write().unwrap();
let nm = io.nm.as_mut().unwrap();
let nv = nm.get(NeedleId(2)).unwrap().unwrap();
nm.put(NeedleId(9), nv.offset, Size(nv.size.0 + 1)).unwrap();
nm.get(NeedleId(9)).unwrap().unwrap()
};
v.fail_next_fsync_for_test(true);
let mut replacement = Needle {
id: NeedleId(9),
cookie: Cookie(0xabcd),
data: b"ninth".to_vec(),
data_size: 5,
..Needle::default()
};
assert!(v.write_needle(&mut replacement, true, true).is_err());
v.fail_next_fsync_for_test(false);
let now = {
let io = v.io.read().unwrap();
io.nm
.as_ref()
.unwrap()
.get(NeedleId(9))
.unwrap()
.expect("a failed write must not drop the index entry it meant to replace")
};
assert_eq!(now.offset, stale_nv.offset);
assert_eq!(now.size, stale_nv.size);
}
// A size disagreeing with the blob's own header indexes the needle at the wrong
// length and, on v3, stamps the append timestamp into the middle of the needle.
#[test]
+2 -2
View File
@@ -1340,11 +1340,11 @@ async fn non_ascii_fid_and_ttl_are_rejected_not_panicked() {
#[tokio::test]
async fn write_queue_upload_returns_same_etag_as_direct_write() {
use seaweed_volume::server::write_queue::{SyncBatchOption, WriteQueue};
use seaweed_volume::server::write_queue::WriteQueue;
let (direct_state, _direct_tmp) = test_state();
let (queued_state, _queued_tmp) = test_state();
let wq = WriteQueue::new(queued_state.clone(), 128, SyncBatchOption::default());
let wq = WriteQueue::new(queued_state.clone(), 128);
let _ = queued_state.write_queue.set(wq);
let compressible = "seaweedfs ".repeat(200).into_bytes();
+23 -68
View File
@@ -3,12 +3,9 @@ package erasure_coding
import (
"bytes"
"context"
"encoding/json"
"fmt"
"io"
"math"
"net"
"net/http"
"os"
"os/exec"
"path/filepath"
@@ -16,7 +13,6 @@ import (
"testing"
"time"
"github.com/seaweedfs/seaweedfs/test/testutil"
"github.com/seaweedfs/seaweedfs/weed/operation"
"github.com/seaweedfs/seaweedfs/weed/pb"
"github.com/seaweedfs/seaweedfs/weed/pb/volume_server_pb"
@@ -51,15 +47,17 @@ func TestECEncodingVolumeLocationTimingBug(t *testing.T) {
defer cluster.Stop()
// Wait for servers to be ready
require.NoError(t, waitForServer(cluster.masterAddr, 30*time.Second))
for _, addr := range cluster.volumeAddrs {
require.NoError(t, waitForServer(addr, 30*time.Second))
}
require.NoError(t, waitForMasterLeader(cluster.masterAddr, 30*time.Second))
require.NoError(t, waitForServer("127.0.0.1:9333", 30*time.Second))
require.NoError(t, waitForServer("127.0.0.1:8080", 30*time.Second))
require.NoError(t, waitForServer("127.0.0.1:8081", 30*time.Second))
require.NoError(t, waitForServer("127.0.0.1:8082", 30*time.Second))
require.NoError(t, waitForServer("127.0.0.1:8083", 30*time.Second))
require.NoError(t, waitForServer("127.0.0.1:8084", 30*time.Second))
require.NoError(t, waitForServer("127.0.0.1:8085", 30*time.Second))
// Create command environment
options := &shell.ShellOptions{
Masters: stringPtr(cluster.masterAddr),
Masters: stringPtr("127.0.0.1:9333"),
GrpcDialOption: grpc.WithInsecure(),
FilerGroup: stringPtr("default"),
}
@@ -73,7 +71,7 @@ func TestECEncodingVolumeLocationTimingBug(t *testing.T) {
// Upload some test data to create volumes
testData := []byte("This is test data for EC encoding integration test")
volumeId, err := uploadTestData(testData, cluster.masterAddr)
volumeId, err := uploadTestData(testData, "127.0.0.1:9333")
require.NoError(t, err)
t.Logf("Created volume %d with test data", volumeId)
@@ -259,13 +257,12 @@ func TestECEncodingMasterTimingRaceCondition(t *testing.T) {
defer cluster.Stop()
// Wait for servers to be ready
require.NoError(t, waitForServer(cluster.masterAddr, 30*time.Second))
require.NoError(t, waitForServer(cluster.volumeAddrs[0], 30*time.Second))
require.NoError(t, waitForMasterLeader(cluster.masterAddr, 30*time.Second))
require.NoError(t, waitForServer("127.0.0.1:9333", 30*time.Second))
require.NoError(t, waitForServer("127.0.0.1:8080", 30*time.Second))
// Create command environment
options := &shell.ShellOptions{
Masters: stringPtr(cluster.masterAddr),
Masters: stringPtr("127.0.0.1:9333"),
GrpcDialOption: grpc.WithInsecure(),
FilerGroup: stringPtr("default"),
}
@@ -279,7 +276,7 @@ func TestECEncodingMasterTimingRaceCondition(t *testing.T) {
// Upload test data
testData := []byte("Race condition test data")
volumeId, err := uploadTestData(testData, cluster.masterAddr)
volumeId, err := uploadTestData(testData, "127.0.0.1:9333")
require.NoError(t, err)
t.Logf("Created volume %d for race condition test", volumeId)
@@ -369,8 +366,6 @@ func TestECEncodingMasterTimingRaceCondition(t *testing.T) {
type TestCluster struct {
masterCmd *exec.Cmd
volumeServers []*exec.Cmd
masterAddr string
volumeAddrs []string
}
func (c *TestCluster) Stop() {
@@ -407,23 +402,9 @@ func startSeaweedFSCluster(ctx context.Context, dataDir string) (*TestCluster, e
securityToml := filepath.Join(dataDir, "security.toml")
os.WriteFile(securityToml, []byte("# Empty security config for testing\n"), 0644)
// Allocate ports instead of using the defaults: on a shared or reused
// runner a stale weed process can still hold 9333/808x, which kills the
// freshly started master at bind time and leaves the test talking to a
// raft follower that answers every assign with "not current leader".
// Mini-safe ports guarantee each HTTP port's derived gRPC port
// (port + 10000) is free as well.
ports, err := testutil.AllocateMiniPorts(7)
if err != nil {
return nil, fmt.Errorf("failed to allocate ports: %v", err)
}
masterPort := ports[0]
masterAddr := fmt.Sprintf("127.0.0.1:%d", masterPort)
cluster.masterAddr = masterAddr
// Start master server
masterCmd := exec.CommandContext(ctx, weedBinary, "master",
"-port", fmt.Sprintf("%d", masterPort),
"-port", "9333",
"-mdir", masterDir,
"-volumeSizeLimitMB", "10", // Small volumes for testing
"-ip", "127.0.0.1",
@@ -451,14 +432,13 @@ func startSeaweedFSCluster(ctx context.Context, dataDir string) (*TestCluster, e
volumeDir := filepath.Join(dataDir, fmt.Sprintf("volume%d", i))
os.MkdirAll(volumeDir, 0755)
port := fmt.Sprintf("%d", ports[1+i])
cluster.volumeAddrs = append(cluster.volumeAddrs, fmt.Sprintf("127.0.0.1:%d", ports[1+i]))
port := fmt.Sprintf("808%d", i)
rack := fmt.Sprintf("rack%d", i)
volumeCmd := exec.CommandContext(ctx, weedBinary, "volume",
"-port", port,
"-dir", volumeDir,
"-max", "10",
"-master", masterAddr,
"-master", "127.0.0.1:9333",
"-volume.allowUntrustedRemoteEndpoints",
"-ip", "127.0.0.1",
"-dataCenter", "dc1",
@@ -518,8 +498,7 @@ func findWeedBinary() string {
func waitForServer(address string, timeout time.Duration) error {
start := time.Now()
for time.Since(start) < timeout {
conn, err := net.DialTimeout("tcp", address, time.Second)
if err == nil {
if conn, err := grpc.NewClient(address, grpc.WithInsecure()); err == nil {
conn.Close()
return nil
}
@@ -528,29 +507,6 @@ func waitForServer(address string, timeout time.Duration) error {
return fmt.Errorf("timeout waiting for server %s", address)
}
// waitForMasterLeader polls the master's /cluster/status endpoint until it
// reports itself as the raft leader. A goraft master can take up to the
// configured election timeout (10s by default) before it is ready to serve
// raft-gated requests like volume assign, so a bare port check is not enough.
func waitForMasterLeader(masterAddr string, timeout time.Duration) error {
start := time.Now()
for time.Since(start) < timeout {
resp, err := http.Get(fmt.Sprintf("http://%s/cluster/status", masterAddr))
if err == nil {
var status struct {
IsLeader bool `json:"IsLeader"`
}
err = json.NewDecoder(resp.Body).Decode(&status)
resp.Body.Close()
if err == nil && status.IsLeader {
return nil
}
}
time.Sleep(500 * time.Millisecond)
}
return fmt.Errorf("timeout waiting for master %s to become raft leader", masterAddr)
}
func uploadTestData(data []byte, masterAddress string) (needle.VolumeId, error) {
// Upload data to get a file ID
assignResult, err := operation.Assign(context.Background(), func(ctx context.Context) pb.ServerAddress {
@@ -2253,15 +2209,14 @@ func TestECEncodeReplicatedVolumeSync(t *testing.T) {
defer cluster.Stop()
// Wait for servers to be ready
require.NoError(t, waitForServer(cluster.masterAddr, 30*time.Second))
for _, addr := range cluster.volumeAddrs {
require.NoError(t, waitForServer(addr, 30*time.Second))
require.NoError(t, waitForServer("127.0.0.1:9333", 30*time.Second))
for i := 0; i < 6; i++ {
require.NoError(t, waitForServer(fmt.Sprintf("127.0.0.1:%d", 8080+i), 30*time.Second))
}
require.NoError(t, waitForMasterLeader(cluster.masterAddr, 30*time.Second))
// Create command environment
options := &shell.ShellOptions{
Masters: stringPtr(cluster.masterAddr),
Masters: stringPtr("127.0.0.1:9333"),
GrpcDialOption: grpc.WithInsecure(),
FilerGroup: stringPtr("default"),
}
@@ -2285,7 +2240,7 @@ func TestECEncodeReplicatedVolumeSync(t *testing.T) {
var volumeId needle.VolumeId
var uploadErr error
for retry := 0; retry < 5; retry++ {
volumeId, uploadErr = uploadTestDataWithReplication(t, cluster.masterAddr, testCollection, testReplication)
volumeId, uploadErr = uploadTestDataWithReplication(t, "127.0.0.1:9333", testCollection, testReplication)
if uploadErr == nil {
break
}
@@ -2335,7 +2290,7 @@ func TestECEncodeReplicatedVolumeSync(t *testing.T) {
var volumeId needle.VolumeId
var uploadErr error
for retry := 0; retry < 5; retry++ {
volumeId, uploadErr = uploadTestDataWithReplication(t, cluster.masterAddr, testCollection, testReplication)
volumeId, uploadErr = uploadTestDataWithReplication(t, "127.0.0.1:9333", testCollection, testReplication)
if uploadErr == nil {
break
}
-15
View File
@@ -110,21 +110,6 @@ func startFailoverCluster(t testing.TB, numVolumes, numMounts int) *failoverClus
"mount %d not ready\n%s", i, c.tailLog(fmt.Sprintf("mount%d", i)))
}
require.NoError(t, c.WaitForVolumeServers(60*time.Second))
// A mounted filesystem and registered volume servers do not yet guarantee
// that the lock ring has settled or the first replicated volume is ready.
// Exercise the write/read path during setup so the append latency checks
// measure failover on a running cluster rather than its cold start.
probeName := ".failover-ready"
probeData := []byte("ready\n")
probePath := filepath.Join(c.MountDir(0), probeName)
require.NoError(t, os.WriteFile(probePath, probeData, 0644),
"readiness write failed\n%s", c.tailLog("mount0"))
for i := 0; i < numMounts; i++ {
_, ready := waitForContent(filepath.Join(c.MountDir(i), probeName), probeData, convergeTimeout)
require.True(t, ready, "mount %d did not observe readiness write\n%s", i, c.tailLog(fmt.Sprintf("mount%d", i)))
}
require.NoError(t, os.Remove(probePath))
return c
}
-645
View File
@@ -1,645 +0,0 @@
# SeaweedFS FUSE POSIX Compliance Testing
## Overview
This comprehensive test suite provides full POSIX compliance testing for SeaweedFS FUSE mounts. It includes basic POSIX operations, extended features, external test suite integration, and performance benchmarking to ensure SeaweedFS meets filesystem standards.
## 🎯 Key Features
### ✅ **Comprehensive Test Coverage**
- **Basic POSIX Operations**: File/directory create, read, write, delete, rename
- **Advanced Features**: Extended attributes, file locking, memory mapping *(see roadmap)*
- **I/O Operations**: Synchronous/asynchronous I/O, direct I/O, vectored I/O *(see roadmap)*
- **Error Handling**: Comprehensive error condition testing
- **Concurrent Operations**: Multi-threaded stress testing
- **External Integration**: pjdfstest, nfstest, FIO integration
> **📋 Implementation Status**: Some advanced features are currently skipped pending platform-specific implementations. See [`POSIX_IMPLEMENTATION_ROADMAP.md`](./POSIX_IMPLEMENTATION_ROADMAP.md) for a detailed implementation plan and current status.
### 📊 **Performance Analysis**
- **Benchmarking**: Built-in performance benchmarks
- **Profiling**: CPU and memory profiling support
- **Coverage Analysis**: Code coverage reporting
- **Stress Testing**: High-load concurrent operation testing
### 🔧 **External Tool Integration**
- **pjdfstest**: Industry-standard POSIX filesystem test suite
- **nfstest_posix**: Network filesystem POSIX API verification
- **FIO**: Flexible I/O performance testing
- **Custom Litmus Tests**: Focused edge case testing
## 📋 Test Categories
### 1. Basic POSIX Compliance (`posix_compliance_test.go`)
#### File Operations
- ✅ File creation with O_CREAT, O_EXCL flags
- ✅ File truncation and size management
- ✅ File deletion and unlinking
- ✅ Atomic rename operations
- ✅ File permission management
#### Directory Operations
- ✅ Directory creation and removal
- ✅ Directory listing and traversal
- ✅ Directory rename operations
- ✅ Non-empty directory handling
#### Symlink Operations
- ✅ Symbolic link creation and resolution
- ✅ Broken symlink handling
- ✅ Symlink target verification
- ✅ Symlink permission handling
#### Permission Tests
- ✅ File permission setting and verification
- ✅ Directory permission enforcement
- ✅ Permission inheritance
- ✅ Access control validation
#### Timestamp Tests
- ✅ Access time (atime) updates
- ✅ Modification time (mtime) tracking
- ✅ Change time (ctime) management
- ✅ Timestamp precision and setting
#### I/O Operations
- ✅ Read/write operations
- ✅ File seeking and positioning
- ✅ Append mode operations
- ✅ Buffer management
#### File Descriptors
- ✅ File descriptor duplication
- ✅ File descriptor flags
- ✅ Close-on-exec handling
- ✅ File descriptor limits
#### Atomic Operations
- ✅ Atomic file operations
- ✅ Rename atomicity
- ✅ Link/unlink atomicity
#### Concurrent Access
- ✅ Multi-reader scenarios
- ✅ Concurrent write handling
- ✅ Reader-writer coordination
- ✅ Race condition prevention
#### Error Handling
- ✅ ENOENT (file not found) handling
- ✅ EACCES (permission denied) handling
- ✅ EBADF (bad file descriptor) handling
- ✅ ENOTEMPTY (directory not empty) handling
- ✅ Error code compliance
### 2. Extended POSIX Features (`posix_extended_test.go`)
#### Extended Attributes
- ✅ Setting extended attributes (setxattr)
- ✅ Getting extended attributes (getxattr)
- ✅ Listing extended attributes (listxattr)
- ✅ Removing extended attributes (removexattr)
- ✅ Extended attribute namespaces
#### File Locking
- ✅ Advisory locking (fcntl)
- ✅ Exclusive locks (F_WRLCK)
- ✅ Shared locks (F_RDLCK)
- ✅ Lock conflict detection
- ✅ Lock release and cleanup
#### Advanced I/O
- ✅ Vectored I/O (readv/writev)
- ✅ Positioned I/O (pread/pwrite)
- ✅ Asynchronous I/O patterns
- ✅ Scatter-gather operations
#### Sparse Files
- ✅ Sparse file creation
- ✅ Hole detection and handling
- ✅ Sparse file efficiency
- ✅ Seek beyond EOF
#### Large Files (>2GB)
- ✅ Large file operations
- ✅ 64-bit offset handling
- ✅ Large file seeking
- ✅ Large file truncation
#### Memory Mapping
- ✅ File memory mapping (mmap)
- ✅ Memory-mapped I/O
- ✅ Memory synchronization (msync)
- ✅ Memory unmapping (munmap)
- ✅ Shared/private mappings
#### Direct I/O
- ✅ Direct I/O operations (O_DIRECT)
- ✅ Buffer alignment requirements
- ✅ Direct I/O performance
- ✅ Bypass buffer cache
#### File Preallocation
- ✅ Space preallocation (fallocate)
- ✅ File hole punching
- ✅ Space reservation
- ✅ Allocation efficiency
#### Zero-Copy Operations
- ✅ Zero-copy file transfer (sendfile)
- ✅ Efficient data movement
- ✅ Cross-filesystem transfers
### 3. External Test Suite Integration (`posix_external_test.go`)
#### pjdfstest Integration
- ✅ Comprehensive POSIX filesystem test suite
- ✅ Industry-standard compliance verification
- ✅ Automated test execution
- ✅ Result analysis and reporting
#### nfstest_posix Integration
- ✅ POSIX API verification
- ✅ Network filesystem compliance
- ✅ API behavior validation
- ✅ Comprehensive coverage
#### Custom Litmus Tests
- ✅ Atomic rename verification
- ✅ Link count accuracy
- ✅ Symlink cycle detection
- ✅ Concurrent operation safety
- ✅ Directory consistency
#### Stress Testing
- ✅ High-frequency operations
- ✅ Concurrent access patterns
- ✅ Resource exhaustion scenarios
- ✅ Error recovery testing
#### Edge Case Testing
- ✅ Empty filenames
- ✅ Very long filenames
- ✅ Special characters
- ✅ Deep directory nesting
- ✅ Path length limits
## 🚀 Quick Start
### Prerequisites
```bash
# Install required tools
sudo apt-get install fuse # Linux
brew install macfuse # macOS
# Install Go 1.21+
go version
# Build SeaweedFS
cd /path/to/seaweedfs
make
```
### Basic Usage
```bash
# Navigate to test directory
cd test/fuse_integration
# Check prerequisites
make -f posix_Makefile check-prereqs
# Run basic POSIX compliance tests
make -f posix_Makefile test-posix-basic
# Run complete test suite
make -f posix_Makefile test-posix-full
# Generate compliance report
make -f posix_Makefile generate-report
```
## 🔧 Advanced Usage
### Test Categories
#### Critical Tests Only
```bash
# Run only critical POSIX compliance tests (faster)
make -f posix_Makefile test-posix-critical
```
#### Extended Feature Tests
```bash
# Test advanced POSIX features
make -f posix_Makefile test-posix-extended
```
#### External Tool Integration
```bash
# Setup and run external test suites
make -f posix_Makefile setup-external-tools
make -f posix_Makefile test-posix-external
```
#### Stress Testing
```bash
# Run stress tests for concurrent operations
make -f posix_Makefile test-posix-stress
```
### Performance Analysis
#### Benchmarking
```bash
# Run performance benchmarks
make -f posix_Makefile benchmark-posix
```
#### Profiling
```bash
# Profile CPU and memory usage
make -f posix_Makefile profile-posix
# View CPU profile
go tool pprof reports/posix.cpu.prof
```
#### Coverage Analysis
```bash
# Generate test coverage report
make -f posix_Makefile coverage-posix
# View: reports/posix_coverage.html
```
#### FIO Performance Testing
```bash
# Run FIO-based I/O performance tests
make -f posix_Makefile test-fio-posix
```
### External Test Suites
#### pjdfstest
```bash
# Setup and run pjdfstest
make -f posix_Makefile setup-pjdfstest
make -f posix_Makefile test-pjdfstest
```
#### nfstest_posix
```bash
# Install and run nfstest_posix
make -f posix_Makefile setup-nfstest
make -f posix_Makefile test-nfstest-posix
```
## 📊 Understanding Results
### Test Status Indicators
- ✅ **PASS**: Test completed successfully
- ❌ **FAIL**: Test failed - indicates non-compliance
- ⚠️ **SKIP**: Test skipped - feature not supported
- 🔄 **RETRY**: Test retried due to transient failure
### Report Formats
#### HTML Report (`reports/posix_compliance_report.html`)
- Interactive web-based report
- Test category breakdown
- Detailed results with navigation
- Visual status indicators
#### Text Summary (`reports/posix_compliance_summary.txt`)
- Concise text-based summary
- Key findings and recommendations
- Command-line friendly format
#### JSON Report (`reports/posix_compliance_report.json`)
- Machine-readable results
- Integration with CI/CD pipelines
- Automated result processing
### Log Files
- `posix_basic_results.log`: Basic POSIX test detailed output
- `posix_extended_results.log`: Extended feature test output
- `posix_external_results.log`: External test suite output
- `posix_benchmark_results.log`: Performance benchmark results
## 🐛 Debugging Failed Tests
### Common Issues
#### Permission Denied Errors
```bash
# Ensure proper FUSE permissions
sudo usermod -a -G fuse $USER
# Re-login or run: newgrp fuse
# Mount with -allowOthers if other users need access (test config MountOptions)
```
#### Extended Attributes Not Supported
```bash
# Check if xattr is supported by underlying filesystem
getfattr --version
# May need to skip xattr tests on certain filesystems
```
#### File Locking Issues
```bash
# Verify file locking support
# Some network filesystems may not support full locking
```
#### Memory Issues
```bash
# Increase available memory for tests
# Large file tests may require significant RAM
```
### Debug Mode
```bash
# Test targets already run go test -v and keep logs under reports/
make -f posix_Makefile test-posix-basic
# Run against an existing external mount instead of a fresh cluster
make -f posix_Makefile test-posix-basic TEST_MOUNT_POINT=/path/to/mount
# Extend the 45-minute per-suite timeout
make -f posix_Makefile test-posix-full TEST_TIMEOUT=90m
```
### Manual Test Execution
```bash
# Run specific test
cd test/fuse_integration
go test -v -run 'TestPOSIXCompliance/FileOperations' .
# Run with custom timeout
go test -v -timeout 30m -run TestPOSIXExtended .
```
## 🔄 CI/CD Integration
### GitHub Actions
```yaml
name: POSIX Compliance Tests
on: [push, pull_request]
jobs:
posix-compliance:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
- uses: actions/setup-go@v4
with:
go-version: '1.21'
- name: Install FUSE
run: sudo apt-get update && sudo apt-get install -y fuse
- name: Build SeaweedFS
run: make
- name: Run POSIX Compliance Tests
run: |
cd test/fuse_integration
make -f posix_Makefile ci-posix-tests
- name: Upload Test Results
uses: actions/upload-artifact@v4
if: always()
with:
name: posix-test-results
path: test/fuse_integration/reports/
```
### Jenkins Pipeline
```groovy
pipeline {
agent any
stages {
stage('Setup') {
steps {
sh 'make'
}
}
stage('POSIX Compliance') {
steps {
dir('test/fuse_integration') {
sh 'make -f posix_Makefile ci-posix-tests'
}
}
post {
always {
archiveArtifacts 'test/fuse_integration/reports/**'
publishHTML([
allowMissing: false,
alwaysLinkToLastBuild: true,
keepAll: true,
reportDir: 'test/fuse_integration/reports',
reportFiles: 'posix_compliance_report.html',
reportName: 'POSIX Compliance Report'
])
}
}
}
}
}
```
## 🎯 POSIX Compliance Checklist
### Critical POSIX Features
- [ ] File creation, reading, writing, deletion
- [ ] Directory operations (create, remove, list)
- [ ] File permissions and ownership
- [ ] Symbolic links
- [ ] Hard links
- [ ] File timestamps
- [ ] Error handling compliance
- [ ] Atomic operations
### Advanced POSIX Features
- [ ] Extended attributes
- [ ] File locking (advisory)
- [ ] Memory-mapped I/O
- [ ] Vectored I/O (readv/writev)
- [ ] Positioned I/O (pread/pwrite)
- [ ] Direct I/O
- [ ] File preallocation
- [ ] Sparse files
### Performance Requirements
- [ ] Reasonable performance under load
- [ ] Concurrent access handling
- [ ] Memory usage optimization
- [ ] CPU efficiency
- [ ] I/O throughput
### Reliability Requirements
- [ ] Data consistency
- [ ] Error recovery
- [ ] Race condition prevention
- [ ] Resource cleanup
- [ ] Crash recovery
## 📈 Performance Benchmarks
### Expected Performance Characteristics
| Operation | Expected Range | Notes |
|-----------|----------------|-------|
| File Create | 1000-10000 ops/sec | Depends on chunk size |
| File Read | 100-1000 MB/sec | Network limited |
| File Write | 50-500 MB/sec | Depends on replication |
| Directory Ops | 500-5000 ops/sec | Metadata operations |
| Concurrent Ops | Scales with cores | Up to hardware limits |
### Benchmarking Commands
```bash
# Basic I/O performance
make -f posix_Makefile benchmark-posix
# FIO comprehensive testing
make -f posix_Makefile test-fio-posix
# Custom benchmark with specific parameters
go test -v -bench=. -benchtime=30s posix_compliance_test.go
```
## 🔍 Troubleshooting
### Common Test Failures
#### 1. Mount Permission Issues
```bash
# Error: Permission denied mounting
# Solution: Check FUSE permissions — add the user to the fuse group and
# re-login rather than widening permissions on the device node
sudo usermod -a -G fuse $USER
```
#### 2. Extended Attribute Tests Fail
```bash
# Error: Operation not supported (ENOTSUP)
# Solution: Check filesystem xattr support
mount | grep $(df . | tail -1 | awk '{print $1}')
# May need to remount with xattr support
```
#### 3. File Locking Tests Fail
```bash
# Error: Function not implemented (ENOSYS)
# Solution: Some network filesystems don't support locking
# This may be expected behavior
```
#### 4. Large File Tests Fail
```bash
# Error: No space left on device
# Solution: Ensure sufficient disk space
df -h /tmp # Check temp directory space
# May need to clean up or use different temp directory
```
#### 5. Performance Tests Timeout
```bash
# Error: Test timeout exceeded
# Solution: Increase timeout or reduce test scope
go test -timeout 60m -run TestPOSIXCompliance
```
### Debug Information Collection
```bash
# Collect system information
uname -a > debug_info.txt
mount >> debug_info.txt
df -h >> debug_info.txt
free -h >> debug_info.txt
# Collect SeaweedFS information
./weed version >> debug_info.txt
ps aux | grep weed >> debug_info.txt
# Collect test logs
cp reports/*.log debug_logs/
```
## 📚 Additional Resources
### POSIX Standards
- [POSIX.1-2017 (IEEE Std 1003.1-2017)](https://pubs.opengroup.org/onlinepubs/9699919799/)
- [Single UNIX Specification](https://www.opengroup.org/openbrand/register/)
### External Test Suites
- [pjdfstest](https://github.com/pjd/pjdfstest) - Comprehensive POSIX filesystem tests
- [nfstest](https://nfstest.readthedocs.io/) - Network filesystem testing
- [FIO](https://fio.readthedocs.io/) - Flexible I/O tester
### SeaweedFS Documentation
- [SeaweedFS Wiki](https://github.com/seaweedfs/seaweedfs/wiki)
- [FUSE Mount Documentation](https://github.com/seaweedfs/seaweedfs/wiki/FUSE-Mount)
- [Performance Tuning Guide](https://github.com/seaweedfs/seaweedfs/wiki/Optimization)
## 🤝 Contributing
### Adding New Tests
1. **Basic Tests**: Add to `posix_compliance_test.go`
2. **Extended Tests**: Add to `posix_extended_test.go`
3. **External Integration**: Add to `posix_external_test.go`
### Test Guidelines
```go
func (s *POSIXTestSuite) TestNewFeature(t *testing.T) {
mountPoint := s.framework.GetMountPoint()
t.Run("SubTestName", func(t *testing.T) {
// Test setup
testFile := filepath.Join(mountPoint, "test.txt")
// Test execution
err := someOperation(testFile)
// Assertions
require.NoError(t, err)
// Cleanup (if needed)
defer os.Remove(testFile)
})
}
```
### Submitting Improvements
1. Fork the repository
2. Create a feature branch
3. Add comprehensive tests
4. Ensure all existing tests pass
5. Submit a pull request with detailed description
## 📞 Support
For questions, issues, or contributions:
- **GitHub Issues**: [SeaweedFS Issues](https://github.com/seaweedfs/seaweedfs/issues)
- **Discussions**: [SeaweedFS Discussions](https://github.com/seaweedfs/seaweedfs/discussions)
- **Documentation**: [SeaweedFS Wiki](https://github.com/seaweedfs/seaweedfs/wiki)
---
*This comprehensive POSIX compliance test suite ensures SeaweedFS FUSE mounts meet industry standards for filesystem behavior, performance, and reliability.*
@@ -1,272 +0,0 @@
# POSIX Compliance Implementation Roadmap
This document tracks the implementation status of POSIX features in the SeaweedFS FUSE mount compliance test suite and provides a roadmap for completing comprehensive POSIX compliance testing.
## Overview
The platform-specific tests listed below are implemented; this document records the implementation plan that produced them, plus the one feature still pending.
## Current Status
### ✅ Implemented Features
- Basic file operations (create, read, write, delete)
- Directory operations (mkdir, rmdir, rename)
- File permissions and ownership
- Symbolic and hard links
- Basic I/O operations (seek, append, positioned I/O)
- File descriptors and atomic operations
- Concurrent access patterns
- Error handling compliance
- Timestamp operations
### 🚧 Partially Implemented
- Cross-platform compatibility (basic implementation with platform-specific access time handling)
- File sealing (still skipped; Linux-specific, pending)
### ✅ Implemented Platform-Specific Features
#### 1. Extended Attributes (xattr)
**Priority: High**
**Platforms: Linux, macOS, FreeBSD**
**Status:** Implemented
- `TestExtendedAttributes/SetExtendedAttribute`
- `TestExtendedAttributes/ListExtendedAttributes`
- `TestExtendedAttributes/RemoveExtendedAttribute`
**Implementation Plan:**
```go
// Linux implementation
//go:build linux
func setXattr(path, name string, value []byte) error {
return syscall.Setxattr(path, name, value, 0)
}
// macOS implementation
//go:build darwin
func setXattr(path, name string, value []byte) error {
return syscall.Setxattr(path, name, value, 0, 0)
}
```
**Required Files:**
- `xattr_linux.go` - Linux syscall implementations
- `xattr_darwin.go` - macOS syscall implementations
- `xattr_freebsd.go` - FreeBSD syscall implementations
- `xattr_unsupported.go` - Fallback for unsupported platforms
#### 2. Memory Mapping (mmap)
**Priority: High**
**Platforms: All POSIX systems**
**Status:** Implemented
- `TestMemoryMapping/MemoryMappedRead`
- `TestMemoryMapping/MemoryMappedWrite`
**Implementation Plan:**
```go
func testMmap(t *testing.T, filePath string) {
fd, err := syscall.Open(filePath, syscall.O_RDWR, 0)
require.NoError(t, err)
defer syscall.Close(fd)
// Platform-specific mmap implementation
data, err := syscall.Mmap(fd, 0, size, syscall.PROT_READ|syscall.PROT_WRITE, syscall.MAP_SHARED)
require.NoError(t, err)
defer syscall.Munmap(data)
}
```
**Required Files:**
- `mmap_unix.go` - Unix-like systems implementation
- `mmap_windows.go` - Windows implementation (if needed)
#### 3. Vectored I/O (readv/writev)
**Priority: Medium**
**Platforms: All POSIX systems**
**Status:** Implemented
- `TestAdvancedIO/VectoredIO`
**Implementation Plan:**
```go
func testVectoredIO(t *testing.T, filePath string) {
// Use syscall.Syscall for readv/writev
// Linux: SYS_READV, SYS_WRITEV
// Other platforms: platform-specific syscall numbers
}
```
#### 4. Direct I/O
**Priority: Medium**
**Platforms: Linux, some Unix variants**
**Status:** Implemented
- `TestDirectIO/DirectIO`
**Implementation Plan:**
```go
//go:build linux
func testDirectIO(t *testing.T, filePath string) {
fd, err := syscall.Open(filePath, syscall.O_RDWR|syscall.O_DIRECT, 0)
// Test with aligned buffers
}
```
#### 5. File Preallocation (fallocate)
**Priority: Medium**
**Platforms: Linux, some Unix variants**
**Status:** Implemented
- `TestFilePreallocation/Fallocate`
**Implementation Plan:**
```go
//go:build linux
func testFallocate(t *testing.T, filePath string) {
fd, err := syscall.Open(filePath, syscall.O_RDWR, 0)
err = syscall.Syscall6(syscall.SYS_FALLOCATE, uintptr(fd), 0, 0, uintptr(size), 0, 0)
}
```
#### 6. Zero-Copy Transfer (sendfile)
**Priority: Low**
**Platforms: Linux, FreeBSD, some Unix variants**
**Status:** Implemented
- `TestZeroCopyTransfer/Sendfile`
**Implementation Plan:**
```go
//go:build linux
func testSendfile(t *testing.T, srcPath, dstPath string) {
// Use syscall.Syscall for sendfile
// Platform-specific implementations
}
```
#### 7. File Sealing (Linux-specific)
**Priority: Low**
**Platforms: Linux only**
**Current Status:** File sealing test is still skipped
- `TestFileSealing`
**Implementation Plan:**
```go
//go:build linux
func testFileSealing(t *testing.T) {
// Use fcntl with F_ADD_SEALS, F_GET_SEALS
// Test various seal types: F_SEAL_WRITE, F_SEAL_SHRINK, etc.
}
```
## Implementation Strategy
### Phase 1: Core Features (High Priority)
1. **Extended Attributes** - Essential for many applications
2. **Memory Mapping** - Critical for performance-sensitive applications
### Phase 2: Advanced I/O (Medium Priority)
3. **Vectored I/O** - Important for efficient bulk operations
4. **Direct I/O** - Needed for database and high-performance applications
5. **File Preallocation** - Important for preventing fragmentation
### Phase 3: Specialized Features (Low Priority)
6. **Zero-Copy Transfer** - Performance optimization feature
7. **File Sealing** - Security feature, Linux-specific
## Platform Support Matrix
| Feature | Linux | macOS | FreeBSD | Windows | Notes |
|---------|-------|-------|---------|---------|-------|
| Extended Attributes | ✅ | ✅ | ✅ | ❌ | Different syscall signatures |
| Memory Mapping | ✅ | ✅ | ✅ | ✅ | Standard POSIX |
| Vectored I/O | ✅ | ✅ | ✅ | ❌ | readv/writev syscalls |
| Direct I/O | ✅ | ❌ | ✅ | ❌ | O_DIRECT flag |
| fallocate | ✅ | ❌ | ❌ | ❌ | Linux-specific |
| sendfile | ✅ | ❌ | ✅ | ❌ | Platform-specific |
| File Sealing | ✅ | ❌ | ❌ | ❌ | Linux-only |
## Development Guidelines
### 1. Platform-Specific Implementation Pattern
```go
// feature_linux.go
//go:build linux
package fuse
func platformSpecificFunction() { /* Linux implementation */ }
// feature_darwin.go
//go:build darwin
package fuse
func platformSpecificFunction() { /* macOS implementation */ }
// feature_unsupported.go
//go:build !linux && !darwin
package fuse
func platformSpecificFunction() { /* Skip or error */ }
```
### 2. Test Structure Pattern
```go
func (s *POSIXExtendedTestSuite) TestFeature(t *testing.T) {
if !isFeatureSupported() {
t.Skip("Feature not supported on this platform")
return
}
t.Run("FeatureTest", func(t *testing.T) {
// Actual test implementation
})
}
```
### 3. Error Handling
- Use platform-specific error checking
- Provide meaningful error messages
- Gracefully handle unsupported features
## Testing Strategy
### 1. Continuous Integration
- Run tests on multiple platforms (Linux, macOS)
- Use build tags to enable/disable platform-specific tests
- Ensure graceful degradation on unsupported platforms
### 2. Feature Detection
- Implement runtime feature detection where possible
- Skip tests gracefully when features are unavailable
- Log clear messages about skipped functionality
### 3. Documentation
- Document platform-specific behavior
- Provide examples for each implemented feature
- Maintain compatibility matrices
## Contributing
When implementing these features:
1. **Start with high-priority items** (Extended Attributes, Memory Mapping)
2. **Follow the platform-specific pattern** outlined above
3. **Add comprehensive tests** for each feature
4. **Update this roadmap** as features are implemented
5. **Document any platform-specific quirks** or limitations
## Future Enhancements
Beyond the current roadmap, consider:
- **File locking** (flock, fcntl locks)
- **Asynchronous I/O** (aio_read, aio_write)
- **File change notifications** (inotify, kqueue)
- **POSIX ACLs** (Access Control Lists)
- **Sparse file operations**
- **File hole punching**
## References
- [POSIX.1-2017 Standard](https://pubs.opengroup.org/onlinepubs/9699919799/)
- [Linux Programmer's Manual](https://man7.org/linux/man-pages/)
- [FreeBSD System Calls](https://www.freebsd.org/cgi/man.cgi)
- [macOS System Calls](https://developer.apple.com/library/archive/documentation/System/Conceptual/ManPages_iPhoneOS/)
-12
View File
@@ -1,12 +0,0 @@
//go:build linux
package fuse
import (
"syscall"
)
// getAtimeNano returns the access time in nanoseconds for Linux systems
func getAtimeNano(stat *syscall.Stat_t) int64 {
return stat.Atim.Sec*1e9 + stat.Atim.Nsec
}
-12
View File
@@ -1,12 +0,0 @@
//go:build !linux && !windows
package fuse
import (
"syscall"
)
// getAtimeNano returns the access time in nanoseconds for non-Linux systems (macOS, BSD, etc.)
func getAtimeNano(stat *syscall.Stat_t) int64 {
return stat.Atimespec.Sec*1e9 + stat.Atimespec.Nsec
}
-12
View File
@@ -1,12 +0,0 @@
//go:build windows
package fuse
import (
"syscall"
)
// getAtimeNano returns the access time in nanoseconds for Windows systems
func getAtimeNano(stat *syscall.Win32FileAttributeData) int64 {
return stat.LastAccessTime.Nanoseconds()
}
-39
View File
@@ -1,39 +0,0 @@
//go:build darwin
package fuse
import (
"syscall"
)
// Direct I/O support for macOS
const (
// macOS doesn't have O_DIRECT, but we can use fcntl with F_NOCACHE
F_NOCACHE = 48
)
func openDirectIO(path string, flags int, mode uint32) (int, error) {
// Open file normally first
fd, err := syscall.Open(path, flags, mode)
if err != nil {
return -1, err
}
// Set F_NOCACHE to bypass buffer cache (similar to O_DIRECT)
_, _, errno := syscall.Syscall(syscall.SYS_FCNTL,
uintptr(fd),
F_NOCACHE,
1) // enable
if errno != 0 {
syscall.Close(fd)
return -1, errno
}
return fd, nil
}
func isDirectIOSupported() bool {
return true
}
Loaded 100 of 484 files, more files were not shown because too many files have changed in this diff. Show more