mirror of
https://github.com/seaweedfs/seaweedfs.git
synced 2026-11-20 11:23:50 +01:00
Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
e7d12a1c04 | ||
|
|
11fb65571f | ||
|
|
6c2a11f1d6 | ||
|
|
851011a32b | ||
|
|
a9a9c732b9 | ||
|
|
6328141019 | ||
|
|
b508e00de7 | ||
|
|
e486817499 | ||
|
|
aec8542b6e | ||
|
|
8fb9df72ae | ||
|
|
af25f92e36 | ||
|
|
66829fa7c8 | ||
|
|
5c24bfc795 | ||
|
|
1d1c717493 | ||
|
|
7661449f82 | ||
|
|
a5a6b5a4d4 | ||
|
|
8121fdeec3 | ||
|
|
f63f40a3d4 | ||
|
|
2d6532d80e | ||
|
|
dce0672ae0 | ||
|
|
40d9f4fa9c | ||
|
|
8cc99ca121 | ||
|
|
c5628b000b | ||
|
|
f419271299 | ||
|
|
a0ab227e07 | ||
|
|
2ad4a924be | ||
|
|
1e1806c7a1 | ||
|
|
eaaef569ce | ||
|
|
1807f4e94d | ||
|
|
7203c78e4d | ||
|
|
1bc503530a | ||
|
|
9f88fd2ea5 | ||
|
|
0123abe49b | ||
|
|
4b92f03b65 | ||
|
|
1e7a3c09d4 | ||
|
|
ddb2105f00 | ||
|
|
bbcd19f6c6 | ||
|
|
fc2fcae543 | ||
|
|
4801f641f1 | ||
|
|
770888df81 | ||
|
|
1dfdd3c84d | ||
|
|
b63855560a | ||
|
|
9f8e0257b0 | ||
|
|
e53cd29eb2 | ||
|
|
eeeb595346 | ||
|
|
678f962ded | ||
|
|
a906358c59 | ||
|
|
12f34f965f | ||
|
|
1db149ecc2 | ||
|
|
59f8444a90 | ||
|
|
bcc7a74a25 | ||
|
|
0d4eeb67be | ||
|
|
85e34df214 | ||
|
|
bb05e6cade | ||
|
|
bc46214445 | ||
|
|
121f6ef2c0 | ||
|
|
22673fee76 | ||
|
|
3fbab20b83 | ||
|
|
b8cbe9b6b2 | ||
|
|
8bb47f3c03 | ||
|
|
7bec2ef94b | ||
|
|
f9cbed7ee1 |
No files matched your search
@@ -1,24 +0,0 @@
|
||||
[codespell]
|
||||
# Ref: https://github.com/codespell-project/codespell#using-a-config-file
|
||||
skip = .git,.git-meta,.gitignore,.gitattributes,*.svg,go.sum,vendor,*.lock,*.css,*.min.*,.codespellrc,*_templ.go
|
||||
check-hidden = true
|
||||
# Ignore camelCase and PascalCase identifiers (very common in Go/Rust/JS
|
||||
# source, e.g. allLocations, publishErr, ReadInside, FlushInterval).
|
||||
ignore-regex = \b[a-z]+[A-Z]\w*\b|\b[A-Z][a-z]+[A-Z]\w*\b
|
||||
# visibles: variable name for VisibleInterval collections in filer/mount code
|
||||
# fo: `*FilerOptions` receiver name (e.g. `func (fo *FilerOptions) ...`)
|
||||
# te: "truncate error" local variable (e.g. `if te := w.Truncate(end); te != nil`)
|
||||
# ser: Rust serde serializer variable (`serde_json::ser`, `let mut ser = ...`)
|
||||
# bject: intentional wildcard test data (e.g. `s3:Get?bject` matching `s3:GetObject`)
|
||||
# unparseable: accepted alternate spelling used throughout the codebase
|
||||
# keep-alives: correct plural of the technical term (SSH/HTTP keep-alive)
|
||||
# tread: valid English word in the idiom "tread carefully" (help text)
|
||||
# anc: variable abbreviation for "ancestor" in tree/path tests
|
||||
# ue: appears inside JSON test fixtures with embedded escaped quotes (Bl\"ue)
|
||||
# auther: local variable meaning "authenticator" (tls.go: `auther := Authenticator{}`)
|
||||
# thirdparty: literal Maven groupId `org.apache.hadoop.thirdparty` (external, cannot rename)
|
||||
# unknwon: GitHub username / Go module path (`github.com/unknwon/goconfig`)
|
||||
# atleast: CLI mode literal string in test/benchmark/fuse_db/bin/sqlite_verify.py
|
||||
# sme: local variable for a *streamMutateError in mount tests
|
||||
# nin: AWS S3 Vectors "$nin" filter operator literal (API-defined, cannot rename)
|
||||
ignore-words-list = visibles,fo,te,ser,bject,unparseable,keep-alives,tread,anc,ue,auther,thirdparty,unknwon,atleast,sme,nin
|
||||
@@ -1,11 +0,0 @@
|
||||
name: Clean up stale SeaweedFS test mounts
|
||||
description: Detach disconnected FUSE test mounts left by earlier jobs on a shared runner.
|
||||
runs:
|
||||
using: composite
|
||||
steps:
|
||||
- name: Unmount disconnected SeaweedFS test mounts
|
||||
if: runner.os == 'Linux' && runner.environment == 'self-hosted'
|
||||
shell: bash
|
||||
env:
|
||||
CLEANUP_SCRIPT: ${{ github.action_path }}/cleanup.py
|
||||
run: python3 "$CLEANUP_SCRIPT"
|
||||
@@ -1,89 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Detach disconnected SeaweedFS integration-test mounts without touching live ones."""
|
||||
|
||||
import errno
|
||||
import os
|
||||
from pathlib import Path
|
||||
import re
|
||||
import shutil
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
|
||||
|
||||
def test_mounts(mountinfo):
|
||||
"""Read the kernel table directly: traversing a stale mount may block."""
|
||||
roots = {tempfile.gettempdir(), "/tmp", "/var/tmp"}
|
||||
for line in mountinfo.splitlines():
|
||||
fields, filesystem = line.split(" - ", 1)
|
||||
if filesystem.split()[0].split(".")[0] != "fuse":
|
||||
continue
|
||||
fields = fields.split()
|
||||
# mountinfo escapes whitespace and backslashes with octal sequences.
|
||||
target = re.sub(r"\\([0-7]{3})", lambda m: chr(int(m[1], 8)), fields[4])
|
||||
path = Path(target)
|
||||
# Test mounts are a mount* leaf under a seaweedfs_fuse_* scratch dir
|
||||
# (seaweedfs_fuse_test_*/mount, seaweedfs_fuse_failover_*/mount0, ...)
|
||||
# or the posix-compliance workflow's fixed /tmp/seaweedfs/mount.
|
||||
if (not path.name.startswith("mount")
|
||||
or str(path.parent.parent) not in roots
|
||||
or not (path.parent.name.startswith("seaweedfs_fuse")
|
||||
or path.parent.name == "seaweedfs")):
|
||||
continue
|
||||
yield fields[0], target
|
||||
|
||||
|
||||
def mounted(mount_id, target):
|
||||
return (mount_id, target) in test_mounts(Path("/proc/self/mountinfo").read_text())
|
||||
|
||||
|
||||
def cleanup():
|
||||
failed = False
|
||||
for mount_id, target in test_mounts(Path("/proc/self/mountinfo").read_text()):
|
||||
try:
|
||||
# A separate, bounded probe prevents a wedged FUSE daemon from
|
||||
# hanging cleanup. Only ENOTCONN proves the daemon disconnected;
|
||||
# a timeout or another error is not enough to detach a live mount.
|
||||
probe = subprocess.run(
|
||||
[sys.executable, __file__, "--probe", target], timeout=5,
|
||||
capture_output=True, check=False,
|
||||
)
|
||||
except subprocess.TimeoutExpired:
|
||||
print(f"::warning::FUSE probe timed out; leaving {target} mounted")
|
||||
continue
|
||||
if probe.returncode != errno.ENOTCONN or not mounted(mount_id, target):
|
||||
continue
|
||||
|
||||
print(f"Unmounting disconnected FUSE test mount: {target}", flush=True)
|
||||
# Lazy unmount detaches a dead mount even when an old process still
|
||||
# holds it open. Prefer the unprivileged FUSE helper; sudo is fallback.
|
||||
for command in (
|
||||
["fusermount3", "-uz", "--", target],
|
||||
["fusermount", "-uz", "--", target],
|
||||
["sudo", "-n", "umount", "-l", "--", target],
|
||||
):
|
||||
try:
|
||||
subprocess.run(command, timeout=10, check=False)
|
||||
except (OSError, subprocess.TimeoutExpired) as error:
|
||||
print(f"{command[0]}: {error}", file=sys.stderr)
|
||||
if not mounted(mount_id, target):
|
||||
break
|
||||
else:
|
||||
print(f"::error::Could not unmount disconnected FUSE test mount: {target}")
|
||||
failed = True
|
||||
continue
|
||||
# The detached mount's parent still holds the crashed run's data and
|
||||
# logs; test_mounts only yields targets under a SeaweedFS test dir in
|
||||
# a temp root, so the parent is safe to reclaim here.
|
||||
shutil.rmtree(Path(target).parent, ignore_errors=True)
|
||||
return int(failed)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
if len(sys.argv) == 3 and sys.argv[1] == "--probe":
|
||||
try:
|
||||
os.stat(sys.argv[2])
|
||||
except OSError as error:
|
||||
sys.exit(error.errno)
|
||||
sys.exit(0)
|
||||
sys.exit(cleanup())
|
||||
@@ -1,137 +0,0 @@
|
||||
import errno
|
||||
import importlib.util
|
||||
from pathlib import Path
|
||||
import subprocess
|
||||
import unittest
|
||||
from unittest.mock import patch
|
||||
|
||||
spec = importlib.util.spec_from_file_location("cleanup", Path(__file__).with_name("cleanup.py"))
|
||||
cleanup = importlib.util.module_from_spec(spec)
|
||||
spec.loader.exec_module(cleanup)
|
||||
|
||||
TARGET = "/tmp/seaweedfs_fuse_test_123/mount"
|
||||
MOUNTINFO = f"42 1 0:9 / {TARGET} rw - fuse.seaweedfs seaweedfs rw\n"
|
||||
|
||||
|
||||
class CleanupTests(unittest.TestCase):
|
||||
def run_cleanup(self, results, mounted=True):
|
||||
with patch.object(cleanup.Path, "read_text", return_value=MOUNTINFO), \
|
||||
patch.object(cleanup, "mounted", side_effect=mounted if isinstance(mounted, list) else None,
|
||||
return_value=mounted) as check, \
|
||||
patch.object(cleanup.shutil, "rmtree") as rmtree, \
|
||||
patch.object(cleanup.subprocess, "run", side_effect=results) as run:
|
||||
status = cleanup.cleanup()
|
||||
return status, run, check, rmtree
|
||||
|
||||
def result(self, status):
|
||||
return subprocess.CompletedProcess([], status)
|
||||
|
||||
def test_only_fuse_integration_test_mounts_are_candidates(self):
|
||||
rows = MOUNTINFO
|
||||
rows += MOUNTINFO.replace("fuse.seaweedfs", "ext4")
|
||||
rows += MOUNTINFO.replace(TARGET, "/mnt/production")
|
||||
rows += MOUNTINFO.replace(TARGET, "/tmp/seaweedfs_fuse_test_123/data")
|
||||
rows += MOUNTINFO.replace(TARGET, "/home/user/seaweedfs_fuse_test_123/mount")
|
||||
rows += MOUNTINFO.replace(TARGET, "/tmp/seaweedfs/data")
|
||||
self.assertEqual(list(cleanup.test_mounts(rows)), [("42", TARGET)])
|
||||
|
||||
def test_all_test_mount_layouts_are_candidates(self):
|
||||
targets = (
|
||||
"/tmp/seaweedfs_fuse_test_123/mount",
|
||||
"/tmp/seaweedfs_fuse_failover_456/mount0",
|
||||
"/tmp/seaweedfs_fuse_failover_456/mount1",
|
||||
"/var/tmp/seaweedfs_fuse_p2p_test_789/mount2",
|
||||
"/tmp/seaweedfs/mount",
|
||||
)
|
||||
rows = "".join(
|
||||
MOUNTINFO.replace("42", str(40 + i)).replace(TARGET, target)
|
||||
for i, target in enumerate(targets)
|
||||
)
|
||||
self.assertEqual(
|
||||
list(cleanup.test_mounts(rows)),
|
||||
[(str(40 + i), target) for i, target in enumerate(targets)],
|
||||
)
|
||||
|
||||
def test_mountinfo_paths_are_unescaped_once(self):
|
||||
target = "/tmp/seaweedfs_fuse_test_space and\\slash/mount"
|
||||
encoded = target.replace("\\", r"\134").replace(" ", r"\040")
|
||||
rows = MOUNTINFO.replace(TARGET, encoded)
|
||||
self.assertEqual(list(cleanup.test_mounts(rows)), [("42", target)])
|
||||
|
||||
def test_no_mounts_is_a_noop(self):
|
||||
with patch.object(cleanup.Path, "read_text", return_value=""), \
|
||||
patch.object(cleanup.subprocess, "run") as run:
|
||||
self.assertEqual(cleanup.cleanup(), 0)
|
||||
run.assert_not_called()
|
||||
|
||||
def test_healthy_mount_is_untouched(self):
|
||||
status, run, _, _ = self.run_cleanup([self.result(0)])
|
||||
self.assertEqual(status, 0)
|
||||
self.assertEqual(run.call_count, 1)
|
||||
|
||||
def test_other_probe_errors_are_untouched(self):
|
||||
for error in (errno.EACCES, errno.ENOENT, errno.EIO):
|
||||
with self.subTest(error=error):
|
||||
status, run, _, _ = self.run_cleanup([self.result(error)])
|
||||
self.assertEqual(status, 0)
|
||||
self.assertEqual(run.call_count, 1)
|
||||
|
||||
def test_probe_timeout_does_not_detach_a_possibly_live_mount(self):
|
||||
status, run, _, _ = self.run_cleanup([subprocess.TimeoutExpired("probe", 5)])
|
||||
self.assertEqual(status, 0)
|
||||
self.assertEqual(run.call_count, 1)
|
||||
|
||||
def test_disconnected_mount_is_lazily_unmounted(self):
|
||||
status, run, _, _ = self.run_cleanup([self.result(errno.ENOTCONN), self.result(0)], [True, False])
|
||||
self.assertEqual(status, 0)
|
||||
self.assertEqual(run.call_args_list[1].args[0], ["fusermount3", "-uz", "--", TARGET])
|
||||
self.assertEqual(run.call_args_list[0].kwargs["timeout"], 5)
|
||||
self.assertEqual(run.call_args_list[1].kwargs["timeout"], 10)
|
||||
|
||||
def test_replaced_or_removed_mount_is_untouched(self):
|
||||
status, run, _, _ = self.run_cleanup([self.result(errno.ENOTCONN)], False)
|
||||
self.assertEqual(status, 0)
|
||||
self.assertEqual(run.call_count, 1)
|
||||
|
||||
def test_mount_identity_is_checked_without_stat(self):
|
||||
with patch.object(cleanup.Path, "read_text", return_value=MOUNTINFO):
|
||||
self.assertTrue(cleanup.mounted("42", TARGET))
|
||||
self.assertFalse(cleanup.mounted("43", TARGET))
|
||||
|
||||
def test_missing_or_timed_out_helper_uses_fallback(self):
|
||||
for error in (FileNotFoundError(), subprocess.TimeoutExpired("fusermount3", 10)):
|
||||
with self.subTest(error=error):
|
||||
status, run, _, _ = self.run_cleanup(
|
||||
[self.result(errno.ENOTCONN), error, self.result(0)], [True, True, False])
|
||||
self.assertEqual(status, 0)
|
||||
self.assertEqual(run.call_args.args[0], ["fusermount", "-uz", "--", TARGET])
|
||||
|
||||
def test_failed_helpers_fall_back_to_sudo(self):
|
||||
status, run, _, _ = self.run_cleanup(
|
||||
[self.result(errno.ENOTCONN), self.result(1), self.result(1), self.result(0)],
|
||||
[True, True, True, False])
|
||||
self.assertEqual(status, 0)
|
||||
self.assertEqual(run.call_args.args[0], ["sudo", "-n", "umount", "-l", "--", TARGET])
|
||||
|
||||
def test_cleanup_fails_when_stale_mount_remains(self):
|
||||
status, run, _, rmtree = self.run_cleanup(
|
||||
[self.result(errno.ENOTCONN), self.result(1), self.result(1), self.result(1)])
|
||||
self.assertEqual(status, 1)
|
||||
self.assertEqual(run.call_count, 4)
|
||||
rmtree.assert_not_called()
|
||||
|
||||
def test_unmounted_test_dir_is_reclaimed(self):
|
||||
status, _, _, rmtree = self.run_cleanup(
|
||||
[self.result(errno.ENOTCONN), self.result(0)], [True, False])
|
||||
self.assertEqual(status, 0)
|
||||
rmtree.assert_called_once_with(Path(TARGET).parent, ignore_errors=True)
|
||||
|
||||
def test_probe_process_reports_errno(self):
|
||||
for target, expected in ((str(Path(__file__)), 0), (str(Path(__file__) / "absent"), errno.ENOTDIR)):
|
||||
with self.subTest(target=target):
|
||||
result = subprocess.run([cleanup.sys.executable, cleanup.__file__, "--probe", target], check=False)
|
||||
self.assertEqual(result.returncode, expected)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -1,72 +0,0 @@
|
||||
name: Fix fusermount3 setuid
|
||||
description: >
|
||||
Make sure the fusermount3 an unprivileged mount will find can actually mount.
|
||||
Some runner images carry a second, source-built fusermount3 in /usr/local/bin
|
||||
that shadows the distro one in PATH; it is neither setuid nor root-owned, so
|
||||
every mount fails with "mount failed: Operation not permitted". Both the Go
|
||||
mount and sw-fuse resolve the helper through PATH.
|
||||
Run it after the step that apt-installs fuse3.
|
||||
|
||||
runs:
|
||||
using: composite
|
||||
steps:
|
||||
- name: Point PATH at a fusermount3 that can mount
|
||||
shell: bash
|
||||
run: |
|
||||
set -euo pipefail
|
||||
|
||||
# setuid only grants root when root owns the file, and exec follows
|
||||
# symlinks, so judge the target.
|
||||
can_mount() {
|
||||
[ -u "$1" ] && [ "$(stat -Lc %u "$1")" = 0 ]
|
||||
}
|
||||
|
||||
bin=$(command -v fusermount3 || true)
|
||||
if [ -z "$bin" ]; then
|
||||
# Self-hosted runners are not guaranteed to carry fuse3.
|
||||
sudo apt-get update
|
||||
sudo apt-get install -y fuse3
|
||||
bin=$(command -v fusermount3 || true)
|
||||
if [ -z "$bin" ]; then
|
||||
echo "no fusermount3 in PATH after installing fuse3" >&2
|
||||
exit 1
|
||||
fi
|
||||
fi
|
||||
if can_mount "$bin"; then
|
||||
ls -l "$bin"
|
||||
exit 0
|
||||
fi
|
||||
echo "$bin cannot mount unprivileged:"
|
||||
ls -l "$bin"
|
||||
|
||||
# The distro fusermount3 is setuid root and is the one meant to be used.
|
||||
# Reach it through a symlink earlier in PATH: exec resolves the link, so
|
||||
# the target keeps its setuid bit, and nothing on the image is modified.
|
||||
for distro in /usr/bin/fusermount3 /bin/fusermount3; do
|
||||
if [ "$distro" != "$bin" ] && can_mount "$distro"; then
|
||||
mkdir -p "$RUNNER_TEMP/fuse-bin"
|
||||
ln -sf "$distro" "$RUNNER_TEMP/fuse-bin/fusermount3"
|
||||
echo "$RUNNER_TEMP/fuse-bin" >> "$GITHUB_PATH"
|
||||
echo "using $distro instead"
|
||||
ls -l "$distro"
|
||||
exit 0
|
||||
fi
|
||||
done
|
||||
|
||||
# No usable distro binary, so setting the bit is the only way out - but
|
||||
# the target came from PATH: do that only for a root-owned system binary,
|
||||
# never for anything else that happens to sit there.
|
||||
case "$bin" in
|
||||
/bin/*|/sbin/*|/usr/bin/*|/usr/sbin/*|/usr/local/bin/*|/usr/local/sbin/*) ;;
|
||||
*) echo "refusing to setuid $bin: outside the system bin paths" >&2; exit 1 ;;
|
||||
esac
|
||||
if [ -L "$bin" ] || [ ! -f "$bin" ] || [ ! -x "$bin" ]; then
|
||||
echo "refusing to setuid $bin: not a regular executable file" >&2
|
||||
exit 1
|
||||
fi
|
||||
if [ "$(stat -c %u "$bin")" != 0 ] || [ "$(stat -c %g "$bin")" != 0 ]; then
|
||||
echo "refusing to setuid $bin: not owned by root:root" >&2
|
||||
exit 1
|
||||
fi
|
||||
sudo chmod u+s "$bin"
|
||||
ls -l "$bin"
|
||||
@@ -1,57 +0,0 @@
|
||||
name: Set up a musl Rust target
|
||||
description: Install the stable toolchain for a static Linux musl target, with the C toolchain and the link shims aws-lc-sys needs.
|
||||
inputs:
|
||||
target:
|
||||
description: x86_64-unknown-linux-musl or aarch64-unknown-linux-musl
|
||||
required: true
|
||||
runs:
|
||||
using: composite
|
||||
steps:
|
||||
- name: Install Rust toolchain
|
||||
uses: dtolnay/rust-toolchain@stable
|
||||
with:
|
||||
targets: ${{ inputs.target }}
|
||||
|
||||
- name: Install musl tools (amd64)
|
||||
if: inputs.target == 'x86_64-unknown-linux-musl'
|
||||
shell: bash
|
||||
run: |
|
||||
sudo apt-get install -y musl-tools
|
||||
# C deps (aws-lc-sys) must compile against musl headers, not the host
|
||||
# gcc's glibc headers — glibc's C23 redirects (__isoc23_sscanf,
|
||||
# __isoc23_strtol) don't exist in musl and fail the link.
|
||||
echo "CC_x86_64_unknown_linux_musl=musl-gcc" >> "$GITHUB_ENV"
|
||||
|
||||
- name: Install cross-compilation tools (arm64)
|
||||
if: inputs.target == 'aarch64-unknown-linux-musl'
|
||||
shell: bash
|
||||
run: |
|
||||
sudo apt-get install -y gcc-aarch64-linux-gnu
|
||||
echo "CARGO_TARGET_AARCH64_UNKNOWN_LINUX_MUSL_LINKER=aarch64-linux-gnu-gcc" >> "$GITHUB_ENV"
|
||||
# Disable glibc fortify source — its __memcpy_chk etc. symbols don't exist in musl
|
||||
echo "CFLAGS_aarch64_unknown_linux_musl=-U_FORTIFY_SOURCE" >> "$GITHUB_ENV"
|
||||
# aws-lc-sys compiles against glibc headers, which redirect
|
||||
# sscanf/strtol to C23 __isoc23_* symbols musl does not provide.
|
||||
# Link a tiny forwarding shim so those references resolve to the
|
||||
# musl implementations. Prototypes are declared manually: including
|
||||
# glibc headers here would re-redirect the calls back to __isoc23_*.
|
||||
cat > /tmp/isoc23-shim.c <<'SHIM'
|
||||
#include <stdarg.h>
|
||||
extern int vsscanf(const char *s, const char *fmt, va_list ap);
|
||||
extern long strtol(const char *nptr, char **endptr, int base);
|
||||
int __isoc23_sscanf(const char *s, const char *fmt, ...) {
|
||||
va_list ap;
|
||||
va_start(ap, fmt);
|
||||
int r = vsscanf(s, fmt, ap);
|
||||
va_end(ap);
|
||||
return r;
|
||||
}
|
||||
long __isoc23_strtol(const char *nptr, char **endptr, int base) {
|
||||
return strtol(nptr, endptr, base);
|
||||
}
|
||||
SHIM
|
||||
aarch64-linux-gnu-gcc -O2 -c /tmp/isoc23-shim.c -o /tmp/isoc23-shim.o
|
||||
# link-args land after libc on the link line; rescan musl libc so
|
||||
# the shim's own vsscanf/strtol references resolve, and libgcc for
|
||||
# the fp128 builtins musl's vfscanf/floatscan pull in on aarch64.
|
||||
echo "CARGO_TARGET_AARCH64_UNKNOWN_LINUX_MUSL_RUSTFLAGS=-Clink-arg=/tmp/isoc23-shim.o -Clink-arg=-lc -Clink-arg=-lgcc" >> "$GITHUB_ENV"
|
||||
@@ -1,44 +0,0 @@
|
||||
name: Sign container images
|
||||
description: >
|
||||
Keyless cosign signature on each image, then a verification pass against the
|
||||
identity the signature should carry, so a misconfigured job fails here and not
|
||||
on someone's cluster. That identity is the calling workflow's own,
|
||||
https://github.com/<owner>/<repo>/.github/workflows/<file>@<ref>.
|
||||
The calling job needs `id-token: write` and a registry login for every image.
|
||||
|
||||
inputs:
|
||||
images:
|
||||
description: Image references by digest (name@sha256:...), whitespace separated.
|
||||
required: true
|
||||
|
||||
runs:
|
||||
using: composite
|
||||
steps:
|
||||
- name: Install cosign
|
||||
uses: sigstore/cosign-installer@v4.1.3
|
||||
|
||||
- name: Sign
|
||||
shell: bash
|
||||
env:
|
||||
IMAGES: ${{ inputs.images }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
# The .sig tag layout: the OCI-referrer bundle cosign 3 writes by default
|
||||
# is not read by the Kyverno and policy-controller releases in use today.
|
||||
# Cosign 3 also defaults to --use-signing-config, which insists on a
|
||||
# bundle for its output; turning it off falls back to the default
|
||||
# Fulcio and Rekor URLs, which is all the .sig layout ever used.
|
||||
cosign sign --yes --recursive \
|
||||
--new-bundle-format=false --use-signing-config=false \
|
||||
$IMAGES
|
||||
|
||||
- name: Verify
|
||||
shell: bash
|
||||
env:
|
||||
IMAGES: ${{ inputs.images }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
cosign verify \
|
||||
--certificate-oidc-issuer https://token.actions.githubusercontent.com \
|
||||
--certificate-identity "https://github.com/$GITHUB_WORKFLOW_REF" \
|
||||
$IMAGES
|
||||
@@ -1,14 +1,10 @@
|
||||
version: 2
|
||||
updates:
|
||||
- package-ecosystem: "github-actions"
|
||||
directories:
|
||||
- "/"
|
||||
- "/.github/actions/sign-image"
|
||||
directory: "/"
|
||||
schedule:
|
||||
interval: "weekly"
|
||||
- package-ecosystem: gomod
|
||||
directory: "/"
|
||||
schedule:
|
||||
interval: weekly
|
||||
ignore:
|
||||
- dependency-name: "github.com/seaweedfs/goexif"
|
||||
@@ -5,6 +5,7 @@
|
||||
# How are we solving the problem?
|
||||
|
||||
|
||||
|
||||
# How is the PR tested?
|
||||
|
||||
|
||||
@@ -12,8 +13,3 @@
|
||||
# Checks
|
||||
- [ ] I have added unit tests if possible.
|
||||
- [ ] I will add related wiki document changes and link to this PR after merging.
|
||||
- [ ] All AI code review comments have been addressed. No more comments to fix if reviewed again. Reviewer may request additional gemini and copilot reviews.
|
||||
|
||||
# Checks for AI generated PRs
|
||||
- [ ] I have reviewed every line of code.
|
||||
- [ ] The PR is kept as minimum as possible. Large PRs would not be accepted.
|
||||
@@ -1,240 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Check JWT extraction and, with --context, real Helm upgrade persistence."""
|
||||
|
||||
import argparse
|
||||
import base64
|
||||
import json
|
||||
from pathlib import Path
|
||||
import shutil
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
import uuid
|
||||
|
||||
try:
|
||||
import tomllib
|
||||
except ModuleNotFoundError: # CI also exercises Python 3.10.
|
||||
import tomli as tomllib
|
||||
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[2]
|
||||
CHART = ROOT / "k8s/charts/seaweedfs"
|
||||
SECTIONS = ("jwt.signing", "jwt.signing.read", "jwt.filer_signing", "jwt.filer_signing.read")
|
||||
KEYS = {section: f"active-{index}" for index, section in enumerate(SECTIONS)}
|
||||
ESCAPED_HEADER = '["jw\\u0074".signing]\nkey = "existing"'
|
||||
|
||||
|
||||
def run(*args):
|
||||
return subprocess.run(args, check=True, text=True, capture_output=True).stdout
|
||||
|
||||
|
||||
def keys(raw):
|
||||
document = tomllib.loads(raw)
|
||||
result = {}
|
||||
for section in SECTIONS:
|
||||
table = document
|
||||
for part in section.split("."):
|
||||
table = table.get(part, {})
|
||||
if "key" in table:
|
||||
result[section] = table["key"]
|
||||
return result
|
||||
|
||||
|
||||
def fixtures():
|
||||
canonical = "\n".join(f'[{section}]\nkey = "{value}"' for section, value in KEYS.items())
|
||||
yield "canonical / final line without newline", canonical
|
||||
yield "commented stale keys", canonical.replace("key =", '# key = "stale"\nkey =')
|
||||
yield "commented headers / mixed line endings", "\n".join(
|
||||
f'# [{section}]\r\n# key = "stale"\r\n[{section}]\nkey = "{value}"'
|
||||
for section, value in KEYS.items()
|
||||
)
|
||||
yield "indented headers and keys / trailing comments", "\n".join(
|
||||
f' \t[ {section} ] \t# header [note]\n \tkey \t= \t"{value}" # key comment'
|
||||
for section, value in KEYS.items()
|
||||
)
|
||||
yield "CRLF", canonical.replace("\n", "\r\n")
|
||||
yield "quoted and spaced section names", "\n".join(
|
||||
f'["{section.split(".")[0]}" . \'{section.split(".")[1]}\''
|
||||
+ (f' . "{section.split(".")[2]}"' if section.count(".") == 2 else "")
|
||||
+ f'] # original table\nkey = "{value}"'
|
||||
for section, value in KEYS.items()
|
||||
)
|
||||
yield "unrelated quoted header before JWT keys", '["custom section"]\nkey = "other"\n' + canonical
|
||||
yield "brackets in comments", canonical.replace("key =", "# consult [notes]\nkey =")
|
||||
yield "quoted values and quoted key names", "\n".join((
|
||||
'[jwt.signing]\n"key" = "brackets[inside]#value"',
|
||||
"[jwt.signing.read]\n'key' = 'literal\\path[#value]'",
|
||||
r'[jwt.filer_signing]' + '\n' + r'key = "escaped\"quote\\slash\u0041"',
|
||||
'[jwt.filer_signing.read]\nkey = ""',
|
||||
))
|
||||
yield "absent keys and sections / unrelated key", '\n'.join((
|
||||
'[jwt.signing]\nexpires_after_seconds = 10',
|
||||
'[jwt.signing.read]\nkey = "read-only"',
|
||||
'[unrelated]\nkey = "not-a-jwt-key"',
|
||||
'# [jwt.filer_signing]\n# key = "not-active"',
|
||||
))
|
||||
yield "no existing security.toml", ""
|
||||
|
||||
|
||||
def check_generated(value):
|
||||
decoded = base64.b64decode(value, validate=True).decode("ascii")
|
||||
assert len(decoded) == 10 and decoded.isascii() and decoded.isalnum(), "invalid generated JWT key"
|
||||
|
||||
|
||||
def check_helpers(helm, reference_helm):
|
||||
# Exercise the real helper, not a second implementation of its matching rules.
|
||||
with tempfile.TemporaryDirectory(prefix="helm-jwt-helper-") as directory:
|
||||
chart = Path(directory)
|
||||
(chart / "templates").mkdir()
|
||||
(chart / "Chart.yaml").write_text("apiVersion: v2\nname: jwt-regression\nversion: 0.0.0\n")
|
||||
shutil.copyfile(CHART / "templates/shared/_helpers.tpl", chart / "templates/_helpers.tpl")
|
||||
entries = [
|
||||
json.dumps(section) + ': {{ include "seaweedfs.existingTomlKey" (list '
|
||||
+ json.dumps(section) + ' .Values.raw) | toJson }}'
|
||||
for section in SECTIONS
|
||||
]
|
||||
template = chart / "templates/keys.yaml"
|
||||
prefix = '{"apiVersion":"v1","kind":"ConfigMap","metadata":{"name":"keys"},"data":{'
|
||||
helper_template = prefix + ",".join(entries) + "}}"
|
||||
reference_entries = [
|
||||
json.dumps(section) + ': {{ dig '
|
||||
+ " ".join(json.dumps(part) for part in section.split("."))
|
||||
+ ' "key" "__ABSENT__" (fromToml .Values.raw) | toJson }}'
|
||||
for section in SECTIONS
|
||||
]
|
||||
reference_template = prefix + ",".join(reference_entries) + "}}"
|
||||
|
||||
def render(binary, source, raw):
|
||||
template.write_text(source)
|
||||
values = chart / "input.json"
|
||||
values.write_text(json.dumps({"raw": raw}))
|
||||
output = run(binary, "template", "keys", str(chart), "-f", str(values))
|
||||
return json.loads(output[output.index("{"):])["data"]
|
||||
|
||||
for name, raw in fixtures():
|
||||
expected = keys(raw)
|
||||
tokens = render(helm, helper_template, raw)
|
||||
actual = {section: tomllib.loads("key = " + token)["key"]
|
||||
for section, token in tokens.items() if token != ""}
|
||||
assert actual == expected, f"{name}: extracted keys differ from stored TOML"
|
||||
if reference_helm:
|
||||
reference = render(reference_helm, reference_template, raw)
|
||||
reference = {section: value for section, value in reference.items() if value != "__ABSENT__"}
|
||||
assert actual == reference, f"{name}: keys differ from fromToml/dig"
|
||||
print(f"PASS helper: {name}")
|
||||
|
||||
# A present but unsupported value must not silently become a fresh key.
|
||||
try:
|
||||
render(helm, helper_template, '[jwt.signing]\nkey = """multi\nline"""')
|
||||
except subprocess.CalledProcessError as error:
|
||||
assert "refusing to replace an existing key" in error.stderr, error.stderr
|
||||
else:
|
||||
raise AssertionError("multiline existing key was silently accepted or replaced")
|
||||
print("PASS helper: unsupported existing value fails without rotation")
|
||||
|
||||
try:
|
||||
render(helm, helper_template, ESCAPED_HEADER)
|
||||
except subprocess.CalledProcessError as error:
|
||||
assert "unsupported quoted section header" in error.stderr, error.stderr
|
||||
else:
|
||||
raise AssertionError("unsupported quoted header silently rotated its key")
|
||||
print("PASS helper: unsupported quoted header fails without rotation")
|
||||
|
||||
|
||||
def check_upgrades(helm, context):
|
||||
namespace = "jwt-key-persist-" + uuid.uuid4().hex[:8]
|
||||
current = "jk-seaweedfs-security-config"
|
||||
legacy = "seaweedfs-security-config"
|
||||
kubectl = ["kubectl", "--context", context, "-n", namespace]
|
||||
release_args = ["jk", str(CHART), "--kube-context", context, "-n", namespace]
|
||||
# No workload is needed to exercise Helm's real ConfigMap lookup and update.
|
||||
for setting in (
|
||||
"master.enabled=false", "volume.enabled=false", "filer.enabled=false",
|
||||
"global.seaweedfs.createClusterRole=false",
|
||||
"global.seaweedfs.securityConfig.jwtSigning.volumeWrite=true",
|
||||
"global.seaweedfs.securityConfig.jwtSigning.volumeRead=true",
|
||||
"global.seaweedfs.securityConfig.jwtSigning.filerWrite=true",
|
||||
"global.seaweedfs.securityConfig.jwtSigning.filerRead=true",
|
||||
):
|
||||
release_args += ["--set", setting]
|
||||
|
||||
def stored():
|
||||
cm = json.loads(run(*kubectl, "get", "configmap", current, "-o", "json"))
|
||||
return keys(cm["data"]["security.toml"])
|
||||
|
||||
def upgrade():
|
||||
run(helm, "upgrade", *release_args)
|
||||
return stored()
|
||||
|
||||
def patch(raw):
|
||||
# Seed previous-release content without taking Helm 4's SSA ownership.
|
||||
run(*kubectl, "patch", "configmap", current, "--type=merge", "--field-manager=helm", "-p",
|
||||
json.dumps({"data": {"security.toml": raw}}))
|
||||
|
||||
run(*kubectl, "create", "namespace", namespace)
|
||||
try:
|
||||
run(helm, "install", *release_args)
|
||||
initial = stored()
|
||||
assert set(initial) == set(SECTIONS), "install omitted a JWT section"
|
||||
for value in initial.values():
|
||||
check_generated(value)
|
||||
assert upgrade() == initial, "no-op upgrade changed an existing key"
|
||||
print("PASS upgrade: all four generated keys persist")
|
||||
|
||||
for name, raw in fixtures():
|
||||
patch(raw)
|
||||
actual = upgrade()
|
||||
expected = keys(raw)
|
||||
assert set(actual) == set(SECTIONS), f"{name}: upgrade omitted a JWT section"
|
||||
for section in SECTIONS:
|
||||
if section in expected:
|
||||
assert actual[section] == expected[section], f"{name}: changed {section}"
|
||||
else:
|
||||
check_generated(actual[section])
|
||||
assert upgrade() == actual, f"{name}: subsequent upgrade changed a key"
|
||||
print(f"PASS upgrade: {name}")
|
||||
|
||||
# Migration from the old chart name, followed by precedence of the current name.
|
||||
legacy_raw = "\n".join(f'[{section}]\nkey = "legacy-{index}"'
|
||||
for index, section in enumerate(SECTIONS))
|
||||
run(*kubectl, "create", "configmap", legacy, "--from-literal=security.toml=" + legacy_raw)
|
||||
run(*kubectl, "delete", "configmap", current)
|
||||
assert upgrade() == keys(legacy_raw), "legacy ConfigMap keys were not preserved"
|
||||
print("PASS upgrade: legacy ConfigMap migration")
|
||||
current_raw = next(fixtures())[1]
|
||||
patch(current_raw)
|
||||
assert upgrade() == keys(current_raw), "legacy ConfigMap overrode current ConfigMap"
|
||||
print("PASS upgrade: current ConfigMap takes precedence")
|
||||
|
||||
patch(ESCAPED_HEADER)
|
||||
try:
|
||||
upgrade()
|
||||
except subprocess.CalledProcessError as error:
|
||||
assert "unsupported quoted section header" in error.stderr, error.stderr
|
||||
else:
|
||||
raise AssertionError("unsupported quoted header silently rotated its key")
|
||||
assert keys(run(*kubectl, "get", "configmap", current, "-o",
|
||||
"jsonpath={.data.security\\.toml}"))["jwt.signing"] == "existing"
|
||||
print("PASS upgrade: unsupported quoted header leaves stored key untouched")
|
||||
finally:
|
||||
run(*kubectl, "delete", "namespace", namespace, "--wait=false")
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("--helm", default="helm")
|
||||
parser.add_argument("--reference-helm", help="Helm >=3.17 binary for differential checks")
|
||||
parser.add_argument("--context", help="explicit disposable Kubernetes context for live upgrade checks")
|
||||
args = parser.parse_args()
|
||||
print(run(args.helm, "version", "--short").strip())
|
||||
check_helpers(args.helm, args.reference_helm)
|
||||
if args.context:
|
||||
check_upgrades(args.helm, args.context)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
try:
|
||||
main()
|
||||
except subprocess.CalledProcessError as error:
|
||||
print(error.stderr, file=sys.stderr)
|
||||
sys.exit(error.returncode)
|
||||
@@ -1,307 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Regroup a GitHub release's generated notes by component.
|
||||
|
||||
GitHub's generated notes are one flat list of PRs in merge order. This keeps every
|
||||
line as GitHub wrote it, but sorts the PRs under component headings (S3 API, Filer,
|
||||
Mount, Volume Server, ...), with dependency bumps folded at the end.
|
||||
|
||||
The component comes from the PR title prefix, which nearly every PR has
|
||||
("s3api: ...", "fix(mount): ...", "filer.sync: ...", "rust volume: ..."). A PR with
|
||||
no recognised prefix is placed by the files it changes.
|
||||
|
||||
Usage:
|
||||
release_notes_by_component.py --tag 4.54 # print the regrouped notes
|
||||
release_notes_by_component.py --tag 4.54 --write # also update the release
|
||||
|
||||
Running it again on notes it already regrouped gives the same result.
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import re
|
||||
import subprocess
|
||||
import sys
|
||||
|
||||
# Display order. Each PR lands in exactly one section.
|
||||
SECTIONS = [
|
||||
"S3 API",
|
||||
"S3 Tables and Iceberg",
|
||||
"Vectors",
|
||||
"Filer",
|
||||
"Mount",
|
||||
"Volume Server",
|
||||
"Rust Volume Server",
|
||||
"Erasure Coding",
|
||||
"Master and Topology",
|
||||
"Admin UI and Workers",
|
||||
"Shell",
|
||||
"Security",
|
||||
"Message Queue",
|
||||
"Kubernetes, Helm and Deployment",
|
||||
"Core and CLI",
|
||||
"Documentation",
|
||||
"Tests and CI",
|
||||
"Other",
|
||||
"Dependencies",
|
||||
]
|
||||
|
||||
# Title scope -> section, first match wins. Matched against each lower-cased scope
|
||||
# token, e.g. "filer.sync", "s3 lifecycle", "weed/s3api" (a "weed/" prefix is
|
||||
# dropped first). Order matters: specific scopes come before the general ones
|
||||
# they start with (s3tables before s3, volume.fsck before volume).
|
||||
SCOPE_RULES = [
|
||||
(r"(build\(deps\)|deps|dependabot)\b", "Dependencies"),
|
||||
(r"(test|tests|e2e|integration)\b|.*\btests?$|test/", "Tests and CI"),
|
||||
(r"(rust[ -]?volume|seaweed-volume|volume\(rust\)|rust)\b", "Rust Volume Server"),
|
||||
(r"(s3 ?vectors?|vectors?|seaweed-vector|weed-vector|lance)\b", "Vectors"),
|
||||
(r"(s3 ?tables|iceberg)\b", "S3 Tables and Iceberg"),
|
||||
(r"(s3 sink|remote|remote_storage|filer|webdav|sftp|ftp|notification|replication|"
|
||||
r"redis\d*|leveldb\d*|backup)\b", "Filer"),
|
||||
(r"(s3|s3api|iam|oidc|sts|credential|sse|cors|policy|object ?lock)\b", "S3 API"),
|
||||
(r"(mount|fuse)\b", "Mount"),
|
||||
(r"(ec|ec_\w+|ec\.\w+|erasure[ _-]?coding|worker/ec)\b", "Erasure Coding"),
|
||||
(r"(master|topology|wdclient|cluster|raft|placement)\b", "Master and Topology"),
|
||||
(r"(admin|worker|weed-worker|plugin|plugin worker|maintenance)\b",
|
||||
"Admin UI and Workers"),
|
||||
(r"(shell|fsck|volume\.\w+|fs\.\w+|collection\.\w+|bucket\.\w+)", "Shell"),
|
||||
(r"(volume|volume_\w+|storage|vacuum|operation|compression|needle)\b", "Volume Server"),
|
||||
(r"(security|tls|mtls|jwt|grpc|kms)\b", "Security"),
|
||||
(r"(mq|kafka|messaging)\b", "Message Queue"),
|
||||
(r"(helm|k8s|kubernetes|operator|csi|docker|grafana|install\.sh|chart|snap|terraform)\b",
|
||||
"Kubernetes, Helm and Deployment"),
|
||||
(r"(docs?|readme|wiki)\b", "Documentation"),
|
||||
(r"(ci|release|build|github|workflow|actions)\b", "Tests and CI"),
|
||||
(r"(server|command|mini|weed-db|weed|util|pb|cmd|glog|stats|telemetry|query|config)\b",
|
||||
"Core and CLI"),
|
||||
]
|
||||
|
||||
# Changed path prefix -> section, for PRs whose title has no usable prefix.
|
||||
# Longest prefix wins.
|
||||
PATH_RULES = {
|
||||
"weed/s3api/s3tables/": "S3 Tables and Iceberg",
|
||||
"weed/s3api/iceberg/": "S3 Tables and Iceberg",
|
||||
"weed/s3api/s3vectors/": "Vectors",
|
||||
"weed/s3api/lance/": "Vectors",
|
||||
"seaweed-vector/": "Vectors",
|
||||
"weed/s3api/": "S3 API",
|
||||
"weed/iam/": "S3 API",
|
||||
"weed/iamapi/": "S3 API",
|
||||
"weed/credential/": "S3 API",
|
||||
"weed/filer/": "Filer",
|
||||
"weed/filer_client/": "Filer",
|
||||
"weed/server/filer": "Filer",
|
||||
"weed/server/webdav": "Filer",
|
||||
"weed/sftpd/": "Filer",
|
||||
"weed/replication/": "Filer",
|
||||
"weed/notification/": "Filer",
|
||||
"weed/remote_storage/": "Filer",
|
||||
"weed/mount/": "Mount",
|
||||
"weed/storage/erasure_coding/": "Erasure Coding",
|
||||
"weed/ec/": "Erasure Coding",
|
||||
"weed/worker/tasks/erasure_coding/": "Erasure Coding",
|
||||
"weed/storage/": "Volume Server",
|
||||
"weed/server/volume": "Volume Server",
|
||||
"weed/operation/": "Volume Server",
|
||||
"seaweed-volume/": "Rust Volume Server",
|
||||
"weed/topology/": "Master and Topology",
|
||||
"weed/server/master": "Master and Topology",
|
||||
"weed/wdclient/": "Master and Topology",
|
||||
"weed/cluster/": "Master and Topology",
|
||||
"weed/placement/": "Master and Topology",
|
||||
"weed/admin/": "Admin UI and Workers",
|
||||
"weed/worker/": "Admin UI and Workers",
|
||||
"weed/plugin/": "Admin UI and Workers",
|
||||
"seaweed-worker/": "Admin UI and Workers",
|
||||
"weed/shell/": "Shell",
|
||||
"weed/security/": "Security",
|
||||
"weed/kms/": "Security",
|
||||
"weed/mq/": "Message Queue",
|
||||
"k8s/": "Kubernetes, Helm and Deployment",
|
||||
"docker/": "Kubernetes, Helm and Deployment",
|
||||
"snap/": "Kubernetes, Helm and Deployment",
|
||||
"terraform/": "Kubernetes, Helm and Deployment",
|
||||
"weed/": "Core and CLI",
|
||||
"test/": "Tests and CI",
|
||||
".github/": "Tests and CI",
|
||||
}
|
||||
|
||||
PR_LINE = re.compile(r"^\* (?P<title>.+?) by @(?P<author>\S+) in https://github\.com/\S+/pull/(?P<num>\d+)\s*$")
|
||||
# Lines regroup() writes between the PRs, so a second run accepts them.
|
||||
SCAFFOLD = re.compile(r"^$|^### (%s)$|^<details><summary>Dependencies \(\d+ updates?\)</summary>$|^</details>$"
|
||||
% "|".join(re.escape(s) for s in SECTIONS))
|
||||
# "fix(s3tables,s3vectors): ...", "s3 lifecycle: ...", "[Volume] ...", "Fix/admin ..."
|
||||
CONVENTIONAL = re.compile(r"^(?P<type>[a-z]+)\((?P<scope>[^)]*)\)!?$")
|
||||
TYPE_ONLY = {"fix", "feat", "feature", "perf", "refactor", "chore", "style", "revert", "improvement"}
|
||||
|
||||
|
||||
def scope_of(title):
|
||||
"""Return the raw scope text of a PR title, or ""."""
|
||||
t = title.strip()
|
||||
m = re.match(r"^\[([^\]]{1,40})\]", t)
|
||||
if m:
|
||||
return m.group(1)
|
||||
m = re.match(r"^(?:fix|feat|feature)/(\S+)", t, re.I)
|
||||
if m:
|
||||
return m.group(1)
|
||||
head, sep, _ = t.partition(":")
|
||||
if not sep or len(head) > 50:
|
||||
return ""
|
||||
head = head.strip()
|
||||
m = CONVENTIONAL.match(head.lower())
|
||||
if m:
|
||||
kind, scope = m.group("type"), m.group("scope")
|
||||
if kind in ("test", "tests", "ci"):
|
||||
return "test"
|
||||
if kind == "docs":
|
||||
return "docs"
|
||||
if kind == "build" and scope == "deps":
|
||||
return "build(deps)"
|
||||
return scope
|
||||
if head.lower() in TYPE_ONLY:
|
||||
return ""
|
||||
return head
|
||||
|
||||
|
||||
def section_by_scope(scope):
|
||||
if scope.strip().lower() == "build(deps)":
|
||||
return "Dependencies"
|
||||
for token in re.split(r"\s*[,+&]\s*|\s+and\s+", scope.lower()):
|
||||
token = token.strip()
|
||||
token = re.sub(r"^weed[/ ]", "", token)
|
||||
if not token:
|
||||
continue
|
||||
for pattern, section in SCOPE_RULES:
|
||||
if re.match(pattern, token):
|
||||
return section
|
||||
return ""
|
||||
|
||||
|
||||
def section_by_paths(paths):
|
||||
votes = {}
|
||||
code = [p for p in paths if not p.endswith(("_test.go", "_test.rs")) and not p.startswith("test/")]
|
||||
for p in code or paths:
|
||||
if p.endswith(".md"):
|
||||
section = "Documentation"
|
||||
else:
|
||||
best = max((pre for pre in PATH_RULES if p.startswith(pre)), key=len, default="")
|
||||
section = PATH_RULES.get(best, "Other")
|
||||
votes[section] = votes.get(section, 0) + 1
|
||||
if not votes:
|
||||
return "Other"
|
||||
# Most files wins; ties go to the earlier section in display order.
|
||||
return max(votes, key=lambda s: (votes[s], -SECTIONS.index(s)))
|
||||
|
||||
|
||||
def classify(title, author, pr_files):
|
||||
if author.startswith("dependabot") or title.lower().startswith("build(deps)"):
|
||||
return "Dependencies"
|
||||
section = section_by_scope(scope_of(title))
|
||||
if section:
|
||||
return section
|
||||
return section_by_paths(pr_files())
|
||||
|
||||
|
||||
def split_notes(body):
|
||||
"""Split generated notes into (PR lines, the text before them, the text after,
|
||||
the non-blank lines of What's Changed)."""
|
||||
lines = body.replace("\r\n", "\n").split("\n")
|
||||
start = next((i for i, l in enumerate(lines) if l.strip() == "## What's Changed"), None)
|
||||
if start is None:
|
||||
return [], body, "", []
|
||||
end = next((i for i in range(start + 1, len(lines))
|
||||
if lines[i].startswith(("## ", "**Full Changelog**"))), len(lines))
|
||||
prs, stray = [], []
|
||||
block = [l.rstrip() for l in lines[start + 1:end] if l.strip()]
|
||||
for line in lines[start + 1:end]:
|
||||
if PR_LINE.match(line):
|
||||
prs.append(line)
|
||||
elif not SCAFFOLD.match(line.strip()):
|
||||
stray.append(line)
|
||||
if prs and stray:
|
||||
raise ValueError("What's Changed has lines that are not PRs: %r" % stray[:3])
|
||||
before = "\n".join(lines[:start]).rstrip()
|
||||
after = "\n".join(lines[end:]).strip()
|
||||
return prs, before, after, block
|
||||
|
||||
|
||||
def regroup(body, pr_files):
|
||||
prs, before, after, block = split_notes(body)
|
||||
if not prs:
|
||||
return body
|
||||
groups = {s: [] for s in SECTIONS}
|
||||
for line in prs:
|
||||
m = PR_LINE.match(line)
|
||||
num = int(m.group("num"))
|
||||
groups[classify(m.group("title"), m.group("author"), lambda: pr_files(num))].append(line)
|
||||
|
||||
out = [before, ""] if before else []
|
||||
out.append("## What's Changed")
|
||||
for section in SECTIONS:
|
||||
items = groups[section]
|
||||
if not items:
|
||||
continue
|
||||
out.append("")
|
||||
if section == "Dependencies":
|
||||
out.append("<details><summary>Dependencies (%d update%s)</summary>" % (len(items), "" if len(items) == 1 else "s"))
|
||||
out.append("")
|
||||
out.extend(items)
|
||||
out.append("")
|
||||
out.append("</details>")
|
||||
else:
|
||||
out.append("### " + section)
|
||||
out.extend(items)
|
||||
if after:
|
||||
out.extend(["", after])
|
||||
notes = "\n".join(out).strip() + "\n"
|
||||
# Headings are accepted only as this script's own grouping: a hand-written
|
||||
# "### Security" over PRs of other components would otherwise be dropped.
|
||||
if len(block) != len(prs) and split_notes(notes)[3] != block:
|
||||
raise ValueError("What's Changed has headings that do not match the component grouping")
|
||||
return notes
|
||||
|
||||
|
||||
def gh(*args, stdin=None):
|
||||
return subprocess.run(["gh", *args], input=stdin, capture_output=True, text=True, check=True).stdout
|
||||
|
||||
|
||||
def pr_files(repo, num):
|
||||
return gh("pr", "view", str(num), "-R", repo, "--json", "files", "--jq", ".files[].path").splitlines()
|
||||
|
||||
|
||||
def main():
|
||||
ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
|
||||
ap.add_argument("--repo", default="seaweedfs/seaweedfs")
|
||||
ap.add_argument("--tag", required=True)
|
||||
ap.add_argument("--write", action="store_true", help="update the release notes on GitHub")
|
||||
args = ap.parse_args()
|
||||
|
||||
body = json.loads(gh("release", "view", args.tag, "-R", args.repo, "--json", "body"))["body"]
|
||||
|
||||
failed = []
|
||||
|
||||
def files_of(num):
|
||||
try:
|
||||
return pr_files(args.repo, num)
|
||||
except subprocess.CalledProcessError as e:
|
||||
print("warning: files of #%d: %s" % (num, e.stderr.strip()), file=sys.stderr)
|
||||
failed.append(num)
|
||||
return []
|
||||
|
||||
try:
|
||||
notes = regroup(body, files_of)
|
||||
except ValueError as e:
|
||||
sys.exit("release %s: %s" % (args.tag, e))
|
||||
sys.stdout.write(notes)
|
||||
if notes.strip() == body.replace("\r\n", "\n").strip():
|
||||
print("release %s: notes already grouped, nothing to change" % args.tag, file=sys.stderr)
|
||||
return
|
||||
if failed and args.write:
|
||||
sys.exit("release %s: not writing, could not read the files of %s"
|
||||
% (args.tag, ", ".join("#%d" % n for n in failed)))
|
||||
if args.write:
|
||||
gh("release", "edit", args.tag, "-R", args.repo, "--notes-file", "-", stdin=notes)
|
||||
print("release %s: notes updated" % args.tag, file=sys.stderr)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -1,111 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Render the repository's star history to note/star_history.svg.
|
||||
|
||||
Uses the GitHub REST stargazers endpoint with the starred-at accept header,
|
||||
which caps at 40,000 entries (400 pages of 100). The chart is regenerated on
|
||||
a schedule; if the repo grows past that cap the script stops at 40,000 and
|
||||
logs a warning rather than under-reporting.
|
||||
"""
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
import time
|
||||
import urllib.error
|
||||
import urllib.request
|
||||
from datetime import datetime
|
||||
|
||||
import matplotlib
|
||||
|
||||
matplotlib.use("Agg")
|
||||
import matplotlib.pyplot as plt # noqa: E402
|
||||
from matplotlib.dates import AutoDateLocator, DateFormatter # noqa: E402
|
||||
|
||||
REPO = "seaweedfs/seaweedfs"
|
||||
TOKEN = os.environ["GITHUB_TOKEN"]
|
||||
OUT = os.environ.get("OUT", "note/star_history.svg")
|
||||
PAGE_CAP = 400 # GitHub's hard limit on stargazer pagination
|
||||
REQUEST_ATTEMPTS = 5
|
||||
RETRYABLE_STATUS_CODES = {429, 500, 502, 503, 504}
|
||||
|
||||
|
||||
def open_url(req):
|
||||
for attempt in range(REQUEST_ATTEMPTS):
|
||||
try:
|
||||
return urllib.request.urlopen(req, timeout=30)
|
||||
except urllib.error.HTTPError as error:
|
||||
if error.code not in RETRYABLE_STATUS_CODES or attempt == REQUEST_ATTEMPTS - 1:
|
||||
raise
|
||||
retry_after = error.headers.get("Retry-After")
|
||||
try:
|
||||
delay = min(float(retry_after), 60) if retry_after else 2**attempt
|
||||
except ValueError:
|
||||
delay = 2**attempt
|
||||
error.close()
|
||||
except urllib.error.URLError:
|
||||
if attempt == REQUEST_ATTEMPTS - 1:
|
||||
raise
|
||||
delay = 2**attempt
|
||||
time.sleep(delay)
|
||||
|
||||
|
||||
def fetch_stargazers():
|
||||
stars = []
|
||||
page = 1
|
||||
while page <= PAGE_CAP:
|
||||
url = f"https://api.github.com/repos/{REPO}/stargazers?per_page=100&page={page}"
|
||||
req = urllib.request.Request(
|
||||
url,
|
||||
headers={
|
||||
"Accept": "application/vnd.github.star+json",
|
||||
"Authorization": f"Bearer {TOKEN}",
|
||||
"X-GitHub-Api-Version": "2022-11-28",
|
||||
"User-Agent": "seaweedfs-star-history",
|
||||
},
|
||||
)
|
||||
with open_url(req) as resp:
|
||||
batch = json.load(resp)
|
||||
if not batch:
|
||||
break
|
||||
for u in batch:
|
||||
sa = u.get("starred_at")
|
||||
if sa:
|
||||
stars.append(datetime.fromisoformat(sa.replace("Z", "+00:00")))
|
||||
if len(batch) < 100:
|
||||
break
|
||||
page += 1
|
||||
if page > PAGE_CAP:
|
||||
print(
|
||||
f"::warning::Hit the {PAGE_CAP}-page stargazer pagination cap; "
|
||||
"chart reflects the first 40,000 stars only."
|
||||
)
|
||||
return stars
|
||||
|
||||
|
||||
def render(stars, out):
|
||||
stars.sort()
|
||||
counts = list(range(1, len(stars) + 1))
|
||||
fig, ax = plt.subplots(figsize=(10, 6), dpi=130)
|
||||
ax.plot(stars, counts, color="#0969da", linewidth=1.6)
|
||||
ax.set_xlabel("Date")
|
||||
ax.set_ylabel("Stars")
|
||||
ax.set_title(f"{REPO} star history")
|
||||
ax.grid(True, linestyle="--", alpha=0.3)
|
||||
ax.xaxis.set_major_locator(AutoDateLocator())
|
||||
ax.xaxis.set_major_formatter(DateFormatter("%Y-%m"))
|
||||
fig.autofmt_xdate()
|
||||
fig.tight_layout()
|
||||
fig.savefig(out, format="svg", transparent=False)
|
||||
plt.close(fig)
|
||||
|
||||
|
||||
def main():
|
||||
stars = fetch_stargazers()
|
||||
if not stars:
|
||||
print("::error::No stargazers fetched; not updating the chart.")
|
||||
sys.exit(1)
|
||||
render(stars, OUT)
|
||||
print(f"Rendered {len(stars)} stars to {OUT}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -1,190 +0,0 @@
|
||||
import importlib.util
|
||||
from pathlib import Path
|
||||
import unittest
|
||||
from unittest import mock
|
||||
|
||||
|
||||
def load():
|
||||
path = Path(__file__).with_name("release_notes_by_component.py")
|
||||
spec = importlib.util.spec_from_file_location("release_notes_by_component", path)
|
||||
module = importlib.util.module_from_spec(spec)
|
||||
spec.loader.exec_module(module)
|
||||
return module
|
||||
|
||||
|
||||
rn = load()
|
||||
|
||||
|
||||
def no_files(num):
|
||||
raise AssertionError("files of #%d should not be needed" % num)
|
||||
|
||||
|
||||
class ClassifyTest(unittest.TestCase):
|
||||
def check(self, title, section, author="someone", files=None):
|
||||
got = rn.classify(title, author, (lambda: files) if files is not None else (lambda: no_files(0)))
|
||||
self.assertEqual(got, section, title)
|
||||
|
||||
def test_title_prefixes(self):
|
||||
cases = {
|
||||
"s3: keep serving for -preStopSeconds after SIGTERM": "S3 API",
|
||||
"s3api: keep a multipart upload's state on the object's owner filer": "S3 API",
|
||||
"fix(s3api): reject an invalid storage class before uploading the body": "S3 API",
|
||||
"s3 lifecycle: count filer entries a walk reads but skips as progress": "S3 API",
|
||||
"iam/oidc: accept a list audience": "S3 API",
|
||||
"fix(s3tables,s3vectors): bind DeleteNamespace to the authorized records": "S3 Tables and Iceberg",
|
||||
"iceberg maintenance: expire snapshots": "S3 Tables and Iceberg",
|
||||
"s3vectors: support TagResource": "Vectors",
|
||||
"s3 vectors: page ListVectors": "Vectors",
|
||||
"vector: compact segments": "Vectors",
|
||||
"filer: fail health check when absent from lock ring": "Filer",
|
||||
"filer.sync: skip a rewritten source version": "Filer",
|
||||
"filer.remote.sync: retry": "Filer",
|
||||
"webdav: honor locks": "Filer",
|
||||
"remote.uncache, remote.meta.sync, remote.cache: keep local writes": "Filer",
|
||||
"mount: cache dir": "Mount",
|
||||
"fix(mount): flush on close": "Mount",
|
||||
"volume: keep reads over the download limit local": "Volume Server",
|
||||
"[Volume] Preserve request semantics in replica proxy reads": "Volume Server",
|
||||
"operation: pad the marginal gzip test": "Volume Server",
|
||||
"fix(storage): update deletion metrics": "Volume Server",
|
||||
"rust volume: bulk-load needle maps": "Rust Volume Server",
|
||||
"seaweed-volume: pin the status": "Rust Volume Server",
|
||||
"volume(rust): fix": "Rust Volume Server",
|
||||
"ec: mount shards through the encode's .vif": "Erasure Coding",
|
||||
"ec.rebuild: pick the right disk": "Erasure Coding",
|
||||
"ec_balance: spread shards": "Erasure Coding",
|
||||
"master: unregister a volume server whose heartbeats stop": "Master and Topology",
|
||||
"topology: never report negative free volume slots": "Master and Topology",
|
||||
"wdclient: let master-pushed adds replace locations": "Master and Topology",
|
||||
"admin: manage filer webhook targets": "Admin UI and Workers",
|
||||
"Fix/admin cluster submenu": "Admin UI and Workers",
|
||||
"weed-worker: retry": "Admin UI and Workers",
|
||||
"shell: fix s3.circuitBreaker help examples": "Shell",
|
||||
"volume.fsck: skip deleted": "Shell",
|
||||
"fix(fsck): inspect references to deleted volumes": "Shell",
|
||||
"security: optional dialed-host check": "Security",
|
||||
"helm: drop the 4.53 vector metrics render guard": "Kubernetes, Helm and Deployment",
|
||||
"install.sh: add --enterprise": "Kubernetes, Helm and Deployment",
|
||||
"mq/kafka: commit offsets": "Message Queue",
|
||||
"server: add -grpc.localSocketDir": "Core and CLI",
|
||||
"pb: recover when a local gRPC socket file is deleted": "Core and CLI",
|
||||
"docs: explain the 3-filer recommendation": "Documentation",
|
||||
"test(fuse): two mounts staging the same files": "Tests and CI",
|
||||
"test/s3vectors: a hardened cluster": "Tests and CI",
|
||||
"s3vectors test: cover tags": "Tests and CI",
|
||||
"ci: run the Python 3.9 jobs on 3.10": "Tests and CI",
|
||||
"build(deps): bump actions/setup-go from 6 to 7": "Dependencies",
|
||||
}
|
||||
for title, section in cases.items():
|
||||
self.check(title, section)
|
||||
|
||||
def test_dependabot_author(self):
|
||||
self.check("Bump x from 1 to 2", "Dependencies", author="dependabot[bot]")
|
||||
|
||||
def test_no_prefix_uses_files(self):
|
||||
self.check("Fix s3 cancelled read relookup", "S3 API",
|
||||
files=["weed/s3api/s3api_object_handlers.go", "weed/s3api/s3api_object_handlers_test.go"])
|
||||
self.check("Retry transient failures in star history generation", "Tests and CI",
|
||||
files=[".github/scripts/star_history.py"])
|
||||
self.check("fix: handle empty chunk list", "Filer",
|
||||
files=["weed/filer/filechunks.go", "weed/storage/needle/needle.go", "weed/filer/reader_at.go"])
|
||||
self.check("Improve everything", "Other", files=[])
|
||||
|
||||
def test_test_files_count_only_when_alone(self):
|
||||
self.check("Make it faster", "Mount",
|
||||
files=["weed/mount/weedfs.go", "test/fuse/a_test.go", "test/fuse/b_test.go"])
|
||||
self.check("Make it right", "S3 API",
|
||||
files=["weed/s3api/auth.go", "test/s3/iam/test_a.py", "test/s3/iam/test_b.py"])
|
||||
self.check("Cover more", "Tests and CI", files=["test/s3/iam/test_a.py"])
|
||||
|
||||
def test_path_with_space(self):
|
||||
with mock.patch.object(rn, "gh", return_value="test/s3/iam/policy cases.json\n"):
|
||||
files = rn.pr_files("seaweedfs/seaweedfs", 1)
|
||||
self.assertEqual(files, ["test/s3/iam/policy cases.json"])
|
||||
self.check("Cover more", "Tests and CI", files=files)
|
||||
|
||||
|
||||
NOTES = """## What's Changed
|
||||
* filer: fail health check when absent from lock ring by @a in https://github.com/seaweedfs/seaweedfs/pull/12709
|
||||
* s3: answer ClientDisconnected by @b in https://github.com/seaweedfs/seaweedfs/pull/12707
|
||||
* build(deps): bump actions/setup-go from 6 to 7 by @dependabot[bot] in https://github.com/seaweedfs/seaweedfs/pull/12685
|
||||
* filer.sync: skip a rewritten source version by @a in https://github.com/seaweedfs/seaweedfs/pull/12366
|
||||
* Fix s3 cancelled read relookup by @c in https://github.com/seaweedfs/seaweedfs/pull/12600
|
||||
|
||||
## New Contributors
|
||||
* @c made their first contribution in https://github.com/seaweedfs/seaweedfs/pull/12600
|
||||
|
||||
**Full Changelog**: https://github.com/seaweedfs/seaweedfs/compare/4.53...4.54
|
||||
"""
|
||||
|
||||
|
||||
class RegroupTest(unittest.TestCase):
|
||||
def files(self, num):
|
||||
self.assertEqual(num, 12600)
|
||||
return ["weed/s3api/s3api_object_handlers.go"]
|
||||
|
||||
def test_regroup(self):
|
||||
out = rn.regroup(NOTES, self.files)
|
||||
self.assertEqual(out, """## What's Changed
|
||||
|
||||
### S3 API
|
||||
* s3: answer ClientDisconnected by @b in https://github.com/seaweedfs/seaweedfs/pull/12707
|
||||
* Fix s3 cancelled read relookup by @c in https://github.com/seaweedfs/seaweedfs/pull/12600
|
||||
|
||||
### Filer
|
||||
* filer: fail health check when absent from lock ring by @a in https://github.com/seaweedfs/seaweedfs/pull/12709
|
||||
* filer.sync: skip a rewritten source version by @a in https://github.com/seaweedfs/seaweedfs/pull/12366
|
||||
|
||||
<details><summary>Dependencies (1 update)</summary>
|
||||
|
||||
* build(deps): bump actions/setup-go from 6 to 7 by @dependabot[bot] in https://github.com/seaweedfs/seaweedfs/pull/12685
|
||||
|
||||
</details>
|
||||
|
||||
## New Contributors
|
||||
* @c made their first contribution in https://github.com/seaweedfs/seaweedfs/pull/12600
|
||||
|
||||
**Full Changelog**: https://github.com/seaweedfs/seaweedfs/compare/4.53...4.54
|
||||
""")
|
||||
|
||||
def test_idempotent(self):
|
||||
once = rn.regroup(NOTES, self.files)
|
||||
self.assertEqual(rn.regroup(once, self.files), once)
|
||||
|
||||
def test_every_pr_kept(self):
|
||||
out = rn.regroup(NOTES, self.files)
|
||||
for n in (12709, 12707, 12685, 12366, 12600):
|
||||
self.assertEqual(out.count("/pull/%d\n" % n), 2 if n == 12600 else 1)
|
||||
|
||||
def test_changelog_kept_without_new_contributors(self):
|
||||
body = NOTES.replace(
|
||||
"## New Contributors\n* @c made their first contribution in https://github.com/seaweedfs/seaweedfs/pull/12600\n\n", "")
|
||||
out = rn.regroup(body, self.files)
|
||||
self.assertTrue(out.endswith("</details>\n\n**Full Changelog**: https://github.com/seaweedfs/seaweedfs/compare/4.53...4.54\n"), out)
|
||||
self.assertEqual(rn.regroup(out, self.files), out)
|
||||
|
||||
def test_unknown_line_refused(self):
|
||||
body = NOTES.replace("## New Contributors", "* a hand-written note\n\n## New Contributors")
|
||||
with self.assertRaises(ValueError):
|
||||
rn.regroup(body, self.files)
|
||||
with self.assertRaises(ValueError):
|
||||
rn.regroup(NOTES.replace("## What's Changed\n", "## What's Changed\n### Known issue\n"), self.files)
|
||||
|
||||
def test_hand_written_component_heading_refused(self):
|
||||
with self.assertRaises(ValueError):
|
||||
rn.regroup(NOTES.replace("## What's Changed\n", "## What's Changed\n### Security\n"), self.files)
|
||||
once = rn.regroup(NOTES, self.files)
|
||||
with self.assertRaises(ValueError):
|
||||
rn.regroup(once.replace("\n### Filer\n", "\n ### Filer\n"), self.files)
|
||||
|
||||
def test_author_from_the_by_clause(self):
|
||||
body = "## What's Changed\n* Bump a by @dependabot[bot] in docs by @alice in https://github.com/seaweedfs/seaweedfs/pull/7\n"
|
||||
self.assertIn("### Documentation", rn.regroup(body, lambda n: ["README.md"]))
|
||||
|
||||
def test_notes_without_prs_unchanged(self):
|
||||
body = "Hand-written notes\n\n**Full Changelog**: x"
|
||||
self.assertEqual(rn.regroup(body, no_files), body)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -1,75 +0,0 @@
|
||||
import importlib.util
|
||||
import os
|
||||
from pathlib import Path
|
||||
import sys
|
||||
import types
|
||||
import unittest
|
||||
from unittest.mock import patch
|
||||
from urllib.error import HTTPError
|
||||
|
||||
|
||||
def load_star_history():
|
||||
matplotlib = types.ModuleType("matplotlib")
|
||||
matplotlib.use = lambda backend: None
|
||||
pyplot = types.ModuleType("matplotlib.pyplot")
|
||||
dates = types.ModuleType("matplotlib.dates")
|
||||
dates.AutoDateLocator = object
|
||||
dates.DateFormatter = object
|
||||
modules = {
|
||||
"matplotlib": matplotlib,
|
||||
"matplotlib.pyplot": pyplot,
|
||||
"matplotlib.dates": dates,
|
||||
}
|
||||
with patch.dict(sys.modules, modules), patch.dict(os.environ, {"GITHUB_TOKEN": "test"}):
|
||||
path = Path(__file__).with_name("star_history.py")
|
||||
spec = importlib.util.spec_from_file_location("star_history", path)
|
||||
module = importlib.util.module_from_spec(spec)
|
||||
spec.loader.exec_module(module)
|
||||
return module
|
||||
|
||||
|
||||
star_history = load_star_history()
|
||||
|
||||
|
||||
class StarHistoryTests(unittest.TestCase):
|
||||
def http_error(self, code):
|
||||
return HTTPError("https://api.github.com", code, "error", {}, None)
|
||||
|
||||
@patch.object(star_history.time, "sleep")
|
||||
@patch.object(star_history.urllib.request, "urlopen")
|
||||
def test_retries_server_error(self, urlopen, sleep):
|
||||
response = object()
|
||||
urlopen.side_effect = [self.http_error(500), response]
|
||||
|
||||
result = star_history.open_url(object())
|
||||
|
||||
self.assertIs(result, response)
|
||||
self.assertEqual(urlopen.call_count, 2)
|
||||
sleep.assert_called_once_with(1)
|
||||
|
||||
@patch.object(star_history.time, "sleep")
|
||||
@patch.object(star_history.urllib.request, "urlopen")
|
||||
def test_does_not_retry_non_transient_http_error(self, urlopen, sleep):
|
||||
error = self.http_error(404)
|
||||
urlopen.side_effect = error
|
||||
|
||||
with self.assertRaises(HTTPError):
|
||||
star_history.open_url(object())
|
||||
|
||||
urlopen.assert_called_once()
|
||||
sleep.assert_not_called()
|
||||
|
||||
@patch.object(star_history.time, "sleep")
|
||||
@patch.object(star_history.urllib.request, "urlopen")
|
||||
def test_stops_after_retry_limit(self, urlopen, sleep):
|
||||
urlopen.side_effect = self.http_error(500)
|
||||
|
||||
with self.assertRaises(HTTPError):
|
||||
star_history.open_url(object())
|
||||
|
||||
self.assertEqual(urlopen.call_count, star_history.REQUEST_ATTEMPTS)
|
||||
self.assertEqual(sleep.call_count, star_history.REQUEST_ATTEMPTS - 1)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -1,41 +0,0 @@
|
||||
name: "actions: unit tests"
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [ master ]
|
||||
paths:
|
||||
- '.github/actions/**'
|
||||
- '.github/scripts/*release_notes_by_component.py'
|
||||
- '.github/workflows/action-tests.yml'
|
||||
pull_request:
|
||||
branches: [ master ]
|
||||
paths:
|
||||
- '.github/actions/**'
|
||||
- '.github/scripts/*release_notes_by_component.py'
|
||||
- '.github/workflows/action-tests.yml'
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.event.pull_request.number || (github.event_name == 'push' && github.ref) || github.run_id }}
|
||||
cancel-in-progress: ${{ github.event_name == 'pull_request' || (github.event_name == 'push' && !startsWith(github.ref, 'refs/tags/')) }}
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
jobs:
|
||||
cleanup-stale-fuse-mounts:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Check out code
|
||||
uses: actions/checkout@v7
|
||||
|
||||
- name: Run cleanup action unit tests
|
||||
run: python3 .github/actions/cleanup-stale-fuse-mounts/test_cleanup.py
|
||||
|
||||
release-notes-by-component:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Check out code
|
||||
uses: actions/checkout@v7
|
||||
|
||||
- name: Run release notes grouping unit tests
|
||||
run: python3 .github/scripts/test_release_notes_by_component.py
|
||||
@@ -3,15 +3,6 @@ name: "go: build dev binaries"
|
||||
on:
|
||||
push:
|
||||
branches: [ master ]
|
||||
paths:
|
||||
- 'weed/**'
|
||||
- 'go.mod'
|
||||
- 'go.sum'
|
||||
- '.github/workflows/binaries_dev.yml'
|
||||
|
||||
concurrency:
|
||||
group: binaries-dev-${{ github.ref }}
|
||||
cancel-in-progress: true
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
@@ -27,7 +18,6 @@ jobs:
|
||||
|
||||
- name: Delete old release assets
|
||||
uses: mknejp/delete-release-assets@v1
|
||||
continue-on-error: true
|
||||
with:
|
||||
token: ${{ github.token }}
|
||||
tag: dev
|
||||
@@ -48,13 +38,13 @@ jobs:
|
||||
steps:
|
||||
|
||||
- name: Check out code into the Go module directory
|
||||
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v2
|
||||
uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v2
|
||||
|
||||
- name: Set BUILD_TIME env
|
||||
run: echo BUILD_TIME=$(date -u +%Y%m%d-%H%M) >> ${GITHUB_ENV}
|
||||
|
||||
- name: Go Release Binaries Large Disk
|
||||
uses: wangyoucao577/go-release-action@279495102627de7960cbc33434ab01a12bae144b # v1.22
|
||||
uses: wangyoucao577/go-release-action@481a2c1a0f1be199722e3e9b74d7199acafc30a8 # v1.22
|
||||
with:
|
||||
github_token: ${{ secrets.GITHUB_TOKEN }}
|
||||
goos: ${{ matrix.goos }}
|
||||
@@ -70,7 +60,7 @@ jobs:
|
||||
asset_name: "weed-large-disk-${{ env.BUILD_TIME }}-${{ matrix.goos }}-${{ matrix.goarch }}"
|
||||
|
||||
- name: Go Release Binaries Normal Volume Size
|
||||
uses: wangyoucao577/go-release-action@279495102627de7960cbc33434ab01a12bae144b # v1.22
|
||||
uses: wangyoucao577/go-release-action@481a2c1a0f1be199722e3e9b74d7199acafc30a8 # v1.22
|
||||
with:
|
||||
github_token: ${{ secrets.GITHUB_TOKEN }}
|
||||
goos: ${{ matrix.goos }}
|
||||
@@ -97,13 +87,13 @@ jobs:
|
||||
steps:
|
||||
|
||||
- name: Check out code into the Go module directory
|
||||
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v2
|
||||
uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v2
|
||||
|
||||
- name: Set BUILD_TIME env
|
||||
run: echo BUILD_TIME=$(date -u +%Y%m%d-%H%M) >> ${GITHUB_ENV}
|
||||
|
||||
- name: Go Release Binaries Large Disk
|
||||
uses: wangyoucao577/go-release-action@279495102627de7960cbc33434ab01a12bae144b # v1.22
|
||||
uses: wangyoucao577/go-release-action@481a2c1a0f1be199722e3e9b74d7199acafc30a8 # v1.22
|
||||
with:
|
||||
github_token: ${{ secrets.GITHUB_TOKEN }}
|
||||
goos: ${{ matrix.goos }}
|
||||
@@ -119,7 +109,7 @@ jobs:
|
||||
asset_name: "weed-large-disk-${{ env.BUILD_TIME }}-${{ matrix.goos }}-${{ matrix.goarch }}"
|
||||
|
||||
- name: Go Release Binaries Normal Volume Size
|
||||
uses: wangyoucao577/go-release-action@279495102627de7960cbc33434ab01a12bae144b # v1.22
|
||||
uses: wangyoucao577/go-release-action@481a2c1a0f1be199722e3e9b74d7199acafc30a8 # v1.22
|
||||
with:
|
||||
github_token: ${{ secrets.GITHUB_TOKEN }}
|
||||
goos: ${{ matrix.goos }}
|
||||
|
||||
@@ -28,9 +28,9 @@ jobs:
|
||||
# Steps represent a sequence of tasks that will be executed as part of the job
|
||||
steps:
|
||||
# Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it
|
||||
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v2
|
||||
- uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v2
|
||||
- name: Go Release Binaries Normal Volume Size
|
||||
uses: wangyoucao577/go-release-action@279495102627de7960cbc33434ab01a12bae144b # v1.22
|
||||
uses: wangyoucao577/go-release-action@481a2c1a0f1be199722e3e9b74d7199acafc30a8 # v1.22
|
||||
with:
|
||||
github_token: ${{ secrets.GITHUB_TOKEN }}
|
||||
goos: ${{ matrix.goos }}
|
||||
@@ -44,7 +44,7 @@ jobs:
|
||||
binary_name: weed
|
||||
asset_name: "${{ matrix.goos }}_${{ matrix.goarch }}"
|
||||
- name: Go Release Large Disk Binaries
|
||||
uses: wangyoucao577/go-release-action@279495102627de7960cbc33434ab01a12bae144b # v1.22
|
||||
uses: wangyoucao577/go-release-action@481a2c1a0f1be199722e3e9b74d7199acafc30a8 # v1.22
|
||||
with:
|
||||
github_token: ${{ secrets.GITHUB_TOKEN }}
|
||||
goos: ${{ matrix.goos }}
|
||||
|
||||
@@ -28,9 +28,9 @@ jobs:
|
||||
# Steps represent a sequence of tasks that will be executed as part of the job
|
||||
steps:
|
||||
# Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it
|
||||
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v2
|
||||
- uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v2
|
||||
- name: Go Release Binaries Normal Volume Size
|
||||
uses: wangyoucao577/go-release-action@279495102627de7960cbc33434ab01a12bae144b # v1.22
|
||||
uses: wangyoucao577/go-release-action@481a2c1a0f1be199722e3e9b74d7199acafc30a8 # v1.22
|
||||
with:
|
||||
github_token: ${{ secrets.GITHUB_TOKEN }}
|
||||
goos: ${{ matrix.goos }}
|
||||
@@ -44,7 +44,7 @@ jobs:
|
||||
binary_name: weed
|
||||
asset_name: "${{ matrix.goos }}_${{ matrix.goarch }}"
|
||||
- name: Go Release Large Disk Binaries
|
||||
uses: wangyoucao577/go-release-action@279495102627de7960cbc33434ab01a12bae144b # v1.22
|
||||
uses: wangyoucao577/go-release-action@481a2c1a0f1be199722e3e9b74d7199acafc30a8 # v1.22
|
||||
with:
|
||||
github_token: ${{ secrets.GITHUB_TOKEN }}
|
||||
goos: ${{ matrix.goos }}
|
||||
|
||||
@@ -28,9 +28,9 @@ jobs:
|
||||
# Steps represent a sequence of tasks that will be executed as part of the job
|
||||
steps:
|
||||
# Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it
|
||||
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v2
|
||||
- uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v2
|
||||
- name: Go Release Binaries Normal Volume Size
|
||||
uses: wangyoucao577/go-release-action@279495102627de7960cbc33434ab01a12bae144b # v1.22
|
||||
uses: wangyoucao577/go-release-action@481a2c1a0f1be199722e3e9b74d7199acafc30a8 # v1.22
|
||||
with:
|
||||
github_token: ${{ secrets.GITHUB_TOKEN }}
|
||||
goos: ${{ matrix.goos }}
|
||||
@@ -44,7 +44,7 @@ jobs:
|
||||
binary_name: weed
|
||||
asset_name: "${{ matrix.goos }}_${{ matrix.goarch }}"
|
||||
- name: Go Release Large Disk Binaries
|
||||
uses: wangyoucao577/go-release-action@279495102627de7960cbc33434ab01a12bae144b # v1.22
|
||||
uses: wangyoucao577/go-release-action@481a2c1a0f1be199722e3e9b74d7199acafc30a8 # v1.22
|
||||
with:
|
||||
github_token: ${{ secrets.GITHUB_TOKEN }}
|
||||
goos: ${{ matrix.goos }}
|
||||
|
||||
@@ -28,9 +28,9 @@ jobs:
|
||||
# Steps represent a sequence of tasks that will be executed as part of the job
|
||||
steps:
|
||||
# Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it
|
||||
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v2
|
||||
- uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v2
|
||||
- name: Go Release Binaries Normal Volume Size
|
||||
uses: wangyoucao577/go-release-action@279495102627de7960cbc33434ab01a12bae144b # v1.22
|
||||
uses: wangyoucao577/go-release-action@481a2c1a0f1be199722e3e9b74d7199acafc30a8 # v1.22
|
||||
with:
|
||||
github_token: ${{ secrets.GITHUB_TOKEN }}
|
||||
goos: ${{ matrix.goos }}
|
||||
@@ -44,7 +44,7 @@ jobs:
|
||||
binary_name: weed
|
||||
asset_name: "${{ matrix.goos }}_${{ matrix.goarch }}"
|
||||
- name: Go Release Large Disk Binaries
|
||||
uses: wangyoucao577/go-release-action@279495102627de7960cbc33434ab01a12bae144b # v1.22
|
||||
uses: wangyoucao577/go-release-action@481a2c1a0f1be199722e3e9b74d7199acafc30a8 # v1.22
|
||||
with:
|
||||
github_token: ${{ secrets.GITHUB_TOKEN }}
|
||||
goos: ${{ matrix.goos }}
|
||||
|
||||
@@ -28,9 +28,9 @@ jobs:
|
||||
# Steps represent a sequence of tasks that will be executed as part of the job
|
||||
steps:
|
||||
# Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it
|
||||
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v2
|
||||
- uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v2
|
||||
- name: Go Release Binaries Normal Volume Size
|
||||
uses: wangyoucao577/go-release-action@279495102627de7960cbc33434ab01a12bae144b # v1.22
|
||||
uses: wangyoucao577/go-release-action@481a2c1a0f1be199722e3e9b74d7199acafc30a8 # v1.22
|
||||
with:
|
||||
github_token: ${{ secrets.GITHUB_TOKEN }}
|
||||
goos: ${{ matrix.goos }}
|
||||
@@ -45,7 +45,7 @@ jobs:
|
||||
binary_name: weed
|
||||
asset_name: "${{ matrix.goos }}_${{ matrix.goarch }}_full"
|
||||
- name: Go Release Large Disk Binaries
|
||||
uses: wangyoucao577/go-release-action@279495102627de7960cbc33434ab01a12bae144b # v1.22
|
||||
uses: wangyoucao577/go-release-action@481a2c1a0f1be199722e3e9b74d7199acafc30a8 # v1.22
|
||||
with:
|
||||
github_token: ${{ secrets.GITHUB_TOKEN }}
|
||||
goos: ${{ matrix.goos }}
|
||||
|
||||
@@ -28,9 +28,9 @@ jobs:
|
||||
# Steps represent a sequence of tasks that will be executed as part of the job
|
||||
steps:
|
||||
# Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it
|
||||
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v2
|
||||
- uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v2
|
||||
- name: Go Release Binaries Normal Volume Size
|
||||
uses: wangyoucao577/go-release-action@279495102627de7960cbc33434ab01a12bae144b # v1.22
|
||||
uses: wangyoucao577/go-release-action@481a2c1a0f1be199722e3e9b74d7199acafc30a8 # v1.22
|
||||
with:
|
||||
github_token: ${{ secrets.GITHUB_TOKEN }}
|
||||
goos: ${{ matrix.goos }}
|
||||
@@ -44,7 +44,7 @@ jobs:
|
||||
binary_name: weed
|
||||
asset_name: "${{ matrix.goos }}_${{ matrix.goarch }}"
|
||||
- name: Go Release Large Disk Binaries
|
||||
uses: wangyoucao577/go-release-action@279495102627de7960cbc33434ab01a12bae144b # v1.22
|
||||
uses: wangyoucao577/go-release-action@481a2c1a0f1be199722e3e9b74d7199acafc30a8 # v1.22
|
||||
with:
|
||||
github_token: ${{ secrets.GITHUB_TOKEN }}
|
||||
goos: ${{ matrix.goos }}
|
||||
|
||||
@@ -1,37 +0,0 @@
|
||||
name: "Cancel runs of closed PRs"
|
||||
|
||||
# Merging or closing a PR leaves its pull_request runs going: concurrency
|
||||
# groups cancel a run only when a newer push arrives. pull_request_target gives
|
||||
# this job a token that can cancel runs, fork PRs included. It checks out and
|
||||
# runs no PR code.
|
||||
on:
|
||||
pull_request_target:
|
||||
types: [closed]
|
||||
|
||||
permissions:
|
||||
actions: write
|
||||
pull-requests: read
|
||||
|
||||
jobs:
|
||||
cancel:
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 5
|
||||
steps:
|
||||
- name: Cancel the PR's queued and running pull_request runs
|
||||
env:
|
||||
GH_TOKEN: ${{ github.token }}
|
||||
GH_REPO: ${{ github.repository }}
|
||||
PR: ${{ github.event.pull_request.number }}
|
||||
HEAD_REF: ${{ github.event.pull_request.head.ref }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
# Match the PR's own commits: forks reuse branch names such as master.
|
||||
shas=$(gh api --paginate "repos/$GH_REPO/pulls/$PR/commits" --jq '.[].sha')
|
||||
for status in queued in_progress waiting pending requested; do
|
||||
gh run list --branch "$HEAD_REF" --event pull_request --status "$status" \
|
||||
--limit 200 --json databaseId,headSha --jq '.[] | "\(.databaseId) \(.headSha)"'
|
||||
done | while read -r id sha; do
|
||||
grep -qx "$sha" <<<"$shas" || continue
|
||||
echo "cancel run $id at $sha"
|
||||
gh run cancel "$id" || true
|
||||
done
|
||||
@@ -2,15 +2,10 @@ name: "Code Scanning - Action"
|
||||
|
||||
on:
|
||||
pull_request:
|
||||
paths:
|
||||
- '**/*.go'
|
||||
- 'go.mod'
|
||||
- 'go.sum'
|
||||
- '.github/workflows/codeql.yml'
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.event.pull_request.number || (github.event_name == 'push' && github.ref) || github.run_id }}
|
||||
cancel-in-progress: ${{ github.event_name == 'pull_request' || (github.event_name == 'push' && !startsWith(github.ref, 'refs/tags/')) }}
|
||||
group: ${{ github.head_ref }}/codeql
|
||||
cancel-in-progress: true
|
||||
|
||||
jobs:
|
||||
CodeQL-Build:
|
||||
@@ -23,17 +18,11 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v7
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
cache: ${{ runner.environment == 'github-hosted' }}
|
||||
uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633
|
||||
|
||||
# Initializes the CodeQL tools for scanning.
|
||||
- name: Initialize CodeQL
|
||||
uses: github/codeql-action/init@v4.39.3
|
||||
uses: github/codeql-action/init@v3
|
||||
# Override language selection by uncommenting this and choosing your languages
|
||||
with:
|
||||
languages: go
|
||||
@@ -41,7 +30,7 @@ jobs:
|
||||
# Autobuild attempts to build any compiled languages (C/C++, C#, or Java).
|
||||
# If this step fails, then you should remove it and run the build manually (see below).
|
||||
- name: Autobuild
|
||||
uses: github/codeql-action/autobuild@v4.39.3
|
||||
uses: github/codeql-action/autobuild@v3
|
||||
|
||||
# ℹ️ Command-line programs to run using the OS shell.
|
||||
# 📚 See https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#jobsjob_idstepsrun
|
||||
@@ -55,4 +44,4 @@ jobs:
|
||||
# make release
|
||||
|
||||
- name: Perform CodeQL Analysis
|
||||
uses: github/codeql-action/analyze@v4.39.3
|
||||
uses: github/codeql-action/analyze@v3
|
||||
@@ -1,27 +0,0 @@
|
||||
# Codespell configuration is within .codespellrc
|
||||
---
|
||||
name: Codespell
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [master]
|
||||
pull_request:
|
||||
branches: [master]
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.event.pull_request.number || (github.event_name == 'push' && github.ref) || github.run_id }}
|
||||
cancel-in-progress: ${{ github.event_name == 'pull_request' || (github.event_name == 'push' && !startsWith(github.ref, 'refs/tags/')) }}
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
jobs:
|
||||
codespell:
|
||||
name: Check for spelling errors
|
||||
runs-on: ubuntu-latest
|
||||
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v7
|
||||
- name: Codespell
|
||||
uses: codespell-project/actions-codespell@8f01853be192eb0f849a5c7d721450e7a467c579 # v2.2
|
||||
@@ -3,154 +3,24 @@ name: "docker: build dev containers"
|
||||
on:
|
||||
push:
|
||||
branches: [ master ]
|
||||
paths:
|
||||
- 'weed/**'
|
||||
- 'seaweed-volume/**'
|
||||
- 'seaweed-common/**'
|
||||
- 'seaweed-worker/**'
|
||||
- 'seaweed-vector/**'
|
||||
- 'docker/**'
|
||||
- 'go.mod'
|
||||
- 'go.sum'
|
||||
- '.github/workflows/container_dev.yml'
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.event.pull_request.number || (github.event_name == 'push' && github.ref) || github.run_id }}
|
||||
cancel-in-progress: ${{ github.event_name == 'pull_request' || (github.event_name == 'push' && !startsWith(github.ref, 'refs/tags/')) }}
|
||||
workflow_dispatch: {}
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
jobs:
|
||||
|
||||
# ── Pre-build the Rust binaries natively ────────────────────────────
|
||||
build-rust-binaries:
|
||||
runs-on: ubuntu-24.04
|
||||
strategy:
|
||||
matrix:
|
||||
include:
|
||||
- target: x86_64-unknown-linux-musl
|
||||
arch: amd64
|
||||
- target: aarch64-unknown-linux-musl
|
||||
arch: arm64
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v7
|
||||
|
||||
- name: Set up the musl target
|
||||
uses: ./.github/actions/rust-musl-target
|
||||
with:
|
||||
target: ${{ matrix.target }}
|
||||
|
||||
- name: Cache cargo registry and target
|
||||
if: runner.environment == 'github-hosted'
|
||||
uses: actions/cache@v6
|
||||
with:
|
||||
path: |
|
||||
~/.cargo/registry
|
||||
~/.cargo/git
|
||||
seaweed-volume/target
|
||||
seaweed-worker/target/${{ matrix.target }}/release
|
||||
seaweed-vector/target/${{ matrix.target }}/release
|
||||
key: rust-docker-dev-${{ matrix.target }}-${{ hashFiles('seaweed-volume/Cargo.lock', 'seaweed-worker/Cargo.lock', 'seaweed-vector/Cargo.lock') }}
|
||||
restore-keys: |
|
||||
rust-docker-dev-${{ matrix.target }}-
|
||||
|
||||
# lance's build scripts compile their own protos and look for a protoc.
|
||||
# Point them at the one protoc-bin-vendored ships, which seaweed-worker's
|
||||
# own build already uses, so no job depends on a system package and every
|
||||
# build sees the same version.
|
||||
- name: Use the vendored protoc
|
||||
run: |
|
||||
cd seaweed-worker
|
||||
cargo fetch
|
||||
# The version from the lock, not whatever else a restored cache holds.
|
||||
version=$(awk '/^name = "protoc-bin-vendored-linux-x86_64"$/{found=1; next} found && /^version = /{gsub(/"/,"",$3); print $3; exit}' Cargo.lock)
|
||||
test -n "$version" || { echo "protoc-bin-vendored-linux-x86_64 is not in Cargo.lock" >&2; exit 1; }
|
||||
protoc=$(find ~/.cargo/registry/src -path "*protoc-bin-vendored-linux-x86_64-$version/bin/protoc" | head -1)
|
||||
test -x "$protoc" || { echo "no vendored protoc $version in the registry" >&2; exit 1; }
|
||||
echo "PROTOC=$protoc" >> "$GITHUB_ENV"
|
||||
|
||||
- name: Build normal variant
|
||||
env:
|
||||
SEAWEEDFS_COMMIT: ${{ github.sha }}
|
||||
run: |
|
||||
cd seaweed-volume
|
||||
cargo build --release --target ${{ matrix.target }} --no-default-features
|
||||
cp target/${{ matrix.target }}/release/weed-volume ../weed-volume-normal-${{ matrix.arch }}
|
||||
|
||||
- name: Build the Rust maintenance worker
|
||||
run: |
|
||||
cd seaweed-worker
|
||||
cargo build --release -p weed-lance-worker --target ${{ matrix.target }}
|
||||
cp target/${{ matrix.target }}/release/weed-worker ../weed-worker-${{ matrix.arch }}
|
||||
|
||||
- name: Build the vector engine
|
||||
run: |
|
||||
cd seaweed-vector
|
||||
cargo build --release -p weed-vector-server --target ${{ matrix.target }}
|
||||
cp target/${{ matrix.target }}/release/weed-vector ../weed-vector-${{ matrix.arch }}
|
||||
|
||||
- name: Upload artifacts
|
||||
uses: actions/upload-artifact@v7
|
||||
with:
|
||||
name: rust-bins-${{ matrix.arch }}
|
||||
path: |
|
||||
weed-volume-normal-${{ matrix.arch }}
|
||||
weed-worker-${{ matrix.arch }}
|
||||
weed-vector-${{ matrix.arch }}
|
||||
|
||||
build-dev-containers:
|
||||
needs: [build-rust-binaries]
|
||||
runs-on: [ubuntu-latest]
|
||||
permissions:
|
||||
contents: read
|
||||
id-token: write
|
||||
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v7
|
||||
|
||||
- name: Download pre-built Rust binaries
|
||||
uses: actions/download-artifact@v8
|
||||
with:
|
||||
pattern: rust-bins-*
|
||||
merge-multiple: true
|
||||
path: ./rust-bins
|
||||
|
||||
- name: Place Rust binaries in Docker context
|
||||
run: |
|
||||
mkdir -p docker/weed-volume-prebuilt
|
||||
for arch in amd64 arm64; do
|
||||
src="./rust-bins/weed-volume-normal-${arch}"
|
||||
if [ -f "$src" ]; then
|
||||
cp "$src" "docker/weed-volume-prebuilt/weed-volume-${arch}"
|
||||
echo "Placed pre-built Rust binary for ${arch}"
|
||||
fi
|
||||
done
|
||||
mkdir -p docker/weed-worker-prebuilt
|
||||
for arch in amd64 arm64; do
|
||||
src="./rust-bins/weed-worker-${arch}"
|
||||
if [ -f "$src" ]; then
|
||||
cp "$src" "docker/weed-worker-prebuilt/weed-worker-${arch}"
|
||||
echo "Placed pre-built Rust worker for ${arch}"
|
||||
fi
|
||||
done
|
||||
mkdir -p docker/weed-vector-prebuilt
|
||||
for arch in amd64 arm64; do
|
||||
src="./rust-bins/weed-vector-${arch}"
|
||||
if [ -f "$src" ]; then
|
||||
cp "$src" "docker/weed-vector-prebuilt/weed-vector-${arch}"
|
||||
echo "Placed pre-built vector engine for ${arch}"
|
||||
fi
|
||||
done
|
||||
ls -la docker/weed-volume-prebuilt/
|
||||
ls -la docker/weed-worker-prebuilt/
|
||||
ls -la docker/weed-vector-prebuilt/
|
||||
|
||||
- name: Docker meta
|
||||
-
|
||||
name: Checkout
|
||||
uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v2
|
||||
-
|
||||
name: Docker meta
|
||||
id: docker_meta
|
||||
uses: docker/metadata-action@v6
|
||||
uses: docker/metadata-action@902fa8ec7d6ecbf8d84d538b9b233a880e428804 # v3
|
||||
with:
|
||||
images: |
|
||||
chrislusf/seaweedfs
|
||||
@@ -161,41 +31,32 @@ jobs:
|
||||
org.opencontainers.image.title=seaweedfs
|
||||
org.opencontainers.image.description=SeaweedFS is a distributed storage system for blobs, objects, files, and data lake, to store and serve billions of files fast!
|
||||
org.opencontainers.image.vendor=Chris Lu
|
||||
|
||||
- name: Set up QEMU
|
||||
uses: docker/setup-qemu-action@v4.4.0
|
||||
|
||||
- name: Create BuildKit config
|
||||
run: |
|
||||
cat > /tmp/buildkitd.toml <<EOF
|
||||
[registry."docker.io"]
|
||||
mirrors = ["https://mirror.gcr.io"]
|
||||
EOF
|
||||
|
||||
- name: Set up Docker Buildx
|
||||
uses: docker/setup-buildx-action@v4
|
||||
-
|
||||
name: Set up QEMU
|
||||
uses: docker/setup-qemu-action@29109295f81e9208d7d86ff1c6c12d2833863392 # v1
|
||||
-
|
||||
name: Set up Docker Buildx
|
||||
uses: docker/setup-buildx-action@e468171a9de216ec08956ac3ada2f0791b6bd435 # v1
|
||||
with:
|
||||
buildkitd-flags: "--debug"
|
||||
buildkitd-config: /tmp/buildkitd.toml
|
||||
|
||||
- name: Login to Docker Hub
|
||||
-
|
||||
name: Login to Docker Hub
|
||||
if: github.event_name != 'pull_request'
|
||||
uses: docker/login-action@v4.8.0
|
||||
uses: docker/login-action@74a5d142397b4f367a81961eba4e8cd7edddf772 # v1
|
||||
with:
|
||||
username: ${{ secrets.DOCKER_USERNAME }}
|
||||
password: ${{ secrets.DOCKER_PASSWORD }}
|
||||
|
||||
- name: Login to GHCR
|
||||
-
|
||||
name: Login to GHCR
|
||||
if: github.event_name != 'pull_request'
|
||||
uses: docker/login-action@v4.8.0
|
||||
uses: docker/login-action@74a5d142397b4f367a81961eba4e8cd7edddf772 # v1
|
||||
with:
|
||||
registry: ghcr.io
|
||||
username: ${{ secrets.GHCR_USERNAME }}
|
||||
password: ${{ secrets.GHCR_TOKEN }}
|
||||
|
||||
- name: Build
|
||||
id: build
|
||||
uses: docker/build-push-action@v7
|
||||
-
|
||||
name: Build
|
||||
uses: docker/build-push-action@263435318d21b8e681c14492fe198d362a7d2c83 # v2
|
||||
with:
|
||||
context: ./docker
|
||||
push: ${{ github.event_name != 'pull_request' }}
|
||||
@@ -203,17 +64,3 @@ jobs:
|
||||
platforms: linux/amd64, linux/arm64
|
||||
tags: ${{ steps.docker_meta.outputs.tags }}
|
||||
labels: ${{ steps.docker_meta.outputs.labels }}
|
||||
cache-from: type=gha
|
||||
cache-to: type=gha,mode=max
|
||||
# Key the checkout layer on the pushed sha so cached layers can't
|
||||
# serve stale code; the Dockerfile fetch fallback picks it up.
|
||||
build-args: |
|
||||
BRANCH=${{ github.sha }}
|
||||
|
||||
- name: Sign
|
||||
if: github.event_name != 'pull_request'
|
||||
uses: ./.github/actions/sign-image
|
||||
with:
|
||||
images: >-
|
||||
chrislusf/seaweedfs@${{ steps.build.outputs.digest }}
|
||||
ghcr.io/chrislusf/seaweedfs@${{ steps.build.outputs.digest }}
|
||||
@@ -1,187 +0,0 @@
|
||||
name: "docker: build foundationdb image by version"
|
||||
|
||||
on:
|
||||
pull_request:
|
||||
branches: [ master, main ]
|
||||
paths:
|
||||
- 'weed/filer/foundationdb/**'
|
||||
- 'test/foundationdb/**'
|
||||
- 'docker/Dockerfile.foundationdb_large'
|
||||
- 'docker/filer_foundationdb.toml'
|
||||
- '.github/workflows/container_foundationdb_version.yml'
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
fdb_version:
|
||||
description: 'FoundationDB version to build (e.g. 7.4.5)'
|
||||
required: true
|
||||
default: '7.4.5'
|
||||
seaweedfs_ref:
|
||||
description: 'SeaweedFS git tag, branch, or commit to build'
|
||||
required: true
|
||||
default: 'master'
|
||||
image_tag:
|
||||
description: 'Optional Docker tag suffix (defaults to foundationdb_<fdb>_seaweedfs_<ref>)'
|
||||
required: false
|
||||
default: ''
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.event.pull_request.number || (github.event_name == 'push' && github.ref) || github.event.inputs.image_tag || format('foundationdb_{0}_seaweedfs_{1}', github.event.inputs.fdb_version || '7.4.5', github.event.inputs.seaweedfs_ref || github.ref_name) }}
|
||||
cancel-in-progress: ${{ github.event_name == 'pull_request' || (github.event_name == 'push' && !startsWith(github.ref, 'refs/tags/')) }}
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
jobs:
|
||||
build-foundationdb-image:
|
||||
runs-on: ubuntu-latest
|
||||
permissions:
|
||||
contents: read
|
||||
id-token: write
|
||||
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v7
|
||||
with:
|
||||
fetch-depth: 0
|
||||
|
||||
- name: Install FoundationDB client libraries
|
||||
run: |
|
||||
set -euo pipefail
|
||||
sudo apt-get update
|
||||
sudo apt-get install -y ca-certificates wget
|
||||
FDB_VERSION="${{ inputs.fdb_version || '7.4.5' }}"
|
||||
case "${FDB_VERSION}_amd64" in
|
||||
"7.4.5_amd64") EXPECTED_SHA256="eea6b98cf386a0848655b2e196d18633662a7440a7ee061c10e32153c7e7e112" ;;
|
||||
"7.3.43_amd64") EXPECTED_SHA256="c3fa0a59c7355b914a1455dac909238d5ea3b6c6bc7b530af8597e6487c1651a" ;;
|
||||
*)
|
||||
echo "Unsupported FoundationDB version ${FDB_VERSION} for CI client install" >&2
|
||||
exit 1 ;;
|
||||
esac
|
||||
PACKAGE="foundationdb-clients_${FDB_VERSION}-1_amd64.deb"
|
||||
wget --timeout=30 --tries=3 -O "${PACKAGE}" "https://github.com/apple/foundationdb/releases/download/${FDB_VERSION}/${PACKAGE}"
|
||||
echo "${EXPECTED_SHA256} ${PACKAGE}" | sha256sum -c -
|
||||
sudo dpkg -i "${PACKAGE}"
|
||||
rm "${PACKAGE}"
|
||||
sudo ldconfig
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v7
|
||||
with:
|
||||
go-version-file: go.mod
|
||||
cache: ${{ runner.environment == 'github-hosted' }}
|
||||
|
||||
- name: Run FoundationDB tagged tests
|
||||
env:
|
||||
CGO_ENABLED: 1
|
||||
run: |
|
||||
go test ./weed/filer/foundationdb -tags foundationdb -count=1
|
||||
|
||||
- name: Prepare Docker tag
|
||||
id: tag
|
||||
env:
|
||||
FDB_VERSION_INPUT: ${{ inputs.fdb_version }}
|
||||
SEAWEEDFS_REF_INPUT: ${{ inputs.seaweedfs_ref }}
|
||||
CUSTOM_TAG_INPUT: ${{ inputs.image_tag }}
|
||||
EVENT_NAME: ${{ github.event_name }}
|
||||
HEAD_REF: ${{ github.head_ref }}
|
||||
REF_NAME: ${{ github.ref_name }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
sanitize() {
|
||||
local value="$1"
|
||||
value="${value,,}"
|
||||
value="${value// /-}"
|
||||
value="${value//[^a-z0-9_.-]/-}"
|
||||
value="${value#-}"
|
||||
value="${value%-}"
|
||||
printf '%s' "$value"
|
||||
}
|
||||
version="${FDB_VERSION_INPUT}"
|
||||
seaweed="${SEAWEEDFS_REF_INPUT}"
|
||||
tag="${CUSTOM_TAG_INPUT}"
|
||||
# Use defaults for PR builds
|
||||
if [ -z "$version" ]; then
|
||||
version="7.4.5"
|
||||
fi
|
||||
if [ -z "$seaweed" ]; then
|
||||
if [ "$EVENT_NAME" = "pull_request" ]; then
|
||||
seaweed="${HEAD_REF}"
|
||||
else
|
||||
seaweed="${REF_NAME}"
|
||||
fi
|
||||
fi
|
||||
sanitized_version="$(sanitize "$version")"
|
||||
if [ -z "$sanitized_version" ]; then
|
||||
echo "Unable to sanitize FoundationDB version '$version'." >&2
|
||||
exit 1
|
||||
fi
|
||||
sanitized_seaweed="$(sanitize "$seaweed")"
|
||||
if [ -z "$sanitized_seaweed" ]; then
|
||||
echo "Unable to sanitize SeaweedFS ref '$seaweed'." >&2
|
||||
exit 1
|
||||
fi
|
||||
if [ -z "$tag" ]; then
|
||||
tag="foundationdb_${sanitized_version}_seaweedfs_${sanitized_seaweed}"
|
||||
else
|
||||
tag="$(sanitize "$tag")"
|
||||
fi
|
||||
if [ -z "$tag" ]; then
|
||||
echo "Resulting Docker tag is empty." >&2
|
||||
exit 1
|
||||
fi
|
||||
echo "docker_tag=$tag" >> "$GITHUB_OUTPUT"
|
||||
echo "full_image=chrislusf/seaweedfs:$tag" >> "$GITHUB_OUTPUT"
|
||||
echo "seaweedfs_ref=$seaweed" >> "$GITHUB_OUTPUT"
|
||||
|
||||
- name: Set up QEMU
|
||||
uses: docker/setup-qemu-action@v4.4.0
|
||||
|
||||
- name: Set up Docker Buildx
|
||||
uses: docker/setup-buildx-action@v4
|
||||
|
||||
- name: Login to Docker Hub
|
||||
if: github.event_name != 'pull_request'
|
||||
uses: docker/login-action@v4.8.0
|
||||
with:
|
||||
username: ${{ secrets.DOCKER_USERNAME }}
|
||||
password: ${{ secrets.DOCKER_PASSWORD }}
|
||||
|
||||
- name: Determine branch to build
|
||||
id: branch
|
||||
env:
|
||||
INPUT_REF: ${{ inputs.seaweedfs_ref }}
|
||||
HEAD_REF: ${{ github.head_ref }}
|
||||
REF_NAME: ${{ github.ref_name }}
|
||||
run: |
|
||||
if [ -n "$INPUT_REF" ]; then
|
||||
echo "branch=$INPUT_REF" >> "$GITHUB_OUTPUT"
|
||||
elif [ "${{ github.event_name }}" = "pull_request" ]; then
|
||||
echo "branch=$HEAD_REF" >> "$GITHUB_OUTPUT"
|
||||
else
|
||||
echo "branch=$REF_NAME" >> "$GITHUB_OUTPUT"
|
||||
fi
|
||||
|
||||
- name: Build and push image
|
||||
id: build
|
||||
uses: docker/build-push-action@v7
|
||||
with:
|
||||
context: ./docker
|
||||
push: ${{ github.event_name != 'pull_request' }}
|
||||
file: ./docker/Dockerfile.foundationdb_large
|
||||
build-args: |
|
||||
FDB_VERSION=${{ inputs.fdb_version || '7.4.5' }}
|
||||
BRANCH=${{ steps.branch.outputs.branch }}
|
||||
# Note: ARM64 support requires FoundationDB ARM64 packages which are not available for all versions
|
||||
# Currently only building for amd64. To enable ARM64, verify package availability and add checksums.
|
||||
platforms: linux/amd64
|
||||
tags: ${{ steps.tag.outputs.full_image || 'seaweedfs:foundationdb-test' }}
|
||||
labels: |
|
||||
org.opencontainers.image.title=seaweedfs
|
||||
org.opencontainers.image.description=SeaweedFS is a distributed storage system for blobs, objects, files, and data lake, to store and serve billions of files fast!
|
||||
org.opencontainers.image.vendor=Chris Lu
|
||||
|
||||
- name: Sign
|
||||
if: github.event_name != 'pull_request'
|
||||
uses: ./.github/actions/sign-image
|
||||
with:
|
||||
images: chrislusf/seaweedfs@${{ steps.build.outputs.digest }}
|
||||
@@ -1,599 +1,67 @@
|
||||
name: "docker: build latest container"
|
||||
|
||||
# Manual fallback only. On tag push, container_release_unified.yml already
|
||||
# re-tags the released versioned image as `latest` / `latest_large_disk`,
|
||||
# so a full rebuild here is unnecessary. Run this manually if you need to
|
||||
# rebuild `latest` from an arbitrary ref.
|
||||
on:
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
source_ref:
|
||||
description: 'Git ref to build (branch, tag, or commit SHA)'
|
||||
required: true
|
||||
default: 'master'
|
||||
image_tag:
|
||||
description: 'Docker tag to publish (without variant suffix)'
|
||||
required: true
|
||||
default: 'latest'
|
||||
variant:
|
||||
description: 'Variant to build manually'
|
||||
required: true
|
||||
type: choice
|
||||
default: all
|
||||
options:
|
||||
- all
|
||||
- standard
|
||||
- large_disk
|
||||
publish:
|
||||
description: 'Publish images and manifests'
|
||||
required: true
|
||||
type: boolean
|
||||
default: false
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.event.pull_request.number || (github.event_name == 'push' && github.ref) || github.event.inputs.image_tag || github.run_id }}
|
||||
cancel-in-progress: false
|
||||
queue: max
|
||||
push:
|
||||
tags:
|
||||
- '*'
|
||||
workflow_dispatch: {}
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
security-events: write
|
||||
|
||||
jobs:
|
||||
setup:
|
||||
runs-on: ubuntu-latest
|
||||
outputs:
|
||||
variants: ${{ steps.set-variants.outputs.variants }}
|
||||
publish: ${{ steps.set-publish.outputs.publish }}
|
||||
steps:
|
||||
- name: Select variants for this run
|
||||
id: set-variants
|
||||
run: |
|
||||
if [ "${{ github.event_name }}" = "workflow_dispatch" ] && [ "${{ github.event.inputs.variant }}" != "all" ]; then
|
||||
variants="[\"${{ github.event.inputs.variant }}\"]"
|
||||
else
|
||||
variants='["standard","large_disk"]'
|
||||
fi
|
||||
echo "variants=$variants" >> "$GITHUB_OUTPUT"
|
||||
- name: Select publish mode
|
||||
id: set-publish
|
||||
run: |
|
||||
if [ "${{ github.event_name }}" = "workflow_dispatch" ]; then
|
||||
echo "publish=${{ github.event.inputs.publish }}" >> "$GITHUB_OUTPUT"
|
||||
else
|
||||
echo "publish=true" >> "$GITHUB_OUTPUT"
|
||||
fi
|
||||
|
||||
# ── Pre-build the Rust binaries natively ────────────────────────────
|
||||
build-rust-binaries:
|
||||
runs-on: ubuntu-24.04
|
||||
strategy:
|
||||
matrix:
|
||||
include:
|
||||
- target: x86_64-unknown-linux-musl
|
||||
arch: amd64
|
||||
- target: aarch64-unknown-linux-musl
|
||||
arch: arm64
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v7
|
||||
with:
|
||||
ref: ${{ github.event_name == 'workflow_dispatch' && github.event.inputs.source_ref || github.ref }}
|
||||
|
||||
# The checkout above is the source ref, which may predate these
|
||||
# actions; take them from master, the ref this fallback runs from.
|
||||
- name: Set up the musl target
|
||||
uses: seaweedfs/seaweedfs/.github/actions/rust-musl-target@master
|
||||
with:
|
||||
target: ${{ matrix.target }}
|
||||
|
||||
- name: Cache cargo registry and target
|
||||
if: runner.environment == 'github-hosted'
|
||||
uses: actions/cache@v6
|
||||
with:
|
||||
path: |
|
||||
~/.cargo/registry
|
||||
~/.cargo/git
|
||||
seaweed-volume/target
|
||||
seaweed-worker/target/${{ matrix.target }}/release
|
||||
seaweed-vector/target/${{ matrix.target }}/release
|
||||
key: rust-docker-${{ matrix.target }}-${{ hashFiles('seaweed-volume/Cargo.lock', 'seaweed-worker/Cargo.lock', 'seaweed-vector/Cargo.lock') }}
|
||||
restore-keys: |
|
||||
rust-docker-${{ matrix.target }}-
|
||||
|
||||
# lance's build scripts compile their own protos and look for a protoc.
|
||||
# Point them at the one protoc-bin-vendored ships, which seaweed-worker's
|
||||
# own build already uses, so no job depends on a system package and every
|
||||
# build sees the same version.
|
||||
- name: Use the vendored protoc
|
||||
run: |
|
||||
cd seaweed-worker
|
||||
cargo fetch
|
||||
# The version from the lock, not whatever else a restored cache holds.
|
||||
version=$(awk '/^name = "protoc-bin-vendored-linux-x86_64"$/{found=1; next} found && /^version = /{gsub(/"/,"",$3); print $3; exit}' Cargo.lock)
|
||||
test -n "$version" || { echo "protoc-bin-vendored-linux-x86_64 is not in Cargo.lock" >&2; exit 1; }
|
||||
protoc=$(find ~/.cargo/registry/src -path "*protoc-bin-vendored-linux-x86_64-$version/bin/protoc" | head -1)
|
||||
test -x "$protoc" || { echo "no vendored protoc $version in the registry" >&2; exit 1; }
|
||||
echo "PROTOC=$protoc" >> "$GITHUB_ENV"
|
||||
|
||||
- name: Build large-disk variant
|
||||
env:
|
||||
SEAWEEDFS_COMMIT: ${{ github.sha }}
|
||||
run: |
|
||||
cd seaweed-volume
|
||||
cargo build --release --target ${{ matrix.target }}
|
||||
cp target/${{ matrix.target }}/release/weed-volume ../weed-volume-large-disk-${{ matrix.arch }}
|
||||
|
||||
- name: Build normal variant
|
||||
env:
|
||||
SEAWEEDFS_COMMIT: ${{ github.sha }}
|
||||
run: |
|
||||
cd seaweed-volume
|
||||
cargo build --release --target ${{ matrix.target }} --no-default-features
|
||||
cp target/${{ matrix.target }}/release/weed-volume ../weed-volume-normal-${{ matrix.arch }}
|
||||
|
||||
- name: Build the Rust maintenance worker
|
||||
run: |
|
||||
cd seaweed-worker
|
||||
cargo build --release -p weed-lance-worker --target ${{ matrix.target }}
|
||||
cp target/${{ matrix.target }}/release/weed-worker ../weed-worker-${{ matrix.arch }}
|
||||
|
||||
- name: Build the vector engine
|
||||
run: |
|
||||
# An older source_ref predates the engine; its image goes without one.
|
||||
if [ ! -f seaweed-vector/crates/server/Cargo.toml ]; then
|
||||
echo "No vector engine in this source ref"
|
||||
exit 0
|
||||
fi
|
||||
cd seaweed-vector
|
||||
cargo build --release -p weed-vector-server --target ${{ matrix.target }}
|
||||
cp target/${{ matrix.target }}/release/weed-vector ../weed-vector-${{ matrix.arch }}
|
||||
|
||||
- name: Upload artifacts
|
||||
uses: actions/upload-artifact@v7
|
||||
with:
|
||||
name: rust-bins-${{ matrix.arch }}
|
||||
path: |
|
||||
weed-volume-large-disk-${{ matrix.arch }}
|
||||
weed-volume-normal-${{ matrix.arch }}
|
||||
weed-worker-${{ matrix.arch }}
|
||||
weed-vector-${{ matrix.arch }}
|
||||
|
||||
build:
|
||||
needs: [setup, build-rust-binaries]
|
||||
runs-on: ubuntu-latest
|
||||
strategy:
|
||||
matrix:
|
||||
platform: [amd64, arm64, arm, 386, ppc64le, s390x]
|
||||
variant: ${{ fromJSON(needs.setup.outputs.variants) }}
|
||||
build-latest-container:
|
||||
runs-on: [ubuntu-latest]
|
||||
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v7
|
||||
with:
|
||||
ref: ${{ github.event_name == 'workflow_dispatch' && github.event.inputs.source_ref || github.ref }}
|
||||
- name: Clean up stale FUSE mounts
|
||||
uses: seaweedfs/seaweedfs/.github/actions/cleanup-stale-fuse-mounts@master
|
||||
|
||||
- name: Free Disk Space
|
||||
run: |
|
||||
echo "Available disk space before cleanup:"
|
||||
timeout 15 df -h || true
|
||||
# Remove pre-installed tools
|
||||
sudo rm -rf /usr/share/dotnet /usr/local/lib/android /opt/ghc /opt/hostedtoolcache/CodeQL
|
||||
# Clean package managers
|
||||
sudo apt-get clean
|
||||
sudo rm -rf /var/lib/apt/lists/*
|
||||
# Clean Docker aggressively
|
||||
sudo docker system prune -af --volumes
|
||||
# Clean Go cache if it exists
|
||||
[ -d ~/.cache/go-build ] && rm -rf ~/.cache/go-build || true
|
||||
[ -d /go/pkg ] && rm -rf /go/pkg || true
|
||||
echo "Available disk space after cleanup:"
|
||||
timeout 15 df -h || true
|
||||
|
||||
- name: Configure variant
|
||||
id: config
|
||||
run: |
|
||||
if [ "${{ matrix.variant }}" == "large_disk" ]; then
|
||||
echo "tag_suffix=_large_disk" >> $GITHUB_OUTPUT
|
||||
echo "build_args=TAGS=5BytesOffset" >> $GITHUB_OUTPUT
|
||||
echo "rust_variant=large-disk" >> $GITHUB_OUTPUT
|
||||
else
|
||||
echo "tag_suffix=" >> $GITHUB_OUTPUT
|
||||
echo "build_args=" >> $GITHUB_OUTPUT
|
||||
echo "rust_variant=normal" >> $GITHUB_OUTPUT
|
||||
fi
|
||||
|
||||
- name: Download pre-built Rust binaries
|
||||
uses: actions/download-artifact@v8
|
||||
with:
|
||||
pattern: rust-bins-*
|
||||
merge-multiple: true
|
||||
path: ./rust-bins
|
||||
|
||||
- name: Place Rust binaries in Docker context
|
||||
run: |
|
||||
mkdir -p docker/weed-volume-prebuilt
|
||||
for arch in amd64 arm64; do
|
||||
src="./rust-bins/weed-volume-${{ steps.config.outputs.rust_variant }}-${arch}"
|
||||
if [ -f "$src" ]; then
|
||||
cp "$src" "docker/weed-volume-prebuilt/weed-volume-${arch}"
|
||||
echo "Placed pre-built Rust binary for ${arch}"
|
||||
fi
|
||||
done
|
||||
mkdir -p docker/weed-worker-prebuilt
|
||||
for arch in amd64 arm64; do
|
||||
src="./rust-bins/weed-worker-${arch}"
|
||||
if [ -f "$src" ]; then
|
||||
cp "$src" "docker/weed-worker-prebuilt/weed-worker-${arch}"
|
||||
echo "Placed pre-built Rust worker for ${arch}"
|
||||
fi
|
||||
done
|
||||
mkdir -p docker/weed-vector-prebuilt
|
||||
for arch in amd64 arm64; do
|
||||
src="./rust-bins/weed-vector-${arch}"
|
||||
if [ -f "$src" ]; then
|
||||
cp "$src" "docker/weed-vector-prebuilt/weed-vector-${arch}"
|
||||
echo "Placed pre-built vector engine for ${arch}"
|
||||
fi
|
||||
done
|
||||
ls -la docker/weed-volume-prebuilt/
|
||||
ls -la docker/weed-worker-prebuilt/
|
||||
ls -la docker/weed-vector-prebuilt/
|
||||
|
||||
- name: Docker meta
|
||||
-
|
||||
name: Checkout
|
||||
uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v2
|
||||
-
|
||||
name: Docker meta
|
||||
id: docker_meta
|
||||
uses: docker/metadata-action@v6
|
||||
uses: docker/metadata-action@902fa8ec7d6ecbf8d84d538b9b233a880e428804 # v3
|
||||
with:
|
||||
images: |
|
||||
chrislusf/seaweedfs
|
||||
ghcr.io/chrislusf/seaweedfs
|
||||
tags: type=raw,value=${{ github.event_name == 'workflow_dispatch' && github.event.inputs.image_tag || 'latest' }},suffix=${{ steps.config.outputs.tag_suffix }}
|
||||
tags: |
|
||||
type=raw,value=latest
|
||||
labels: |
|
||||
org.opencontainers.image.title=seaweedfs
|
||||
org.opencontainers.image.description=SeaweedFS is a distributed storage system for blobs, objects, files, and data lake, to store and serve billions of files fast!
|
||||
org.opencontainers.image.vendor=Chris Lu
|
||||
- name: Set up QEMU
|
||||
if: matrix.platform != 'amd64'
|
||||
uses: docker/setup-qemu-action@v4.4.0
|
||||
- name: Create BuildKit config
|
||||
run: |
|
||||
cat > /tmp/buildkitd.toml <<EOF
|
||||
[registry."docker.io"]
|
||||
mirrors = ["https://mirror.gcr.io"]
|
||||
EOF
|
||||
- name: Set up Docker Buildx
|
||||
uses: docker/setup-buildx-action@v4
|
||||
-
|
||||
name: Set up QEMU
|
||||
uses: docker/setup-qemu-action@29109295f81e9208d7d86ff1c6c12d2833863392 # v1
|
||||
-
|
||||
name: Set up Docker Buildx
|
||||
uses: docker/setup-buildx-action@e468171a9de216ec08956ac3ada2f0791b6bd435 # v1
|
||||
with:
|
||||
buildkitd-flags: "--debug"
|
||||
buildkitd-config: /tmp/buildkitd.toml
|
||||
- name: Login to Docker Hub
|
||||
if: needs.setup.outputs.publish == 'true'
|
||||
uses: docker/login-action@v4.8.0
|
||||
-
|
||||
name: Login to Docker Hub
|
||||
if: github.event_name != 'pull_request'
|
||||
uses: docker/login-action@74a5d142397b4f367a81961eba4e8cd7edddf772 # v1
|
||||
with:
|
||||
username: ${{ secrets.DOCKER_USERNAME }}
|
||||
password: ${{ secrets.DOCKER_PASSWORD }}
|
||||
- name: Login to GHCR
|
||||
if: needs.setup.outputs.publish == 'true'
|
||||
uses: docker/login-action@v4.8.0
|
||||
-
|
||||
name: Login to GHCR
|
||||
if: github.event_name != 'pull_request'
|
||||
uses: docker/login-action@74a5d142397b4f367a81961eba4e8cd7edddf772 # v1
|
||||
with:
|
||||
registry: ghcr.io
|
||||
username: ${{ secrets.GHCR_USERNAME }}
|
||||
password: ${{ secrets.GHCR_TOKEN }}
|
||||
- name: Build ${{ matrix.platform }} ${{ matrix.variant }}
|
||||
uses: docker/build-push-action@v7
|
||||
env:
|
||||
DOCKER_BUILDKIT: 1
|
||||
-
|
||||
name: Build
|
||||
uses: docker/build-push-action@263435318d21b8e681c14492fe198d362a7d2c83 # v2
|
||||
with:
|
||||
context: ./docker
|
||||
push: ${{ needs.setup.outputs.publish == 'true' }}
|
||||
push: ${{ github.event_name != 'pull_request' }}
|
||||
file: ./docker/Dockerfile.go_build
|
||||
platforms: linux/${{ matrix.platform }}
|
||||
# Push to GHCR only during build to avoid Docker Hub rate limits
|
||||
tags: ghcr.io/chrislusf/seaweedfs:${{ github.event_name == 'workflow_dispatch' && github.event.inputs.image_tag || 'latest' }}${{ steps.config.outputs.tag_suffix }}-${{ matrix.platform }}
|
||||
platforms: linux/amd64, linux/arm, linux/arm64, linux/386
|
||||
tags: ${{ steps.docker_meta.outputs.tags }}
|
||||
labels: ${{ steps.docker_meta.outputs.labels }}
|
||||
cache-from: type=gha,scope=${{ matrix.variant }}-${{ matrix.platform }}
|
||||
cache-to: type=gha,mode=max,scope=${{ matrix.variant }}-${{ matrix.platform }}
|
||||
build-args: |
|
||||
BUILDKIT_INLINE_CACHE=1
|
||||
BRANCH=${{ github.event_name == 'workflow_dispatch' && github.event.inputs.source_ref || github.sha }}
|
||||
${{ steps.config.outputs.build_args }}
|
||||
- name: Clean up build artifacts
|
||||
if: always()
|
||||
run: |
|
||||
# Clean up Docker build cache and temporary files
|
||||
sudo docker system prune -f
|
||||
# Remove Go build cache
|
||||
sudo rm -rf /tmp/go-build*
|
||||
|
||||
trivy-scan:
|
||||
runs-on: ubuntu-latest
|
||||
needs: [setup, build, build-rust-binaries]
|
||||
strategy:
|
||||
matrix:
|
||||
variant: ${{ fromJSON(needs.setup.outputs.variants) }}
|
||||
steps:
|
||||
- name: Configure variant
|
||||
id: config
|
||||
run: |
|
||||
if [ "${{ matrix.variant }}" == "large_disk" ]; then
|
||||
echo "tag_suffix=_large_disk" >> $GITHUB_OUTPUT
|
||||
else
|
||||
echo "tag_suffix=" >> $GITHUB_OUTPUT
|
||||
fi
|
||||
- name: Login to GHCR
|
||||
if: needs.setup.outputs.publish == 'true'
|
||||
uses: docker/login-action@v4.8.0
|
||||
with:
|
||||
registry: ghcr.io
|
||||
username: ${{ secrets.GHCR_USERNAME }}
|
||||
password: ${{ secrets.GHCR_TOKEN }}
|
||||
- name: Checkout for local scan build
|
||||
if: needs.setup.outputs.publish != 'true'
|
||||
uses: actions/checkout@v7
|
||||
with:
|
||||
ref: ${{ github.event_name == 'workflow_dispatch' && github.event.inputs.source_ref || github.ref }}
|
||||
- name: Download pre-built Rust binaries for local scan
|
||||
if: needs.setup.outputs.publish != 'true'
|
||||
uses: actions/download-artifact@v8
|
||||
with:
|
||||
pattern: rust-bins-*
|
||||
merge-multiple: true
|
||||
path: ./rust-bins
|
||||
- name: Place Rust binaries in Docker context for local scan
|
||||
if: needs.setup.outputs.publish != 'true'
|
||||
run: |
|
||||
rust_variant="normal"
|
||||
if [ "${{ matrix.variant }}" == "large_disk" ]; then
|
||||
rust_variant="large-disk"
|
||||
fi
|
||||
mkdir -p docker/weed-volume-prebuilt
|
||||
for arch in amd64 arm64; do
|
||||
src="./rust-bins/weed-volume-${rust_variant}-${arch}"
|
||||
if [ -f "$src" ]; then
|
||||
cp "$src" "docker/weed-volume-prebuilt/weed-volume-${arch}"
|
||||
echo "Placed pre-built Rust binary for ${arch}"
|
||||
fi
|
||||
done
|
||||
mkdir -p docker/weed-worker-prebuilt
|
||||
for arch in amd64 arm64; do
|
||||
src="./rust-bins/weed-worker-${arch}"
|
||||
if [ -f "$src" ]; then
|
||||
cp "$src" "docker/weed-worker-prebuilt/weed-worker-${arch}"
|
||||
echo "Placed pre-built Rust worker for ${arch}"
|
||||
fi
|
||||
done
|
||||
mkdir -p docker/weed-vector-prebuilt
|
||||
for arch in amd64 arm64; do
|
||||
src="./rust-bins/weed-vector-${arch}"
|
||||
if [ -f "$src" ]; then
|
||||
cp "$src" "docker/weed-vector-prebuilt/weed-vector-${arch}"
|
||||
echo "Placed pre-built vector engine for ${arch}"
|
||||
fi
|
||||
done
|
||||
ls -la docker/weed-volume-prebuilt/
|
||||
ls -la docker/weed-worker-prebuilt/
|
||||
ls -la docker/weed-vector-prebuilt/
|
||||
- name: Create BuildKit config for local scan build
|
||||
if: needs.setup.outputs.publish != 'true'
|
||||
run: |
|
||||
cat > /tmp/buildkitd.toml <<EOF
|
||||
[registry."docker.io"]
|
||||
mirrors = ["https://mirror.gcr.io"]
|
||||
EOF
|
||||
- name: Set up Docker Buildx for local scan build
|
||||
if: needs.setup.outputs.publish != 'true'
|
||||
uses: docker/setup-buildx-action@v4
|
||||
with:
|
||||
buildkitd-flags: "--debug"
|
||||
buildkitd-config: /tmp/buildkitd.toml
|
||||
- name: Build local scan image tarball
|
||||
if: needs.setup.outputs.publish != 'true'
|
||||
uses: docker/build-push-action@v7
|
||||
env:
|
||||
DOCKER_BUILDKIT: 1
|
||||
with:
|
||||
context: ./docker
|
||||
file: ./docker/Dockerfile.go_build
|
||||
platforms: linux/amd64
|
||||
outputs: type=docker,dest=/tmp/seaweedfs${{ steps.config.outputs.tag_suffix }}-amd64.tar
|
||||
build-args: |
|
||||
BUILDKIT_INLINE_CACHE=1
|
||||
BRANCH=${{ github.event_name == 'workflow_dispatch' && github.event.inputs.source_ref || github.sha }}
|
||||
${{ matrix.variant == 'large_disk' && 'TAGS=5BytesOffset' || '' }}
|
||||
- name: Trivy report (published image)
|
||||
if: needs.setup.outputs.publish == 'true'
|
||||
# Pin to SHA - mutable tags were compromised (GHSA-69fq-xp46-6x23)
|
||||
uses: aquasecurity/trivy-action@ed142fd0673e97e23eac54620cfb913e5ce36c25 # v0.36.0
|
||||
with:
|
||||
scan-type: image
|
||||
# Scan amd64 only - OS packages are identical across architectures
|
||||
# since they all use the same alpine base, so a single-arch scan
|
||||
# provides sufficient coverage without multiplying CI time.
|
||||
image-ref: ghcr.io/chrislusf/seaweedfs:${{ github.event_name == 'workflow_dispatch' && github.event.inputs.image_tag || 'latest' }}${{ steps.config.outputs.tag_suffix }}-amd64
|
||||
scanners: vuln
|
||||
vuln-type: os,library
|
||||
severity: HIGH,CRITICAL
|
||||
ignore-unfixed: true
|
||||
limit-severities-for-sarif: true
|
||||
format: sarif
|
||||
output: trivy-results.sarif
|
||||
exit-code: '0'
|
||||
- name: Trivy report (local tarball)
|
||||
if: needs.setup.outputs.publish != 'true'
|
||||
uses: aquasecurity/trivy-action@ed142fd0673e97e23eac54620cfb913e5ce36c25 # v0.36.0
|
||||
with:
|
||||
input: /tmp/seaweedfs${{ steps.config.outputs.tag_suffix }}-amd64.tar
|
||||
scanners: vuln
|
||||
vuln-type: os,library
|
||||
severity: HIGH,CRITICAL
|
||||
ignore-unfixed: true
|
||||
limit-severities-for-sarif: true
|
||||
format: sarif
|
||||
output: trivy-results.sarif
|
||||
exit-code: '0'
|
||||
- name: Upload Trivy scan results to GitHub Security
|
||||
uses: github/codeql-action/upload-sarif@v4.39.3
|
||||
if: always()
|
||||
with:
|
||||
sarif_file: trivy-results.sarif
|
||||
- name: Trivy gate (published image)
|
||||
if: needs.setup.outputs.publish == 'true'
|
||||
# Gate only on fixable high/critical vulnerabilities. Non-fixable
|
||||
# findings are still visible in the SARIF upload above.
|
||||
uses: aquasecurity/trivy-action@ed142fd0673e97e23eac54620cfb913e5ce36c25 # v0.36.0
|
||||
with:
|
||||
scan-type: image
|
||||
image-ref: ghcr.io/chrislusf/seaweedfs:${{ github.event_name == 'workflow_dispatch' && github.event.inputs.image_tag || 'latest' }}${{ steps.config.outputs.tag_suffix }}-amd64
|
||||
scanners: vuln
|
||||
vuln-type: os,library
|
||||
severity: HIGH,CRITICAL
|
||||
ignore-unfixed: true
|
||||
format: table
|
||||
exit-code: '1'
|
||||
skip-setup-trivy: true
|
||||
- name: Trivy gate (local tarball)
|
||||
if: needs.setup.outputs.publish != 'true'
|
||||
uses: aquasecurity/trivy-action@ed142fd0673e97e23eac54620cfb913e5ce36c25 # v0.36.0
|
||||
with:
|
||||
input: /tmp/seaweedfs${{ steps.config.outputs.tag_suffix }}-amd64.tar
|
||||
scanners: vuln
|
||||
vuln-type: os,library
|
||||
severity: HIGH,CRITICAL
|
||||
ignore-unfixed: true
|
||||
format: table
|
||||
exit-code: '1'
|
||||
skip-setup-trivy: true
|
||||
|
||||
create-manifest:
|
||||
runs-on: ubuntu-latest
|
||||
needs: [setup, build, trivy-scan]
|
||||
if: needs.setup.outputs.publish == 'true' && github.event_name != 'pull_request'
|
||||
permissions:
|
||||
contents: read
|
||||
id-token: write
|
||||
strategy:
|
||||
matrix:
|
||||
variant: ${{ fromJSON(needs.setup.outputs.variants) }}
|
||||
steps:
|
||||
- name: Checkout the signing action
|
||||
uses: actions/checkout@v7
|
||||
with:
|
||||
sparse-checkout: .github/actions
|
||||
persist-credentials: false
|
||||
|
||||
- name: Configure variant
|
||||
id: config
|
||||
run: |
|
||||
if [ "${{ matrix.variant }}" == "large_disk" ]; then
|
||||
echo "tag_suffix=_large_disk" >> $GITHUB_OUTPUT
|
||||
else
|
||||
echo "tag_suffix=" >> $GITHUB_OUTPUT
|
||||
fi
|
||||
|
||||
- name: Docker meta
|
||||
id: docker_meta
|
||||
uses: docker/metadata-action@v6
|
||||
with:
|
||||
images: |
|
||||
chrislusf/seaweedfs
|
||||
ghcr.io/chrislusf/seaweedfs
|
||||
tags: type=raw,value=${{ github.event_name == 'workflow_dispatch' && github.event.inputs.image_tag || 'latest' }},suffix=${{ steps.config.outputs.tag_suffix }}
|
||||
- name: Login to Docker Hub
|
||||
uses: docker/login-action@v4.8.0
|
||||
with:
|
||||
username: ${{ secrets.DOCKER_USERNAME }}
|
||||
password: ${{ secrets.DOCKER_PASSWORD }}
|
||||
- name: Login to GHCR
|
||||
uses: docker/login-action@v4.8.0
|
||||
with:
|
||||
registry: ghcr.io
|
||||
username: ${{ secrets.GHCR_USERNAME }}
|
||||
password: ${{ secrets.GHCR_TOKEN }}
|
||||
- name: Install crane
|
||||
run: |
|
||||
# Install crane for efficient multi-arch image copying
|
||||
cd $(mktemp -d)
|
||||
curl -sLO https://github.com/google/go-containerregistry/releases/download/v0.22.0/go-containerregistry_Linux_x86_64.tar.gz
|
||||
echo "edb74d53fad9a596860f59d1c5d04a43dfb5f441dc71f57060dd0bf39483c833 go-containerregistry_Linux_x86_64.tar.gz" | sha256sum -c -
|
||||
tar xzf go-containerregistry_Linux_x86_64.tar.gz crane
|
||||
sudo mv crane /usr/local/bin/
|
||||
crane version
|
||||
- name: Create and push manifest
|
||||
id: manifest
|
||||
env:
|
||||
BASE_TAG: ${{ github.event_name == 'workflow_dispatch' && github.event.inputs.image_tag || 'latest' }}
|
||||
run: |
|
||||
SUFFIX="${{ steps.config.outputs.tag_suffix }}"
|
||||
|
||||
# Create manifest on GHCR first (no rate limits)
|
||||
echo "Creating GHCR manifest (no rate limits)..."
|
||||
docker buildx imagetools create -t ghcr.io/chrislusf/seaweedfs:${BASE_TAG}${SUFFIX} \
|
||||
--metadata-file /tmp/manifest.json \
|
||||
ghcr.io/chrislusf/seaweedfs:${BASE_TAG}${SUFFIX}-amd64 \
|
||||
ghcr.io/chrislusf/seaweedfs:${BASE_TAG}${SUFFIX}-arm64 \
|
||||
ghcr.io/chrislusf/seaweedfs:${BASE_TAG}${SUFFIX}-arm \
|
||||
ghcr.io/chrislusf/seaweedfs:${BASE_TAG}${SUFFIX}-386 \
|
||||
ghcr.io/chrislusf/seaweedfs:${BASE_TAG}${SUFFIX}-ppc64le \
|
||||
ghcr.io/chrislusf/seaweedfs:${BASE_TAG}${SUFFIX}-s390x
|
||||
# The copy and the signature below use the digest this run pushed, not whatever the tag points at by then.
|
||||
DIGEST=$(jq -er '."containerimage.descriptor".digest' /tmp/manifest.json)
|
||||
echo "digest=${DIGEST}" >> "$GITHUB_OUTPUT"
|
||||
|
||||
# Copy the complete multi-arch image from GHCR to Docker Hub
|
||||
# This only requires one pull from GHCR (no rate limit) and one push to Docker Hub
|
||||
echo "Copying manifest from GHCR to Docker Hub..."
|
||||
|
||||
# Function to retry with exponential backoff for Docker Hub operations
|
||||
retry_with_backoff() {
|
||||
local max_attempts=5
|
||||
local timeout=1
|
||||
local attempt=1
|
||||
local exit_code=0
|
||||
|
||||
while [ $attempt -le $max_attempts ]; do
|
||||
if "$@"; then
|
||||
return 0
|
||||
else
|
||||
exit_code=$?
|
||||
fi
|
||||
|
||||
if [ $attempt -lt $max_attempts ]; then
|
||||
echo "Attempt $attempt failed. Retrying in ${timeout}s..." >&2
|
||||
sleep $timeout
|
||||
timeout=$((timeout * 2))
|
||||
fi
|
||||
|
||||
attempt=$((attempt + 1))
|
||||
done
|
||||
|
||||
echo "Command failed after $max_attempts attempts" >&2
|
||||
return $exit_code
|
||||
}
|
||||
|
||||
# Use crane or skopeo to copy, fallback to docker if not available
|
||||
if command -v crane &> /dev/null; then
|
||||
echo "Using crane to copy..."
|
||||
retry_with_backoff crane copy ghcr.io/chrislusf/seaweedfs@${DIGEST} chrislusf/seaweedfs:${BASE_TAG}${SUFFIX}
|
||||
elif command -v skopeo &> /dev/null; then
|
||||
echo "Using skopeo to copy..."
|
||||
retry_with_backoff skopeo copy --all docker://ghcr.io/chrislusf/seaweedfs@${DIGEST} docker://chrislusf/seaweedfs:${BASE_TAG}${SUFFIX}
|
||||
else
|
||||
echo "Using docker buildx imagetools (pulling 6 images from Docker Hub)..."
|
||||
# Fallback: create manifest directly on Docker Hub (pulls from Docker Hub - rate limited)
|
||||
retry_with_backoff docker buildx imagetools create -t chrislusf/seaweedfs:${BASE_TAG}${SUFFIX} \
|
||||
ghcr.io/chrislusf/seaweedfs:${BASE_TAG}${SUFFIX}-amd64 \
|
||||
ghcr.io/chrislusf/seaweedfs:${BASE_TAG}${SUFFIX}-arm64 \
|
||||
ghcr.io/chrislusf/seaweedfs:${BASE_TAG}${SUFFIX}-arm \
|
||||
ghcr.io/chrislusf/seaweedfs:${BASE_TAG}${SUFFIX}-386 \
|
||||
ghcr.io/chrislusf/seaweedfs:${BASE_TAG}${SUFFIX}-ppc64le \
|
||||
ghcr.io/chrislusf/seaweedfs:${BASE_TAG}${SUFFIX}-s390x
|
||||
fi
|
||||
|
||||
- name: Sign
|
||||
uses: ./.github/actions/sign-image
|
||||
with:
|
||||
images: >-
|
||||
ghcr.io/chrislusf/seaweedfs@${{ steps.manifest.outputs.digest }}
|
||||
chrislusf/seaweedfs@${{ steps.manifest.outputs.digest }}
|
||||
@@ -0,0 +1,57 @@
|
||||
name: "docker: build release containers for normal volume"
|
||||
|
||||
on:
|
||||
push:
|
||||
tags:
|
||||
- '*'
|
||||
workflow_dispatch: {}
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
jobs:
|
||||
build-default-release-container:
|
||||
runs-on: [ubuntu-latest]
|
||||
|
||||
steps:
|
||||
-
|
||||
name: Checkout
|
||||
uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v2
|
||||
-
|
||||
name: Docker meta
|
||||
id: docker_meta
|
||||
uses: docker/metadata-action@902fa8ec7d6ecbf8d84d538b9b233a880e428804 # v3
|
||||
with:
|
||||
images: |
|
||||
chrislusf/seaweedfs
|
||||
tags: |
|
||||
type=ref,event=tag
|
||||
flavor: |
|
||||
latest=false
|
||||
labels: |
|
||||
org.opencontainers.image.title=seaweedfs
|
||||
org.opencontainers.image.description=SeaweedFS is a distributed storage system for blobs, objects, files, and data lake, to store and serve billions of files fast!
|
||||
org.opencontainers.image.vendor=Chris Lu
|
||||
-
|
||||
name: Set up QEMU
|
||||
uses: docker/setup-qemu-action@29109295f81e9208d7d86ff1c6c12d2833863392 # v1
|
||||
-
|
||||
name: Set up Docker Buildx
|
||||
uses: docker/setup-buildx-action@e468171a9de216ec08956ac3ada2f0791b6bd435 # v1
|
||||
-
|
||||
name: Login to Docker Hub
|
||||
if: github.event_name != 'pull_request'
|
||||
uses: docker/login-action@74a5d142397b4f367a81961eba4e8cd7edddf772 # v1
|
||||
with:
|
||||
username: ${{ secrets.DOCKER_USERNAME }}
|
||||
password: ${{ secrets.DOCKER_PASSWORD }}
|
||||
-
|
||||
name: Build
|
||||
uses: docker/build-push-action@263435318d21b8e681c14492fe198d362a7d2c83 # v2
|
||||
with:
|
||||
context: ./docker
|
||||
push: ${{ github.event_name != 'pull_request' }}
|
||||
file: ./docker/Dockerfile.go_build
|
||||
platforms: linux/amd64, linux/arm, linux/arm64, linux/386
|
||||
tags: ${{ steps.docker_meta.outputs.tags }}
|
||||
labels: ${{ steps.docker_meta.outputs.labels }}
|
||||
@@ -0,0 +1,59 @@
|
||||
name: "docker: build release containers for large volume"
|
||||
|
||||
on:
|
||||
push:
|
||||
tags:
|
||||
- '*'
|
||||
workflow_dispatch: {}
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
jobs:
|
||||
|
||||
build-large-release-container:
|
||||
runs-on: [ubuntu-latest]
|
||||
|
||||
steps:
|
||||
-
|
||||
name: Checkout
|
||||
uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v2
|
||||
-
|
||||
name: Docker meta
|
||||
id: docker_meta
|
||||
uses: docker/metadata-action@902fa8ec7d6ecbf8d84d538b9b233a880e428804 # v3
|
||||
with:
|
||||
images: |
|
||||
chrislusf/seaweedfs
|
||||
tags: |
|
||||
type=ref,event=tag,suffix=_large_disk
|
||||
flavor: |
|
||||
latest=false
|
||||
labels: |
|
||||
org.opencontainers.image.title=seaweedfs
|
||||
org.opencontainers.image.description=SeaweedFS is a distributed storage system for blobs, objects, files, and data lake, to store and serve billions of files fast!
|
||||
org.opencontainers.image.vendor=Chris Lu
|
||||
-
|
||||
name: Set up QEMU
|
||||
uses: docker/setup-qemu-action@29109295f81e9208d7d86ff1c6c12d2833863392 # v1
|
||||
-
|
||||
name: Set up Docker Buildx
|
||||
uses: docker/setup-buildx-action@e468171a9de216ec08956ac3ada2f0791b6bd435 # v1
|
||||
-
|
||||
name: Login to Docker Hub
|
||||
if: github.event_name != 'pull_request'
|
||||
uses: docker/login-action@74a5d142397b4f367a81961eba4e8cd7edddf772 # v1
|
||||
with:
|
||||
username: ${{ secrets.DOCKER_USERNAME }}
|
||||
password: ${{ secrets.DOCKER_PASSWORD }}
|
||||
-
|
||||
name: Build
|
||||
uses: docker/build-push-action@263435318d21b8e681c14492fe198d362a7d2c83 # v2
|
||||
with:
|
||||
context: ./docker
|
||||
push: ${{ github.event_name != 'pull_request' }}
|
||||
file: ./docker/Dockerfile.go_build
|
||||
build-args: TAGS=5BytesOffset
|
||||
platforms: linux/amd64, linux/arm, linux/arm64, linux/386
|
||||
tags: ${{ steps.docker_meta.outputs.tags }}
|
||||
labels: ${{ steps.docker_meta.outputs.labels }}
|
||||
@@ -0,0 +1,58 @@
|
||||
name: "docker: build release containers for rocksdb"
|
||||
|
||||
on:
|
||||
push:
|
||||
tags:
|
||||
- '*'
|
||||
workflow_dispatch: {}
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
jobs:
|
||||
|
||||
build-large-release-container_rocksdb:
|
||||
runs-on: [ubuntu-latest]
|
||||
|
||||
steps:
|
||||
-
|
||||
name: Checkout
|
||||
uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v2
|
||||
-
|
||||
name: Docker meta
|
||||
id: docker_meta
|
||||
uses: docker/metadata-action@902fa8ec7d6ecbf8d84d538b9b233a880e428804 # v3
|
||||
with:
|
||||
images: |
|
||||
chrislusf/seaweedfs
|
||||
tags: |
|
||||
type=ref,event=tag,suffix=_large_disk_rocksdb
|
||||
flavor: |
|
||||
latest=false
|
||||
labels: |
|
||||
org.opencontainers.image.title=seaweedfs
|
||||
org.opencontainers.image.description=SeaweedFS is a distributed storage system for blobs, objects, files, and data lake, to store and serve billions of files fast!
|
||||
org.opencontainers.image.vendor=Chris Lu
|
||||
-
|
||||
name: Set up QEMU
|
||||
uses: docker/setup-qemu-action@29109295f81e9208d7d86ff1c6c12d2833863392 # v1
|
||||
-
|
||||
name: Set up Docker Buildx
|
||||
uses: docker/setup-buildx-action@e468171a9de216ec08956ac3ada2f0791b6bd435 # v1
|
||||
-
|
||||
name: Login to Docker Hub
|
||||
if: github.event_name != 'pull_request'
|
||||
uses: docker/login-action@74a5d142397b4f367a81961eba4e8cd7edddf772 # v1
|
||||
with:
|
||||
username: ${{ secrets.DOCKER_USERNAME }}
|
||||
password: ${{ secrets.DOCKER_PASSWORD }}
|
||||
-
|
||||
name: Build
|
||||
uses: docker/build-push-action@263435318d21b8e681c14492fe198d362a7d2c83 # v2
|
||||
with:
|
||||
context: ./docker
|
||||
push: ${{ github.event_name != 'pull_request' }}
|
||||
file: ./docker/Dockerfile.rocksdb_large
|
||||
platforms: linux/amd64
|
||||
tags: ${{ steps.docker_meta.outputs.tags }}
|
||||
labels: ${{ steps.docker_meta.outputs.labels }}
|
||||
@@ -0,0 +1,58 @@
|
||||
name: "docker: build release containers for all tags"
|
||||
|
||||
on:
|
||||
push:
|
||||
tags:
|
||||
- '*'
|
||||
workflow_dispatch: {}
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
jobs:
|
||||
build-default-release-container:
|
||||
runs-on: [ubuntu-latest]
|
||||
|
||||
steps:
|
||||
-
|
||||
name: Checkout
|
||||
uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v2
|
||||
-
|
||||
name: Docker meta
|
||||
id: docker_meta
|
||||
uses: docker/metadata-action@902fa8ec7d6ecbf8d84d538b9b233a880e428804 # v3
|
||||
with:
|
||||
images: |
|
||||
chrislusf/seaweedfs
|
||||
tags: |
|
||||
type=ref,event=tag,suffix=_full
|
||||
flavor: |
|
||||
latest=false
|
||||
labels: |
|
||||
org.opencontainers.image.title=seaweedfs
|
||||
org.opencontainers.image.description=SeaweedFS is a distributed storage system for blobs, objects, files, and data lake, to store and serve billions of files fast!
|
||||
org.opencontainers.image.vendor=Chris Lu
|
||||
-
|
||||
name: Set up QEMU
|
||||
uses: docker/setup-qemu-action@29109295f81e9208d7d86ff1c6c12d2833863392 # v1
|
||||
-
|
||||
name: Set up Docker Buildx
|
||||
uses: docker/setup-buildx-action@e468171a9de216ec08956ac3ada2f0791b6bd435 # v1
|
||||
-
|
||||
name: Login to Docker Hub
|
||||
if: github.event_name != 'pull_request'
|
||||
uses: docker/login-action@74a5d142397b4f367a81961eba4e8cd7edddf772 # v1
|
||||
with:
|
||||
username: ${{ secrets.DOCKER_USERNAME }}
|
||||
password: ${{ secrets.DOCKER_PASSWORD }}
|
||||
-
|
||||
name: Build
|
||||
uses: docker/build-push-action@263435318d21b8e681c14492fe198d362a7d2c83 # v2
|
||||
with:
|
||||
context: ./docker
|
||||
push: ${{ github.event_name != 'pull_request' }}
|
||||
file: ./docker/Dockerfile.go_build
|
||||
build-args: TAGS=elastic,gocdk,rclone,sqlite,tarantool,tikv,ydb
|
||||
platforms: linux/amd64
|
||||
tags: ${{ steps.docker_meta.outputs.tags }}
|
||||
labels: ${{ steps.docker_meta.outputs.labels }}
|
||||
@@ -0,0 +1,58 @@
|
||||
name: "docker: build release containers for all tags and large volume"
|
||||
|
||||
on:
|
||||
push:
|
||||
tags:
|
||||
- '*'
|
||||
workflow_dispatch: {}
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
jobs:
|
||||
build-default-release-container:
|
||||
runs-on: [ubuntu-latest]
|
||||
|
||||
steps:
|
||||
-
|
||||
name: Checkout
|
||||
uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v2
|
||||
-
|
||||
name: Docker meta
|
||||
id: docker_meta
|
||||
uses: docker/metadata-action@902fa8ec7d6ecbf8d84d538b9b233a880e428804 # v3
|
||||
with:
|
||||
images: |
|
||||
chrislusf/seaweedfs
|
||||
tags: |
|
||||
type=ref,event=tag,suffix=_large_disk_full
|
||||
flavor: |
|
||||
latest=false
|
||||
labels: |
|
||||
org.opencontainers.image.title=seaweedfs
|
||||
org.opencontainers.image.description=SeaweedFS is a distributed storage system for blobs, objects, files, and data lake, to store and serve billions of files fast!
|
||||
org.opencontainers.image.vendor=Chris Lu
|
||||
-
|
||||
name: Set up QEMU
|
||||
uses: docker/setup-qemu-action@29109295f81e9208d7d86ff1c6c12d2833863392 # v1
|
||||
-
|
||||
name: Set up Docker Buildx
|
||||
uses: docker/setup-buildx-action@e468171a9de216ec08956ac3ada2f0791b6bd435 # v1
|
||||
-
|
||||
name: Login to Docker Hub
|
||||
if: github.event_name != 'pull_request'
|
||||
uses: docker/login-action@74a5d142397b4f367a81961eba4e8cd7edddf772 # v1
|
||||
with:
|
||||
username: ${{ secrets.DOCKER_USERNAME }}
|
||||
password: ${{ secrets.DOCKER_PASSWORD }}
|
||||
-
|
||||
name: Build
|
||||
uses: docker/build-push-action@263435318d21b8e681c14492fe198d362a7d2c83 # v2
|
||||
with:
|
||||
context: ./docker
|
||||
push: ${{ github.event_name != 'pull_request' }}
|
||||
file: ./docker/Dockerfile.go_build
|
||||
build-args: TAGS=5BytesOffset,elastic,gocdk,rclone,sqlite,tarantool,tikv,ydb
|
||||
platforms: linux/amd64
|
||||
tags: ${{ steps.docker_meta.outputs.tags }}
|
||||
labels: ${{ steps.docker_meta.outputs.labels }}
|
||||
@@ -1,94 +0,0 @@
|
||||
name: "docker: build release containers for foundationdb"
|
||||
|
||||
on:
|
||||
push:
|
||||
tags:
|
||||
- '*'
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
release_tag:
|
||||
description: 'Release tag to publish (e.g. 3.93)'
|
||||
required: true
|
||||
default: ''
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.event.pull_request.number || (github.event_name == 'push' && github.ref) || format('refs/tags/{0}', github.event.inputs.release_tag) || github.run_id }}
|
||||
cancel-in-progress: false
|
||||
queue: max
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
env:
|
||||
RELEASE_TAG: ${{ github.event_name == 'workflow_dispatch' && github.event.inputs.release_tag || github.ref_name }}
|
||||
|
||||
jobs:
|
||||
|
||||
build-large-release-container_foundationdb:
|
||||
runs-on: [ubuntu-latest]
|
||||
permissions:
|
||||
contents: read
|
||||
id-token: write
|
||||
|
||||
steps:
|
||||
-
|
||||
name: Checkout
|
||||
uses: actions/checkout@v7
|
||||
-
|
||||
name: Docker meta
|
||||
id: docker_meta
|
||||
uses: docker/metadata-action@v6
|
||||
with:
|
||||
images: |
|
||||
chrislusf/seaweedfs
|
||||
tags: |
|
||||
type=raw,value=${{ env.RELEASE_TAG }}_large_disk_foundationdb
|
||||
flavor: |
|
||||
latest=false
|
||||
labels: |
|
||||
org.opencontainers.image.title=seaweedfs
|
||||
org.opencontainers.image.description=SeaweedFS is a distributed storage system for blobs, objects, files, and data lake, to store and serve billions of files fast!
|
||||
org.opencontainers.image.vendor=Chris Lu
|
||||
-
|
||||
name: Set up QEMU
|
||||
uses: docker/setup-qemu-action@v4.4.0
|
||||
-
|
||||
name: Set up Docker Buildx
|
||||
uses: docker/setup-buildx-action@v4
|
||||
-
|
||||
name: Login to Docker Hub
|
||||
if: github.event_name != 'pull_request'
|
||||
uses: docker/login-action@v4.8.0
|
||||
with:
|
||||
username: ${{ secrets.DOCKER_USERNAME }}
|
||||
password: ${{ secrets.DOCKER_PASSWORD }}
|
||||
-
|
||||
name: Determine branch to build
|
||||
id: branch
|
||||
run: |
|
||||
if [ "${{ github.event_name }}" = "push" ] && [ -n "${{ github.ref_name }}" ]; then
|
||||
echo "branch=${{ github.ref_name }}" >> "$GITHUB_OUTPUT"
|
||||
else
|
||||
echo "branch=master" >> "$GITHUB_OUTPUT"
|
||||
fi
|
||||
-
|
||||
name: Build
|
||||
id: build
|
||||
uses: docker/build-push-action@v7
|
||||
with:
|
||||
context: ./docker
|
||||
push: ${{ github.event_name != 'pull_request' }}
|
||||
file: ./docker/Dockerfile.foundationdb_large
|
||||
build-args: |
|
||||
BRANCH=${{ steps.branch.outputs.branch }}
|
||||
# Note: ARM64 support requires FoundationDB ARM64 packages which are not available for all versions
|
||||
platforms: linux/amd64
|
||||
tags: ${{ steps.docker_meta.outputs.tags }}
|
||||
labels: ${{ steps.docker_meta.outputs.labels }}
|
||||
-
|
||||
name: Sign
|
||||
if: github.event_name != 'pull_request'
|
||||
uses: ./.github/actions/sign-image
|
||||
with:
|
||||
images: chrislusf/seaweedfs@${{ steps.build.outputs.digest }}
|
||||
|
||||
@@ -1,612 +0,0 @@
|
||||
name: "docker: build all release containers (unified)"
|
||||
|
||||
on:
|
||||
push:
|
||||
tags:
|
||||
- '*'
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
variant:
|
||||
description: 'Variant to build manually'
|
||||
required: true
|
||||
type: choice
|
||||
default: all
|
||||
options:
|
||||
- all
|
||||
- normal
|
||||
- large_disk
|
||||
- full
|
||||
- large_disk_full
|
||||
- rocksdb
|
||||
release_tag:
|
||||
description: 'Release tag to publish (e.g. 3.93)'
|
||||
required: true
|
||||
default: ''
|
||||
rocksdb_version:
|
||||
description: 'RocksDB git tag to use when variant=rocksdb'
|
||||
required: false
|
||||
default: 'v11.1.2'
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
security-events: write
|
||||
|
||||
env:
|
||||
RELEASE_TAG: ${{ github.event_name == 'workflow_dispatch' && github.event.inputs.release_tag || github.ref_name }}
|
||||
IMAGE: ghcr.io/chrislusf/seaweedfs
|
||||
|
||||
# Limit concurrent builds to avoid rate limits
|
||||
concurrency:
|
||||
group: release-${{ github.ref }}
|
||||
cancel-in-progress: false
|
||||
|
||||
jobs:
|
||||
|
||||
# ── Pre-build the Rust binaries natively ────────────────────────────
|
||||
# The volume server, the Rust maintenance worker and the vector engine,
|
||||
# cross-compiled for amd64 and arm64 without QEMU, turning a 5-hour emulated
|
||||
# cargo build into ~15 minutes of native compilation.
|
||||
build-rust-binaries:
|
||||
runs-on: ubuntu-24.04
|
||||
strategy:
|
||||
matrix:
|
||||
include:
|
||||
- target: x86_64-unknown-linux-musl
|
||||
arch: amd64
|
||||
- target: aarch64-unknown-linux-musl
|
||||
arch: arm64
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v7
|
||||
|
||||
- name: Set up the musl target
|
||||
uses: ./.github/actions/rust-musl-target
|
||||
with:
|
||||
target: ${{ matrix.target }}
|
||||
|
||||
- name: Cache cargo registry and target
|
||||
if: runner.environment == 'github-hosted'
|
||||
uses: actions/cache@v6
|
||||
with:
|
||||
path: |
|
||||
~/.cargo/registry
|
||||
~/.cargo/git
|
||||
seaweed-volume/target
|
||||
seaweed-worker/target/${{ matrix.target }}/release
|
||||
seaweed-vector/target/${{ matrix.target }}/release
|
||||
key: rust-docker-${{ matrix.target }}-${{ hashFiles('seaweed-volume/Cargo.lock', 'seaweed-worker/Cargo.lock', 'seaweed-vector/Cargo.lock') }}
|
||||
restore-keys: |
|
||||
rust-docker-${{ matrix.target }}-
|
||||
|
||||
# lance's build scripts compile their own protos and look for a protoc.
|
||||
# Point them at the one protoc-bin-vendored ships, which seaweed-worker's
|
||||
# own build already uses, so no job depends on a system package and every
|
||||
# build sees the same version.
|
||||
- name: Use the vendored protoc
|
||||
run: |
|
||||
cd seaweed-worker
|
||||
cargo fetch
|
||||
# The version from the lock, not whatever else a restored cache holds.
|
||||
version=$(awk '/^name = "protoc-bin-vendored-linux-x86_64"$/{found=1; next} found && /^version = /{gsub(/"/,"",$3); print $3; exit}' Cargo.lock)
|
||||
test -n "$version" || { echo "protoc-bin-vendored-linux-x86_64 is not in Cargo.lock" >&2; exit 1; }
|
||||
protoc=$(find ~/.cargo/registry/src -path "*protoc-bin-vendored-linux-x86_64-$version/bin/protoc" | head -1)
|
||||
test -x "$protoc" || { echo "no vendored protoc $version in the registry" >&2; exit 1; }
|
||||
echo "PROTOC=$protoc" >> "$GITHUB_ENV"
|
||||
|
||||
- name: Build large-disk variant
|
||||
env:
|
||||
SEAWEEDFS_COMMIT: ${{ github.sha }}
|
||||
run: |
|
||||
cd seaweed-volume
|
||||
cargo build --release --target ${{ matrix.target }}
|
||||
cp target/${{ matrix.target }}/release/weed-volume ../weed-volume-large-disk-${{ matrix.arch }}
|
||||
|
||||
- name: Build normal variant
|
||||
env:
|
||||
SEAWEEDFS_COMMIT: ${{ github.sha }}
|
||||
run: |
|
||||
cd seaweed-volume
|
||||
cargo build --release --target ${{ matrix.target }} --no-default-features
|
||||
cp target/${{ matrix.target }}/release/weed-volume ../weed-volume-normal-${{ matrix.arch }}
|
||||
|
||||
- name: Build the Rust maintenance worker
|
||||
run: |
|
||||
cd seaweed-worker
|
||||
cargo build --release -p weed-lance-worker --target ${{ matrix.target }}
|
||||
cp target/${{ matrix.target }}/release/weed-worker ../weed-worker-${{ matrix.arch }}
|
||||
|
||||
- name: Build the vector engine
|
||||
run: |
|
||||
cd seaweed-vector
|
||||
cargo build --release -p weed-vector-server --target ${{ matrix.target }}
|
||||
cp target/${{ matrix.target }}/release/weed-vector ../weed-vector-${{ matrix.arch }}
|
||||
|
||||
- name: Upload artifacts
|
||||
uses: actions/upload-artifact@v7
|
||||
with:
|
||||
name: rust-bins-${{ matrix.arch }}
|
||||
path: |
|
||||
weed-volume-large-disk-${{ matrix.arch }}
|
||||
weed-volume-normal-${{ matrix.arch }}
|
||||
weed-worker-${{ matrix.arch }}
|
||||
weed-vector-${{ matrix.arch }}
|
||||
|
||||
# One job per (variant, platform) on a native runner, pushed by digest;
|
||||
# the merge job stitches the digests into one multi-arch tag.
|
||||
build:
|
||||
needs: [build-rust-binaries]
|
||||
runs-on: ${{ matrix.runner }}
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
include:
|
||||
# Normal volume - multi-arch
|
||||
- { variant: normal, tag_suffix: "", dockerfile: ./docker/Dockerfile.go_build, build_args: "", rust_variant: normal, platform: linux/amd64, arch: amd64, runner: ubuntu-latest, qemu: false }
|
||||
- { variant: normal, tag_suffix: "", dockerfile: ./docker/Dockerfile.go_build, build_args: "", rust_variant: normal, platform: linux/arm64, arch: arm64, runner: ubuntu-24.04-arm, qemu: false }
|
||||
- { variant: normal, tag_suffix: "", dockerfile: ./docker/Dockerfile.go_build, build_args: "", rust_variant: normal, platform: linux/arm/v7, arch: armv7, runner: ubuntu-latest, qemu: true }
|
||||
- { variant: normal, tag_suffix: "", dockerfile: ./docker/Dockerfile.go_build, build_args: "", rust_variant: normal, platform: linux/386, arch: i386, runner: ubuntu-latest, qemu: false }
|
||||
- { variant: normal, tag_suffix: "", dockerfile: ./docker/Dockerfile.go_build, build_args: "", rust_variant: normal, platform: linux/ppc64le, arch: ppc64le, runner: ubuntu-latest, qemu: true }
|
||||
- { variant: normal, tag_suffix: "", dockerfile: ./docker/Dockerfile.go_build, build_args: "", rust_variant: normal, platform: linux/s390x, arch: s390x, runner: ubuntu-latest, qemu: true }
|
||||
|
||||
# Large disk - multi-arch
|
||||
- { variant: large_disk, tag_suffix: _large_disk, dockerfile: ./docker/Dockerfile.go_build, build_args: TAGS=5BytesOffset, rust_variant: large-disk, platform: linux/amd64, arch: amd64, runner: ubuntu-latest, qemu: false }
|
||||
- { variant: large_disk, tag_suffix: _large_disk, dockerfile: ./docker/Dockerfile.go_build, build_args: TAGS=5BytesOffset, rust_variant: large-disk, platform: linux/arm64, arch: arm64, runner: ubuntu-24.04-arm, qemu: false }
|
||||
- { variant: large_disk, tag_suffix: _large_disk, dockerfile: ./docker/Dockerfile.go_build, build_args: TAGS=5BytesOffset, rust_variant: large-disk, platform: linux/arm/v7, arch: armv7, runner: ubuntu-latest, qemu: true }
|
||||
- { variant: large_disk, tag_suffix: _large_disk, dockerfile: ./docker/Dockerfile.go_build, build_args: TAGS=5BytesOffset, rust_variant: large-disk, platform: linux/386, arch: i386, runner: ubuntu-latest, qemu: false }
|
||||
- { variant: large_disk, tag_suffix: _large_disk, dockerfile: ./docker/Dockerfile.go_build, build_args: TAGS=5BytesOffset, rust_variant: large-disk, platform: linux/ppc64le, arch: ppc64le, runner: ubuntu-latest, qemu: true }
|
||||
- { variant: large_disk, tag_suffix: _large_disk, dockerfile: ./docker/Dockerfile.go_build, build_args: TAGS=5BytesOffset, rust_variant: large-disk, platform: linux/s390x, arch: s390x, runner: ubuntu-latest, qemu: true }
|
||||
|
||||
# Full tags - multi-arch
|
||||
- { variant: full, tag_suffix: _full, dockerfile: ./docker/Dockerfile.go_build, build_args: "TAGS=elastic,gocdk,rclone,sqlite,tarantool,tikv,ydb", rust_variant: normal, platform: linux/amd64, arch: amd64, runner: ubuntu-latest, qemu: false }
|
||||
- { variant: full, tag_suffix: _full, dockerfile: ./docker/Dockerfile.go_build, build_args: "TAGS=elastic,gocdk,rclone,sqlite,tarantool,tikv,ydb", rust_variant: normal, platform: linux/arm64, arch: arm64, runner: ubuntu-24.04-arm, qemu: false }
|
||||
|
||||
# Large disk + full tags - multi-arch
|
||||
- { variant: large_disk_full, tag_suffix: _large_disk_full, dockerfile: ./docker/Dockerfile.go_build, build_args: "TAGS=5BytesOffset,elastic,gocdk,rclone,sqlite,tarantool,tikv,ydb", rust_variant: large-disk, platform: linux/amd64, arch: amd64, runner: ubuntu-latest, qemu: false }
|
||||
- { variant: large_disk_full, tag_suffix: _large_disk_full, dockerfile: ./docker/Dockerfile.go_build, build_args: "TAGS=5BytesOffset,elastic,gocdk,rclone,sqlite,tarantool,tikv,ydb", rust_variant: large-disk, platform: linux/arm64, arch: arm64, runner: ubuntu-24.04-arm, qemu: false }
|
||||
|
||||
# RocksDB large disk - amd64 only
|
||||
- { variant: rocksdb, tag_suffix: _large_disk_rocksdb, dockerfile: ./docker/Dockerfile.rocksdb_large, build_args: "", rust_variant: large-disk, platform: linux/amd64, arch: amd64, runner: ubuntu-latest, qemu: false }
|
||||
steps:
|
||||
- name: Skip unselected variant
|
||||
if: github.event_name == 'workflow_dispatch' && github.event.inputs.variant != 'all' && github.event.inputs.variant != matrix.variant
|
||||
run: echo "Skipping ${{ matrix.variant }} (${{ matrix.platform }})" && exit 0
|
||||
|
||||
- name: Checkout
|
||||
if: github.event_name != 'workflow_dispatch' || github.event.inputs.variant == 'all' || github.event.inputs.variant == matrix.variant
|
||||
uses: actions/checkout@v7
|
||||
|
||||
- name: Clean up stale FUSE mounts
|
||||
if: github.event_name != 'workflow_dispatch' || github.event.inputs.variant == 'all' || github.event.inputs.variant == matrix.variant
|
||||
uses: ./.github/actions/cleanup-stale-fuse-mounts
|
||||
|
||||
- name: Download pre-built Rust binaries
|
||||
if: github.event_name != 'workflow_dispatch' || github.event.inputs.variant == 'all' || github.event.inputs.variant == matrix.variant
|
||||
uses: actions/download-artifact@v8
|
||||
with:
|
||||
pattern: rust-bins-*
|
||||
merge-multiple: true
|
||||
path: ./rust-bins
|
||||
|
||||
- name: Place Rust binaries in Docker context
|
||||
if: github.event_name != 'workflow_dispatch' || github.event.inputs.variant == 'all' || github.event.inputs.variant == matrix.variant
|
||||
run: |
|
||||
mkdir -p docker/weed-volume-prebuilt
|
||||
for arch in amd64 arm64; do
|
||||
src="./rust-bins/weed-volume-${{ matrix.rust_variant }}-${arch}"
|
||||
if [ -f "$src" ]; then
|
||||
cp "$src" "docker/weed-volume-prebuilt/weed-volume-${arch}"
|
||||
echo "Placed pre-built Rust binary for ${arch}"
|
||||
fi
|
||||
done
|
||||
mkdir -p docker/weed-worker-prebuilt
|
||||
for arch in amd64 arm64; do
|
||||
src="./rust-bins/weed-worker-${arch}"
|
||||
if [ -f "$src" ]; then
|
||||
cp "$src" "docker/weed-worker-prebuilt/weed-worker-${arch}"
|
||||
echo "Placed pre-built Rust worker for ${arch}"
|
||||
fi
|
||||
done
|
||||
mkdir -p docker/weed-vector-prebuilt
|
||||
for arch in amd64 arm64; do
|
||||
src="./rust-bins/weed-vector-${arch}"
|
||||
if [ -f "$src" ]; then
|
||||
cp "$src" "docker/weed-vector-prebuilt/weed-vector-${arch}"
|
||||
echo "Placed pre-built vector engine for ${arch}"
|
||||
fi
|
||||
done
|
||||
ls -la docker/weed-volume-prebuilt/
|
||||
ls -la docker/weed-worker-prebuilt/
|
||||
ls -la docker/weed-vector-prebuilt/
|
||||
|
||||
- name: Free Disk Space
|
||||
if: github.event_name != 'workflow_dispatch' || github.event.inputs.variant == 'all' || github.event.inputs.variant == matrix.variant
|
||||
run: |
|
||||
sudo rm -rf /usr/share/dotnet /usr/local/lib/android /opt/ghc /opt/hostedtoolcache/CodeQL
|
||||
sudo apt-get clean
|
||||
sudo rm -rf /var/lib/apt/lists/*
|
||||
sudo docker system prune -af --volumes
|
||||
[ -d ~/.cache/go-build ] && rm -rf ~/.cache/go-build || true
|
||||
[ -d /go/pkg ] && rm -rf /go/pkg || true
|
||||
timeout 15 df -h || true
|
||||
|
||||
- name: Docker meta
|
||||
if: github.event_name != 'workflow_dispatch' || github.event.inputs.variant == 'all' || github.event.inputs.variant == matrix.variant
|
||||
id: docker_meta
|
||||
uses: docker/metadata-action@v6
|
||||
with:
|
||||
images: ${{ env.IMAGE }}
|
||||
tags: type=raw,value=${{ env.RELEASE_TAG }}${{ matrix.tag_suffix }}
|
||||
flavor: latest=false
|
||||
labels: |
|
||||
org.opencontainers.image.title=seaweedfs
|
||||
org.opencontainers.image.description=SeaweedFS is a distributed storage system for blobs, objects, files, and data lake, to store and serve billions of files fast!
|
||||
org.opencontainers.image.vendor=Chris Lu
|
||||
|
||||
- name: Set up QEMU
|
||||
if: (github.event_name != 'workflow_dispatch' || github.event.inputs.variant == 'all' || github.event.inputs.variant == matrix.variant) && matrix.qemu
|
||||
uses: docker/setup-qemu-action@v4.4.0
|
||||
|
||||
- name: Create BuildKit config
|
||||
if: github.event_name != 'workflow_dispatch' || github.event.inputs.variant == 'all' || github.event.inputs.variant == matrix.variant
|
||||
run: |
|
||||
cat > /tmp/buildkitd.toml <<EOF
|
||||
[registry."docker.io"]
|
||||
mirrors = ["https://mirror.gcr.io"]
|
||||
EOF
|
||||
|
||||
- name: Set up Docker Buildx
|
||||
if: github.event_name != 'workflow_dispatch' || github.event.inputs.variant == 'all' || github.event.inputs.variant == matrix.variant
|
||||
uses: docker/setup-buildx-action@v4
|
||||
with:
|
||||
buildkitd-config: /tmp/buildkitd.toml
|
||||
|
||||
- name: Login to GHCR
|
||||
if: github.event_name != 'workflow_dispatch' || github.event.inputs.variant == 'all' || github.event.inputs.variant == matrix.variant
|
||||
uses: docker/login-action@v4.8.0
|
||||
with:
|
||||
registry: ghcr.io
|
||||
username: ${{ secrets.GHCR_USERNAME }}
|
||||
password: ${{ secrets.GHCR_TOKEN }}
|
||||
|
||||
- name: Build and push ${{ matrix.variant }} (${{ matrix.platform }})
|
||||
if: github.event_name != 'workflow_dispatch' || github.event.inputs.variant == 'all' || github.event.inputs.variant == matrix.variant
|
||||
id: build
|
||||
uses: docker/build-push-action@v7
|
||||
env:
|
||||
DOCKER_BUILDKIT: 1
|
||||
with:
|
||||
context: ./docker
|
||||
file: ${{ matrix.dockerfile }}
|
||||
platforms: ${{ matrix.platform }}
|
||||
labels: ${{ steps.docker_meta.outputs.labels }}
|
||||
# mode=max keeps resolvedDependencies complete and records the
|
||||
# builder identity; attestations ride inside each pushed index and
|
||||
# survive the imagetools merge into the multi-arch manifest.
|
||||
provenance: mode=max
|
||||
sbom: true
|
||||
outputs: type=image,name=${{ env.IMAGE }},push-by-digest=true,name-canonical=true,push=true
|
||||
cache-from: type=gha,scope=${{ matrix.variant }}-${{ matrix.arch }}
|
||||
# max only for rocksdb: its RocksDB compile is sha-independent and worth
|
||||
# keeping; go-build layers are sha-busted every release, so min elsewhere.
|
||||
cache-to: type=gha,mode=${{ matrix.variant == 'rocksdb' && 'max' || 'min' }},scope=${{ matrix.variant }}-${{ matrix.arch }}
|
||||
build-args: |
|
||||
${{ matrix.build_args }}
|
||||
BUILDKIT_INLINE_CACHE=1
|
||||
BRANCH=${{ github.sha }}
|
||||
${{ matrix.variant == 'rocksdb' && format('ROCKSDB_VERSION={0}', github.event.inputs.rocksdb_version || 'v11.1.2') || '' }}
|
||||
|
||||
- name: Export digest
|
||||
if: github.event_name != 'workflow_dispatch' || github.event.inputs.variant == 'all' || github.event.inputs.variant == matrix.variant
|
||||
run: |
|
||||
mkdir -p /tmp/digests
|
||||
digest="${{ steps.build.outputs.digest }}"
|
||||
touch "/tmp/digests/${digest#sha256:}"
|
||||
|
||||
- name: Upload digest
|
||||
if: github.event_name != 'workflow_dispatch' || github.event.inputs.variant == 'all' || github.event.inputs.variant == matrix.variant
|
||||
uses: actions/upload-artifact@v7
|
||||
with:
|
||||
name: digest-${{ matrix.variant }}-${{ matrix.arch }}
|
||||
path: /tmp/digests/*
|
||||
if-no-files-found: error
|
||||
retention-days: 1
|
||||
|
||||
# Assemble each variant's per-platform digests into one tag, mirror it to
|
||||
# Docker Hub, and sign the result on both registries.
|
||||
merge:
|
||||
needs: [build]
|
||||
runs-on: ubuntu-latest
|
||||
permissions:
|
||||
contents: read
|
||||
id-token: write
|
||||
attestations: write
|
||||
packages: write
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
include:
|
||||
- { variant: normal, tag_suffix: "" }
|
||||
- { variant: large_disk, tag_suffix: _large_disk }
|
||||
- { variant: full, tag_suffix: _full }
|
||||
- { variant: large_disk_full, tag_suffix: _large_disk_full }
|
||||
- { variant: rocksdb, tag_suffix: _large_disk_rocksdb }
|
||||
steps:
|
||||
- name: Checkout the signing action
|
||||
if: github.event_name != 'workflow_dispatch' || github.event.inputs.variant == 'all' || github.event.inputs.variant == matrix.variant
|
||||
uses: actions/checkout@v7
|
||||
with:
|
||||
sparse-checkout: .github/actions
|
||||
persist-credentials: false
|
||||
|
||||
- name: Download digests
|
||||
if: github.event_name != 'workflow_dispatch' || github.event.inputs.variant == 'all' || github.event.inputs.variant == matrix.variant
|
||||
uses: actions/download-artifact@v8
|
||||
with:
|
||||
pattern: digest-${{ matrix.variant }}-*
|
||||
merge-multiple: true
|
||||
path: /tmp/digests
|
||||
|
||||
- name: Set up Docker Buildx
|
||||
if: github.event_name != 'workflow_dispatch' || github.event.inputs.variant == 'all' || github.event.inputs.variant == matrix.variant
|
||||
uses: docker/setup-buildx-action@v4
|
||||
|
||||
- name: Login to GHCR
|
||||
if: github.event_name != 'workflow_dispatch' || github.event.inputs.variant == 'all' || github.event.inputs.variant == matrix.variant
|
||||
uses: docker/login-action@v4.8.0
|
||||
with:
|
||||
registry: ghcr.io
|
||||
username: ${{ secrets.GHCR_USERNAME }}
|
||||
password: ${{ secrets.GHCR_TOKEN }}
|
||||
|
||||
- name: Create multi-arch tag on GHCR
|
||||
if: github.event_name != 'workflow_dispatch' || github.event.inputs.variant == 'all' || github.event.inputs.variant == matrix.variant
|
||||
id: manifest
|
||||
working-directory: /tmp/digests
|
||||
run: |
|
||||
docker buildx imagetools create \
|
||||
-t ${{ env.IMAGE }}:${{ env.RELEASE_TAG }}${{ matrix.tag_suffix }} \
|
||||
--metadata-file /tmp/manifest.json \
|
||||
$(printf '${{ env.IMAGE }}@sha256:%s ' *)
|
||||
docker buildx imagetools inspect ${{ env.IMAGE }}:${{ env.RELEASE_TAG }}${{ matrix.tag_suffix }}
|
||||
# The copy and the signature below use the digest this run pushed, not whatever the tag points at by then.
|
||||
digest=$(jq -er '."containerimage.descriptor".digest' /tmp/manifest.json)
|
||||
echo "digest=${digest}" >> "$GITHUB_OUTPUT"
|
||||
|
||||
- name: Login to Docker Hub
|
||||
if: github.event_name != 'workflow_dispatch' || github.event.inputs.variant == 'all' || github.event.inputs.variant == matrix.variant
|
||||
uses: docker/login-action@v4.8.0
|
||||
with:
|
||||
username: ${{ secrets.DOCKER_USERNAME }}
|
||||
password: ${{ secrets.DOCKER_PASSWORD }}
|
||||
|
||||
- name: Install crane
|
||||
if: github.event_name != 'workflow_dispatch' || github.event.inputs.variant == 'all' || github.event.inputs.variant == matrix.variant
|
||||
run: |
|
||||
cd $(mktemp -d)
|
||||
curl -sLO https://github.com/google/go-containerregistry/releases/download/v0.22.0/go-containerregistry_Linux_x86_64.tar.gz
|
||||
echo "edb74d53fad9a596860f59d1c5d04a43dfb5f441dc71f57060dd0bf39483c833 go-containerregistry_Linux_x86_64.tar.gz" | sha256sum -c -
|
||||
tar xzf go-containerregistry_Linux_x86_64.tar.gz crane
|
||||
sudo mv crane /usr/local/bin/
|
||||
crane version
|
||||
|
||||
- name: Copy ${{ matrix.variant }} to Docker Hub
|
||||
if: github.event_name != 'workflow_dispatch' || github.event.inputs.variant == 'all' || github.event.inputs.variant == matrix.variant
|
||||
run: |
|
||||
retry_with_backoff() {
|
||||
local max_attempts=5
|
||||
local timeout=1
|
||||
local attempt=1
|
||||
local exit_code=0
|
||||
while [ $attempt -le $max_attempts ]; do
|
||||
if "$@"; then
|
||||
return 0
|
||||
else
|
||||
exit_code=$?
|
||||
fi
|
||||
if [ $attempt -lt $max_attempts ]; then
|
||||
echo "Attempt $attempt failed. Retrying in ${timeout}s..." >&2
|
||||
sleep $timeout
|
||||
timeout=$((timeout * 2))
|
||||
fi
|
||||
attempt=$((attempt + 1))
|
||||
done
|
||||
echo "Command failed after $max_attempts attempts" >&2
|
||||
return $exit_code
|
||||
}
|
||||
|
||||
echo "Copying ${{ matrix.variant }} from GHCR to Docker Hub..."
|
||||
retry_with_backoff crane copy \
|
||||
${{ env.IMAGE }}@${{ steps.manifest.outputs.digest }} \
|
||||
chrislusf/seaweedfs:${{ env.RELEASE_TAG }}${{ matrix.tag_suffix }}
|
||||
echo "Copied ${{ matrix.variant }} to Docker Hub"
|
||||
|
||||
- name: Sign ${{ matrix.variant }}
|
||||
if: github.event_name != 'workflow_dispatch' || github.event.inputs.variant == 'all' || github.event.inputs.variant == matrix.variant
|
||||
uses: ./.github/actions/sign-image
|
||||
with:
|
||||
images: >-
|
||||
${{ env.IMAGE }}@${{ steps.manifest.outputs.digest }}
|
||||
chrislusf/seaweedfs@${{ steps.manifest.outputs.digest }}
|
||||
|
||||
# Attest the merged multi-arch digest itself, so digest-pinning consumers
|
||||
# can verify provenance against the exact index they pull. Both steps
|
||||
# reuse the GHCR login above and push as OCI referrers.
|
||||
- name: Attest build provenance for ${{ matrix.variant }}
|
||||
if: github.event_name != 'workflow_dispatch' || github.event.inputs.variant == 'all' || github.event.inputs.variant == matrix.variant
|
||||
uses: actions/attest-build-provenance@v4
|
||||
with:
|
||||
subject-name: ${{ env.IMAGE }}
|
||||
subject-digest: ${{ steps.manifest.outputs.digest }}
|
||||
push-to-registry: true
|
||||
|
||||
# SBOMs are generated and attested per platform manifest: an index-wide
|
||||
# scan only describes syft's default platform, while the manifests it
|
||||
# merges differ (some ship the real binary, others placeholders), so the
|
||||
# SBOM has to be bound to the platform digest a consumer actually pulls.
|
||||
- name: Install syft
|
||||
if: github.event_name != 'workflow_dispatch' || github.event.inputs.variant == 'all' || github.event.inputs.variant == matrix.variant
|
||||
uses: anchore/sbom-action/download-syft@v0
|
||||
|
||||
- name: Attest per-platform SBOMs for ${{ matrix.variant }}
|
||||
if: github.event_name != 'workflow_dispatch' || github.event.inputs.variant == 'all' || github.event.inputs.variant == matrix.variant
|
||||
working-directory: /tmp/digests
|
||||
run: |
|
||||
set -euo pipefail
|
||||
for f in *; do
|
||||
# Each saved digest is an OCI index (image + attestation manifests);
|
||||
# scan and attest the image manifest a same-platform pull resolves.
|
||||
ref="${{ env.IMAGE }}@sha256:${f}"
|
||||
image=$(crane manifest "$ref" | jq -r '
|
||||
[.manifests[]? | select((.annotations["vnd.docker.reference.type"] // "") != "attestation-manifest"
|
||||
and (.platform.os // "unknown") != "unknown") | .digest][0] // empty')
|
||||
ref="${{ env.IMAGE }}@${image:-sha256:${f}}"
|
||||
syft "registry:${ref}" -o "spdx-json=/tmp/sbom-${f}.spdx.json"
|
||||
cosign attest --yes \
|
||||
--predicate "/tmp/sbom-${f}.spdx.json" \
|
||||
--type spdxjson \
|
||||
"$ref"
|
||||
done
|
||||
|
||||
# Report-only trivy scan: uploads fixable HIGH/CRITICAL findings to GitHub
|
||||
# Security for visibility, but never blocks the release. Releases (including
|
||||
# `latest`) ship regardless — vulnerabilities are tracked, not gated, since
|
||||
# we sometimes need to publish through known findings (e.g. unfixed upstream
|
||||
# CVE, base-image lag).
|
||||
trivy-scan:
|
||||
runs-on: ubuntu-latest
|
||||
needs: [merge]
|
||||
if: github.event_name == 'push'
|
||||
continue-on-error: true
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
include:
|
||||
- source_suffix: ""
|
||||
variant: normal
|
||||
- source_suffix: _large_disk
|
||||
variant: large_disk
|
||||
steps:
|
||||
- name: Login to GHCR
|
||||
uses: docker/login-action@v4.8.0
|
||||
with:
|
||||
registry: ghcr.io
|
||||
username: ${{ secrets.GHCR_USERNAME }}
|
||||
password: ${{ secrets.GHCR_TOKEN }}
|
||||
|
||||
- name: Trivy report (${{ matrix.variant }})
|
||||
# Pin to SHA - mutable tags were compromised (GHSA-69fq-xp46-6x23)
|
||||
uses: aquasecurity/trivy-action@ed142fd0673e97e23eac54620cfb913e5ce36c25 # v0.36.0
|
||||
with:
|
||||
scan-type: image
|
||||
# Scan the multi-arch tag on GHCR (already pushed by the build job).
|
||||
# Trivy scans the runner's native platform; OS packages are identical
|
||||
# across architectures since they all share the same alpine base.
|
||||
image-ref: ${{ env.IMAGE }}:${{ env.RELEASE_TAG }}${{ matrix.source_suffix }}
|
||||
scanners: vuln
|
||||
vuln-type: os,library
|
||||
severity: HIGH,CRITICAL
|
||||
# Keep unfixed findings in the report so digest-pinning consumers see
|
||||
# the complete picture, not just what a patch could resolve.
|
||||
ignore-unfixed: false
|
||||
limit-severities-for-sarif: true
|
||||
format: sarif
|
||||
output: trivy-results.sarif
|
||||
exit-code: '0'
|
||||
|
||||
- name: Upload Trivy scan results to GitHub Security
|
||||
if: always()
|
||||
uses: github/codeql-action/upload-sarif@v4.39.3
|
||||
with:
|
||||
sarif_file: trivy-results.sarif
|
||||
category: trivy-${{ matrix.variant }}
|
||||
|
||||
# Point `latest` (and `latest_large_disk`) at the just-released versioned
|
||||
# image. crane tag adds an extra tag to an existing manifest — no rebuild,
|
||||
# no QEMU, no separate workflow. Replaces the old container_latest.yml
|
||||
# rebuild that often failed or lagged behind the release. Independent of
|
||||
# trivy-scan: vuln findings are reported but do not block `latest`. The
|
||||
# cosign signature is attached to the digest, so `latest` carries it too.
|
||||
tag-latest:
|
||||
runs-on: ubuntu-latest
|
||||
needs: [merge]
|
||||
if: github.event_name == 'push'
|
||||
strategy:
|
||||
matrix:
|
||||
include:
|
||||
- source_suffix: ""
|
||||
latest_tag: latest
|
||||
- source_suffix: _large_disk
|
||||
latest_tag: latest_large_disk
|
||||
steps:
|
||||
- name: Login to Docker Hub
|
||||
uses: docker/login-action@v4.8.0
|
||||
with:
|
||||
username: ${{ secrets.DOCKER_USERNAME }}
|
||||
password: ${{ secrets.DOCKER_PASSWORD }}
|
||||
|
||||
- name: Login to GHCR
|
||||
uses: docker/login-action@v4.8.0
|
||||
with:
|
||||
registry: ghcr.io
|
||||
username: ${{ secrets.GHCR_USERNAME }}
|
||||
password: ${{ secrets.GHCR_TOKEN }}
|
||||
|
||||
- name: Install crane
|
||||
run: |
|
||||
cd $(mktemp -d)
|
||||
curl -sLO https://github.com/google/go-containerregistry/releases/download/v0.22.0/go-containerregistry_Linux_x86_64.tar.gz
|
||||
echo "edb74d53fad9a596860f59d1c5d04a43dfb5f441dc71f57060dd0bf39483c833 go-containerregistry_Linux_x86_64.tar.gz" | sha256sum -c -
|
||||
tar xzf go-containerregistry_Linux_x86_64.tar.gz crane
|
||||
sudo mv crane /usr/local/bin/
|
||||
crane version
|
||||
|
||||
- name: Re-tag ${{ env.RELEASE_TAG }}${{ matrix.source_suffix }} as ${{ matrix.latest_tag }}
|
||||
run: |
|
||||
retry_with_backoff() {
|
||||
local max_attempts=5
|
||||
local timeout=1
|
||||
local attempt=1
|
||||
local exit_code=0
|
||||
while [ $attempt -le $max_attempts ]; do
|
||||
if "$@"; then
|
||||
return 0
|
||||
else
|
||||
exit_code=$?
|
||||
fi
|
||||
if [ $attempt -lt $max_attempts ]; then
|
||||
echo "Attempt $attempt failed. Retrying in ${timeout}s..." >&2
|
||||
sleep $timeout
|
||||
timeout=$((timeout * 2))
|
||||
fi
|
||||
attempt=$((attempt + 1))
|
||||
done
|
||||
echo "Command failed after $max_attempts attempts" >&2
|
||||
return $exit_code
|
||||
}
|
||||
|
||||
SRC_TAG="${{ env.RELEASE_TAG }}${{ matrix.source_suffix }}"
|
||||
DST_TAG="${{ matrix.latest_tag }}"
|
||||
|
||||
echo "Tagging ${{ env.IMAGE }}:${SRC_TAG} as ${DST_TAG}"
|
||||
retry_with_backoff crane tag "${{ env.IMAGE }}:${SRC_TAG}" "${DST_TAG}"
|
||||
|
||||
echo "Tagging chrislusf/seaweedfs:${SRC_TAG} as ${DST_TAG}"
|
||||
retry_with_backoff crane tag "chrislusf/seaweedfs:${SRC_TAG}" "${DST_TAG}"
|
||||
|
||||
helm-release:
|
||||
runs-on: ubuntu-latest
|
||||
needs: [build]
|
||||
if: github.event_name == 'push' || github.event_name == 'workflow_dispatch'
|
||||
permissions:
|
||||
contents: write
|
||||
pages: write
|
||||
steps:
|
||||
- uses: actions/checkout@v7
|
||||
- name: Publish Helm charts
|
||||
uses: stefanprodan/helm-gh-pages@v1.7.0
|
||||
with:
|
||||
token: ${{ secrets.GITHUB_TOKEN }}
|
||||
charts_dir: k8s/charts
|
||||
target_dir: helm
|
||||
branch: gh-pages
|
||||
helm_version: "3.18.4"
|
||||
@@ -1,124 +0,0 @@
|
||||
name: "docker: build rocksdb image by version"
|
||||
|
||||
on:
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
rocksdb_version:
|
||||
description: 'RocksDB git tag or branch to build (e.g. v11.1.2)'
|
||||
required: true
|
||||
default: 'v11.1.2'
|
||||
seaweedfs_ref:
|
||||
description: 'SeaweedFS git tag, branch, or commit to build'
|
||||
required: true
|
||||
default: 'master'
|
||||
image_tag:
|
||||
description: 'Optional Docker tag suffix (defaults to rocksdb_<rocksdb>_seaweedfs_<ref>)'
|
||||
required: false
|
||||
default: ''
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.event.inputs.image_tag || format('rocksdb_{0}_seaweedfs_{1}', github.event.inputs.rocksdb_version, github.event.inputs.seaweedfs_ref) }}
|
||||
cancel-in-progress: false
|
||||
queue: max
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
jobs:
|
||||
build-rocksdb-image:
|
||||
runs-on: ubuntu-latest
|
||||
permissions:
|
||||
contents: read
|
||||
id-token: write
|
||||
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v2
|
||||
|
||||
- name: Prepare Docker tag
|
||||
id: tag
|
||||
env:
|
||||
ROCKSDB_VERSION_INPUT: ${{ inputs.rocksdb_version }}
|
||||
SEAWEEDFS_REF_INPUT: ${{ inputs.seaweedfs_ref }}
|
||||
CUSTOM_TAG_INPUT: ${{ inputs.image_tag }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
sanitize() {
|
||||
local value="$1"
|
||||
value="${value,,}"
|
||||
value="${value// /-}"
|
||||
value="${value//[^a-z0-9_.-]/-}"
|
||||
value="${value#-}"
|
||||
value="${value%-}"
|
||||
printf '%s' "$value"
|
||||
}
|
||||
version="${ROCKSDB_VERSION_INPUT}"
|
||||
seaweed="${SEAWEEDFS_REF_INPUT}"
|
||||
tag="${CUSTOM_TAG_INPUT}"
|
||||
if [ -z "$version" ]; then
|
||||
echo "RocksDB version input is required." >&2
|
||||
exit 1
|
||||
fi
|
||||
if [ -z "$seaweed" ]; then
|
||||
echo "SeaweedFS ref input is required." >&2
|
||||
exit 1
|
||||
fi
|
||||
sanitized_version="$(sanitize "$version")"
|
||||
if [ -z "$sanitized_version" ]; then
|
||||
echo "Unable to sanitize RocksDB version '$version'." >&2
|
||||
exit 1
|
||||
fi
|
||||
sanitized_seaweed="$(sanitize "$seaweed")"
|
||||
if [ -z "$sanitized_seaweed" ]; then
|
||||
echo "Unable to sanitize SeaweedFS ref '$seaweed'." >&2
|
||||
exit 1
|
||||
fi
|
||||
if [ -z "$tag" ]; then
|
||||
tag="rocksdb_${sanitized_version}_seaweedfs_${sanitized_seaweed}"
|
||||
fi
|
||||
tag="${tag,,}"
|
||||
tag="${tag// /-}"
|
||||
tag="${tag//[^a-z0-9_.-]/-}"
|
||||
tag="${tag#-}"
|
||||
tag="${tag%-}"
|
||||
if [ -z "$tag" ]; then
|
||||
echo "Resulting Docker tag is empty." >&2
|
||||
exit 1
|
||||
fi
|
||||
echo "docker_tag=$tag" >> "$GITHUB_OUTPUT"
|
||||
echo "full_image=chrislusf/seaweedfs:$tag" >> "$GITHUB_OUTPUT"
|
||||
echo "seaweedfs_ref=$seaweed" >> "$GITHUB_OUTPUT"
|
||||
|
||||
- name: Set up QEMU
|
||||
uses: docker/setup-qemu-action@99012661954931238ded8c8b007157a8430204e1 # v4.4.0
|
||||
|
||||
- name: Set up Docker Buildx
|
||||
uses: docker/setup-buildx-action@4d04d5d9486b7bd6fa91e7baf45bbb4f8b9deedd # v4.0.0
|
||||
|
||||
- name: Login to Docker Hub
|
||||
uses: docker/login-action@8aa5831fb8d48d8f76594c7ddde61ae58019f550 # v4.8.0
|
||||
with:
|
||||
username: ${{ secrets.DOCKER_USERNAME }}
|
||||
password: ${{ secrets.DOCKER_PASSWORD }}
|
||||
|
||||
- name: Build and push image
|
||||
id: build
|
||||
uses: docker/build-push-action@bcafcacb16a39f128d818304e6c9c0c18556b85f # v2
|
||||
with:
|
||||
context: ./docker
|
||||
push: true
|
||||
file: ./docker/Dockerfile.rocksdb_large
|
||||
build-args: |
|
||||
ROCKSDB_VERSION=${{ inputs.rocksdb_version }}
|
||||
BRANCH=${{ inputs.seaweedfs_ref }}
|
||||
platforms: linux/amd64
|
||||
tags: ${{ steps.tag.outputs.full_image }}
|
||||
labels: |
|
||||
org.opencontainers.image.title=seaweedfs
|
||||
org.opencontainers.image.description=SeaweedFS is a distributed storage system for blobs, objects, files, and data lake, to store and serve billions of files fast!
|
||||
org.opencontainers.image.vendor=Chris Lu
|
||||
|
||||
- name: Sign
|
||||
uses: ./.github/actions/sign-image
|
||||
with:
|
||||
images: chrislusf/seaweedfs@${{ steps.build.outputs.digest }}
|
||||
@@ -21,63 +21,22 @@ jobs:
|
||||
deploy:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v7
|
||||
- uses: actions/checkout@v4
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v7
|
||||
uses: actions/setup-go@v5.5.0
|
||||
with:
|
||||
go-version-file: 'telemetry/server/go.mod'
|
||||
cache: ${{ runner.environment == 'github-hosted' }}
|
||||
go-version: '1.24'
|
||||
|
||||
- name: Build Telemetry Server
|
||||
if: github.event_name == 'workflow_dispatch' && inputs.deploy
|
||||
run: |
|
||||
# telemetry/server is its own Go module; build from within it
|
||||
cd telemetry/server
|
||||
go mod tidy
|
||||
echo "Building telemetry server..."
|
||||
CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -o ../../telemetry-server .
|
||||
cd ../..
|
||||
GOOS=linux GOARCH=amd64 go build -o telemetry-server ./telemetry/server/main.go
|
||||
ls -la telemetry-server
|
||||
echo "Build completed successfully"
|
||||
|
||||
- name: Generate Service Configuration
|
||||
if: github.event_name == 'workflow_dispatch' && (inputs.setup || inputs.deploy)
|
||||
env:
|
||||
REMOTE_USER: ${{ secrets.TELEMETRY_USER }}
|
||||
run: |
|
||||
# Create systemd service file
|
||||
echo "
|
||||
[Unit]
|
||||
Description=SeaweedFS Telemetry Server
|
||||
After=network.target
|
||||
|
||||
[Service]
|
||||
Type=simple
|
||||
User=$REMOTE_USER
|
||||
WorkingDirectory=/home/$REMOTE_USER/seaweedfs-telemetry
|
||||
ExecStart=/bin/sh -c 'exec /home/$REMOTE_USER/seaweedfs-telemetry/bin/telemetry-server -port=8353 >>/home/$REMOTE_USER/seaweedfs-telemetry/logs/telemetry.log 2>>/home/$REMOTE_USER/seaweedfs-telemetry/logs/telemetry.error.log'
|
||||
Restart=always
|
||||
RestartSec=5
|
||||
|
||||
[Install]
|
||||
WantedBy=multi-user.target" > telemetry.service
|
||||
|
||||
# Setup logrotate configuration
|
||||
echo "# SeaweedFS Telemetry service log rotation
|
||||
/home/$REMOTE_USER/seaweedfs-telemetry/logs/*.log {
|
||||
daily
|
||||
rotate 30
|
||||
compress
|
||||
delaycompress
|
||||
missingok
|
||||
notifempty
|
||||
create 644 $REMOTE_USER $REMOTE_USER
|
||||
postrotate
|
||||
systemctl restart telemetry.service
|
||||
endscript
|
||||
}" > telemetry_logrotate
|
||||
|
||||
- name: First-time Server Setup
|
||||
if: github.event_name == 'workflow_dispatch' && inputs.setup
|
||||
env:
|
||||
@@ -99,6 +58,40 @@ jobs:
|
||||
touch ~/seaweedfs-telemetry/logs/telemetry.log ~/seaweedfs-telemetry/logs/telemetry.error.log && \
|
||||
chmod 644 ~/seaweedfs-telemetry/logs/*.log"
|
||||
|
||||
# Create systemd service file
|
||||
echo "
|
||||
[Unit]
|
||||
Description=SeaweedFS Telemetry Server
|
||||
After=network.target
|
||||
|
||||
[Service]
|
||||
Type=simple
|
||||
User=$REMOTE_USER
|
||||
WorkingDirectory=/home/$REMOTE_USER/seaweedfs-telemetry
|
||||
ExecStart=/home/$REMOTE_USER/seaweedfs-telemetry/bin/telemetry-server -port=8353
|
||||
Restart=always
|
||||
RestartSec=5
|
||||
StandardOutput=append:/home/$REMOTE_USER/seaweedfs-telemetry/logs/telemetry.log
|
||||
StandardError=append:/home/$REMOTE_USER/seaweedfs-telemetry/logs/telemetry.error.log
|
||||
|
||||
[Install]
|
||||
WantedBy=multi-user.target" > telemetry.service
|
||||
|
||||
# Setup logrotate configuration
|
||||
echo "# SeaweedFS Telemetry service log rotation
|
||||
/home/$REMOTE_USER/seaweedfs-telemetry/logs/*.log {
|
||||
daily
|
||||
rotate 30
|
||||
compress
|
||||
delaycompress
|
||||
missingok
|
||||
notifempty
|
||||
create 644 $REMOTE_USER $REMOTE_USER
|
||||
postrotate
|
||||
systemctl restart telemetry.service
|
||||
endscript
|
||||
}" > telemetry_logrotate
|
||||
|
||||
# Copy configuration files
|
||||
scp -i ~/.ssh/deploy_key telemetry/grafana-dashboard.json $REMOTE_USER@$REMOTE_HOST:~/seaweedfs-telemetry/
|
||||
scp -i ~/.ssh/deploy_key telemetry/prometheus.yml $REMOTE_USER@$REMOTE_HOST:~/seaweedfs-telemetry/
|
||||
@@ -141,18 +134,11 @@ jobs:
|
||||
scp -i ~/.ssh/deploy_key telemetry/grafana-dashboard.json $REMOTE_USER@$REMOTE_HOST:~/seaweedfs-telemetry/
|
||||
scp -i ~/.ssh/deploy_key telemetry/prometheus.yml $REMOTE_USER@$REMOTE_HOST:~/seaweedfs-telemetry/
|
||||
|
||||
# Copy updated service and logrotate files
|
||||
scp -i ~/.ssh/deploy_key telemetry.service telemetry_logrotate $REMOTE_USER@$REMOTE_HOST:~/seaweedfs-telemetry/
|
||||
|
||||
# Check if service exists and deploy accordingly
|
||||
ssh -i ~/.ssh/deploy_key $REMOTE_USER@$REMOTE_HOST "
|
||||
if systemctl list-unit-files telemetry.service >/dev/null 2>&1; then
|
||||
echo 'Service exists, performing update...'
|
||||
set -e
|
||||
sudo systemctl stop telemetry.service
|
||||
sudo mv ~/seaweedfs-telemetry/telemetry.service /etc/systemd/system/
|
||||
sudo mv ~/seaweedfs-telemetry/telemetry_logrotate /etc/logrotate.d/seaweedfs-telemetry
|
||||
sudo systemctl daemon-reload
|
||||
mkdir -p ~/seaweedfs-telemetry/bin
|
||||
mv ~/seaweedfs-telemetry/tmp/telemetry-server ~/seaweedfs-telemetry/bin/
|
||||
chmod +x ~/seaweedfs-telemetry/bin/telemetry-server
|
||||
|
||||
@@ -1,10 +1,6 @@
|
||||
name: 'Dependency Review'
|
||||
on: [pull_request]
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.event.pull_request.number || (github.event_name == 'push' && github.ref) || github.run_id }}
|
||||
cancel-in-progress: ${{ github.event_name == 'pull_request' || (github.event_name == 'push' && !startsWith(github.ref, 'refs/tags/')) }}
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
@@ -13,6 +9,6 @@ jobs:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: 'Checkout Repository'
|
||||
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1
|
||||
uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633
|
||||
- name: 'Dependency Review'
|
||||
uses: actions/dependency-review-action@a1d282b36b6f3519aa1f3fc636f609c47dddb294
|
||||
uses: actions/dependency-review-action@da24556b548a50705dd671f47852072ea4c105d9
|
||||
+15
-82
@@ -3,24 +3,12 @@ name: "End to End"
|
||||
on:
|
||||
push:
|
||||
branches: [ master ]
|
||||
paths:
|
||||
- 'weed/**'
|
||||
- 'docker/**'
|
||||
- 'go.mod'
|
||||
- 'go.sum'
|
||||
- '.github/workflows/e2e.yml'
|
||||
pull_request:
|
||||
branches: [ master ]
|
||||
paths:
|
||||
- 'weed/**'
|
||||
- 'docker/**'
|
||||
- 'go.mod'
|
||||
- 'go.sum'
|
||||
- '.github/workflows/e2e.yml'
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.event.pull_request.number || (github.event_name == 'push' && github.ref) || github.run_id }}
|
||||
cancel-in-progress: ${{ github.event_name == 'pull_request' || (github.event_name == 'push' && !startsWith(github.ref, 'refs/tags/')) }}
|
||||
group: ${{ github.head_ref }}/e2e
|
||||
cancel-in-progress: true
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
@@ -32,81 +20,26 @@ defaults:
|
||||
jobs:
|
||||
e2e:
|
||||
name: FUSE Mount
|
||||
runs-on: ubuntu-24.04
|
||||
runs-on: ubuntu-22.04
|
||||
timeout-minutes: 30
|
||||
steps:
|
||||
- name: Set up Go 1.x
|
||||
uses: actions/setup-go@fa96338abe5531f6e34c5cc0bbe28c1a533d5505 # v2
|
||||
with:
|
||||
go-version: ^1.13
|
||||
id: go
|
||||
|
||||
- name: Check out code into the Go module directory
|
||||
uses: actions/checkout@v7
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v7
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
cache: ${{ runner.environment == 'github-hosted' }}
|
||||
|
||||
- name: Configure Docker Hub mirror
|
||||
run: |
|
||||
echo '{"registry-mirrors": ["https://mirror.gcr.io"]}' | sudo tee /etc/docker/daemon.json
|
||||
sudo systemctl restart docker
|
||||
|
||||
- name: Set up Docker Buildx
|
||||
uses: docker/setup-buildx-action@v4
|
||||
|
||||
- name: Cache Docker layers
|
||||
if: runner.environment == 'github-hosted'
|
||||
uses: actions/cache@v6
|
||||
with:
|
||||
path: /tmp/.buildx-cache
|
||||
key: ${{ runner.os }}-buildx-e2e-${{ github.sha }}
|
||||
restore-keys: |
|
||||
${{ runner.os }}-buildx-e2e-
|
||||
|
||||
- name: Test APT mirror fallback
|
||||
run: python3 test_apt_install.py
|
||||
uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v2
|
||||
|
||||
- name: Install dependencies
|
||||
run: |
|
||||
sudo rm -f /etc/apt/sources.list.d/azure-cli.list /etc/apt/sources.list.d/microsoft-prod.list
|
||||
# Same helper the e2e image installs through: the runner's own list is
|
||||
# azure-only too, and an outage there fails this step outright.
|
||||
sudo ./apt-install fuse
|
||||
|
||||
# Verify FUSE installation
|
||||
echo "FUSE version: $(fusermount --version 2>&1 || echo 'fusermount not found')"
|
||||
echo "FUSE device: $(ls -la /dev/fuse 2>&1 || echo '/dev/fuse not found')"
|
||||
sudo apt-get update
|
||||
sudo apt-get install -y fuse
|
||||
|
||||
- name: Start SeaweedFS
|
||||
timeout-minutes: 15
|
||||
run: |
|
||||
# Enable Docker buildkit for better caching
|
||||
export DOCKER_BUILDKIT=1
|
||||
export COMPOSE_DOCKER_CLI_BUILD=1
|
||||
|
||||
# Build with retry logic
|
||||
for i in {1..3}; do
|
||||
echo "Build attempt $i/3"
|
||||
if make build_e2e; then
|
||||
echo "Build successful on attempt $i"
|
||||
break
|
||||
elif [ $i -eq 3 ]; then
|
||||
echo "Build failed after 3 attempts"
|
||||
exit 1
|
||||
else
|
||||
echo "Build attempt $i failed, retrying in 30 seconds..."
|
||||
sleep 30
|
||||
fi
|
||||
done
|
||||
|
||||
# Start services with wait
|
||||
docker compose -f ./compose/e2e-mount.yml up --wait
|
||||
|
||||
- name: Rotate buildx cache
|
||||
if: always()
|
||||
run: |
|
||||
# Without this, --cache-to writes to .buildx-cache-new but actions/cache only
|
||||
# uploads .buildx-cache, so layers (notably the slow apt RUN) never persist.
|
||||
rm -rf /tmp/.buildx-cache
|
||||
if [ -d /tmp/.buildx-cache-new ]; then mv /tmp/.buildx-cache-new /tmp/.buildx-cache; fi
|
||||
timeout-minutes: 5
|
||||
run: make build_e2e && docker compose -f ./compose/e2e-mount.yml up --wait
|
||||
|
||||
- name: Run FIO 4k
|
||||
timeout-minutes: 15
|
||||
@@ -161,7 +94,7 @@ jobs:
|
||||
|
||||
- name: Archive logs
|
||||
if: always()
|
||||
uses: actions/upload-artifact@v7
|
||||
uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: output-logs
|
||||
path: docker/output.log
|
||||
|
||||
@@ -1,81 +0,0 @@
|
||||
name: "EC Integration Tests"
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [ master ]
|
||||
paths:
|
||||
- 'weed/**'
|
||||
- 'test/erasure_coding/**'
|
||||
- 'go.mod'
|
||||
- 'go.sum'
|
||||
- '.github/workflows/ec-integration-tests.yml'
|
||||
pull_request:
|
||||
branches: [ master ]
|
||||
paths:
|
||||
- 'weed/**'
|
||||
- 'test/erasure_coding/**'
|
||||
- 'go.mod'
|
||||
- 'go.sum'
|
||||
- '.github/workflows/ec-integration-tests.yml'
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.event.pull_request.number || (github.event_name == 'push' && github.ref) || github.run_id }}
|
||||
cancel-in-progress: ${{ github.event_name == 'pull_request' || (github.event_name == 'push' && !startsWith(github.ref, 'refs/tags/')) }}
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
jobs:
|
||||
ec-integration-tests:
|
||||
name: EC Integration Tests
|
||||
runs-on: ubuntu-24.04
|
||||
timeout-minutes: 30
|
||||
steps:
|
||||
- name: Set up Go 1.x
|
||||
uses: actions/setup-go@v7
|
||||
with:
|
||||
cache: ${{ runner.environment == 'github-hosted' }}
|
||||
go-version: ^1.26
|
||||
id: go
|
||||
|
||||
- name: Check out code into the Go module directory
|
||||
uses: actions/checkout@v7
|
||||
|
||||
- name: Build weed binary
|
||||
run: |
|
||||
cd weed && go build -o weed .
|
||||
|
||||
- name: Run EC Integration Tests
|
||||
working-directory: test/erasure_coding
|
||||
run: |
|
||||
# The suite now includes the interruption matrix and runs close to Go's
|
||||
# default 10m binary timeout on slower runners; bound it by the job's
|
||||
# 30m budget instead.
|
||||
go test -v -timeout 25m
|
||||
|
||||
- name: Collect server logs on failure
|
||||
if: failure()
|
||||
run: |
|
||||
echo "Collecting server logs from temp directories..."
|
||||
mkdir -p /tmp/ec-test-logs
|
||||
# Find all temp directories created by the tests (they persist on failure with t.TempDir())
|
||||
find /tmp -maxdepth 1 -type d \( -name "TestEC*" -o -name "TestDisk*" -o -name "TestCross*" -o -name "TestEvacuation*" \) 2>/dev/null | while read dir; do
|
||||
if [ -d "$dir" ]; then
|
||||
echo "Found test directory: $dir"
|
||||
# Copy the entire directory structure to preserve organization
|
||||
cp -r "$dir" /tmp/ec-test-logs/ 2>/dev/null || true
|
||||
fi
|
||||
done
|
||||
# List what we collected
|
||||
echo "Collected logs:"
|
||||
find /tmp/ec-test-logs -type f -name "*.log" 2>/dev/null || echo "No logs found"
|
||||
|
||||
- name: Archive logs
|
||||
if: failure()
|
||||
uses: actions/upload-artifact@v7
|
||||
with:
|
||||
name: ec-integration-test-logs
|
||||
path: |
|
||||
/tmp/ec-test-logs/
|
||||
test/erasure_coding/
|
||||
if-no-files-found: warn
|
||||
@@ -1,70 +0,0 @@
|
||||
name: "FUSE DLM Integration Tests"
|
||||
|
||||
on:
|
||||
pull_request:
|
||||
paths:
|
||||
- 'weed/command/mount*.go'
|
||||
- 'weed/mount/**'
|
||||
- 'weed/cluster/**'
|
||||
- 'test/fuse_dlm/**'
|
||||
- '.github/workflows/fuse-dlm-integration.yml'
|
||||
- '.github/actions/fix-fusermount-setuid/**'
|
||||
push:
|
||||
branches: [master]
|
||||
paths:
|
||||
- 'weed/command/mount*.go'
|
||||
- 'weed/mount/**'
|
||||
- 'weed/cluster/**'
|
||||
- 'test/fuse_dlm/**'
|
||||
- '.github/actions/fix-fusermount-setuid/**'
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.event.pull_request.number || (github.event_name == 'push' && github.ref) || github.run_id }}
|
||||
cancel-in-progress: ${{ github.event_name == 'pull_request' || (github.event_name == 'push' && !startsWith(github.ref, 'refs/tags/')) }}
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
jobs:
|
||||
fuse-dlm-integration:
|
||||
name: FUSE DLM Integration Tests
|
||||
runs-on: ubuntu-24.04
|
||||
timeout-minutes: 30
|
||||
|
||||
steps:
|
||||
- name: Check out code
|
||||
uses: actions/checkout@v7
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v7
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
cache: ${{ runner.environment == 'github-hosted' }}
|
||||
|
||||
- name: Configure FUSE
|
||||
run: |
|
||||
# Nothing to install: fuse3 ships fusermount3 and is pre-installed,
|
||||
# and go-fuse is pure Go, so the libfuse headers were never linked
|
||||
# against.
|
||||
echo 'user_allow_other' | sudo tee -a /etc/fuse.conf
|
||||
sudo chmod 644 /etc/fuse.conf
|
||||
|
||||
- name: Repair the fusermount3 setuid bit
|
||||
uses: ./.github/actions/fix-fusermount-setuid
|
||||
|
||||
- name: Build SeaweedFS
|
||||
run: go build -o weed/weed -buildvcs=false ./weed
|
||||
|
||||
- name: Run DLM integration tests
|
||||
timeout-minutes: 25
|
||||
env:
|
||||
WEED_BINARY: ${{ github.workspace }}/weed/weed
|
||||
run: go test -v -count=1 -timeout=20m ./test/fuse_dlm/...
|
||||
|
||||
- name: Upload logs on failure
|
||||
if: failure()
|
||||
uses: actions/upload-artifact@v7
|
||||
with:
|
||||
name: fuse-dlm-test-logs
|
||||
path: /tmp/seaweedfs-fuse-dlm-logs/
|
||||
retention-days: 3
|
||||
@@ -1,80 +0,0 @@
|
||||
name: "FUSE Volume Server Failover Tests"
|
||||
|
||||
on:
|
||||
pull_request:
|
||||
paths:
|
||||
- 'weed/command/mount*.go'
|
||||
- 'weed/mount/**'
|
||||
- 'weed/filer/**'
|
||||
- 'weed/wdclient/**'
|
||||
- 'weed/operation/upload_content.go'
|
||||
- 'test/fuse_failover/**'
|
||||
- '.github/workflows/fuse-failover.yml'
|
||||
- '.github/actions/fix-fusermount-setuid/**'
|
||||
push:
|
||||
branches: [master]
|
||||
paths:
|
||||
- 'weed/command/mount*.go'
|
||||
- 'weed/mount/**'
|
||||
- 'weed/filer/**'
|
||||
- 'weed/wdclient/**'
|
||||
- 'weed/operation/upload_content.go'
|
||||
- 'test/fuse_failover/**'
|
||||
- '.github/workflows/fuse-failover.yml'
|
||||
- '.github/actions/fix-fusermount-setuid/**'
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.event.pull_request.number || (github.event_name == 'push' && github.ref) || github.run_id }}
|
||||
cancel-in-progress: ${{ github.event_name == 'pull_request' || (github.event_name == 'push' && !startsWith(github.ref, 'refs/tags/')) }}
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
jobs:
|
||||
fuse-failover:
|
||||
name: FUSE Volume Server Failover
|
||||
runs-on: ubuntu-24.04
|
||||
timeout-minutes: 40
|
||||
|
||||
steps:
|
||||
- name: Check out code
|
||||
uses: actions/checkout@v7
|
||||
with:
|
||||
persist-credentials: false
|
||||
|
||||
- name: Clean up stale FUSE mounts
|
||||
uses: ./.github/actions/cleanup-stale-fuse-mounts
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v7
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
cache: ${{ runner.environment == 'github-hosted' }}
|
||||
|
||||
- name: Configure FUSE
|
||||
run: |
|
||||
# Nothing to install: fuse3 ships fusermount3 and is pre-installed,
|
||||
# and go-fuse is pure Go, so the libfuse headers were never linked
|
||||
# against.
|
||||
echo 'user_allow_other' | sudo tee -a /etc/fuse.conf
|
||||
sudo chmod 644 /etc/fuse.conf
|
||||
|
||||
- name: Repair the fusermount3 setuid bit
|
||||
uses: ./.github/actions/fix-fusermount-setuid
|
||||
|
||||
- name: Build SeaweedFS
|
||||
run: go build -o weed/weed -buildvcs=false ./weed
|
||||
|
||||
- name: Run failover integration tests
|
||||
timeout-minutes: 35
|
||||
env:
|
||||
WEED_BINARY: ${{ github.workspace }}/weed/weed
|
||||
run: go test -v -count=1 -timeout=30m ./test/fuse_failover/...
|
||||
|
||||
- name: Upload logs on failure
|
||||
if: failure()
|
||||
uses: actions/upload-artifact@v7
|
||||
with:
|
||||
name: fuse-failover-test-logs
|
||||
path: /tmp/seaweedfs-fuse-failover-logs/
|
||||
retention-days: 3
|
||||
@@ -7,79 +7,228 @@ on:
|
||||
- 'weed/**'
|
||||
- 'test/fuse_integration/**'
|
||||
- '.github/workflows/fuse-integration.yml'
|
||||
- '.github/actions/fix-fusermount-setuid/**'
|
||||
pull_request:
|
||||
branches: [ master, main ]
|
||||
paths:
|
||||
- 'weed/**'
|
||||
- 'test/fuse_integration/**'
|
||||
- '.github/workflows/fuse-integration.yml'
|
||||
- '.github/actions/fix-fusermount-setuid/**'
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.event.pull_request.number || (github.event_name == 'push' && github.ref) || github.run_id }}
|
||||
cancel-in-progress: ${{ github.event_name == 'pull_request' || (github.event_name == 'push' && !startsWith(github.ref, 'refs/tags/')) }}
|
||||
group: ${{ github.head_ref }}/fuse-integration
|
||||
cancel-in-progress: true
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
env:
|
||||
GO_VERSION: '1.21'
|
||||
TEST_TIMEOUT: '45m'
|
||||
|
||||
jobs:
|
||||
fuse-integration:
|
||||
name: FUSE Integration Testing
|
||||
runs-on: ubuntu-24.04
|
||||
runs-on: ubuntu-22.04
|
||||
timeout-minutes: 50
|
||||
|
||||
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v7
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v7
|
||||
uses: actions/checkout@v4
|
||||
|
||||
- name: Set up Go ${{ env.GO_VERSION }}
|
||||
uses: actions/setup-go@v4
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
cache: ${{ runner.environment == 'github-hosted' }}
|
||||
|
||||
- name: Configure FUSE
|
||||
go-version: ${{ env.GO_VERSION }}
|
||||
|
||||
- name: Install FUSE and dependencies
|
||||
run: |
|
||||
# Nothing to install: fuse3 ships fusermount3 and is pre-installed, and
|
||||
# go-fuse is pure Go, so the libfuse headers were never linked against.
|
||||
# Allow non-root FUSE mounts with allow_other
|
||||
echo 'user_allow_other' | sudo tee -a /etc/fuse.conf
|
||||
sudo chmod 644 /etc/fuse.conf
|
||||
sudo apt-get update
|
||||
sudo apt-get install -y fuse libfuse-dev
|
||||
# Verify FUSE installation
|
||||
fusermount3 --version || fusermount --version || true
|
||||
ls -la /dev/fuse
|
||||
|
||||
- name: Repair the fusermount3 setuid bit
|
||||
uses: ./.github/actions/fix-fusermount-setuid
|
||||
|
||||
fusermount --version || true
|
||||
ls -la /dev/fuse || true
|
||||
|
||||
- name: Build SeaweedFS
|
||||
run: |
|
||||
cd weed
|
||||
go build -tags "elastic gocdk sqlite ydb tarantool tikv rclone" -o weed .
|
||||
go build -tags "elastic gocdk sqlite ydb tarantool tikv rclone" -v .
|
||||
chmod +x weed
|
||||
# Verify binary
|
||||
./weed version
|
||||
# Make weed binary available in PATH for the test framework
|
||||
sudo cp weed /usr/local/bin/weed
|
||||
|
||||
- name: Install test dependencies
|
||||
|
||||
- name: Prepare FUSE Integration Tests
|
||||
run: |
|
||||
cd test/fuse_integration
|
||||
go mod download
|
||||
|
||||
# Create isolated test directory to avoid Go module conflicts
|
||||
mkdir -p /tmp/seaweedfs-fuse-tests
|
||||
|
||||
# Copy only the working test files to avoid Go module conflicts
|
||||
# These are the files we've verified work without package name issues
|
||||
cp test/fuse_integration/simple_test.go /tmp/seaweedfs-fuse-tests/ 2>/dev/null || echo "⚠️ simple_test.go not found"
|
||||
cp test/fuse_integration/working_demo_test.go /tmp/seaweedfs-fuse-tests/ 2>/dev/null || echo "⚠️ working_demo_test.go not found"
|
||||
|
||||
# Note: Other test files (framework.go, basic_operations_test.go, etc.)
|
||||
# have Go module conflicts and are skipped until resolved
|
||||
|
||||
echo "📁 Working test files copied:"
|
||||
ls -la /tmp/seaweedfs-fuse-tests/*.go 2>/dev/null || echo "ℹ️ No test files found"
|
||||
|
||||
# Initialize Go module in isolated directory
|
||||
cd /tmp/seaweedfs-fuse-tests
|
||||
go mod init seaweedfs-fuse-tests
|
||||
go mod tidy
|
||||
|
||||
# Verify setup
|
||||
echo "✅ FUSE integration test environment prepared"
|
||||
ls -la /tmp/seaweedfs-fuse-tests/
|
||||
|
||||
echo ""
|
||||
echo "ℹ️ Current Status: Running working subset of FUSE tests"
|
||||
echo " • simple_test.go: Package structure verification"
|
||||
echo " • working_demo_test.go: Framework capability demonstration"
|
||||
echo " • Full framework: Available in test/fuse_integration/ (module conflicts pending resolution)"
|
||||
|
||||
- name: Run FUSE Integration Tests
|
||||
run: |
|
||||
set -o pipefail
|
||||
cd test/fuse_integration
|
||||
echo "Running full FUSE integration test suite..."
|
||||
go test -v -count=1 -timeout=45m ./... 2>&1 | tee /tmp/fuse-test-output.log
|
||||
|
||||
- name: Upload Test Logs
|
||||
cd /tmp/seaweedfs-fuse-tests
|
||||
|
||||
echo "🧪 Running FUSE integration tests..."
|
||||
echo "============================================"
|
||||
|
||||
# Run available working test files
|
||||
TESTS_RUN=0
|
||||
|
||||
if [ -f "simple_test.go" ]; then
|
||||
echo "📋 Running simple_test.go..."
|
||||
go test -v -timeout=${{ env.TEST_TIMEOUT }} simple_test.go
|
||||
TESTS_RUN=$((TESTS_RUN + 1))
|
||||
fi
|
||||
|
||||
if [ -f "working_demo_test.go" ]; then
|
||||
echo "📋 Running working_demo_test.go..."
|
||||
go test -v -timeout=${{ env.TEST_TIMEOUT }} working_demo_test.go
|
||||
TESTS_RUN=$((TESTS_RUN + 1))
|
||||
fi
|
||||
|
||||
# Run combined test if multiple files exist
|
||||
if [ -f "simple_test.go" ] && [ -f "working_demo_test.go" ]; then
|
||||
echo "📋 Running combined tests..."
|
||||
go test -v -timeout=${{ env.TEST_TIMEOUT }} simple_test.go working_demo_test.go
|
||||
fi
|
||||
|
||||
if [ $TESTS_RUN -eq 0 ]; then
|
||||
echo "⚠️ No working test files found, running module verification only"
|
||||
go version
|
||||
go mod verify
|
||||
else
|
||||
echo "✅ Successfully ran $TESTS_RUN test file(s)"
|
||||
fi
|
||||
|
||||
echo "============================================"
|
||||
echo "✅ FUSE integration tests completed"
|
||||
|
||||
- name: Run Extended Framework Validation
|
||||
run: |
|
||||
cd /tmp/seaweedfs-fuse-tests
|
||||
|
||||
echo "🔍 Running extended framework validation..."
|
||||
echo "============================================"
|
||||
|
||||
# Test individual components (only run tests that exist)
|
||||
if [ -f "simple_test.go" ]; then
|
||||
echo "Testing simple verification..."
|
||||
go test -v simple_test.go
|
||||
fi
|
||||
|
||||
if [ -f "working_demo_test.go" ]; then
|
||||
echo "Testing framework demo..."
|
||||
go test -v working_demo_test.go
|
||||
fi
|
||||
|
||||
# Test combined execution if both files exist
|
||||
if [ -f "simple_test.go" ] && [ -f "working_demo_test.go" ]; then
|
||||
echo "Testing combined execution..."
|
||||
go test -v simple_test.go working_demo_test.go
|
||||
elif [ -f "simple_test.go" ] || [ -f "working_demo_test.go" ]; then
|
||||
echo "✅ Individual tests already validated above"
|
||||
else
|
||||
echo "⚠️ No working test files found for combined testing"
|
||||
fi
|
||||
|
||||
echo "============================================"
|
||||
echo "✅ Extended validation completed"
|
||||
|
||||
- name: Generate Test Coverage Report
|
||||
run: |
|
||||
cd /tmp/seaweedfs-fuse-tests
|
||||
|
||||
echo "📊 Generating test coverage report..."
|
||||
go test -v -coverprofile=coverage.out .
|
||||
go tool cover -html=coverage.out -o coverage.html
|
||||
|
||||
echo "Coverage report generated: coverage.html"
|
||||
|
||||
- name: Verify SeaweedFS Binary Integration
|
||||
run: |
|
||||
# Test that SeaweedFS binary is accessible from test environment
|
||||
WEED_BINARY=$(pwd)/weed/weed
|
||||
|
||||
if [ -f "$WEED_BINARY" ]; then
|
||||
echo "✅ SeaweedFS binary found at: $WEED_BINARY"
|
||||
$WEED_BINARY version
|
||||
echo "Binary is ready for full integration testing"
|
||||
else
|
||||
echo "❌ SeaweedFS binary not found"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
- name: Upload Test Artifacts
|
||||
if: always()
|
||||
uses: actions/upload-artifact@v7
|
||||
uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: fuse-integration-test-results
|
||||
path: |
|
||||
/tmp/fuse-test-output.log
|
||||
/tmp/seaweedfs-fuse-logs/
|
||||
/tmp/seaweedfs-fuse-tests/coverage.out
|
||||
/tmp/seaweedfs-fuse-tests/coverage.html
|
||||
/tmp/seaweedfs-fuse-tests/*.log
|
||||
retention-days: 7
|
||||
|
||||
- name: Test Summary
|
||||
if: always()
|
||||
run: |
|
||||
echo "## 🚀 FUSE Integration Test Summary" >> $GITHUB_STEP_SUMMARY
|
||||
echo "" >> $GITHUB_STEP_SUMMARY
|
||||
echo "### Framework Status" >> $GITHUB_STEP_SUMMARY
|
||||
echo "- ✅ **Framework Design**: Complete and validated" >> $GITHUB_STEP_SUMMARY
|
||||
echo "- ✅ **Working Tests**: Core framework demonstration functional" >> $GITHUB_STEP_SUMMARY
|
||||
echo "- ⚠️ **Full Framework**: Available but requires Go module resolution" >> $GITHUB_STEP_SUMMARY
|
||||
echo "- ✅ **CI/CD Integration**: Automated testing pipeline established" >> $GITHUB_STEP_SUMMARY
|
||||
echo "" >> $GITHUB_STEP_SUMMARY
|
||||
echo "### Test Capabilities" >> $GITHUB_STEP_SUMMARY
|
||||
echo "- 📁 **File Operations**: Create, read, write, delete, permissions" >> $GITHUB_STEP_SUMMARY
|
||||
echo "- 📂 **Directory Operations**: Create, list, delete, nested structures" >> $GITHUB_STEP_SUMMARY
|
||||
echo "- 📊 **Large Files**: Multi-megabyte file handling" >> $GITHUB_STEP_SUMMARY
|
||||
echo "- 🔄 **Concurrent Operations**: Multi-threaded stress testing" >> $GITHUB_STEP_SUMMARY
|
||||
echo "- ⚠️ **Error Scenarios**: Comprehensive error handling validation" >> $GITHUB_STEP_SUMMARY
|
||||
echo "" >> $GITHUB_STEP_SUMMARY
|
||||
echo "### Comparison with Current Tests" >> $GITHUB_STEP_SUMMARY
|
||||
echo "| Aspect | Current (FIO) | This Framework |" >> $GITHUB_STEP_SUMMARY
|
||||
echo "|--------|---------------|----------------|" >> $GITHUB_STEP_SUMMARY
|
||||
echo "| **Scope** | Performance only | Functional + Performance |" >> $GITHUB_STEP_SUMMARY
|
||||
echo "| **Operations** | Read/Write only | All FUSE operations |" >> $GITHUB_STEP_SUMMARY
|
||||
echo "| **Concurrency** | Single-threaded | Multi-threaded stress tests |" >> $GITHUB_STEP_SUMMARY
|
||||
echo "| **Automation** | Manual setup | Fully automated |" >> $GITHUB_STEP_SUMMARY
|
||||
echo "| **Validation** | Speed metrics | Correctness + Performance |" >> $GITHUB_STEP_SUMMARY
|
||||
echo "" >> $GITHUB_STEP_SUMMARY
|
||||
echo "### Current Working Tests" >> $GITHUB_STEP_SUMMARY
|
||||
echo "- ✅ **Framework Structure**: Package and module verification" >> $GITHUB_STEP_SUMMARY
|
||||
echo "- ✅ **Configuration Management**: Test config validation" >> $GITHUB_STEP_SUMMARY
|
||||
echo "- ✅ **File Operations Demo**: Basic file create/read/write simulation" >> $GITHUB_STEP_SUMMARY
|
||||
echo "- ✅ **Large File Handling**: 1MB+ file processing demonstration" >> $GITHUB_STEP_SUMMARY
|
||||
echo "- ✅ **Concurrency Simulation**: Multi-file operation testing" >> $GITHUB_STEP_SUMMARY
|
||||
echo "" >> $GITHUB_STEP_SUMMARY
|
||||
echo "### Next Steps" >> $GITHUB_STEP_SUMMARY
|
||||
echo "1. **Module Resolution**: Fix Go package conflicts for full framework" >> $GITHUB_STEP_SUMMARY
|
||||
echo "2. **SeaweedFS Integration**: Connect with real cluster for end-to-end testing" >> $GITHUB_STEP_SUMMARY
|
||||
echo "3. **Performance Benchmarks**: Add performance regression testing" >> $GITHUB_STEP_SUMMARY
|
||||
echo "" >> $GITHUB_STEP_SUMMARY
|
||||
echo "📈 **Total Framework Size**: ~1,500 lines of comprehensive testing infrastructure" >> $GITHUB_STEP_SUMMARY
|
||||
@@ -1,79 +0,0 @@
|
||||
name: "FUSE P2P Peer Chunk Sharing Integration Tests"
|
||||
|
||||
on:
|
||||
pull_request:
|
||||
paths:
|
||||
- 'weed/command/mount*.go'
|
||||
- 'weed/mount/**'
|
||||
- 'weed/filer/mount_peer_registry*.go'
|
||||
- 'weed/server/filer_grpc_server_mount_peer.go'
|
||||
- 'weed/pb/mount_peer.proto'
|
||||
- 'weed/pb/filer.proto'
|
||||
- 'test/fuse_p2p/**'
|
||||
- '.github/workflows/fuse-p2p-integration.yml'
|
||||
- '.github/actions/fix-fusermount-setuid/**'
|
||||
push:
|
||||
branches: [master]
|
||||
paths:
|
||||
- 'weed/command/mount*.go'
|
||||
- 'weed/mount/**'
|
||||
- 'weed/filer/mount_peer_registry*.go'
|
||||
- 'weed/server/filer_grpc_server_mount_peer.go'
|
||||
- 'weed/pb/mount_peer.proto'
|
||||
- 'weed/pb/filer.proto'
|
||||
- 'test/fuse_p2p/**'
|
||||
- '.github/actions/fix-fusermount-setuid/**'
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.event.pull_request.number || (github.event_name == 'push' && github.ref) || github.run_id }}
|
||||
cancel-in-progress: ${{ github.event_name == 'pull_request' || (github.event_name == 'push' && !startsWith(github.ref, 'refs/tags/')) }}
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
jobs:
|
||||
fuse-p2p-integration:
|
||||
name: FUSE P2P Peer Chunk Sharing
|
||||
runs-on: ubuntu-24.04
|
||||
timeout-minutes: 20
|
||||
|
||||
steps:
|
||||
- name: Check out code
|
||||
uses: actions/checkout@v7
|
||||
|
||||
- name: Clean up stale FUSE mounts
|
||||
uses: ./.github/actions/cleanup-stale-fuse-mounts
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v7
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
cache: ${{ runner.environment == 'github-hosted' }}
|
||||
|
||||
- name: Configure FUSE
|
||||
run: |
|
||||
# Nothing to install: fuse3 ships fusermount3 and is pre-installed,
|
||||
# and go-fuse is pure Go, so the libfuse headers were never linked
|
||||
# against.
|
||||
echo 'user_allow_other' | sudo tee -a /etc/fuse.conf
|
||||
sudo chmod 644 /etc/fuse.conf
|
||||
|
||||
- name: Repair the fusermount3 setuid bit
|
||||
uses: ./.github/actions/fix-fusermount-setuid
|
||||
|
||||
- name: Build SeaweedFS
|
||||
run: go build -o weed/weed -buildvcs=false ./weed
|
||||
|
||||
- name: Run P2P integration tests
|
||||
timeout-minutes: 15
|
||||
env:
|
||||
WEED_BINARY: ${{ github.workspace }}/weed/weed
|
||||
run: go test -v -count=1 -timeout=12m ./test/fuse_p2p/...
|
||||
|
||||
- name: Upload logs on failure
|
||||
if: failure()
|
||||
uses: actions/upload-artifact@v7
|
||||
with:
|
||||
name: fuse-p2p-test-logs
|
||||
path: /tmp/seaweedfs-fuse-p2p-logs/
|
||||
retention-days: 3
|
||||
+15
-150
@@ -3,173 +3,38 @@ name: "go: build binary"
|
||||
on:
|
||||
push:
|
||||
branches: [ master ]
|
||||
paths:
|
||||
- '**/*.go'
|
||||
- 'go.mod'
|
||||
- 'go.sum'
|
||||
- '.github/workflows/go.yml'
|
||||
pull_request:
|
||||
branches: [ master ]
|
||||
paths:
|
||||
- '**/*.go'
|
||||
- 'go.mod'
|
||||
- 'go.sum'
|
||||
- '.github/workflows/go.yml'
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.event.pull_request.number || (github.event_name == 'push' && github.ref) || github.run_id }}
|
||||
cancel-in-progress: ${{ github.event_name == 'pull_request' || (github.event_name == 'push' && !startsWith(github.ref, 'refs/tags/')) }}
|
||||
group: ${{ github.head_ref }}/go
|
||||
cancel-in-progress: true
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
jobs:
|
||||
|
||||
vet:
|
||||
name: Go Vet
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Check out code into the Go module directory
|
||||
uses: actions/checkout@v7
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v7
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
cache: ${{ runner.environment == 'github-hosted' }}
|
||||
- name: Get dependencies
|
||||
run: |
|
||||
cd weed; go get -v -t -d ./...
|
||||
- name: Go Vet (excluding protobuf lock copying)
|
||||
run: |
|
||||
cd weed
|
||||
# Run go vet and filter out known protobuf MessageState lock copying warnings
|
||||
# These are expected in generated protobuf code with embedded sync.Mutex and are safe in practice
|
||||
go vet -v ./... 2>&1 | grep -v "MessageState contains sync.Mutex" | grep -v "IdentityAccessManagement contains sync.RWMutex" | tee vet-output.txt
|
||||
# Fail only if there are actual vet errors (not counting the filtered lock warnings)
|
||||
if grep -q "vet:" vet-output.txt; then exit 1; fi
|
||||
|
||||
vet-32bit:
|
||||
name: Go Vet 32-bit
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Check out code into the Go module directory
|
||||
uses: actions/checkout@v7
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v7
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
cache: ${{ runner.environment == 'github-hosted' }}
|
||||
- name: Go Vet linux/386 (type-checks code and tests for 32-bit int overflows)
|
||||
run: |
|
||||
GOOS=linux GOARCH=386 go vet ./... 2>&1 | grep -v "MessageState contains sync.Mutex" | grep -v "IdentityAccessManagement contains sync.RWMutex" | tee vet-32bit-output.txt
|
||||
if grep -q "vet:" vet-32bit-output.txt; then exit 1; fi
|
||||
|
||||
build:
|
||||
name: Build
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Check out code into the Go module directory
|
||||
uses: actions/checkout@v7
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v7
|
||||
|
||||
- name: Set up Go 1.x
|
||||
uses: actions/setup-go@fa96338abe5531f6e34c5cc0bbe28c1a533d5505 # v2
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
cache: ${{ runner.environment == 'github-hosted' }}
|
||||
go-version: ^1.13
|
||||
id: go
|
||||
|
||||
- name: Check out code into the Go module directory
|
||||
uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v2
|
||||
|
||||
- name: Get dependencies
|
||||
run: |
|
||||
cd weed; go get -v -t -d ./...
|
||||
|
||||
- name: Build
|
||||
run: cd weed; go build -tags "elastic gocdk sqlite ydb tarantool tikv rclone" -v .
|
||||
|
||||
build-cross:
|
||||
name: Build cross-platform (${{ matrix.goos }}/${{ matrix.goarch }})
|
||||
runs-on: ubuntu-latest
|
||||
strategy:
|
||||
# One target's breakage should not hide the other three.
|
||||
fail-fast: false
|
||||
matrix:
|
||||
include:
|
||||
- { goos: windows, goarch: amd64 }
|
||||
- { goos: windows, goarch: arm64 }
|
||||
- { goos: freebsd, goarch: amd64 }
|
||||
- { goos: darwin, goarch: arm64 }
|
||||
steps:
|
||||
- name: Check out code into the Go module directory
|
||||
uses: actions/checkout@v7
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v7
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
cache: ${{ runner.environment == 'github-hosted' }}
|
||||
# Nothing else on a PR compiles these, so per-OS syscall constants creep
|
||||
# back into shared files unnoticed and only a release build catches it.
|
||||
- name: Cross-compile
|
||||
env:
|
||||
GOOS: ${{ matrix.goos }}
|
||||
GOARCH: ${{ matrix.goarch }}
|
||||
run: |
|
||||
go build ./weed/...
|
||||
# Tests too: they reach for per-OS syscall constants the build does
|
||||
# not, and only compiling them catches an untagged one.
|
||||
go vet ./weed/mount/... ./weed/command/... 2>&1 |
|
||||
grep -v "MessageState contains sync.Mutex" | tee /tmp/vet.txt
|
||||
if grep -q "vet:" /tmp/vet.txt; then exit 1; fi
|
||||
|
||||
test:
|
||||
name: Test
|
||||
runs-on: ubuntu-latest
|
||||
services:
|
||||
redis:
|
||||
image: redis:8
|
||||
ports:
|
||||
- 6379:6379
|
||||
options: >-
|
||||
--health-cmd "redis-cli ping"
|
||||
--health-interval 10s
|
||||
--health-timeout 5s
|
||||
--health-retries 5
|
||||
steps:
|
||||
- name: Check out code into the Go module directory
|
||||
uses: actions/checkout@v7
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v7
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
cache: ${{ runner.environment == 'github-hosted' }}
|
||||
- name: Test
|
||||
env:
|
||||
RUN_REDIS_TESTS: "1"
|
||||
run: cd weed; go test -tags "elastic gocdk sqlite ydb tarantool tikv rclone" -v ./...
|
||||
|
||||
test-32bit:
|
||||
name: Test 32-bit
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Check out code into the Go module directory
|
||||
uses: actions/checkout@v7
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v7
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
cache: ${{ runner.environment == 'github-hosted' }}
|
||||
# 386 test binaries run natively on the amd64 runner. This catches what vet
|
||||
# can't: unaligned 64-bit atomics and arithmetic that wraps at runtime.
|
||||
# -short skips the e2e suites already covered on amd64.
|
||||
- name: Test linux/386
|
||||
run: cd weed; GOOS=linux GOARCH=386 go test -short ./...
|
||||
|
||||
build-azurekms:
|
||||
name: Build and test with azurekms tag
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Check out code into the Go module directory
|
||||
uses: actions/checkout@v7
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v7
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
cache: ${{ runner.environment == 'github-hosted' }}
|
||||
# weed/kms/azure is excluded from every default build, so without this
|
||||
# nothing compiles it and tag-gated code silently rots.
|
||||
- name: Build and test with azurekms
|
||||
run: |
|
||||
cd weed
|
||||
go build -tags azurekms ./...
|
||||
go test -tags azurekms ./kms/...
|
||||
@@ -0,0 +1,22 @@
|
||||
name: "helm: publish charts"
|
||||
on:
|
||||
push:
|
||||
tags:
|
||||
- '*'
|
||||
|
||||
permissions:
|
||||
contents: write
|
||||
pages: write
|
||||
|
||||
jobs:
|
||||
release:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633
|
||||
- name: Publish Helm charts
|
||||
uses: stefanprodan/helm-gh-pages@master
|
||||
with:
|
||||
token: ${{ secrets.GITHUB_TOKEN }}
|
||||
charts_dir: k8s/charts
|
||||
target_dir: helm
|
||||
branch: gh-pages
|
||||
+9
-2478
File diff suppressed because it is too large.
Load diff
@@ -1,41 +0,0 @@
|
||||
name: "helm: release"
|
||||
|
||||
on:
|
||||
push:
|
||||
tags:
|
||||
- '*'
|
||||
workflow_dispatch:
|
||||
|
||||
permissions:
|
||||
contents: write
|
||||
pages: write
|
||||
packages: write
|
||||
|
||||
jobs:
|
||||
helm-release:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v7
|
||||
with:
|
||||
fetch-depth: 0
|
||||
|
||||
- name: Publish Helm charts to github pages
|
||||
uses: stefanprodan/helm-gh-pages@v1.7.0
|
||||
with:
|
||||
token: ${{ secrets.GITHUB_TOKEN }}
|
||||
charts_dir: k8s/charts
|
||||
target_dir: helm
|
||||
branch: gh-pages
|
||||
helm_version: "3.18.4"
|
||||
|
||||
- name: Publish Helm charts to github container registry (ghcr.io)
|
||||
uses: bitdeps/helm-oci-charts-releaser@v0.1.5
|
||||
with:
|
||||
charts_dir: k8s/charts
|
||||
github_token: ${{ secrets.GITHUB_TOKEN }}
|
||||
oci_registry: ghcr.io/${{ github.repository_owner }}
|
||||
oci_username: github-actions
|
||||
oci_password: ${{ secrets.GITHUB_TOKEN }}
|
||||
skip_dependencies: true
|
||||
skip_helm_install: true
|
||||
skip_gh_release: true
|
||||
@@ -1,199 +0,0 @@
|
||||
name: Java Client Integration Tests
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [ master ]
|
||||
paths:
|
||||
- 'other/java/**'
|
||||
- 'weed/**'
|
||||
- '.github/workflows/java_integration_tests.yml'
|
||||
pull_request:
|
||||
branches: [ master ]
|
||||
paths:
|
||||
- 'other/java/**'
|
||||
- 'weed/**'
|
||||
- '.github/workflows/java_integration_tests.yml'
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.event.pull_request.number || (github.event_name == 'push' && github.ref) || github.run_id }}
|
||||
cancel-in-progress: ${{ github.event_name == 'pull_request' || (github.event_name == 'push' && !startsWith(github.ref, 'refs/tags/')) }}
|
||||
|
||||
jobs:
|
||||
test:
|
||||
name: Java Integration Tests
|
||||
runs-on: ubuntu-latest
|
||||
|
||||
strategy:
|
||||
matrix:
|
||||
java: ['11', '17']
|
||||
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v7
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v7
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
cache: ${{ runner.environment == 'github-hosted' }}
|
||||
id: go
|
||||
|
||||
- name: Set up Java
|
||||
uses: actions/setup-java@v6
|
||||
with:
|
||||
java-version: ${{ matrix.java }}
|
||||
distribution: 'temurin'
|
||||
cache: ${{ runner.environment == 'github-hosted' && 'maven' || '' }}
|
||||
|
||||
- name: Build SeaweedFS
|
||||
run: |
|
||||
cd weed
|
||||
go install -buildvcs=false
|
||||
weed version
|
||||
|
||||
- name: Start SeaweedFS Server
|
||||
run: |
|
||||
# Create clean data directory
|
||||
export WEED_DATA_DIR="/tmp/seaweedfs-java-tests-$(date +%s)"
|
||||
mkdir -p "$WEED_DATA_DIR"
|
||||
|
||||
# Start SeaweedFS with optimized settings for CI
|
||||
# Include S3 API for s3copier integration tests
|
||||
weed server -dir="$WEED_DATA_DIR" \
|
||||
-ip=127.0.0.1 -ip.bind=127.0.0.1 \
|
||||
-master.raftHashicorp \
|
||||
-master.electionTimeout=1s \
|
||||
-master.volumeSizeLimitMB=100 \
|
||||
-volume.max=100 \
|
||||
-volume.preStopSeconds=1 \
|
||||
-master.peers=none \
|
||||
-filer -filer.maxMB=64 \
|
||||
-s3 -s3.port=8333 \
|
||||
-s3.config="$GITHUB_WORKSPACE/docker/compose/s3.json" \
|
||||
-s3.allowDeleteBucketNotEmpty=true \
|
||||
-master.port=9333 \
|
||||
-volume.port=8080 \
|
||||
-filer.port=8888 \
|
||||
-metricsPort=9324 > seaweedfs.log 2>&1 &
|
||||
|
||||
SERVER_PID=$!
|
||||
echo "SERVER_PID=$SERVER_PID" >> $GITHUB_ENV
|
||||
echo "WEED_DATA_DIR=$WEED_DATA_DIR" >> $GITHUB_ENV
|
||||
echo "SeaweedFS server started with PID: $SERVER_PID"
|
||||
|
||||
- name: Wait for SeaweedFS Components
|
||||
run: |
|
||||
echo "Waiting for SeaweedFS components to start..."
|
||||
|
||||
# Wait for master
|
||||
for i in {1..30}; do
|
||||
if curl -s http://localhost:9333/cluster/status > /dev/null 2>&1; then
|
||||
echo "✓ Master server is ready"
|
||||
break
|
||||
fi
|
||||
echo "Waiting for master server... ($i/30)"
|
||||
sleep 2
|
||||
done
|
||||
|
||||
# Wait for volume
|
||||
for i in {1..30}; do
|
||||
if curl -s http://localhost:8080/status > /dev/null 2>&1; then
|
||||
echo "✓ Volume server is ready"
|
||||
break
|
||||
fi
|
||||
echo "Waiting for volume server... ($i/30)"
|
||||
sleep 2
|
||||
done
|
||||
|
||||
# Wait for filer
|
||||
for i in {1..30}; do
|
||||
if curl -s http://localhost:8888/ > /dev/null 2>&1; then
|
||||
echo "✓ Filer is ready"
|
||||
break
|
||||
fi
|
||||
echo "Waiting for filer... ($i/30)"
|
||||
sleep 2
|
||||
done
|
||||
|
||||
# Wait for S3 API
|
||||
for i in {1..30}; do
|
||||
if curl -s http://localhost:8333/healthz > /dev/null 2>&1; then
|
||||
echo "✓ S3 API is ready"
|
||||
break
|
||||
fi
|
||||
echo "Waiting for S3 API... ($i/30)"
|
||||
sleep 2
|
||||
done
|
||||
|
||||
echo "✓ All SeaweedFS components are ready!"
|
||||
|
||||
# Display cluster status
|
||||
echo "Cluster status:"
|
||||
curl -s http://localhost:9333/cluster/status | head -20
|
||||
|
||||
- name: Build and Install SeaweedFS Client
|
||||
working-directory: other/java/client
|
||||
run: |
|
||||
mvn clean install -DskipTests -Dmaven.javadoc.skip=true -Dgpg.skip=true
|
||||
|
||||
- name: Run Client Unit Tests
|
||||
working-directory: other/java/client
|
||||
run: |
|
||||
mvn test -Dtest=SeaweedReadTest,SeaweedCipherTest
|
||||
|
||||
- name: Run Client Integration Tests
|
||||
working-directory: other/java/client
|
||||
env:
|
||||
SEAWEEDFS_TEST_ENABLED: true
|
||||
run: |
|
||||
mvn test -Dtest=*IntegrationTest
|
||||
|
||||
- name: Run HDFS3 Configuration Tests
|
||||
working-directory: other/java/hdfs3
|
||||
run: |
|
||||
mvn test -Dtest=SeaweedFileSystemConfigTest -Dmaven.javadoc.skip=true -Dgpg.skip=true
|
||||
|
||||
- name: Run S3 ETag Validation Tests (Issue #7768)
|
||||
working-directory: other/java/s3copier
|
||||
env:
|
||||
S3_ENDPOINT: http://127.0.0.1:8333
|
||||
S3_ACCESS_KEY: some_access_key1
|
||||
S3_SECRET_KEY: some_secret_key1
|
||||
run: |
|
||||
echo "Running S3 ETag validation tests against $S3_ENDPOINT"
|
||||
mvn test -Dtest=ETagValidationTest \
|
||||
-DS3_ENDPOINT=$S3_ENDPOINT \
|
||||
-DS3_ACCESS_KEY=$S3_ACCESS_KEY \
|
||||
-DS3_SECRET_KEY=$S3_SECRET_KEY \
|
||||
-Dmaven.javadoc.skip=true -Dgpg.skip=true
|
||||
|
||||
- name: Display logs on failure
|
||||
if: failure()
|
||||
run: |
|
||||
echo "=== SeaweedFS Server Log ==="
|
||||
tail -100 seaweedfs.log || echo "No server log"
|
||||
echo ""
|
||||
echo "=== Cluster Status ==="
|
||||
curl -s http://localhost:9333/cluster/status || echo "Cannot reach cluster"
|
||||
echo ""
|
||||
echo "=== Process Status ==="
|
||||
ps aux | grep weed || echo "No weed processes"
|
||||
|
||||
- name: Cleanup
|
||||
if: always()
|
||||
run: |
|
||||
# Stop server using stored PID
|
||||
if [ -n "$SERVER_PID" ]; then
|
||||
echo "Stopping SeaweedFS server (PID: $SERVER_PID)"
|
||||
kill -9 $SERVER_PID 2>/dev/null || true
|
||||
fi
|
||||
|
||||
# Fallback: kill any remaining weed processes
|
||||
pkill -f "weed server" || true
|
||||
|
||||
# Clean up data directory
|
||||
if [ -n "$WEED_DATA_DIR" ]; then
|
||||
echo "Cleaning up data directory: $WEED_DATA_DIR"
|
||||
rm -rf "$WEED_DATA_DIR" || true
|
||||
fi
|
||||
|
||||
@@ -1,183 +0,0 @@
|
||||
name: "release: java clients"
|
||||
|
||||
# Publishes the SeaweedFS Java clients (seaweedfs-client and
|
||||
# seaweedfs-hadoop3-client) to Maven Central via the Sonatype Central Portal.
|
||||
#
|
||||
# Required repository secrets:
|
||||
# MAVEN_CENTRAL_USERNAME - Central Portal user token username (central.sonatype.com -> Account -> Generate User Token)
|
||||
# MAVEN_CENTRAL_PASSWORD - Central Portal user token password
|
||||
# MAVEN_GPG_PRIVATE_KEY - ASCII-armored signing key:
|
||||
# gpg --homedir <keyring> --armor --export-secret-keys <KEYID> > key.asc
|
||||
# MAVEN_GPG_PASSPHRASE - passphrase for that key
|
||||
#
|
||||
# A plain run publishes to Central. Check dry_run to exercise secrets, GPG
|
||||
# signing, and the build without committing or publishing.
|
||||
|
||||
on:
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
version:
|
||||
description: "Version to publish, e.g. 4.39. Leave blank to use the current SeaweedFS version."
|
||||
type: string
|
||||
required: false
|
||||
dry_run:
|
||||
description: "Build and GPG-sign only; skip the version commit and the Central upload"
|
||||
type: boolean
|
||||
default: false
|
||||
|
||||
permissions:
|
||||
contents: write
|
||||
|
||||
concurrency:
|
||||
group: java-release
|
||||
cancel-in-progress: false
|
||||
|
||||
env:
|
||||
# Java 17 needs these opens for the Maven publishing plugins' reflective access.
|
||||
MAVEN_OPTS: >-
|
||||
--add-opens=java.base/java.util=ALL-UNNAMED
|
||||
--add-opens=java.base/java.lang.reflect=ALL-UNNAMED
|
||||
--add-opens=java.base/java.text=ALL-UNNAMED
|
||||
--add-opens=java.desktop/java.awt.font=ALL-UNNAMED
|
||||
|
||||
jobs:
|
||||
publish:
|
||||
name: Publish to Maven Central
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v7
|
||||
with:
|
||||
# Full history so the version-bump commit can rebase onto a moved master.
|
||||
fetch-depth: 0
|
||||
|
||||
# An explicit input wins; otherwise fall back to the current SeaweedFS
|
||||
# version (MAJOR.MINOR) from constants.go, matching its %d.%02d formatting.
|
||||
- name: Resolve version
|
||||
id: resolve
|
||||
env:
|
||||
INPUT_VERSION: ${{ inputs.version }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
VERSION="$INPUT_VERSION"
|
||||
if [ -z "$VERSION" ]; then
|
||||
CONST=weed/util/version/constants.go
|
||||
MAJOR=$(grep -oP 'MAJOR_VERSION\s*=\s*int32\(\K[0-9]+' "$CONST")
|
||||
MINOR=$(grep -oP 'MINOR_VERSION\s*=\s*int32\(\K[0-9]+' "$CONST")
|
||||
VERSION=$(printf '%d.%02d' "$MAJOR" "$MINOR")
|
||||
echo "No version given; using current SeaweedFS version ${VERSION}."
|
||||
fi
|
||||
if ! [[ "$VERSION" =~ ^[0-9]+\.[0-9]+$ ]]; then
|
||||
echo "::error::version must be MAJOR.MINOR, e.g. 4.39 (got '$VERSION')"
|
||||
exit 1
|
||||
fi
|
||||
echo "version=${VERSION}" >> "$GITHUB_OUTPUT"
|
||||
|
||||
- name: Set up JDK 17
|
||||
uses: actions/setup-java@v6
|
||||
with:
|
||||
java-version: '17'
|
||||
distribution: 'temurin'
|
||||
cache: ${{ runner.environment == 'github-hosted' && 'maven' || '' }}
|
||||
server-id: central
|
||||
server-username: MAVEN_CENTRAL_USERNAME
|
||||
server-password: MAVEN_CENTRAL_PASSWORD
|
||||
gpg-private-key: ${{ secrets.MAVEN_GPG_PRIVATE_KEY }}
|
||||
gpg-passphrase: MAVEN_GPG_PASSPHRASE
|
||||
|
||||
# client carries the version literally; hdfs3 derives its own version and
|
||||
# its seaweedfs-client dependency from the seaweedfs.client.version property.
|
||||
- name: Set versions in poms
|
||||
env:
|
||||
VERSION: ${{ steps.resolve.outputs.version }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
V=org.codehaus.mojo:versions-maven-plugin:2.16.2
|
||||
mvn -B -ntp -f other/java/client/pom.xml "${V}:set" \
|
||||
-DnewVersion="$VERSION" -DgenerateBackupPoms=false
|
||||
mvn -B -ntp -f other/java/hdfs3/pom.xml "${V}:set-property" \
|
||||
-Dproperty=seaweedfs.client.version -DnewVersion="$VERSION" -DgenerateBackupPoms=false
|
||||
|
||||
- name: Commit version bump
|
||||
if: ${{ !inputs.dry_run }}
|
||||
env:
|
||||
VERSION: ${{ steps.resolve.outputs.version }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
if git diff --quiet; then
|
||||
echo "Poms already at ${VERSION}; nothing to commit."
|
||||
exit 0
|
||||
fi
|
||||
git config user.name "github-actions[bot]"
|
||||
git config user.email "41898282+github-actions[bot]@users.noreply.github.com"
|
||||
git add other/java/client/pom.xml other/java/hdfs3/pom.xml
|
||||
git commit -m "java ${VERSION}"
|
||||
# Rebase and retry so a concurrent push to master doesn't lose the bump.
|
||||
for attempt in 1 2 3 4 5; do
|
||||
if git push origin HEAD:master; then
|
||||
exit 0
|
||||
fi
|
||||
echo "push rejected (attempt ${attempt}); rebasing onto latest master"
|
||||
git pull --rebase origin master
|
||||
done
|
||||
echo "::error::could not push version bump after retries"
|
||||
exit 1
|
||||
|
||||
# Tests are skipped here: the integration tests need a live filer, and the
|
||||
# offline unit tests already run on every push in java_unit_tests.yml.
|
||||
# dry_run stops at `install` (build + GPG sign, no upload). A real run deploys.
|
||||
# client goes first either way so its install populates the local repo for hdfs3.
|
||||
- name: Publish seaweedfs-client
|
||||
working-directory: other/java/client
|
||||
env:
|
||||
MAVEN_CENTRAL_USERNAME: ${{ secrets.MAVEN_CENTRAL_USERNAME }}
|
||||
MAVEN_CENTRAL_PASSWORD: ${{ secrets.MAVEN_CENTRAL_PASSWORD }}
|
||||
MAVEN_GPG_PASSPHRASE: ${{ secrets.MAVEN_GPG_PASSPHRASE }}
|
||||
DRY_RUN: ${{ inputs.dry_run }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
[ "$DRY_RUN" = "true" ] && GOAL="clean install" || GOAL="clean deploy"
|
||||
mvn -B -ntp $GOAL -DskipTests
|
||||
|
||||
- name: Publish seaweedfs-hadoop3-client
|
||||
working-directory: other/java/hdfs3
|
||||
env:
|
||||
MAVEN_CENTRAL_USERNAME: ${{ secrets.MAVEN_CENTRAL_USERNAME }}
|
||||
MAVEN_CENTRAL_PASSWORD: ${{ secrets.MAVEN_CENTRAL_PASSWORD }}
|
||||
MAVEN_GPG_PASSPHRASE: ${{ secrets.MAVEN_GPG_PASSPHRASE }}
|
||||
DRY_RUN: ${{ inputs.dry_run }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
[ "$DRY_RUN" = "true" ] && GOAL="clean install" || GOAL="clean deploy"
|
||||
mvn -B -ntp $GOAL -DskipTests
|
||||
|
||||
# Bump every version the wiki quotes for the Java clients. Runs only after
|
||||
# a successful real publish. Uses GITHUB_TOKEN; set a WIKI_TOKEN secret if
|
||||
# the token can't push to the wiki.
|
||||
- name: Update wiki client versions
|
||||
if: ${{ !inputs.dry_run }}
|
||||
env:
|
||||
VERSION: ${{ steps.resolve.outputs.version }}
|
||||
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
WIKI_TOKEN: ${{ secrets.WIKI_TOKEN }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
TOKEN="${WIKI_TOKEN:-$GH_TOKEN}"
|
||||
git clone -q "https://x-access-token:${TOKEN}@github.com/${{ github.repository }}.wiki.git" wiki
|
||||
cd wiki
|
||||
for f in $(grep -rlE "seaweedfs-(client|hadoop3-client)" --include="*.md" .); do
|
||||
NEWV="$VERSION" perl -0777 -i -pe '
|
||||
my $v = $ENV{NEWV};
|
||||
s{(<artifactId>seaweedfs-(?:client|hadoop3-client)</artifactId>\s*<version>)[0-9]+\.[0-9]+(</version>)}{$1$v$2}g;
|
||||
s{(seaweedfs-(?:client|hadoop3-client):)[0-9]+\.[0-9]+}{$1$v}g;
|
||||
s{(seaweedfs-(?:client|hadoop3-client)-)[0-9]+\.[0-9]+(\.jar)}{$1$v$2}g;
|
||||
s{(seaweedfs-(?:client|hadoop3-client)/)[0-9]+\.[0-9]+(/)}{$1$v$2}g;
|
||||
' "$f"
|
||||
done
|
||||
if git diff --quiet; then
|
||||
echo "Wiki already references ${VERSION}."
|
||||
exit 0
|
||||
fi
|
||||
git config user.name "github-actions[bot]"
|
||||
git config user.email "41898282+github-actions[bot]@users.noreply.github.com"
|
||||
git commit -am "java clients ${VERSION}"
|
||||
git push
|
||||
@@ -1,62 +0,0 @@
|
||||
name: Java Client Unit Tests
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [ master ]
|
||||
paths:
|
||||
- 'other/java/**'
|
||||
- '.github/workflows/java_unit_tests.yml'
|
||||
pull_request:
|
||||
branches: [ master ]
|
||||
paths:
|
||||
- 'other/java/**'
|
||||
- '.github/workflows/java_unit_tests.yml'
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.event.pull_request.number || (github.event_name == 'push' && github.ref) || github.run_id }}
|
||||
cancel-in-progress: ${{ github.event_name == 'pull_request' || (github.event_name == 'push' && !startsWith(github.ref, 'refs/tags/')) }}
|
||||
|
||||
jobs:
|
||||
test:
|
||||
name: Java Unit Tests
|
||||
runs-on: ubuntu-latest
|
||||
|
||||
strategy:
|
||||
matrix:
|
||||
java: ['8', '11', '17', '21']
|
||||
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v7
|
||||
|
||||
- name: Set up Java
|
||||
uses: actions/setup-java@v6
|
||||
with:
|
||||
java-version: ${{ matrix.java }}
|
||||
distribution: 'temurin'
|
||||
cache: ${{ runner.environment == 'github-hosted' && 'maven' || '' }}
|
||||
|
||||
- name: Build and Install SeaweedFS Client
|
||||
working-directory: other/java/client
|
||||
run: |
|
||||
mvn clean install -DskipTests -Dmaven.javadoc.skip=true -Dgpg.skip=true
|
||||
|
||||
- name: Run Client Unit Tests
|
||||
working-directory: other/java/client
|
||||
run: |
|
||||
mvn test -Dtest=SeaweedReadTest,SeaweedCipherTest
|
||||
|
||||
- name: Run HDFS3 Configuration Tests
|
||||
working-directory: other/java/hdfs3
|
||||
run: |
|
||||
mvn test -Dtest=SeaweedFileSystemConfigTest -Dmaven.javadoc.skip=true -Dgpg.skip=true
|
||||
|
||||
- name: Upload Test Reports
|
||||
if: always()
|
||||
uses: actions/upload-artifact@v7
|
||||
with:
|
||||
name: test-reports-java-${{ matrix.java }}
|
||||
path: |
|
||||
other/java/client/target/surefire-reports/
|
||||
other/java/hdfs3/target/surefire-reports/
|
||||
|
||||
@@ -1,145 +0,0 @@
|
||||
name: "Kafka Quick Test (Load Test with Schema Registry)"
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [ master ]
|
||||
paths:
|
||||
- 'weed/mq/**'
|
||||
- 'weed/pb/mq_pb/**'
|
||||
- 'weed/pb/schema_pb/**'
|
||||
- 'test/kafka/**'
|
||||
- 'go.mod'
|
||||
- 'go.sum'
|
||||
- '.github/workflows/kafka-quicktest.yml'
|
||||
pull_request:
|
||||
branches: [ master ]
|
||||
paths:
|
||||
- 'weed/mq/**'
|
||||
- 'weed/pb/mq_pb/**'
|
||||
- 'weed/pb/schema_pb/**'
|
||||
- 'test/kafka/**'
|
||||
- 'go.mod'
|
||||
- 'go.sum'
|
||||
- '.github/workflows/kafka-quicktest.yml'
|
||||
workflow_dispatch: # Allow manual trigger
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.event.pull_request.number || (github.event_name == 'push' && github.ref) || github.run_id }}
|
||||
cancel-in-progress: ${{ github.event_name == 'pull_request' || (github.event_name == 'push' && !startsWith(github.ref, 'refs/tags/')) }}
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
jobs:
|
||||
kafka-client-quicktest:
|
||||
name: Kafka Client Load Test (Quick)
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 15
|
||||
steps:
|
||||
- name: Check out code
|
||||
uses: actions/checkout@v7
|
||||
|
||||
- name: Set up Go 1.x
|
||||
uses: actions/setup-go@v7
|
||||
with:
|
||||
go-version: ^1.26
|
||||
cache: ${{ runner.environment == 'github-hosted' }}
|
||||
cache-dependency-path: |
|
||||
**/go.sum
|
||||
id: go
|
||||
|
||||
- name: Configure Docker Hub mirror
|
||||
run: |
|
||||
echo '{"registry-mirrors": ["https://mirror.gcr.io"]}' | sudo tee /etc/docker/daemon.json
|
||||
sudo systemctl restart docker
|
||||
|
||||
- name: Set up Docker Buildx
|
||||
uses: docker/setup-buildx-action@v4
|
||||
|
||||
- name: Install dependencies
|
||||
run: |
|
||||
# Ensure make is available
|
||||
sudo apt-get update -qq
|
||||
sudo apt-get install -y make
|
||||
|
||||
- name: Validate test setup
|
||||
working-directory: test/kafka/kafka-client-loadtest
|
||||
run: |
|
||||
make validate-setup
|
||||
|
||||
- name: Run quick-test
|
||||
working-directory: test/kafka/kafka-client-loadtest
|
||||
run: |
|
||||
# Run the quick-test target which includes:
|
||||
# 1. Building the gateway
|
||||
# 2. Starting all services (SeaweedFS, MQ broker, Schema Registry)
|
||||
# 3. Registering Avro schemas
|
||||
# 4. Running a 1-minute load test with Avro messages
|
||||
# Override GOARCH to build for AMD64 (GitHub Actions runners are x86_64)
|
||||
GOARCH=amd64 make quick-test
|
||||
env:
|
||||
# Docker Compose settings
|
||||
COMPOSE_HTTP_TIMEOUT: 300
|
||||
DOCKER_CLIENT_TIMEOUT: 300
|
||||
# Test parameters (set by quick-test, but can override)
|
||||
TEST_DURATION: 60s
|
||||
PRODUCER_COUNT: 1
|
||||
CONSUMER_COUNT: 1
|
||||
MESSAGE_RATE: 10
|
||||
VALUE_TYPE: avro
|
||||
|
||||
- name: Show test results
|
||||
if: always()
|
||||
working-directory: test/kafka/kafka-client-loadtest
|
||||
run: |
|
||||
echo "========================================="
|
||||
echo "Test Results"
|
||||
echo "========================================="
|
||||
make show-results || echo "Could not retrieve results"
|
||||
|
||||
- name: Show service logs on failure
|
||||
if: failure()
|
||||
working-directory: test/kafka/kafka-client-loadtest
|
||||
run: |
|
||||
echo "========================================="
|
||||
echo "Service Logs"
|
||||
echo "========================================="
|
||||
|
||||
echo "Checking running containers..."
|
||||
docker compose ps || true
|
||||
|
||||
echo "========================================="
|
||||
echo "Master Logs"
|
||||
echo "========================================="
|
||||
docker compose logs --tail=100 seaweedfs-master 2>&1 || echo "No master logs available"
|
||||
|
||||
echo "========================================="
|
||||
echo "MQ Broker Logs (Last 100 lines)"
|
||||
echo "========================================="
|
||||
docker compose logs --tail=100 seaweedfs-mq-broker 2>&1 || echo "No broker logs available"
|
||||
|
||||
echo "========================================="
|
||||
echo "Kafka Gateway Logs (FULL - Critical for debugging)"
|
||||
echo "========================================="
|
||||
docker compose logs kafka-gateway 2>&1 || echo "ERROR: Could not retrieve kafka-gateway logs"
|
||||
|
||||
echo "========================================="
|
||||
echo "Schema Registry Logs (FULL)"
|
||||
echo "========================================="
|
||||
docker compose logs schema-registry 2>&1 || echo "ERROR: Could not retrieve schema-registry logs"
|
||||
|
||||
echo "========================================="
|
||||
echo "Load Test Logs"
|
||||
echo "========================================="
|
||||
docker compose logs --tail=100 kafka-client-loadtest 2>&1 || echo "No loadtest logs available"
|
||||
|
||||
- name: Cleanup
|
||||
if: always()
|
||||
working-directory: test/kafka/kafka-client-loadtest
|
||||
run: |
|
||||
# Stop containers first
|
||||
docker compose --profile loadtest --profile monitoring down -v --remove-orphans || true
|
||||
# Clean up data with sudo to handle Docker root-owned files
|
||||
sudo rm -rf data/* || true
|
||||
# Clean up binary
|
||||
rm -f weed-linux-* || true
|
||||
@@ -1,844 +0,0 @@
|
||||
name: "Kafka Gateway Tests"
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [ master ]
|
||||
paths:
|
||||
- 'weed/mq/**'
|
||||
- 'weed/pb/mq_pb/**'
|
||||
- 'weed/pb/schema_pb/**'
|
||||
- 'test/kafka/**'
|
||||
- 'go.mod'
|
||||
- 'go.sum'
|
||||
- '.github/workflows/kafka-tests.yml'
|
||||
pull_request:
|
||||
branches: [ master ]
|
||||
paths:
|
||||
- 'weed/mq/**'
|
||||
- 'weed/pb/mq_pb/**'
|
||||
- 'weed/pb/schema_pb/**'
|
||||
- 'test/kafka/**'
|
||||
- 'go.mod'
|
||||
- 'go.sum'
|
||||
- '.github/workflows/kafka-tests.yml'
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.event.pull_request.number || (github.event_name == 'push' && github.ref) || github.run_id }}
|
||||
cancel-in-progress: ${{ github.event_name == 'pull_request' || (github.event_name == 'push' && !startsWith(github.ref, 'refs/tags/')) }}
|
||||
|
||||
# Force different runners for better isolation
|
||||
env:
|
||||
FORCE_RUNNER_SEPARATION: true
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
jobs:
|
||||
kafka-unit-tests:
|
||||
name: Kafka Unit Tests
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 5
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
container-id: [unit-tests-1]
|
||||
container:
|
||||
image: golang:1.26-alpine
|
||||
options: --cpus 1.0 --memory 1g --hostname kafka-unit-${{ matrix.container-id }}
|
||||
env:
|
||||
GOMAXPROCS: 1
|
||||
CGO_ENABLED: 0
|
||||
CONTAINER_ID: ${{ matrix.container-id }}
|
||||
steps:
|
||||
- name: Set up Go 1.x
|
||||
uses: actions/setup-go@v7
|
||||
with:
|
||||
cache: ${{ runner.environment == 'github-hosted' }}
|
||||
go-version: ^1.26
|
||||
id: go
|
||||
|
||||
- name: Check out code
|
||||
uses: actions/checkout@v7
|
||||
|
||||
- name: Setup Container Environment
|
||||
run: |
|
||||
apk add --no-cache git
|
||||
ulimit -n 1024 || echo "Warning: Could not set file descriptor limit"
|
||||
|
||||
- name: Get dependencies
|
||||
run: |
|
||||
cd test/kafka
|
||||
go mod download
|
||||
go mod tidy
|
||||
|
||||
- name: Run Kafka Gateway Unit Tests
|
||||
run: |
|
||||
cd test/kafka
|
||||
# Set process limits for container isolation
|
||||
ulimit -n 512 || echo "Warning: Could not set file descriptor limit"
|
||||
ulimit -u 100 || echo "Warning: Could not set process limit"
|
||||
go test -v -timeout 10s ./unit/...
|
||||
|
||||
kafka-integration-tests:
|
||||
name: Kafka Integration Tests (Critical)
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 5
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
container-id: [integration-1]
|
||||
container:
|
||||
image: golang:1.26-alpine
|
||||
options: --cpus 2.0 --memory 2g --ulimit nofile=1024:1024 --hostname kafka-integration-${{ matrix.container-id }}
|
||||
env:
|
||||
GOMAXPROCS: 2
|
||||
CGO_ENABLED: 0
|
||||
KAFKA_TEST_ISOLATION: "true"
|
||||
CONTAINER_ID: ${{ matrix.container-id }}
|
||||
steps:
|
||||
- name: Set up Go 1.x
|
||||
uses: actions/setup-go@v7
|
||||
with:
|
||||
cache: ${{ runner.environment == 'github-hosted' }}
|
||||
go-version: ^1.26
|
||||
id: go
|
||||
|
||||
- name: Check out code
|
||||
uses: actions/checkout@v7
|
||||
|
||||
- name: Setup Integration Container Environment
|
||||
run: |
|
||||
apk add --no-cache git procps
|
||||
ulimit -n 2048 || echo "Warning: Could not set file descriptor limit"
|
||||
|
||||
- name: Get dependencies
|
||||
run: |
|
||||
cd test/kafka
|
||||
go mod download
|
||||
go mod tidy
|
||||
|
||||
- name: Run Integration Tests
|
||||
run: |
|
||||
cd test/kafka
|
||||
# Higher limits for integration tests
|
||||
ulimit -n 1024 || echo "Warning: Could not set file descriptor limit"
|
||||
ulimit -u 200 || echo "Warning: Could not set process limit"
|
||||
go test -v -timeout 90s ./integration/...
|
||||
env:
|
||||
GOMAXPROCS: 2
|
||||
|
||||
kafka-e2e-tests:
|
||||
name: Kafka End-to-End Tests (with SMQ)
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 20
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
container-id: [e2e-1]
|
||||
container:
|
||||
image: golang:1.26-alpine
|
||||
options: --cpus 2.0 --memory 2g --hostname kafka-e2e-${{ matrix.container-id }}
|
||||
env:
|
||||
GOMAXPROCS: 2
|
||||
CGO_ENABLED: 0
|
||||
KAFKA_E2E_ISOLATION: "true"
|
||||
CONTAINER_ID: ${{ matrix.container-id }}
|
||||
steps:
|
||||
- name: Check out code
|
||||
uses: actions/checkout@v7
|
||||
|
||||
- name: Set up Go 1.x
|
||||
uses: actions/setup-go@v7
|
||||
with:
|
||||
go-version: ^1.26
|
||||
cache: ${{ runner.environment == 'github-hosted' }}
|
||||
cache-dependency-path: |
|
||||
**/go.sum
|
||||
id: go
|
||||
|
||||
- name: Setup E2E Container Environment
|
||||
run: |
|
||||
apk add --no-cache git procps curl netcat-openbsd
|
||||
ulimit -n 2048 || echo "Warning: Could not set file descriptor limit"
|
||||
|
||||
- name: Warm Go module cache
|
||||
run: |
|
||||
# Warm cache for root module
|
||||
go mod download || true
|
||||
# Warm cache for kafka test module
|
||||
cd test/kafka
|
||||
go mod download || true
|
||||
|
||||
- name: Get dependencies
|
||||
run: |
|
||||
cd test/kafka
|
||||
# Use go mod download with timeout to prevent hanging
|
||||
timeout 90s go mod download || echo "Warning: Dependency download timed out, continuing with cached modules"
|
||||
go mod tidy
|
||||
|
||||
- name: Build and start SeaweedFS MQ
|
||||
run: |
|
||||
set -e
|
||||
cd $GITHUB_WORKSPACE
|
||||
# Build weed binary
|
||||
go build -o /usr/local/bin/weed ./weed
|
||||
# Start SeaweedFS components with MQ brokers
|
||||
export WEED_DATA_DIR=/tmp/seaweedfs-e2e-$RANDOM
|
||||
mkdir -p "$WEED_DATA_DIR"
|
||||
|
||||
# Start SeaweedFS server (master, volume, filer) with consistent IP advertising
|
||||
nohup weed -v 1 server \
|
||||
-ip="127.0.0.1" \
|
||||
-ip.bind="0.0.0.0" \
|
||||
-dir="$WEED_DATA_DIR" \
|
||||
-master.raftHashicorp \
|
||||
-master.port=9333 \
|
||||
-volume.port=8081 \
|
||||
-filer.port=8888 \
|
||||
-filer=true \
|
||||
-metricsPort=9325 \
|
||||
-master.peers=none \
|
||||
> /tmp/weed-server.log 2>&1 &
|
||||
|
||||
# Wait for master to be ready
|
||||
for i in $(seq 1 30); do
|
||||
if curl -s http://127.0.0.1:9333/cluster/status >/dev/null; then
|
||||
echo "SeaweedFS master HTTP is up"; break
|
||||
fi
|
||||
echo "Waiting for SeaweedFS master HTTP... ($i/30)"; sleep 1
|
||||
done
|
||||
|
||||
# Wait for master gRPC to be ready (this is what broker discovery uses)
|
||||
echo "Waiting for master gRPC port..."
|
||||
for i in $(seq 1 30); do
|
||||
if nc -z 127.0.0.1 19333; then
|
||||
echo "✓ SeaweedFS master gRPC is up (port 19333)"
|
||||
break
|
||||
fi
|
||||
echo " Waiting for master gRPC... ($i/30)"; sleep 1
|
||||
done
|
||||
|
||||
# Give server time to initialize all components including gRPC services
|
||||
echo "Waiting for SeaweedFS components to initialize..."
|
||||
sleep 15
|
||||
|
||||
# Additional wait specifically for gRPC services to be ready for streaming
|
||||
echo "Allowing extra time for master gRPC streaming services to initialize..."
|
||||
sleep 10
|
||||
|
||||
# Start MQ broker with maximum verbosity for debugging
|
||||
echo "Starting MQ broker..."
|
||||
nohup weed -v 3 mq.broker \
|
||||
-master="127.0.0.1:9333" \
|
||||
-ip="127.0.0.1" \
|
||||
-port=17777 \
|
||||
-logFlushInterval=0 \
|
||||
> /tmp/weed-mq-broker.log 2>&1 &
|
||||
|
||||
# Wait for broker to be ready with better error reporting
|
||||
sleep 15
|
||||
broker_ready=false
|
||||
for i in $(seq 1 20); do
|
||||
if nc -z 127.0.0.1 17777; then
|
||||
echo "SeaweedFS MQ broker is up"
|
||||
broker_ready=true
|
||||
break
|
||||
fi
|
||||
echo "Waiting for MQ broker... ($i/20)"; sleep 1
|
||||
done
|
||||
|
||||
# Give broker additional time to register with master
|
||||
if [ "$broker_ready" = true ]; then
|
||||
echo "Allowing broker to register with master..."
|
||||
sleep 30
|
||||
|
||||
# Check if broker is properly registered by querying cluster nodes
|
||||
echo "Cluster status after broker registration:"
|
||||
curl -s "http://127.0.0.1:9333/cluster/status" || echo "Could not check cluster status"
|
||||
|
||||
echo "Checking cluster topology (includes registered components):"
|
||||
curl -s "http://127.0.0.1:9333/dir/status" | head -20 || echo "Could not check dir status"
|
||||
|
||||
echo "Verifying broker discovery via master client debug:"
|
||||
echo "If broker registration is successful, it should appear in dir status"
|
||||
|
||||
echo "Testing gRPC connectivity with weed binary:"
|
||||
echo "This simulates what the gateway does during broker discovery..."
|
||||
timeout 10s weed shell -master=127.0.0.1:9333 -filer=127.0.0.1:8888 > /tmp/shell-test.log 2>&1 || echo "weed shell test completed or timed out - checking logs..."
|
||||
echo "Shell test results:"
|
||||
cat /tmp/shell-test.log 2>/dev/null | head -10 || echo "No shell test logs"
|
||||
fi
|
||||
|
||||
# Check if broker failed to start and show logs
|
||||
if [ "$broker_ready" = false ]; then
|
||||
echo "ERROR: MQ broker failed to start. Broker logs:"
|
||||
cat /tmp/weed-mq-broker.log || echo "No broker logs found"
|
||||
echo "Server logs:"
|
||||
tail -20 /tmp/weed-server.log || echo "No server logs found"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
- name: Run End-to-End Tests
|
||||
run: |
|
||||
cd test/kafka
|
||||
# Higher limits for E2E tests
|
||||
ulimit -n 1024 || echo "Warning: Could not set file descriptor limit"
|
||||
ulimit -u 200 || echo "Warning: Could not set process limit"
|
||||
|
||||
# Allow additional time for all background processes to settle
|
||||
echo "Allowing additional settlement time for SeaweedFS ecosystem..."
|
||||
sleep 15
|
||||
|
||||
# Run tests and capture result
|
||||
if ! go test -v -timeout 180s ./e2e/...; then
|
||||
echo "========================================="
|
||||
echo "Tests failed! Showing debug information:"
|
||||
echo "========================================="
|
||||
echo "Server logs (last 50 lines):"
|
||||
tail -50 /tmp/weed-server.log || echo "No server logs"
|
||||
echo "========================================="
|
||||
echo "Broker logs (last 50 lines):"
|
||||
tail -50 /tmp/weed-mq-broker.log || echo "No broker logs"
|
||||
echo "========================================="
|
||||
exit 1
|
||||
fi
|
||||
env:
|
||||
GOMAXPROCS: 2
|
||||
SEAWEEDFS_MASTERS: 127.0.0.1:9333
|
||||
|
||||
kafka-consumer-group-tests:
|
||||
name: Kafka Consumer Group Tests (Highly Isolated)
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 20
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
container-id: [consumer-group-1]
|
||||
container:
|
||||
image: golang:1.26-alpine
|
||||
options: --cpus 1.0 --memory 2g --ulimit nofile=512:512 --hostname kafka-consumer-${{ matrix.container-id }}
|
||||
env:
|
||||
GOMAXPROCS: 1
|
||||
CGO_ENABLED: 0
|
||||
KAFKA_CONSUMER_ISOLATION: "true"
|
||||
CONTAINER_ID: ${{ matrix.container-id }}
|
||||
steps:
|
||||
- name: Check out code
|
||||
uses: actions/checkout@v7
|
||||
|
||||
- name: Set up Go 1.x
|
||||
uses: actions/setup-go@v7
|
||||
with:
|
||||
go-version: ^1.26
|
||||
cache: ${{ runner.environment == 'github-hosted' }}
|
||||
cache-dependency-path: |
|
||||
**/go.sum
|
||||
id: go
|
||||
|
||||
- name: Setup Consumer Group Container Environment
|
||||
run: |
|
||||
apk add --no-cache git procps curl netcat-openbsd
|
||||
ulimit -n 256 || echo "Warning: Could not set file descriptor limit"
|
||||
|
||||
- name: Warm Go module cache
|
||||
run: |
|
||||
# Warm cache for root module
|
||||
go mod download || true
|
||||
# Warm cache for kafka test module
|
||||
cd test/kafka
|
||||
go mod download || true
|
||||
|
||||
- name: Get dependencies
|
||||
run: |
|
||||
cd test/kafka
|
||||
# Use go mod download with timeout to prevent hanging
|
||||
timeout 90s go mod download || echo "Warning: Dependency download timed out, continuing with cached modules"
|
||||
go mod tidy
|
||||
|
||||
- name: Build and start SeaweedFS MQ
|
||||
run: |
|
||||
set -e
|
||||
cd $GITHUB_WORKSPACE
|
||||
# Build weed binary
|
||||
go build -o /usr/local/bin/weed ./weed
|
||||
# Start SeaweedFS components with MQ brokers
|
||||
export WEED_DATA_DIR=/tmp/seaweedfs-mq-$RANDOM
|
||||
mkdir -p "$WEED_DATA_DIR"
|
||||
|
||||
# Start SeaweedFS server (master, volume, filer) with consistent IP advertising
|
||||
nohup weed -v 1 server \
|
||||
-ip="127.0.0.1" \
|
||||
-ip.bind="0.0.0.0" \
|
||||
-dir="$WEED_DATA_DIR" \
|
||||
-master.raftHashicorp \
|
||||
-master.port=9333 \
|
||||
-volume.port=8081 \
|
||||
-filer.port=8888 \
|
||||
-filer=true \
|
||||
-metricsPort=9325 \
|
||||
-master.peers=none \
|
||||
> /tmp/weed-server.log 2>&1 &
|
||||
|
||||
# Wait for master to be ready
|
||||
for i in $(seq 1 30); do
|
||||
if curl -s http://127.0.0.1:9333/cluster/status >/dev/null; then
|
||||
echo "SeaweedFS master HTTP is up"; break
|
||||
fi
|
||||
echo "Waiting for SeaweedFS master HTTP... ($i/30)"; sleep 1
|
||||
done
|
||||
|
||||
# Wait for master gRPC to be ready (this is what broker discovery uses)
|
||||
echo "Waiting for master gRPC port..."
|
||||
for i in $(seq 1 30); do
|
||||
if nc -z 127.0.0.1 19333; then
|
||||
echo "✓ SeaweedFS master gRPC is up (port 19333)"
|
||||
break
|
||||
fi
|
||||
echo " Waiting for master gRPC... ($i/30)"; sleep 1
|
||||
done
|
||||
|
||||
# Give server time to initialize all components including gRPC services
|
||||
echo "Waiting for SeaweedFS components to initialize..."
|
||||
sleep 15
|
||||
|
||||
# Additional wait specifically for gRPC services to be ready for streaming
|
||||
echo "Allowing extra time for master gRPC streaming services to initialize..."
|
||||
sleep 10
|
||||
|
||||
# Start MQ broker with maximum verbosity for debugging
|
||||
echo "Starting MQ broker..."
|
||||
nohup weed -v 3 mq.broker \
|
||||
-master="127.0.0.1:9333" \
|
||||
-ip="127.0.0.1" \
|
||||
-port=17777 \
|
||||
-logFlushInterval=0 \
|
||||
> /tmp/weed-mq-broker.log 2>&1 &
|
||||
|
||||
# Wait for broker to be ready with better error reporting
|
||||
sleep 15
|
||||
broker_ready=false
|
||||
for i in $(seq 1 20); do
|
||||
if nc -z 127.0.0.1 17777; then
|
||||
echo "SeaweedFS MQ broker is up"
|
||||
broker_ready=true
|
||||
break
|
||||
fi
|
||||
echo "Waiting for MQ broker... ($i/20)"; sleep 1
|
||||
done
|
||||
|
||||
# Give broker additional time to register with master
|
||||
if [ "$broker_ready" = true ]; then
|
||||
echo "Allowing broker to register with master..."
|
||||
sleep 30
|
||||
|
||||
# Check if broker is properly registered by querying cluster nodes
|
||||
echo "Cluster status after broker registration:"
|
||||
curl -s "http://127.0.0.1:9333/cluster/status" || echo "Could not check cluster status"
|
||||
|
||||
echo "Checking cluster topology (includes registered components):"
|
||||
curl -s "http://127.0.0.1:9333/dir/status" | head -20 || echo "Could not check dir status"
|
||||
|
||||
echo "Verifying broker discovery via master client debug:"
|
||||
echo "If broker registration is successful, it should appear in dir status"
|
||||
|
||||
echo "Testing gRPC connectivity with weed binary:"
|
||||
echo "This simulates what the gateway does during broker discovery..."
|
||||
timeout 10s weed shell -master=127.0.0.1:9333 -filer=127.0.0.1:8888 > /tmp/shell-test.log 2>&1 || echo "weed shell test completed or timed out - checking logs..."
|
||||
echo "Shell test results:"
|
||||
cat /tmp/shell-test.log 2>/dev/null | head -10 || echo "No shell test logs"
|
||||
fi
|
||||
|
||||
# Check if broker failed to start and show logs
|
||||
if [ "$broker_ready" = false ]; then
|
||||
echo "ERROR: MQ broker failed to start. Broker logs:"
|
||||
cat /tmp/weed-mq-broker.log || echo "No broker logs found"
|
||||
echo "Server logs:"
|
||||
tail -20 /tmp/weed-server.log || echo "No server logs found"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
- name: Run Consumer Group Tests
|
||||
run: |
|
||||
cd test/kafka
|
||||
# Test consumer group functionality with explicit timeout
|
||||
ulimit -n 512 || echo "Warning: Could not set file descriptor limit"
|
||||
ulimit -u 100 || echo "Warning: Could not set process limit"
|
||||
timeout 240s go test -v -run "^TestConsumerGroups" -timeout 180s ./integration/...
|
||||
env:
|
||||
GOMAXPROCS: 1
|
||||
SEAWEEDFS_MASTERS: 127.0.0.1:9333
|
||||
|
||||
kafka-client-compatibility:
|
||||
name: Kafka Client Compatibility (with SMQ)
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 25
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
container-id: [client-compat-1]
|
||||
container:
|
||||
image: golang:1.26-alpine
|
||||
options: --cpus 1.0 --memory 1.5g --shm-size 256m --hostname kafka-client-${{ matrix.container-id }}
|
||||
env:
|
||||
GOMAXPROCS: 1
|
||||
CGO_ENABLED: 0
|
||||
KAFKA_CLIENT_ISOLATION: "true"
|
||||
CONTAINER_ID: ${{ matrix.container-id }}
|
||||
steps:
|
||||
- name: Check out code
|
||||
uses: actions/checkout@v7
|
||||
|
||||
- name: Set up Go 1.x
|
||||
uses: actions/setup-go@v7
|
||||
with:
|
||||
go-version: ^1.26
|
||||
cache: ${{ runner.environment == 'github-hosted' }}
|
||||
cache-dependency-path: |
|
||||
**/go.sum
|
||||
id: go
|
||||
|
||||
- name: Setup Client Container Environment
|
||||
run: |
|
||||
apk add --no-cache git procps curl netcat-openbsd
|
||||
ulimit -n 1024 || echo "Warning: Could not set file descriptor limit"
|
||||
|
||||
- name: Warm Go module cache
|
||||
run: |
|
||||
# Warm cache for root module
|
||||
go mod download || true
|
||||
# Warm cache for kafka test module
|
||||
cd test/kafka
|
||||
go mod download || true
|
||||
|
||||
- name: Get dependencies
|
||||
run: |
|
||||
cd test/kafka
|
||||
timeout 90s go mod download || echo "Warning: Dependency download timed out, continuing with cached modules"
|
||||
go mod tidy
|
||||
|
||||
- name: Build and start SeaweedFS MQ
|
||||
run: |
|
||||
set -e
|
||||
cd $GITHUB_WORKSPACE
|
||||
# Build weed binary
|
||||
go build -o /usr/local/bin/weed ./weed
|
||||
# Start SeaweedFS components with MQ brokers
|
||||
export WEED_DATA_DIR=/tmp/seaweedfs-client-$RANDOM
|
||||
mkdir -p "$WEED_DATA_DIR"
|
||||
|
||||
# Start SeaweedFS server (master, volume, filer) with consistent IP advertising
|
||||
nohup weed -v 1 server \
|
||||
-ip="127.0.0.1" \
|
||||
-ip.bind="0.0.0.0" \
|
||||
-dir="$WEED_DATA_DIR" \
|
||||
-master.raftHashicorp \
|
||||
-master.port=9333 \
|
||||
-volume.port=8081 \
|
||||
-filer.port=8888 \
|
||||
-filer=true \
|
||||
-metricsPort=9325 \
|
||||
-master.peers=none \
|
||||
> /tmp/weed-server.log 2>&1 &
|
||||
|
||||
# Wait for master to be ready
|
||||
for i in $(seq 1 30); do
|
||||
if curl -s http://127.0.0.1:9333/cluster/status >/dev/null; then
|
||||
echo "SeaweedFS master HTTP is up"; break
|
||||
fi
|
||||
echo "Waiting for SeaweedFS master HTTP... ($i/30)"; sleep 1
|
||||
done
|
||||
|
||||
# Wait for master gRPC to be ready (this is what broker discovery uses)
|
||||
echo "Waiting for master gRPC port..."
|
||||
for i in $(seq 1 30); do
|
||||
if nc -z 127.0.0.1 19333; then
|
||||
echo "✓ SeaweedFS master gRPC is up (port 19333)"
|
||||
break
|
||||
fi
|
||||
echo " Waiting for master gRPC... ($i/30)"; sleep 1
|
||||
done
|
||||
|
||||
# Give server time to initialize all components including gRPC services
|
||||
echo "Waiting for SeaweedFS components to initialize..."
|
||||
sleep 15
|
||||
|
||||
# Additional wait specifically for gRPC services to be ready for streaming
|
||||
echo "Allowing extra time for master gRPC streaming services to initialize..."
|
||||
sleep 10
|
||||
|
||||
# Start MQ broker with maximum verbosity for debugging
|
||||
echo "Starting MQ broker..."
|
||||
nohup weed -v 3 mq.broker \
|
||||
-master="127.0.0.1:9333" \
|
||||
-ip="127.0.0.1" \
|
||||
-port=17777 \
|
||||
-logFlushInterval=0 \
|
||||
> /tmp/weed-mq-broker.log 2>&1 &
|
||||
|
||||
# Wait for broker to be ready with better error reporting
|
||||
sleep 15
|
||||
broker_ready=false
|
||||
for i in $(seq 1 20); do
|
||||
if nc -z 127.0.0.1 17777; then
|
||||
echo "SeaweedFS MQ broker is up"
|
||||
broker_ready=true
|
||||
break
|
||||
fi
|
||||
echo "Waiting for MQ broker... ($i/20)"; sleep 1
|
||||
done
|
||||
|
||||
# Give broker additional time to register with master
|
||||
if [ "$broker_ready" = true ]; then
|
||||
echo "Allowing broker to register with master..."
|
||||
sleep 30
|
||||
|
||||
# Check if broker is properly registered by querying cluster nodes
|
||||
echo "Cluster status after broker registration:"
|
||||
curl -s "http://127.0.0.1:9333/cluster/status" || echo "Could not check cluster status"
|
||||
|
||||
echo "Checking cluster topology (includes registered components):"
|
||||
curl -s "http://127.0.0.1:9333/dir/status" | head -20 || echo "Could not check dir status"
|
||||
|
||||
echo "Verifying broker discovery via master client debug:"
|
||||
echo "If broker registration is successful, it should appear in dir status"
|
||||
|
||||
echo "Testing gRPC connectivity with weed binary:"
|
||||
echo "This simulates what the gateway does during broker discovery..."
|
||||
timeout 10s weed shell -master=127.0.0.1:9333 -filer=127.0.0.1:8888 > /tmp/shell-test.log 2>&1 || echo "weed shell test completed or timed out - checking logs..."
|
||||
echo "Shell test results:"
|
||||
cat /tmp/shell-test.log 2>/dev/null | head -10 || echo "No shell test logs"
|
||||
fi
|
||||
|
||||
# Check if broker failed to start and show logs
|
||||
if [ "$broker_ready" = false ]; then
|
||||
echo "ERROR: MQ broker failed to start. Broker logs:"
|
||||
cat /tmp/weed-mq-broker.log || echo "No broker logs found"
|
||||
echo "Server logs:"
|
||||
tail -20 /tmp/weed-server.log || echo "No server logs found"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
- name: Run Client Compatibility Tests
|
||||
run: |
|
||||
cd test/kafka
|
||||
go test -v -run "^TestClientCompatibility" -timeout 180s ./integration/...
|
||||
env:
|
||||
GOMAXPROCS: 1
|
||||
SEAWEEDFS_MASTERS: 127.0.0.1:9333
|
||||
|
||||
kafka-smq-integration-tests:
|
||||
name: Kafka SMQ Integration Tests (Full Stack)
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 20
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
container-id: [smq-integration-1]
|
||||
container:
|
||||
image: golang:1.26-alpine
|
||||
options: --cpus 1.0 --memory 2g --hostname kafka-smq-${{ matrix.container-id }}
|
||||
env:
|
||||
GOMAXPROCS: 1
|
||||
CGO_ENABLED: 0
|
||||
KAFKA_SMQ_INTEGRATION: "true"
|
||||
CONTAINER_ID: ${{ matrix.container-id }}
|
||||
steps:
|
||||
- name: Check out code
|
||||
uses: actions/checkout@v7
|
||||
|
||||
- name: Set up Go 1.x
|
||||
uses: actions/setup-go@v7
|
||||
with:
|
||||
go-version: ^1.26
|
||||
cache: ${{ runner.environment == 'github-hosted' }}
|
||||
cache-dependency-path: |
|
||||
**/go.sum
|
||||
id: go
|
||||
|
||||
- name: Setup SMQ Integration Container Environment
|
||||
run: |
|
||||
apk add --no-cache git procps curl netcat-openbsd
|
||||
ulimit -n 1024 || echo "Warning: Could not set file descriptor limit"
|
||||
|
||||
- name: Warm Go module cache
|
||||
run: |
|
||||
# Warm cache for root module
|
||||
go mod download || true
|
||||
# Warm cache for kafka test module
|
||||
cd test/kafka
|
||||
go mod download || true
|
||||
|
||||
- name: Get dependencies
|
||||
run: |
|
||||
cd test/kafka
|
||||
timeout 90s go mod download || echo "Warning: Dependency download timed out, continuing with cached modules"
|
||||
go mod tidy
|
||||
|
||||
- name: Build and start SeaweedFS MQ
|
||||
run: |
|
||||
set -e
|
||||
cd $GITHUB_WORKSPACE
|
||||
# Build weed binary
|
||||
go build -o /usr/local/bin/weed ./weed
|
||||
# Start SeaweedFS components with MQ brokers
|
||||
export WEED_DATA_DIR=/tmp/seaweedfs-smq-$RANDOM
|
||||
mkdir -p "$WEED_DATA_DIR"
|
||||
|
||||
# Start SeaweedFS server (master, volume, filer) with consistent IP advertising
|
||||
nohup weed -v 1 server \
|
||||
-ip="127.0.0.1" \
|
||||
-ip.bind="0.0.0.0" \
|
||||
-dir="$WEED_DATA_DIR" \
|
||||
-master.raftHashicorp \
|
||||
-master.port=9333 \
|
||||
-volume.port=8081 \
|
||||
-filer.port=8888 \
|
||||
-filer=true \
|
||||
-metricsPort=9325 \
|
||||
-master.peers=none \
|
||||
> /tmp/weed-server.log 2>&1 &
|
||||
|
||||
# Wait for master to be ready
|
||||
for i in $(seq 1 30); do
|
||||
if curl -s http://127.0.0.1:9333/cluster/status >/dev/null; then
|
||||
echo "SeaweedFS master HTTP is up"; break
|
||||
fi
|
||||
echo "Waiting for SeaweedFS master HTTP... ($i/30)"; sleep 1
|
||||
done
|
||||
|
||||
# Wait for master gRPC to be ready (this is what broker discovery uses)
|
||||
echo "Waiting for master gRPC port..."
|
||||
for i in $(seq 1 30); do
|
||||
if nc -z 127.0.0.1 19333; then
|
||||
echo "✓ SeaweedFS master gRPC is up (port 19333)"
|
||||
break
|
||||
fi
|
||||
echo " Waiting for master gRPC... ($i/30)"; sleep 1
|
||||
done
|
||||
|
||||
# Give server time to initialize all components including gRPC services
|
||||
echo "Waiting for SeaweedFS components to initialize..."
|
||||
sleep 15
|
||||
|
||||
# Additional wait specifically for gRPC services to be ready for streaming
|
||||
echo "Allowing extra time for master gRPC streaming services to initialize..."
|
||||
sleep 10
|
||||
|
||||
# Start MQ broker with maximum verbosity for debugging
|
||||
echo "Starting MQ broker..."
|
||||
nohup weed -v 3 mq.broker \
|
||||
-master="127.0.0.1:9333" \
|
||||
-ip="127.0.0.1" \
|
||||
-port=17777 \
|
||||
-logFlushInterval=0 \
|
||||
> /tmp/weed-mq-broker.log 2>&1 &
|
||||
|
||||
# Wait for broker to be ready with better error reporting
|
||||
sleep 15
|
||||
broker_ready=false
|
||||
for i in $(seq 1 20); do
|
||||
if nc -z 127.0.0.1 17777; then
|
||||
echo "SeaweedFS MQ broker is up"
|
||||
broker_ready=true
|
||||
break
|
||||
fi
|
||||
echo "Waiting for MQ broker... ($i/20)"; sleep 1
|
||||
done
|
||||
|
||||
# Give broker additional time to register with master
|
||||
if [ "$broker_ready" = true ]; then
|
||||
echo "Allowing broker to register with master..."
|
||||
sleep 30
|
||||
|
||||
# Check if broker is properly registered by querying cluster nodes
|
||||
echo "Cluster status after broker registration:"
|
||||
curl -s "http://127.0.0.1:9333/cluster/status" || echo "Could not check cluster status"
|
||||
|
||||
echo "Checking cluster topology (includes registered components):"
|
||||
curl -s "http://127.0.0.1:9333/dir/status" | head -20 || echo "Could not check dir status"
|
||||
|
||||
echo "Verifying broker discovery via master client debug:"
|
||||
echo "If broker registration is successful, it should appear in dir status"
|
||||
|
||||
echo "Testing gRPC connectivity with weed binary:"
|
||||
echo "This simulates what the gateway does during broker discovery..."
|
||||
timeout 10s weed shell -master=127.0.0.1:9333 -filer=127.0.0.1:8888 > /tmp/shell-test.log 2>&1 || echo "weed shell test completed or timed out - checking logs..."
|
||||
echo "Shell test results:"
|
||||
cat /tmp/shell-test.log 2>/dev/null | head -10 || echo "No shell test logs"
|
||||
fi
|
||||
|
||||
# Check if broker failed to start and show logs
|
||||
if [ "$broker_ready" = false ]; then
|
||||
echo "ERROR: MQ broker failed to start. Broker logs:"
|
||||
cat /tmp/weed-mq-broker.log || echo "No broker logs found"
|
||||
echo "Server logs:"
|
||||
tail -20 /tmp/weed-server.log || echo "No server logs found"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
- name: Run SMQ Integration Tests
|
||||
run: |
|
||||
cd test/kafka
|
||||
ulimit -n 512 || echo "Warning: Could not set file descriptor limit"
|
||||
ulimit -u 100 || echo "Warning: Could not set process limit"
|
||||
# Run the dedicated SMQ integration tests
|
||||
go test -v -run "^TestSMQIntegration" -timeout 180s ./integration/...
|
||||
env:
|
||||
GOMAXPROCS: 1
|
||||
SEAWEEDFS_MASTERS: 127.0.0.1:9333
|
||||
|
||||
kafka-protocol-tests:
|
||||
name: Kafka Protocol Tests (Isolated)
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 5
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
container-id: [protocol-1]
|
||||
container:
|
||||
image: golang:1.26-alpine
|
||||
options: --cpus 1.0 --memory 1g --tmpfs /tmp:exec --hostname kafka-protocol-${{ matrix.container-id }}
|
||||
env:
|
||||
GOMAXPROCS: 1
|
||||
CGO_ENABLED: 0
|
||||
KAFKA_PROTOCOL_ISOLATION: "true"
|
||||
CONTAINER_ID: ${{ matrix.container-id }}
|
||||
steps:
|
||||
- name: Set up Go 1.x
|
||||
uses: actions/setup-go@v7
|
||||
with:
|
||||
cache: ${{ runner.environment == 'github-hosted' }}
|
||||
go-version: ^1.26
|
||||
id: go
|
||||
|
||||
- name: Check out code
|
||||
uses: actions/checkout@v7
|
||||
|
||||
- name: Setup Protocol Container Environment
|
||||
run: |
|
||||
apk add --no-cache git procps
|
||||
# Ensure proper permissions for test execution
|
||||
chmod -R 755 /tmp || true
|
||||
export TMPDIR=/tmp
|
||||
export GOCACHE=/tmp/go-cache
|
||||
mkdir -p $GOCACHE
|
||||
chmod 755 $GOCACHE
|
||||
|
||||
- name: Get dependencies
|
||||
run: |
|
||||
cd test/kafka
|
||||
go mod download
|
||||
go mod tidy
|
||||
|
||||
- name: Run Protocol Tests
|
||||
run: |
|
||||
cd test/kafka
|
||||
export TMPDIR=/tmp
|
||||
export GOCACHE=/tmp/go-cache
|
||||
# Run protocol tests from the weed/mq/kafka directory since they test the protocol implementation
|
||||
cd ../../weed/mq/kafka
|
||||
go test -v -run "^Test.*" -timeout 10s ./...
|
||||
env:
|
||||
GOMAXPROCS: 1
|
||||
TMPDIR: /tmp
|
||||
GOCACHE: /tmp/go-cache
|
||||
@@ -1,142 +0,0 @@
|
||||
name: "KMS Tests"
|
||||
|
||||
on:
|
||||
pull_request:
|
||||
paths:
|
||||
- 'weed/kms/**'
|
||||
- 'weed/s3api/s3_sse_*.go'
|
||||
- 'weed/s3api/s3api_object_handlers.go'
|
||||
- 'weed/s3api/s3api_object_handlers_put.go'
|
||||
- 'test/kms/**'
|
||||
- '.github/workflows/kms-tests.yml'
|
||||
push:
|
||||
branches: [ master, main ]
|
||||
paths:
|
||||
- 'weed/kms/**'
|
||||
- 'weed/s3api/s3_sse_*.go'
|
||||
- 'weed/s3api/s3api_object_handlers.go'
|
||||
- 'weed/s3api/s3api_object_handlers_put.go'
|
||||
- 'test/kms/**'
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.event.pull_request.number || (github.event_name == 'push' && github.ref) || github.run_id }}
|
||||
cancel-in-progress: ${{ github.event_name == 'pull_request' || (github.event_name == 'push' && !startsWith(github.ref, 'refs/tags/')) }}
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
defaults:
|
||||
run:
|
||||
working-directory: weed
|
||||
|
||||
jobs:
|
||||
kms-provider-tests:
|
||||
name: KMS Provider Integration Tests
|
||||
runs-on: ubuntu-24.04
|
||||
timeout-minutes: 20
|
||||
|
||||
steps:
|
||||
- name: Check out code
|
||||
uses: actions/checkout@v7
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v7
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
cache: ${{ runner.environment == 'github-hosted' }}
|
||||
id: go
|
||||
|
||||
- name: Install SeaweedFS
|
||||
run: |
|
||||
go install -buildvcs=false
|
||||
|
||||
- name: Run KMS provider integration tests
|
||||
timeout-minutes: 15
|
||||
working-directory: test/kms
|
||||
run: |
|
||||
set -x
|
||||
echo "=== System Information ==="
|
||||
uname -a
|
||||
free -h
|
||||
docker --version
|
||||
|
||||
make test-provider-ci
|
||||
|
||||
- name: Show OpenBao logs on failure
|
||||
if: failure()
|
||||
run: |
|
||||
echo "=== OpenBao Container Logs ==="
|
||||
docker logs openbao-ci 2>&1 | tail -50 || echo "No OpenBao container found"
|
||||
echo "=== Setup Logs ==="
|
||||
cat /tmp/openbao-ci-setup.log 2>/dev/null || echo "No setup log found"
|
||||
|
||||
- name: Cleanup
|
||||
if: always()
|
||||
working-directory: test/kms
|
||||
run: |
|
||||
make stop-openbao-ci || true
|
||||
|
||||
s3-kms-e2e-tests:
|
||||
name: S3 KMS End-to-End Tests
|
||||
runs-on: ubuntu-24.04
|
||||
timeout-minutes: 25
|
||||
|
||||
steps:
|
||||
- name: Check out code
|
||||
uses: actions/checkout@v7
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v7
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
cache: ${{ runner.environment == 'github-hosted' }}
|
||||
id: go
|
||||
|
||||
- name: Install SeaweedFS
|
||||
run: |
|
||||
go install -buildvcs=false
|
||||
|
||||
- name: Run S3 KMS end-to-end tests
|
||||
timeout-minutes: 20
|
||||
working-directory: test/kms
|
||||
run: |
|
||||
set -x
|
||||
echo "=== System Information ==="
|
||||
uname -a
|
||||
free -h
|
||||
docker --version
|
||||
aws --version
|
||||
|
||||
make test-s3-kms-ci
|
||||
|
||||
- name: Show logs on failure
|
||||
if: failure()
|
||||
working-directory: test/kms
|
||||
run: |
|
||||
echo "=== OpenBao Container Logs ==="
|
||||
cat /tmp/openbao-ci-container.log 2>/dev/null || docker logs openbao-ci 2>&1 | tail -50 || echo "No OpenBao logs found"
|
||||
echo "=== SeaweedFS Server Logs ==="
|
||||
tail -100 /tmp/seaweedfs-kms-mini.log 2>/dev/null || echo "No server log found"
|
||||
echo "=== Setup Logs ==="
|
||||
cat /tmp/weed-kms-ci-setup.log 2>/dev/null || echo "No weed setup log"
|
||||
echo "=== Process Information ==="
|
||||
ps aux | grep -E "(weed|test)" || true
|
||||
|
||||
- name: Upload test logs on failure
|
||||
if: failure()
|
||||
uses: actions/upload-artifact@v7
|
||||
with:
|
||||
name: s3-kms-e2e-logs
|
||||
path: |
|
||||
/tmp/seaweedfs-kms-mini.log
|
||||
/tmp/openbao-ci-container.log
|
||||
/tmp/weed-kms-ci-setup.log
|
||||
retention-days: 3
|
||||
|
||||
- name: Cleanup
|
||||
if: always()
|
||||
working-directory: test/kms
|
||||
run: |
|
||||
make stop-seaweedfs-ci || true
|
||||
make stop-openbao-ci || true
|
||||
make clean-ci || true
|
||||
@@ -1,80 +0,0 @@
|
||||
name: "Master Cold Start Tests"
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [ master ]
|
||||
paths:
|
||||
- 'weed/server/master_*.go'
|
||||
- 'weed/topology/**'
|
||||
- 'weed/operation/**'
|
||||
- 'test/master_cold_start/**'
|
||||
- 'test/testutil/**'
|
||||
- '.github/workflows/master-cold-start-tests.yml'
|
||||
pull_request:
|
||||
branches: [ master ]
|
||||
paths:
|
||||
- 'weed/server/master_*.go'
|
||||
- 'weed/topology/**'
|
||||
- 'weed/operation/**'
|
||||
- 'test/master_cold_start/**'
|
||||
- 'test/testutil/**'
|
||||
- '.github/workflows/master-cold-start-tests.yml'
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.event.pull_request.number || (github.event_name == 'push' && github.ref) || github.run_id }}
|
||||
cancel-in-progress: ${{ github.event_name == 'pull_request' || (github.event_name == 'push' && !startsWith(github.ref, 'refs/tags/')) }}
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
jobs:
|
||||
master-cold-start-tests:
|
||||
name: Master Cold Start Tests
|
||||
runs-on: ubuntu-24.04
|
||||
timeout-minutes: 10
|
||||
steps:
|
||||
- name: Check out code
|
||||
uses: actions/checkout@v7
|
||||
with:
|
||||
persist-credentials: false
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v7
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
cache: ${{ runner.environment == 'github-hosted' }}
|
||||
|
||||
- name: Build weed binary
|
||||
run: |
|
||||
cd weed && go install -buildvcs=false
|
||||
|
||||
- name: Run master cold start tests
|
||||
# test/master_cold_start boots a fresh master plus empty volume
|
||||
# servers and requires the very first assign (HTTP and gRPC, no
|
||||
# client retries) to complete a write: the assign that triggers
|
||||
# volume growth must wait for it instead of failing with
|
||||
# "volume growth in progress".
|
||||
run: |
|
||||
export WEED_BINARY=$(go env GOPATH)/bin/weed
|
||||
go test -v -timeout=8m ./test/master_cold_start/...
|
||||
|
||||
- name: Collect server logs on failure
|
||||
if: failure()
|
||||
run: |
|
||||
# test/master_cold_start/cluster.go keeps failing-test dirs created
|
||||
# via os.MkdirTemp("", "seaweedfs_master_cold_start_it_") with each
|
||||
# process log under <baseDir>/logs/.
|
||||
mkdir -p /tmp/master-cold-start-logs
|
||||
find /tmp -maxdepth 1 -type d -name "seaweedfs_master_cold_start_it_*" 2>/dev/null | while read dir; do
|
||||
echo "Found test directory: $dir"
|
||||
cp -r "$dir" /tmp/master-cold-start-logs/ 2>/dev/null || true
|
||||
done
|
||||
find /tmp/master-cold-start-logs -type f -name "*.log" -print -exec tail -n 100 {} \; 2>/dev/null || echo "No logs found"
|
||||
|
||||
- name: Archive logs
|
||||
if: failure()
|
||||
uses: actions/upload-artifact@v7
|
||||
with:
|
||||
name: master-cold-start-test-logs
|
||||
path: /tmp/master-cold-start-logs/
|
||||
retention-days: 7
|
||||
@@ -1,94 +0,0 @@
|
||||
name: "Metadata Subscribe Integration Tests"
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [ master ]
|
||||
paths:
|
||||
- 'weed/filer/**'
|
||||
- 'weed/pb/filer_pb/**'
|
||||
- 'weed/util/log_buffer/**'
|
||||
- 'weed/server/filer_grpc_server_sub_meta.go'
|
||||
- 'weed/server/master_grpc_server.go'
|
||||
- 'weed/command/filer_backup.go'
|
||||
- 'test/metadata_subscribe/**'
|
||||
- '.github/workflows/metadata-subscribe-tests.yml'
|
||||
pull_request:
|
||||
branches: [ master ]
|
||||
paths:
|
||||
- 'weed/filer/**'
|
||||
- 'weed/pb/filer_pb/**'
|
||||
- 'weed/util/log_buffer/**'
|
||||
- 'weed/server/filer_grpc_server_sub_meta.go'
|
||||
- 'weed/server/master_grpc_server.go'
|
||||
- 'weed/command/filer_backup.go'
|
||||
- 'test/metadata_subscribe/**'
|
||||
- '.github/workflows/metadata-subscribe-tests.yml'
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.event.pull_request.number || (github.event_name == 'push' && github.ref) || github.run_id }}
|
||||
cancel-in-progress: ${{ github.event_name == 'pull_request' || (github.event_name == 'push' && !startsWith(github.ref, 'refs/tags/')) }}
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
env:
|
||||
TEST_TIMEOUT: '10m'
|
||||
|
||||
jobs:
|
||||
metadata-subscribe-integration:
|
||||
name: Metadata Subscribe Integration Tests
|
||||
runs-on: ubuntu-24.04
|
||||
timeout-minutes: 20
|
||||
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v7
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v7
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
cache: ${{ runner.environment == 'github-hosted' }}
|
||||
|
||||
- name: Build SeaweedFS
|
||||
run: |
|
||||
cd weed
|
||||
go build -o weed .
|
||||
chmod +x weed
|
||||
./weed version
|
||||
|
||||
- name: Run Metadata Subscribe Integration Tests
|
||||
run: |
|
||||
cd test/metadata_subscribe
|
||||
|
||||
echo "Running Metadata Subscribe integration tests..."
|
||||
echo "============================================"
|
||||
|
||||
# Run tests with verbose output
|
||||
go test -v -timeout=${{ env.TEST_TIMEOUT }} ./...
|
||||
|
||||
echo "============================================"
|
||||
echo "Metadata Subscribe integration tests completed"
|
||||
|
||||
- name: Archive logs on failure
|
||||
if: failure()
|
||||
uses: actions/upload-artifact@v7
|
||||
with:
|
||||
name: metadata-subscribe-test-logs
|
||||
path: |
|
||||
/tmp/seaweedfs_*
|
||||
retention-days: 7
|
||||
|
||||
- name: Test Summary
|
||||
if: always()
|
||||
run: |
|
||||
echo "## Metadata Subscribe Integration Test Summary" >> $GITHUB_STEP_SUMMARY
|
||||
echo "" >> $GITHUB_STEP_SUMMARY
|
||||
echo "### Test Coverage" >> $GITHUB_STEP_SUMMARY
|
||||
echo "- **Basic Subscription**: Subscribe to metadata changes and receive events" >> $GITHUB_STEP_SUMMARY
|
||||
echo "- **Single-Filer No Stall**: Regression test for issue #4977" >> $GITHUB_STEP_SUMMARY
|
||||
echo "- **Resume from Disk**: Verify subscription can resume from persisted logs" >> $GITHUB_STEP_SUMMARY
|
||||
echo "" >> $GITHUB_STEP_SUMMARY
|
||||
echo "### Related Issues" >> $GITHUB_STEP_SUMMARY
|
||||
echo "- [#4977](https://github.com/seaweedfs/seaweedfs/issues/4977): filer.backup synchronisation stall" >> $GITHUB_STEP_SUMMARY
|
||||
|
||||
@@ -1,205 +0,0 @@
|
||||
name: "mount: benchmark"
|
||||
|
||||
# Manual benchmark: native WinFsp mount vs rclone+WebDAV on the same Windows
|
||||
# runner, with a Linux FUSE mount of the same build as a reference. Numbers
|
||||
# from shared runners are noisy; this is for finding factor-of-N gaps, not
|
||||
# regressions of a few percent.
|
||||
on:
|
||||
workflow_dispatch:
|
||||
push:
|
||||
branches: [ 'winfsp-bench**' ]
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.event.pull_request.number || (github.event_name == 'push' && github.ref) || github.run_id }}
|
||||
cancel-in-progress: ${{ github.event_name == 'pull_request' || (github.event_name == 'push' && !startsWith(github.ref, 'refs/tags/')) }}
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
jobs:
|
||||
bench-windows:
|
||||
name: Windows native vs rclone
|
||||
runs-on: windows-latest
|
||||
timeout-minutes: 60
|
||||
env:
|
||||
CGO_ENABLED: 0
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@v7
|
||||
with:
|
||||
persist-credentials: false
|
||||
|
||||
- uses: actions/setup-go@v7
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
cache: ${{ runner.environment == 'github-hosted' }}
|
||||
|
||||
- name: Install WinFsp and rclone
|
||||
run: choco install winfsp rclone -y --no-progress
|
||||
|
||||
- name: Build weed.exe
|
||||
run: go build -o weed.exe ./weed
|
||||
|
||||
- name: Benchmark both mounts
|
||||
shell: pwsh
|
||||
run: |
|
||||
$ErrorActionPreference = 'Stop'
|
||||
|
||||
function Test-Port($port) {
|
||||
$client = New-Object System.Net.Sockets.TcpClient
|
||||
try { $client.Connect('127.0.0.1', $port); return $client.Connected }
|
||||
catch { return $false }
|
||||
finally { $client.Dispose() }
|
||||
}
|
||||
|
||||
function Wait-Drive($drive, $what) {
|
||||
$deadline = (Get-Date).AddMinutes(2)
|
||||
while ((Get-Date) -lt $deadline) {
|
||||
if (Test-Path "${drive}\") { Write-Host "$what is mounted on $drive"; return }
|
||||
Start-Sleep -Seconds 2
|
||||
}
|
||||
throw "$what never appeared on $drive"
|
||||
}
|
||||
|
||||
function Invoke-Bench($dir, $label, $out) {
|
||||
Write-Host "::group::bench $label"
|
||||
& go run ./test/mount_bench -dir $dir -label $label -filer 127.0.0.1:8888 -out $out
|
||||
$code = $LASTEXITCODE
|
||||
Write-Host "::endgroup::"
|
||||
if ($code -ne 0) { throw "bench $label failed with exit $code" }
|
||||
}
|
||||
|
||||
New-Item -ItemType Directory -Force -Path C:\seaweed-data | Out-Null
|
||||
Start-Process -FilePath .\weed.exe `
|
||||
-ArgumentList '-logtostderr','mini','-dir=C:\seaweed-data','-ip=127.0.0.1' `
|
||||
-RedirectStandardOutput C:\seaweed-mini.log -RedirectStandardError C:\seaweed-mini.err.log
|
||||
|
||||
$deadline = (Get-Date).AddMinutes(3)
|
||||
while ((Get-Date) -lt $deadline) {
|
||||
if ((Test-Port 8888) -and (Test-Port 18888) -and (Test-Port 7333)) { break }
|
||||
Start-Sleep -Seconds 3
|
||||
}
|
||||
if (-not ((Test-Port 8888) -and (Test-Port 18888) -and (Test-Port 7333))) {
|
||||
Get-Content C:\seaweed-mini.log, C:\seaweed-mini.err.log -ErrorAction SilentlyContinue
|
||||
throw "mini cluster never came up"
|
||||
}
|
||||
Write-Host "filer on 8888/18888, webdav on 7333"
|
||||
|
||||
# --- native WinFsp mount ---
|
||||
Start-Process -FilePath .\weed.exe `
|
||||
-ArgumentList '-logtostderr','mount','-filer=127.0.0.1:8888','-dir=S:' `
|
||||
-RedirectStandardOutput C:\seaweed-mount.log -RedirectStandardError C:\seaweed-mount.err.log
|
||||
Wait-Drive 'S:' 'weed mount'
|
||||
|
||||
Invoke-Bench 'S:\bench-native' 'winfsp-native' 'C:\results-native.json'
|
||||
|
||||
Get-CimInstance Win32_Process -Filter "Name = 'weed.exe'" |
|
||||
Where-Object { $_.CommandLine -like '*mount*' } |
|
||||
ForEach-Object { Stop-Process -Id $_.ProcessId -Force }
|
||||
$deadline = (Get-Date).AddMinutes(1)
|
||||
while ((Get-Date) -lt $deadline -and (Test-Path S:\)) { Start-Sleep -Seconds 2 }
|
||||
|
||||
# --- rclone + WebDAV on the same WinFsp ---
|
||||
# rclone serves listings from a directory cache it fills lazily, so the
|
||||
# big-listing files have to exist before it mounts or it never sees them.
|
||||
& go run ./test/mount_bench -filer 127.0.0.1:8888 -seed bench-rclone/biglist
|
||||
if ($LASTEXITCODE -ne 0) { throw "seeding failed" }
|
||||
|
||||
$env:RCLONE_CONFIG_SEAWEED_TYPE = 'webdav'
|
||||
$env:RCLONE_CONFIG_SEAWEED_URL = 'http://127.0.0.1:7333'
|
||||
$env:RCLONE_CONFIG_SEAWEED_VENDOR = 'other'
|
||||
Start-Process -FilePath rclone `
|
||||
-ArgumentList 'mount','seaweed:','T:','--vfs-cache-mode=writes','-v','--log-file=C:\rclone.log'
|
||||
Wait-Drive 'T:' 'rclone mount'
|
||||
|
||||
Invoke-Bench 'T:\bench-rclone' 'rclone-webdav' 'C:\results-rclone.json'
|
||||
|
||||
Stop-Process -Name rclone -Force -ErrorAction SilentlyContinue
|
||||
|
||||
# --- comparison ---
|
||||
$table = & go run ./test/mount_bench -compare C:\results-native.json,C:\results-rclone.json
|
||||
$table | Write-Host
|
||||
"## Windows: native WinFsp vs rclone+WebDAV" | Out-File -Append $env:GITHUB_STEP_SUMMARY
|
||||
$table | Out-File -Append $env:GITHUB_STEP_SUMMARY
|
||||
|
||||
- name: Logs
|
||||
if: always()
|
||||
shell: pwsh
|
||||
run: |
|
||||
foreach ($f in 'C:\seaweed-mount.log','C:\seaweed-mount.err.log','C:\rclone.log','C:\seaweed-mini.log','C:\seaweed-mini.err.log') {
|
||||
if (Test-Path $f) { Write-Host "===== $f"; Get-Content $f -Tail 100 }
|
||||
}
|
||||
|
||||
- name: Results
|
||||
if: always()
|
||||
uses: actions/upload-artifact@v7
|
||||
with:
|
||||
name: results-windows
|
||||
path: C:\results-*.json
|
||||
if-no-files-found: ignore
|
||||
|
||||
bench-linux:
|
||||
name: Linux FUSE reference
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 60
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@v7
|
||||
with:
|
||||
persist-credentials: false
|
||||
|
||||
- uses: actions/setup-go@v7
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
cache: ${{ runner.environment == 'github-hosted' }}
|
||||
|
||||
- name: Repair the fusermount3 setuid bit
|
||||
uses: ./.github/actions/fix-fusermount-setuid
|
||||
|
||||
- name: Allow non-root FUSE mounts with allow_other
|
||||
run: |
|
||||
echo 'user_allow_other' | sudo tee -a /etc/fuse.conf
|
||||
sudo chmod 644 /etc/fuse.conf
|
||||
|
||||
- name: Build weed
|
||||
run: go build -o /tmp/weed ./weed
|
||||
|
||||
- name: Benchmark FUSE mount
|
||||
run: |
|
||||
set -e
|
||||
mkdir -p /tmp/seaweed-data
|
||||
/tmp/weed -logtostderr mini -dir=/tmp/seaweed-data -ip=127.0.0.1 > /tmp/mini.log 2>&1 &
|
||||
|
||||
for i in $(seq 1 60); do
|
||||
if nc -z 127.0.0.1 8888 && nc -z 127.0.0.1 18888; then break; fi
|
||||
sleep 3
|
||||
done
|
||||
nc -z 127.0.0.1 8888 || { cat /tmp/mini.log; echo "filer never came up"; exit 1; }
|
||||
|
||||
mkdir -p "$HOME/mnt"
|
||||
/tmp/weed -logtostderr mount -filer=127.0.0.1:8888 -dir="$HOME/mnt" > /tmp/mount.log 2>&1 &
|
||||
for i in $(seq 1 60); do
|
||||
if mountpoint -q "$HOME/mnt"; then break; fi
|
||||
sleep 2
|
||||
done
|
||||
mountpoint -q "$HOME/mnt" || { cat /tmp/mount.log; echo "mount never appeared"; exit 1; }
|
||||
|
||||
go run ./test/mount_bench -dir "$HOME/mnt/bench-linux" -label linux-fuse -filer 127.0.0.1:8888 -out /tmp/results-linux.json
|
||||
|
||||
{
|
||||
echo "## Linux FUSE reference"
|
||||
go run ./test/mount_bench -compare /tmp/results-linux.json
|
||||
} >> "$GITHUB_STEP_SUMMARY"
|
||||
|
||||
- name: Logs
|
||||
if: always()
|
||||
run: |
|
||||
tail -n 100 /tmp/mount.log /tmp/mini.log 2>/dev/null || true
|
||||
|
||||
- name: Results
|
||||
if: always()
|
||||
uses: actions/upload-artifact@v7
|
||||
with:
|
||||
name: results-linux
|
||||
path: /tmp/results-linux.json
|
||||
if-no-files-found: ignore
|
||||
@@ -1,125 +0,0 @@
|
||||
name: "mount: windows conformance"
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [ master ]
|
||||
paths:
|
||||
- 'weed/mount/**'
|
||||
- 'weed/command/mount*.go'
|
||||
- 'weed/storage/volume_vacuum*.go'
|
||||
- 'weed/storage/volume_loading.go'
|
||||
- 'weed/storage/disk_location.go'
|
||||
- 'test/winfsp-conformance/**'
|
||||
- '.github/workflows/mount-windows-conformance.yml'
|
||||
# No base branch filter: this is the only thing that runs the Windows mount,
|
||||
# so it should cover a pull request stacked on another one too.
|
||||
pull_request:
|
||||
paths:
|
||||
- 'weed/mount/**'
|
||||
- 'weed/command/mount*.go'
|
||||
- 'weed/storage/volume_vacuum*.go'
|
||||
- 'weed/storage/volume_loading.go'
|
||||
- 'weed/storage/disk_location.go'
|
||||
- 'test/winfsp-conformance/**'
|
||||
- '.github/workflows/mount-windows-conformance.yml'
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.event.pull_request.number || (github.event_name == 'push' && github.ref) || github.run_id }}
|
||||
cancel-in-progress: ${{ github.event_name == 'pull_request' || (github.event_name == 'push' && !startsWith(github.ref, 'refs/tags/')) }}
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
jobs:
|
||||
conformance:
|
||||
name: WinFsp conformance
|
||||
runs-on: windows-latest
|
||||
timeout-minutes: 60
|
||||
env:
|
||||
# The runner ships MinGW, so cgo is on by default and cgofuse picks its
|
||||
# cgo variant, which wants WinFsp's headers. The nocgo variant loads
|
||||
# winfsp-x64.dll at run time instead, which is how weed.exe is released.
|
||||
CGO_ENABLED: 0
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@v7
|
||||
with:
|
||||
persist-credentials: false
|
||||
|
||||
- uses: actions/setup-go@v7
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
cache: ${{ runner.environment == 'github-hosted' }}
|
||||
|
||||
# cgofuse loads winfsp-x64.dll at run time, so WinFsp is needed here but
|
||||
# not to build.
|
||||
- name: Install WinFsp
|
||||
run: choco install winfsp -y --no-progress
|
||||
|
||||
- name: Build weed.exe
|
||||
run: go build -o weed.exe ./weed
|
||||
|
||||
# The runner tears down a step's process tree when its shell exits, so a
|
||||
# cluster started in one step is gone by the next. Everything that needs
|
||||
# the cluster and the mount alive has to share a step.
|
||||
- name: Mount and run winfsp-tests
|
||||
shell: pwsh
|
||||
run: |
|
||||
$ErrorActionPreference = 'Stop'
|
||||
|
||||
function Test-Port($port) {
|
||||
# A plain connect, because Test-NetConnection has reported success
|
||||
# here for a port nothing was listening on.
|
||||
$client = New-Object System.Net.Sockets.TcpClient
|
||||
try { $client.Connect('127.0.0.1', $port); return $client.Connected }
|
||||
catch { return $false }
|
||||
finally { $client.Dispose() }
|
||||
}
|
||||
|
||||
function Start-Mount($log) {
|
||||
Start-Process -FilePath .\weed.exe `
|
||||
-ArgumentList '-logtostderr','mount','-filer=127.0.0.1:8888','-dir=S:' `
|
||||
-RedirectStandardOutput "C:\$log.log" -RedirectStandardError "C:\$log.err.log"
|
||||
$deadline = (Get-Date).AddMinutes(2)
|
||||
while ((Get-Date) -lt $deadline) {
|
||||
if (Test-Path S:\) { Write-Host "S: is mounted"; return }
|
||||
Start-Sleep -Seconds 2
|
||||
}
|
||||
Get-Content "C:\$log.log", "C:\$log.err.log" -ErrorAction SilentlyContinue
|
||||
throw "S: never appeared"
|
||||
}
|
||||
|
||||
New-Item -ItemType Directory -Force -Path C:\seaweed-data | Out-Null
|
||||
# -ip pins the cluster to loopback; it otherwise advertises and binds
|
||||
# the runner's LAN address, which 127.0.0.1 cannot reach.
|
||||
Start-Process -FilePath .\weed.exe `
|
||||
-ArgumentList '-logtostderr','mini','-dir=C:\seaweed-data','-ip=127.0.0.1' `
|
||||
-RedirectStandardOutput C:\seaweed-mini.log -RedirectStandardError C:\seaweed-mini.err.log
|
||||
|
||||
$deadline = (Get-Date).AddMinutes(3)
|
||||
while ((Get-Date) -lt $deadline) {
|
||||
# The mount dials grpc, not http, so both ports have to answer.
|
||||
if ((Test-Port 8888) -and (Test-Port 18888)) { break }
|
||||
Start-Sleep -Seconds 3
|
||||
}
|
||||
if (-not ((Test-Port 8888) -and (Test-Port 18888))) {
|
||||
Get-Content C:\seaweed-mini.log, C:\seaweed-mini.err.log -ErrorAction SilentlyContinue
|
||||
throw "filer never came up"
|
||||
}
|
||||
Write-Host "filer is up on http 8888 and grpc 18888"
|
||||
|
||||
Start-Mount 'seaweed-mount'
|
||||
|
||||
Write-Host "::group::winfsp-tests"
|
||||
& pwsh -File test/winfsp-conformance/run.ps1 -MountPoint S:\
|
||||
$code = $LASTEXITCODE
|
||||
Write-Host "::endgroup::"
|
||||
if ($code -ne 0) { throw "winfsp-tests failed with exit $code" }
|
||||
|
||||
- name: Logs
|
||||
if: always()
|
||||
shell: pwsh
|
||||
run: |
|
||||
foreach ($f in 'C:\seaweed-mount.log','C:\seaweed-mount.err.log','C:\seaweed-mini.log','C:\seaweed-mini.err.log') {
|
||||
if (Test-Path $f) { Write-Host "===== $f"; Get-Content $f -Tail 200 }
|
||||
}
|
||||
@@ -1,206 +0,0 @@
|
||||
name: "mount: windows"
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [ master ]
|
||||
paths:
|
||||
- 'weed/mount/**'
|
||||
- 'weed/command/mount*.go'
|
||||
- 'weed/storage/volume_vacuum*.go'
|
||||
- 'weed/storage/volume_loading.go'
|
||||
- 'weed/storage/disk_location.go'
|
||||
- 'test/winfsp/**'
|
||||
- '.github/workflows/mount-windows.yml'
|
||||
# No base branch filter: this is the only thing that runs the Windows mount,
|
||||
# so it should cover a pull request stacked on another one too.
|
||||
pull_request:
|
||||
paths:
|
||||
- 'weed/mount/**'
|
||||
- 'weed/command/mount*.go'
|
||||
- 'weed/storage/volume_vacuum*.go'
|
||||
- 'weed/storage/volume_loading.go'
|
||||
- 'weed/storage/disk_location.go'
|
||||
- 'test/winfsp/**'
|
||||
- '.github/workflows/mount-windows.yml'
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.event.pull_request.number || (github.event_name == 'push' && github.ref) || github.run_id }}
|
||||
cancel-in-progress: ${{ github.event_name == 'pull_request' || (github.event_name == 'push' && !startsWith(github.ref, 'refs/tags/')) }}
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
jobs:
|
||||
mount-windows:
|
||||
name: Mount on Windows
|
||||
runs-on: windows-latest
|
||||
timeout-minutes: 40
|
||||
env:
|
||||
# The runner ships MinGW, so cgo is on by default and cgofuse picks its
|
||||
# cgo variant, which wants WinFsp's headers. The nocgo variant loads
|
||||
# winfsp-x64.dll at run time instead, which is how weed.exe is released.
|
||||
CGO_ENABLED: 0
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@v7
|
||||
with:
|
||||
persist-credentials: false
|
||||
|
||||
- uses: actions/setup-go@v7
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
cache: ${{ runner.environment == 'github-hosted' }}
|
||||
|
||||
# cgofuse loads winfsp-x64.dll at run time, so WinFsp is needed here but
|
||||
# not to build.
|
||||
- name: Install WinFsp
|
||||
run: choco install winfsp -y --no-progress
|
||||
|
||||
- name: Build weed.exe
|
||||
run: go build -o weed.exe ./weed
|
||||
|
||||
# The runner tears down a step's process tree when its shell exits, so a
|
||||
# cluster started in one step is gone by the next. Everything that needs
|
||||
# the cluster and the mount alive has to share a step.
|
||||
- name: Mount and exercise
|
||||
shell: pwsh
|
||||
run: |
|
||||
$ErrorActionPreference = 'Stop'
|
||||
|
||||
function Test-Port($port) {
|
||||
# A plain connect, because Test-NetConnection has reported success
|
||||
# here for a port nothing was listening on.
|
||||
$client = New-Object System.Net.Sockets.TcpClient
|
||||
try { $client.Connect('127.0.0.1', $port); return $client.Connected }
|
||||
catch { return $false }
|
||||
finally { $client.Dispose() }
|
||||
}
|
||||
|
||||
function Start-Mount($log) {
|
||||
Start-Process -FilePath .\weed.exe `
|
||||
-ArgumentList '-logtostderr','mount','-filer=127.0.0.1:8888','-dir=S:' `
|
||||
-RedirectStandardOutput "C:\$log.log" -RedirectStandardError "C:\$log.err.log"
|
||||
$deadline = (Get-Date).AddMinutes(2)
|
||||
while ((Get-Date) -lt $deadline) {
|
||||
if (Test-Path S:\) { Write-Host "S: is mounted"; return }
|
||||
Start-Sleep -Seconds 2
|
||||
}
|
||||
Get-Content "C:\$log.log", "C:\$log.err.log" -ErrorAction SilentlyContinue
|
||||
throw "S: never appeared"
|
||||
}
|
||||
|
||||
function Stop-Mount {
|
||||
Get-CimInstance Win32_Process -Filter "Name = 'weed.exe'" |
|
||||
Where-Object { $_.CommandLine -like '*mount*' } |
|
||||
ForEach-Object { Stop-Process -Id $_.ProcessId -Force }
|
||||
$deadline = (Get-Date).AddMinutes(1)
|
||||
while ((Get-Date) -lt $deadline -and (Test-Path S:\)) { Start-Sleep -Seconds 2 }
|
||||
if (Test-Path S:\) { throw "S: still present after stopping the mount" }
|
||||
Write-Host "unmounted"
|
||||
}
|
||||
|
||||
function Invoke-Tests($label, [string[]]$goArgs) {
|
||||
Write-Host "::group::$label"
|
||||
& go @goArgs
|
||||
$code = $LASTEXITCODE
|
||||
Write-Host "::endgroup::"
|
||||
if ($code -ne 0) { throw "$label failed with exit $code" }
|
||||
}
|
||||
|
||||
New-Item -ItemType Directory -Force -Path C:\seaweed-data | Out-Null
|
||||
# -ip pins the cluster to loopback; it otherwise advertises and binds
|
||||
# the runner's LAN address, which 127.0.0.1 cannot reach.
|
||||
Start-Process -FilePath .\weed.exe `
|
||||
-ArgumentList '-logtostderr','mini','-dir=C:\seaweed-data','-ip=127.0.0.1' `
|
||||
-RedirectStandardOutput C:\seaweed-mini.log -RedirectStandardError C:\seaweed-mini.err.log
|
||||
|
||||
$deadline = (Get-Date).AddMinutes(3)
|
||||
while ((Get-Date) -lt $deadline) {
|
||||
# The mount dials grpc, not http, so both ports have to answer.
|
||||
if ((Test-Port 8888) -and (Test-Port 18888)) { break }
|
||||
Start-Sleep -Seconds 3
|
||||
}
|
||||
if (-not ((Test-Port 8888) -and (Test-Port 18888))) {
|
||||
Get-Content C:\seaweed-mini.log, C:\seaweed-mini.err.log -ErrorAction SilentlyContinue
|
||||
throw "filer never came up"
|
||||
}
|
||||
Write-Host "filer is up on http 8888 and grpc 18888"
|
||||
|
||||
Start-Mount 'seaweed-mount'
|
||||
|
||||
Invoke-Tests 'exercise' @('test','-v','-timeout','20m','./test/winfsp','-mountpoint=S:\')
|
||||
Invoke-Tests 'persist-write' @('test','-v','-timeout','15m','./test/winfsp','-run','TestPersistence','-mountpoint=S:\','-phase=write','-filer=127.0.0.1:8888')
|
||||
|
||||
Stop-Mount
|
||||
Start-Mount 'seaweed-remount'
|
||||
|
||||
Invoke-Tests 'persist-verify' @('test','-v','-timeout','15m','./test/winfsp','-run','TestPersistence','-mountpoint=S:\','-phase=verify')
|
||||
|
||||
# WinFsp creates the mount directory itself, so the path must not
|
||||
# exist; only its parent has to.
|
||||
Write-Host "::group::mount over a directory"
|
||||
Stop-Mount
|
||||
Remove-Item C:\seaweed-mnt -Recurse -Force -ErrorAction SilentlyContinue
|
||||
Start-Process -FilePath .\weed.exe `
|
||||
-ArgumentList '-logtostderr','mount','-filer=127.0.0.1:8888','-dir=C:\seaweed-mnt' `
|
||||
-RedirectStandardOutput C:\seaweed-dirmount.log -RedirectStandardError C:\seaweed-dirmount.err.log
|
||||
$deadline = (Get-Date).AddMinutes(2)
|
||||
$ok = $false
|
||||
while ((Get-Date) -lt $deadline) {
|
||||
# Listing succeeds on the plain empty directory too, so wait for the
|
||||
# reparse point WinFsp turns it into. Otherwise this step passes
|
||||
# without a mount and writes to local disk.
|
||||
$item = Get-Item C:\seaweed-mnt -Force -ErrorAction SilentlyContinue
|
||||
if ($null -ne $item -and $item.Attributes.ToString() -like '*ReparsePoint*') { $ok = $true; break }
|
||||
Start-Sleep -Seconds 2
|
||||
}
|
||||
if (-not $ok) {
|
||||
Get-Content C:\seaweed-dirmount.log, C:\seaweed-dirmount.err.log -ErrorAction SilentlyContinue
|
||||
throw "mounting over a directory failed"
|
||||
}
|
||||
Set-Content -Path C:\seaweed-mnt\dirmount.txt -Value 'via directory mount'
|
||||
if ((Get-Content C:\seaweed-mnt\dirmount.txt) -ne 'via directory mount') { throw "readback through the directory mount differs" }
|
||||
Remove-Item C:\seaweed-mnt\dirmount.txt -Force
|
||||
Get-CimInstance Win32_Process -Filter "Name = 'weed.exe'" |
|
||||
Where-Object { $_.CommandLine -like '*mount*' } |
|
||||
ForEach-Object { Stop-Process -Id $_.ProcessId -Force }
|
||||
Start-Sleep -Seconds 5
|
||||
Write-Host "::endgroup::"
|
||||
|
||||
Start-Mount 'seaweed-remount2'
|
||||
|
||||
Write-Host "::group::explorer-style walk"
|
||||
New-Item -ItemType Directory -Force -Path S:\walk | Out-Null
|
||||
1..200 | ForEach-Object { Set-Content -Path "S:\walk\f$_.txt" -Value "line $_" }
|
||||
$names = @(Get-ChildItem S:\walk | ForEach-Object { $_.Name })
|
||||
if ($names.Count -ne 200) {
|
||||
# Name the strays: a dot entry surfacing here is a different problem
|
||||
# from a missing or duplicated file.
|
||||
$unexpected = $names | Where-Object { $_ -notmatch '^f\d+\.txt$' }
|
||||
throw "listed $($names.Count) entries, expected 200; unexpected: $($unexpected -join ', ')"
|
||||
}
|
||||
$body = Get-Content S:\walk\f42.txt
|
||||
if ($body -ne 'line 42') { throw "unexpected content: $body" }
|
||||
Copy-Item S:\walk\f42.txt S:\walk\copy.txt
|
||||
Remove-Item S:\walk -Recurse -Force
|
||||
if (Test-Path S:\walk) { throw "directory survived recursive delete" }
|
||||
Write-Host "::endgroup::"
|
||||
|
||||
# Tear down everything we started so the runner's later steps (Logs,
|
||||
# post-checkout) launch into a clean environment. A WinFsp mount left
|
||||
# active and a weed.exe holding winfsp-x64.dll have been seen to make
|
||||
# the next step's pwsh.exe fail with STATUS_DLL_INIT_FAILED
|
||||
# (0xC0000142), which fails a job whose actual test step passed.
|
||||
try { Stop-Mount } catch { Write-Host "no mount to stop at teardown: $_" }
|
||||
Get-CimInstance Win32_Process -Filter "Name = 'weed.exe'" |
|
||||
ForEach-Object { Stop-Process -Id $_.ProcessId -Force -ErrorAction SilentlyContinue }
|
||||
Start-Sleep -Seconds 3
|
||||
|
||||
- name: Logs
|
||||
if: always()
|
||||
continue-on-error: true
|
||||
shell: pwsh
|
||||
run: |
|
||||
foreach ($f in 'C:\seaweed-mount.log','C:\seaweed-mount.err.log','C:\seaweed-remount.log','C:\seaweed-remount.err.log','C:\seaweed-remount2.log','C:\seaweed-remount2.err.log','C:\seaweed-dirmount.log','C:\seaweed-dirmount.err.log','C:\seaweed-mini.log','C:\seaweed-mini.err.log') {
|
||||
if (Test-Path $f) { Write-Host "===== $f"; Get-Content $f -Tail 200 }
|
||||
}
|
||||
@@ -1,78 +0,0 @@
|
||||
name: "Multi-Master Tests"
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [ master ]
|
||||
paths:
|
||||
- 'weed/server/master_*.go'
|
||||
- 'weed/server/raft_*.go'
|
||||
- 'weed/topology/**'
|
||||
- 'test/multi_master/**'
|
||||
- 'test/testutil/**'
|
||||
- '.github/workflows/multi-master-tests.yml'
|
||||
pull_request:
|
||||
branches: [ master ]
|
||||
paths:
|
||||
- 'weed/server/master_*.go'
|
||||
- 'weed/server/raft_*.go'
|
||||
- 'weed/topology/**'
|
||||
- 'test/multi_master/**'
|
||||
- 'test/testutil/**'
|
||||
- '.github/workflows/multi-master-tests.yml'
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.event.pull_request.number || (github.event_name == 'push' && github.ref) || github.run_id }}
|
||||
cancel-in-progress: ${{ github.event_name == 'pull_request' || (github.event_name == 'push' && !startsWith(github.ref, 'refs/tags/')) }}
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
jobs:
|
||||
multi-master-failover-tests:
|
||||
name: Multi-Master Failover Tests
|
||||
runs-on: ubuntu-24.04
|
||||
timeout-minutes: 10
|
||||
steps:
|
||||
- name: Check out code
|
||||
uses: actions/checkout@v7
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v7
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
cache: ${{ runner.environment == 'github-hosted' }}
|
||||
|
||||
- name: Install SeaweedFS
|
||||
run: |
|
||||
cd weed && go install -buildvcs=false
|
||||
|
||||
- name: Run multi-master failover tests
|
||||
# The tests in test/multi_master spin up their own 3-node master raft
|
||||
# cluster (using the freshly-installed `weed` binary) and exercise
|
||||
# leader-election, failover and recovery scenarios. The shared
|
||||
# test/testutil port-allocator regression test runs alongside since it
|
||||
# is a prerequisite for the cluster fixtures.
|
||||
run: |
|
||||
go test -v -timeout=8m ./test/multi_master/... ./test/testutil/...
|
||||
|
||||
- name: Collect server logs on failure
|
||||
if: failure()
|
||||
run: |
|
||||
# test/multi_master/cluster.go creates per-test dirs via
|
||||
# os.MkdirTemp("", "seaweedfs_multi_master_it_") and writes each
|
||||
# node's log into <baseDir>/logs/master*.log.
|
||||
echo "Collecting per-node master logs from temp directories..."
|
||||
mkdir -p /tmp/multi-master-logs
|
||||
find /tmp -maxdepth 1 -type d -name "seaweedfs_multi_master_it_*" 2>/dev/null | while read dir; do
|
||||
echo "Found test directory: $dir"
|
||||
cp -r "$dir" /tmp/multi-master-logs/ 2>/dev/null || true
|
||||
done
|
||||
find /tmp/multi-master-logs -type f -name "*.log" -print -exec tail -n 100 {} \; 2>/dev/null || echo "No logs found"
|
||||
|
||||
- name: Archive logs
|
||||
if: failure()
|
||||
uses: actions/upload-artifact@v7
|
||||
with:
|
||||
name: multi-master-test-logs
|
||||
path: /tmp/multi-master-logs/
|
||||
retention-days: 7
|
||||
@@ -1,473 +0,0 @@
|
||||
name: "Performance"
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [ master ]
|
||||
paths:
|
||||
- '**/*.go'
|
||||
- 'go.mod'
|
||||
- 'go.sum'
|
||||
- 'seaweed-volume/**'
|
||||
- 'seaweed-common/**'
|
||||
- 'test/perf/**'
|
||||
- '.github/workflows/performance.yml'
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
profile_duration:
|
||||
description: "CPU profiling duration in seconds"
|
||||
required: false
|
||||
default: "30"
|
||||
type: string
|
||||
benchmark_files:
|
||||
description: "Number of files for the throughput benchmark"
|
||||
required: false
|
||||
default: "100000"
|
||||
type: string
|
||||
benchmark_concurrency:
|
||||
description: "Concurrent read/write workers"
|
||||
required: false
|
||||
default: "16"
|
||||
type: string
|
||||
benchmark_size:
|
||||
description: "Simulated file size in bytes"
|
||||
required: false
|
||||
default: "1024"
|
||||
type: string
|
||||
s3_objects:
|
||||
description: "Number of objects for the S3 benchmark"
|
||||
required: false
|
||||
default: "20000"
|
||||
type: string
|
||||
s3_size:
|
||||
description: "S3 object size in bytes"
|
||||
required: false
|
||||
default: "4096"
|
||||
type: string
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.event.pull_request.number || (github.event_name == 'push' && github.ref) || github.run_id }}
|
||||
cancel-in-progress: ${{ github.event_name == 'pull_request' || (github.event_name == 'push' && !startsWith(github.ref, 'refs/tags/')) }}
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
env:
|
||||
VOL_SIZE_LIMIT: "1024"
|
||||
|
||||
jobs:
|
||||
performance-profile:
|
||||
name: CPU and Heap Profile
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 30
|
||||
steps:
|
||||
- name: Check out code
|
||||
uses: actions/checkout@v7
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v7
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
cache: ${{ runner.environment == 'github-hosted' }}
|
||||
|
||||
- name: Build weed
|
||||
run: go build -o weed_bin ./weed
|
||||
|
||||
- name: Start server with profiling enabled
|
||||
run: |
|
||||
mkdir -p ./perfdata
|
||||
./weed_bin -v=1 server -debug -debug.port=6060 -dir=./perfdata \
|
||||
-s3 -filer -volume.max=0 -master.volumeSizeLimitMB=100 \
|
||||
-s3.port=8000 -s3.config=./docker/compose/s3.json \
|
||||
> weed.log 2>&1 &
|
||||
echo "WEED_PID=$!" >> "$GITHUB_ENV"
|
||||
for i in $(seq 1 60); do
|
||||
if curl -sf http://localhost:9333/dir/status >/dev/null 2>&1; then
|
||||
echo "master is ready"
|
||||
break
|
||||
fi
|
||||
sleep 1
|
||||
done
|
||||
# give the volume server a moment to register with the master
|
||||
sleep 3
|
||||
|
||||
- name: Start memory sampler
|
||||
run: |
|
||||
bash test/perf/mem_sample.sh mem-profile.csv "server=${WEED_PID}" &
|
||||
echo "SAMPLER_PID=$!" >> "$GITHUB_ENV"
|
||||
|
||||
- name: Capture profiles under load
|
||||
run: |
|
||||
DURATION="${{ github.event.inputs.profile_duration || '30' }}"
|
||||
# drive write load so the sampled profile reflects real work
|
||||
./weed_bin benchmark -master=localhost:9333 -writeOnly \
|
||||
-c=16 -n=5000000 -size=1024 > benchmark-load.log 2>&1 &
|
||||
echo "Sampling CPU profile for ${DURATION}s..."
|
||||
curl -s "http://localhost:6060/debug/pprof/profile?seconds=${DURATION}" -o cpu.pprof
|
||||
curl -s "http://localhost:6060/debug/pprof/heap" -o heap.pprof
|
||||
curl -s "http://localhost:6060/debug/pprof/goroutine?debug=1" -o goroutine.txt
|
||||
go tool pprof -top -nodecount=50 cpu.pprof > cpu-top.txt 2>/dev/null || true
|
||||
go tool pprof -top -nodecount=50 -sample_index=inuse_space heap.pprof > heap-top.txt 2>/dev/null || true
|
||||
|
||||
- name: Record memory usage
|
||||
if: always()
|
||||
run: |
|
||||
kill -TERM "${SAMPLER_PID}" 2>/dev/null || true
|
||||
sleep 2
|
||||
{
|
||||
echo "## Memory usage (peak RSS)"
|
||||
echo '```'
|
||||
if [ -f mem-profile.csv.peak ]; then
|
||||
awk -F'\t' '{printf "%-10s %8d KB (%.1f MB)\n", $1, $2, $2/1024}' mem-profile.csv.peak
|
||||
else
|
||||
echo "no memory samples captured"
|
||||
fi
|
||||
echo '```'
|
||||
} >> "$GITHUB_STEP_SUMMARY"
|
||||
|
||||
- name: Profile summary
|
||||
if: always()
|
||||
run: |
|
||||
{
|
||||
echo "## CPU profile (top functions)"
|
||||
echo '```'
|
||||
head -45 cpu-top.txt 2>/dev/null || echo "no cpu profile captured"
|
||||
echo '```'
|
||||
} >> "$GITHUB_STEP_SUMMARY"
|
||||
|
||||
- name: Stop server
|
||||
if: always()
|
||||
run: kill "${WEED_PID}" 2>/dev/null || true
|
||||
|
||||
- name: Show server log on failure
|
||||
if: failure()
|
||||
run: tail -200 weed.log || true
|
||||
|
||||
- name: Upload profiles
|
||||
if: always()
|
||||
uses: actions/upload-artifact@v7
|
||||
with:
|
||||
name: performance-profile-${{ github.run_number }}
|
||||
path: |
|
||||
cpu.pprof
|
||||
heap.pprof
|
||||
cpu-top.txt
|
||||
heap-top.txt
|
||||
goroutine.txt
|
||||
mem-profile.csv
|
||||
mem-profile.csv.peak
|
||||
weed.log
|
||||
retention-days: 30
|
||||
|
||||
benchmark:
|
||||
name: Throughput Benchmark (${{ matrix.impl }} volume)
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 60
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
impl: [go, rust]
|
||||
env:
|
||||
IMPL: ${{ matrix.impl }}
|
||||
steps:
|
||||
- name: Check out code
|
||||
uses: actions/checkout@v7
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v7
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
cache: ${{ runner.environment == 'github-hosted' }}
|
||||
|
||||
- name: Install Rust toolchain
|
||||
if: matrix.impl == 'rust'
|
||||
uses: dtolnay/rust-toolchain@stable
|
||||
|
||||
- name: Cache cargo registry and target
|
||||
if: matrix.impl == 'rust' && runner.environment == 'github-hosted'
|
||||
uses: actions/cache@v6
|
||||
with:
|
||||
path: |
|
||||
~/.cargo/registry
|
||||
~/.cargo/git
|
||||
seaweed-volume/target
|
||||
key: rust-${{ hashFiles('seaweed-volume/Cargo.lock') }}
|
||||
restore-keys: |
|
||||
rust-
|
||||
|
||||
- name: Build weed
|
||||
run: go build -o weed_bin ./weed
|
||||
|
||||
- name: Build Rust volume server
|
||||
if: matrix.impl == 'rust'
|
||||
run: cd seaweed-volume && cargo build --release
|
||||
|
||||
- name: Start master and volume server
|
||||
run: |
|
||||
mkdir -p ./perfdata/master ./perfdata/vol
|
||||
./weed_bin -v=1 master -ip=127.0.0.1 -port=9333 \
|
||||
-mdir=./perfdata/master -peers=none \
|
||||
-volumeSizeLimitMB="${VOL_SIZE_LIMIT}" -defaultReplication=000 \
|
||||
> master.log 2>&1 &
|
||||
echo "MASTER_PID=$!" >> "$GITHUB_ENV"
|
||||
for i in $(seq 1 60); do
|
||||
if curl -sf http://localhost:9333/dir/status >/dev/null 2>&1; then
|
||||
echo "master is ready"
|
||||
break
|
||||
fi
|
||||
sleep 1
|
||||
done
|
||||
if [ "${IMPL}" = "rust" ]; then
|
||||
./seaweed-volume/target/release/weed-volume \
|
||||
--master 127.0.0.1:9333 --ip 127.0.0.1 --ip.bind 127.0.0.1 \
|
||||
--port 8080 --dir ./perfdata/vol --max 100 --preStopSeconds 0 \
|
||||
> volume.log 2>&1 &
|
||||
else
|
||||
./weed_bin -v=1 volume -master=127.0.0.1:9333 -ip=127.0.0.1 \
|
||||
-port=8080 -dir=./perfdata/vol -max=100 \
|
||||
> volume.log 2>&1 &
|
||||
fi
|
||||
echo "VOLUME_PID=$!" >> "$GITHUB_ENV"
|
||||
for i in $(seq 1 60); do
|
||||
if curl -sf http://localhost:8080/status >/dev/null 2>&1; then
|
||||
echo "volume server is ready"
|
||||
break
|
||||
fi
|
||||
sleep 1
|
||||
done
|
||||
# let the volume server register with the master via heartbeat
|
||||
sleep 3
|
||||
|
||||
- name: Start memory sampler
|
||||
run: |
|
||||
bash test/perf/mem_sample.sh mem-benchmark.csv \
|
||||
"master=${MASTER_PID}" "volume=${VOLUME_PID}" &
|
||||
echo "SAMPLER_PID=$!" >> "$GITHUB_ENV"
|
||||
|
||||
- name: Run throughput benchmark
|
||||
run: |
|
||||
N="${{ github.event.inputs.benchmark_files || '100000' }}"
|
||||
C="${{ github.event.inputs.benchmark_concurrency || '16' }}"
|
||||
SIZE="${{ github.event.inputs.benchmark_size || '1024' }}"
|
||||
./weed_bin benchmark -master=localhost:9333 \
|
||||
-c="${C}" -n="${N}" -size="${SIZE}" 2>&1 | tee benchmark-results.txt
|
||||
|
||||
- name: Run Go micro-benchmarks
|
||||
if: matrix.impl == 'go'
|
||||
continue-on-error: true
|
||||
run: |
|
||||
go test -run='^$' -bench=. -benchmem -benchtime=10x \
|
||||
./weed/topology/... ./weed/util/log_buffer/... ./weed/util/buffered_queue/... \
|
||||
2>&1 | tee go-benchmarks.txt
|
||||
|
||||
- name: Record memory usage
|
||||
if: always()
|
||||
run: |
|
||||
kill -TERM "${SAMPLER_PID}" 2>/dev/null || true
|
||||
sleep 2
|
||||
{
|
||||
echo "## Memory usage (peak RSS, ${IMPL} volume)"
|
||||
echo '```'
|
||||
if [ -f mem-benchmark.csv.peak ]; then
|
||||
awk -F'\t' '{printf "%-10s %8d KB (%.1f MB)\n", $1, $2, $2/1024}' mem-benchmark.csv.peak
|
||||
else
|
||||
echo "no memory samples captured"
|
||||
fi
|
||||
echo '```'
|
||||
} >> "$GITHUB_STEP_SUMMARY"
|
||||
|
||||
- name: Benchmark summary
|
||||
if: always()
|
||||
run: |
|
||||
{
|
||||
echo "## Throughput benchmark (${IMPL} volume)"
|
||||
echo '```'
|
||||
grep -E "Concurrency Level|Time taken|Completed requests|Failed requests|Requests per second|Transfer rate" \
|
||||
benchmark-results.txt 2>/dev/null || echo "no benchmark results captured"
|
||||
echo '```'
|
||||
} >> "$GITHUB_STEP_SUMMARY"
|
||||
|
||||
- name: Stop processes
|
||||
if: always()
|
||||
run: |
|
||||
kill "${VOLUME_PID}" "${MASTER_PID}" 2>/dev/null || true
|
||||
|
||||
- name: Show logs on failure
|
||||
if: failure()
|
||||
run: |
|
||||
echo "=== master.log ==="; tail -100 master.log 2>/dev/null || true
|
||||
echo "=== volume.log ==="; tail -200 volume.log 2>/dev/null || true
|
||||
|
||||
- name: Upload benchmark results
|
||||
if: always()
|
||||
uses: actions/upload-artifact@v7
|
||||
with:
|
||||
name: benchmark-results-${{ matrix.impl }}-${{ github.run_number }}
|
||||
path: |
|
||||
benchmark-results.txt
|
||||
go-benchmarks.txt
|
||||
mem-benchmark.csv
|
||||
mem-benchmark.csv.peak
|
||||
master.log
|
||||
volume.log
|
||||
retention-days: 7
|
||||
|
||||
s3-benchmark:
|
||||
name: S3 Read/Write Benchmark (${{ matrix.impl }} volume)
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 60
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
impl: [go, rust]
|
||||
env:
|
||||
IMPL: ${{ matrix.impl }}
|
||||
steps:
|
||||
- name: Check out code
|
||||
uses: actions/checkout@v7
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v7
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
cache: ${{ runner.environment == 'github-hosted' }}
|
||||
|
||||
- name: Install Rust toolchain
|
||||
if: matrix.impl == 'rust'
|
||||
uses: dtolnay/rust-toolchain@stable
|
||||
|
||||
- name: Cache cargo registry and target
|
||||
if: matrix.impl == 'rust' && runner.environment == 'github-hosted'
|
||||
uses: actions/cache@v6
|
||||
with:
|
||||
path: |
|
||||
~/.cargo/registry
|
||||
~/.cargo/git
|
||||
seaweed-volume/target
|
||||
key: rust-${{ hashFiles('seaweed-volume/Cargo.lock') }}
|
||||
restore-keys: |
|
||||
rust-
|
||||
|
||||
- name: Build weed and S3 load tool
|
||||
run: |
|
||||
go build -o weed_bin ./weed
|
||||
go build -o s3bench ./test/s3/benchmark
|
||||
|
||||
- name: Build Rust volume server
|
||||
if: matrix.impl == 'rust'
|
||||
run: cd seaweed-volume && cargo build --release
|
||||
|
||||
- name: Start cluster with S3 gateway
|
||||
run: |
|
||||
mkdir -p ./perfdata/master ./perfdata/vol ./perfdata/filer
|
||||
./weed_bin -v=1 master -ip=127.0.0.1 -port=9333 \
|
||||
-mdir=./perfdata/master -peers=none \
|
||||
-volumeSizeLimitMB="${VOL_SIZE_LIMIT}" -defaultReplication=000 \
|
||||
> master.log 2>&1 &
|
||||
echo "MASTER_PID=$!" >> "$GITHUB_ENV"
|
||||
for i in $(seq 1 60); do
|
||||
if curl -sf http://localhost:9333/dir/status >/dev/null 2>&1; then
|
||||
echo "master is ready"
|
||||
break
|
||||
fi
|
||||
sleep 1
|
||||
done
|
||||
if [ "${IMPL}" = "rust" ]; then
|
||||
./seaweed-volume/target/release/weed-volume \
|
||||
--master 127.0.0.1:9333 --ip 127.0.0.1 --ip.bind 127.0.0.1 \
|
||||
--port 8080 --dir ./perfdata/vol --max 100 --preStopSeconds 0 \
|
||||
> volume.log 2>&1 &
|
||||
else
|
||||
./weed_bin -v=1 volume -master=127.0.0.1:9333 -ip=127.0.0.1 \
|
||||
-port=8080 -dir=./perfdata/vol -max=100 \
|
||||
> volume.log 2>&1 &
|
||||
fi
|
||||
echo "VOLUME_PID=$!" >> "$GITHUB_ENV"
|
||||
for i in $(seq 1 60); do
|
||||
if curl -sf http://localhost:8080/status >/dev/null 2>&1; then
|
||||
echo "volume server is ready"
|
||||
break
|
||||
fi
|
||||
sleep 1
|
||||
done
|
||||
sleep 3
|
||||
./weed_bin -v=1 filer -master=127.0.0.1:9333 -ip=127.0.0.1 -port=8888 \
|
||||
-s3 -s3.port=8000 -s3.config=./docker/compose/s3.json \
|
||||
> filer.log 2>&1 &
|
||||
echo "FILER_PID=$!" >> "$GITHUB_ENV"
|
||||
for i in $(seq 1 30); do
|
||||
if nc -z localhost 8000 2>/dev/null; then
|
||||
echo "s3 gateway is ready"
|
||||
break
|
||||
fi
|
||||
sleep 1
|
||||
done
|
||||
sleep 2
|
||||
|
||||
- name: Start memory sampler
|
||||
run: |
|
||||
bash test/perf/mem_sample.sh mem-s3.csv \
|
||||
"master=${MASTER_PID}" "volume=${VOLUME_PID}" "filer=${FILER_PID}" &
|
||||
echo "SAMPLER_PID=$!" >> "$GITHUB_ENV"
|
||||
|
||||
- name: Run S3 read/write benchmark
|
||||
run: |
|
||||
OBJECTS="${{ github.event.inputs.s3_objects || '20000' }}"
|
||||
C="${{ github.event.inputs.benchmark_concurrency || '16' }}"
|
||||
SIZE="${{ github.event.inputs.s3_size || '4096' }}"
|
||||
./s3bench -endpoint=http://localhost:8000 \
|
||||
-access-key=some_access_key1 -secret-key=some_secret_key1 \
|
||||
-objects="${OBJECTS}" -size="${SIZE}" -concurrency="${C}" -mode=both \
|
||||
2>&1 | tee s3-benchmark-results.txt
|
||||
|
||||
- name: Record memory usage
|
||||
if: always()
|
||||
run: |
|
||||
kill -TERM "${SAMPLER_PID}" 2>/dev/null || true
|
||||
sleep 2
|
||||
{
|
||||
echo "## Memory usage (peak RSS, ${IMPL} volume)"
|
||||
echo '```'
|
||||
if [ -f mem-s3.csv.peak ]; then
|
||||
awk -F'\t' '{printf "%-10s %8d KB (%.1f MB)\n", $1, $2, $2/1024}' mem-s3.csv.peak
|
||||
else
|
||||
echo "no memory samples captured"
|
||||
fi
|
||||
echo '```'
|
||||
} >> "$GITHUB_STEP_SUMMARY"
|
||||
|
||||
- name: S3 benchmark summary
|
||||
if: always()
|
||||
run: |
|
||||
{
|
||||
echo "## S3 read/write benchmark (${IMPL} volume)"
|
||||
echo '```'
|
||||
grep -E "results:|Concurrency Level|Time taken|Completed requests|Failed requests|Requests per second|Transfer rate|Latency" \
|
||||
s3-benchmark-results.txt 2>/dev/null || echo "no S3 benchmark results captured"
|
||||
echo '```'
|
||||
} >> "$GITHUB_STEP_SUMMARY"
|
||||
|
||||
- name: Stop processes
|
||||
if: always()
|
||||
run: |
|
||||
kill "${FILER_PID}" "${VOLUME_PID}" "${MASTER_PID}" 2>/dev/null || true
|
||||
|
||||
- name: Show logs on failure
|
||||
if: failure()
|
||||
run: |
|
||||
echo "=== master.log ==="; tail -100 master.log 2>/dev/null || true
|
||||
echo "=== volume.log ==="; tail -200 volume.log 2>/dev/null || true
|
||||
echo "=== filer.log ==="; tail -200 filer.log 2>/dev/null || true
|
||||
|
||||
- name: Upload S3 benchmark results
|
||||
if: always()
|
||||
uses: actions/upload-artifact@v7
|
||||
with:
|
||||
name: s3-benchmark-results-${{ matrix.impl }}-${{ github.run_number }}
|
||||
path: |
|
||||
s3-benchmark-results.txt
|
||||
mem-s3.csv
|
||||
mem-s3.csv.peak
|
||||
master.log
|
||||
volume.log
|
||||
filer.log
|
||||
retention-days: 7
|
||||
@@ -1,126 +0,0 @@
|
||||
name: "pjdfstest POSIX Compliance"
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [ master, main ]
|
||||
paths:
|
||||
- 'weed/mount/**'
|
||||
- 'weed/filer/**'
|
||||
- 'test/pjdfstest/**'
|
||||
- '.github/workflows/pjdfstest.yml'
|
||||
pull_request:
|
||||
branches: [ master, main ]
|
||||
paths:
|
||||
- 'weed/mount/**'
|
||||
- 'weed/filer/**'
|
||||
- 'test/pjdfstest/**'
|
||||
- '.github/workflows/pjdfstest.yml'
|
||||
workflow_dispatch:
|
||||
|
||||
concurrency:
|
||||
group: pjdfstest/${{ github.ref }}
|
||||
cancel-in-progress: true
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
jobs:
|
||||
pjdfstest:
|
||||
name: pjdfstest
|
||||
runs-on: ubuntu-24.04
|
||||
timeout-minutes: 60
|
||||
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v7
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v7
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
cache: ${{ runner.environment == 'github-hosted' }}
|
||||
|
||||
- name: Start local Docker registry
|
||||
run: |
|
||||
if ! docker start registry 2>/dev/null; then
|
||||
docker rm -f registry >/dev/null 2>&1 || true
|
||||
docker run -d --restart=always -p 5000:5000 --name registry registry:2
|
||||
fi
|
||||
|
||||
- name: Set up Docker Buildx
|
||||
uses: docker/setup-buildx-action@v4
|
||||
with:
|
||||
driver-opts: network=host
|
||||
|
||||
- name: Set up Docker Compose
|
||||
uses: docker/setup-compose-action@v2
|
||||
|
||||
- name: Build weed race binary
|
||||
run: |
|
||||
cd docker
|
||||
make binary_race
|
||||
|
||||
- name: Build SeaweedFS e2e image
|
||||
uses: docker/build-push-action@v7
|
||||
with:
|
||||
context: docker
|
||||
file: docker/Dockerfile.e2e
|
||||
tags: localhost:5000/chrislusf/seaweedfs:e2e
|
||||
push: true
|
||||
cache-from: type=gha,scope=pjdfstest-e2e
|
||||
cache-to: type=gha,mode=max,scope=pjdfstest-e2e
|
||||
|
||||
- name: Tag e2e image for docker compose
|
||||
run: |
|
||||
docker pull localhost:5000/chrislusf/seaweedfs:e2e
|
||||
docker tag localhost:5000/chrislusf/seaweedfs:e2e chrislusf/seaweedfs:e2e
|
||||
|
||||
- name: Build pjdfstest image
|
||||
uses: docker/build-push-action@v7
|
||||
with:
|
||||
context: test/pjdfstest
|
||||
build-contexts: |
|
||||
chrislusf/seaweedfs:e2e=docker-image://localhost:5000/chrislusf/seaweedfs:e2e
|
||||
tags: localhost:5000/chrislusf/seaweedfs:pjdfstest
|
||||
push: true
|
||||
cache-from: type=gha,scope=pjdfstest-harness
|
||||
cache-to: type=gha,mode=max,scope=pjdfstest-harness
|
||||
|
||||
- name: Tag pjdfstest image for docker compose
|
||||
run: |
|
||||
docker pull localhost:5000/chrislusf/seaweedfs:pjdfstest
|
||||
docker tag localhost:5000/chrislusf/seaweedfs:pjdfstest chrislusf/seaweedfs:pjdfstest
|
||||
|
||||
- name: Start SeaweedFS cluster
|
||||
run: |
|
||||
docker compose -f test/pjdfstest/docker-compose.yml up --wait
|
||||
|
||||
- name: Run pjdfstest
|
||||
run: |
|
||||
set -o pipefail
|
||||
docker compose -f test/pjdfstest/docker-compose.yml exec -T mount \
|
||||
/run.sh 2>&1 | tee /tmp/pjdfstest-output.log
|
||||
|
||||
- name: Collect logs
|
||||
if: always()
|
||||
run: |
|
||||
mkdir -p /tmp/pjdfstest-docker-logs
|
||||
for svc in master volume filer mount; do
|
||||
docker compose -f test/pjdfstest/docker-compose.yml logs "$svc" \
|
||||
> "/tmp/pjdfstest-docker-logs/${svc}.log" 2>&1 || true
|
||||
done
|
||||
|
||||
- name: Tear down
|
||||
if: always()
|
||||
run: |
|
||||
docker compose -f test/pjdfstest/docker-compose.yml down -v
|
||||
|
||||
- name: Upload logs
|
||||
if: always()
|
||||
uses: actions/upload-artifact@v7
|
||||
with:
|
||||
name: pjdfstest-results
|
||||
path: |
|
||||
/tmp/pjdfstest-output.log
|
||||
/tmp/pjdfstest-docker-logs/
|
||||
retention-days: 7
|
||||
@@ -1,56 +0,0 @@
|
||||
name: "Plugin Worker Integration Tests"
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [ master ]
|
||||
paths:
|
||||
- 'weed/**'
|
||||
- 'test/plugin_workers/**'
|
||||
- 'go.mod'
|
||||
- 'go.sum'
|
||||
- '.github/workflows/plugin-workers.yml'
|
||||
pull_request:
|
||||
branches: [ master ]
|
||||
paths:
|
||||
- 'weed/**'
|
||||
- 'test/plugin_workers/**'
|
||||
- 'go.mod'
|
||||
- 'go.sum'
|
||||
- '.github/workflows/plugin-workers.yml'
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.event.pull_request.number || (github.event_name == 'push' && github.ref) || github.run_id }}
|
||||
cancel-in-progress: ${{ github.event_name == 'pull_request' || (github.event_name == 'push' && !startsWith(github.ref, 'refs/tags/')) }}
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
jobs:
|
||||
plugin-worker:
|
||||
name: "Plugin Worker: ${{ matrix.worker }}"
|
||||
runs-on: ubuntu-24.04
|
||||
timeout-minutes: 10
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
include:
|
||||
- worker: erasure_coding
|
||||
path: test/plugin_workers/erasure_coding
|
||||
- worker: vacuum
|
||||
path: test/plugin_workers/vacuum
|
||||
- worker: volume_balance
|
||||
path: test/plugin_workers/volume_balance
|
||||
|
||||
steps:
|
||||
- name: Set up Go 1.x
|
||||
uses: actions/setup-go@v7
|
||||
with:
|
||||
cache: ${{ runner.environment == 'github-hosted' }}
|
||||
go-version: ^1.26
|
||||
id: go
|
||||
|
||||
- name: Check out code into the Go module directory
|
||||
uses: actions/checkout@v7
|
||||
|
||||
- name: Run plugin worker tests
|
||||
run: go test -v ./${{ matrix.path }}
|
||||
@@ -1,791 +0,0 @@
|
||||
name: POSIX Compliance Tests
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [ main, master, develop ]
|
||||
paths:
|
||||
- 'weed/mount/**'
|
||||
- 'weed/command/mount*'
|
||||
- 'weed/command/fuse*'
|
||||
- 'test/fuse_integration/**'
|
||||
pull_request:
|
||||
branches: [ main, master, develop ]
|
||||
paths:
|
||||
- 'weed/mount/**'
|
||||
- 'weed/command/mount*'
|
||||
- 'weed/command/fuse*'
|
||||
- 'test/fuse_integration/**'
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
test_type:
|
||||
description: 'Type of POSIX tests to run (default: full for comprehensive testing)'
|
||||
required: true
|
||||
default: 'full'
|
||||
type: choice
|
||||
options:
|
||||
- critical
|
||||
- basic
|
||||
- extended
|
||||
- full
|
||||
enable_external_tests:
|
||||
description: 'Run external test suites (slower)'
|
||||
required: false
|
||||
default: false
|
||||
type: boolean
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
pull-requests: write
|
||||
issues: write
|
||||
|
||||
env:
|
||||
GO_VERSION: '1.26'
|
||||
TIMEOUT: '45m'
|
||||
|
||||
jobs:
|
||||
posix-compliance-ubuntu:
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 60
|
||||
|
||||
strategy:
|
||||
matrix:
|
||||
fuse-version: ['2.9', '3.0']
|
||||
fail-fast: false
|
||||
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v7
|
||||
with:
|
||||
fetch-depth: 1
|
||||
|
||||
- name: Clean up stale FUSE mounts
|
||||
uses: ./.github/actions/cleanup-stale-fuse-mounts
|
||||
|
||||
- name: Set up Go ${{ env.GO_VERSION }}
|
||||
uses: actions/setup-go@v7
|
||||
with:
|
||||
go-version: ${{ env.GO_VERSION }}
|
||||
cache: ${{ runner.environment == 'github-hosted' }}
|
||||
cache-dependency-path: |
|
||||
go.sum
|
||||
test/fuse_integration/go.sum
|
||||
|
||||
- name: Install system dependencies
|
||||
run: |
|
||||
sudo apt-get update
|
||||
sudo apt-get install -y \
|
||||
attr \
|
||||
acl \
|
||||
build-essential \
|
||||
git \
|
||||
python3-pip
|
||||
|
||||
# Install FUSE version specific packages
|
||||
if [ "${{ matrix.fuse-version }}" = "2.9" ]; then
|
||||
echo "Installing FUSE 2.9..."
|
||||
sudo apt-get install -y fuse libfuse-dev
|
||||
elif [ "${{ matrix.fuse-version }}" = "3.0" ]; then
|
||||
echo "Installing FUSE 3.0..."
|
||||
sudo apt-get install -y fuse3 libfuse3-dev
|
||||
# Also install fuse2 for compatibility if needed
|
||||
sudo apt-get install -y fuse libfuse-dev
|
||||
fi
|
||||
|
||||
- name: Set up user permissions for FUSE
|
||||
run: |
|
||||
# Create fuse group if it doesn't exist
|
||||
sudo groupadd -f fuse
|
||||
|
||||
# Add user to fuse group
|
||||
sudo usermod -a -G fuse $USER
|
||||
|
||||
# Set permissions on FUSE device
|
||||
sudo chmod 666 /dev/fuse
|
||||
|
||||
# Allow non-root mounts to pass -o allow_other (weed -allowOthers)
|
||||
echo 'user_allow_other' | sudo tee -a /etc/fuse.conf
|
||||
sudo chmod 644 /etc/fuse.conf
|
||||
|
||||
# Ensure fuse module is loaded
|
||||
sudo modprobe fuse || true
|
||||
|
||||
# Verify setup
|
||||
echo "FUSE setup verification:"
|
||||
ls -la /dev/fuse
|
||||
groups $USER | grep fuse || echo "User not in fuse group yet (will be effective after login)"
|
||||
lsmod | grep fuse || echo "FUSE module not loaded"
|
||||
|
||||
- name: Install external test tools
|
||||
if: ${{ github.event.inputs.enable_external_tests == 'true' }}
|
||||
run: |
|
||||
# Install nfstest for POSIX API verification
|
||||
pip3 install --user nfstest
|
||||
|
||||
# Install FIO for performance testing
|
||||
sudo apt-get install -y fio
|
||||
|
||||
# Install additional test utilities
|
||||
sudo apt-get install -y stress-ng
|
||||
|
||||
- name: Build SeaweedFS
|
||||
run: |
|
||||
make
|
||||
# Verify binary exists and is executable (it's installed to GOPATH/bin)
|
||||
weed version
|
||||
|
||||
# Make weed binary available in PATH (it's already there from go install)
|
||||
sudo cp $(which weed) /usr/local/bin/weed
|
||||
which weed
|
||||
weed version
|
||||
|
||||
- name: Set up SeaweedFS cluster
|
||||
run: |
|
||||
start_and_wait() {
|
||||
local name=$1
|
||||
local url=$2
|
||||
local pidfile=$3
|
||||
shift 3
|
||||
local cmd="$@"
|
||||
|
||||
echo "Starting $name..."
|
||||
$cmd > "/tmp/seaweedfs/$name.log" 2>&1 &
|
||||
local pid=$!
|
||||
echo $pid > "$pidfile"
|
||||
|
||||
echo "Waiting for $name to start..."
|
||||
for i in {1..30}; do
|
||||
if curl -sf "$url" > /dev/null 2>&1; then
|
||||
echo "$name is ready"
|
||||
return 0
|
||||
fi
|
||||
if [ $i -eq 30 ]; then
|
||||
echo "$name failed to start"
|
||||
cat "/tmp/seaweedfs/$name.log"
|
||||
exit 1
|
||||
fi
|
||||
sleep 2
|
||||
done
|
||||
}
|
||||
|
||||
mkdir -p /tmp/seaweedfs/{master,volume,filer,mount}
|
||||
|
||||
start_and_wait "master" "http://127.0.0.1:9333/cluster/status" "/tmp/seaweedfs/master.pid" \
|
||||
"weed master -ip=127.0.0.1 -port=9333 -mdir=/tmp/seaweedfs/master -raftBootstrap=true"
|
||||
|
||||
start_and_wait "volume" "http://127.0.0.1:8080/status" "/tmp/seaweedfs/volume.pid" \
|
||||
"weed volume -mserver=127.0.0.1:9333 -ip=127.0.0.1 -port=8080 -dir=/tmp/seaweedfs/volume -max=100"
|
||||
|
||||
start_and_wait "filer" "http://127.0.0.1:8888/" "/tmp/seaweedfs/filer.pid" \
|
||||
"weed filer -master=127.0.0.1:9333 -ip=127.0.0.1 -port=8888"
|
||||
|
||||
echo "SeaweedFS cluster status:"
|
||||
curl -s http://127.0.0.1:9333/cluster/status || true
|
||||
|
||||
- name: Set up FUSE mount
|
||||
run: |
|
||||
# Create mount point
|
||||
MOUNT_POINT="/tmp/seaweedfs/mount"
|
||||
mkdir -p $MOUNT_POINT
|
||||
|
||||
echo "Mounting SeaweedFS FUSE filesystem..."
|
||||
# Mount SeaweedFS FUSE filesystem in background
|
||||
weed mount \
|
||||
-filer=127.0.0.1:8888 \
|
||||
-dir=$MOUNT_POINT \
|
||||
-filer.path=/ \
|
||||
-dirAutoCreate=true \
|
||||
-allowOthers=true \
|
||||
-nonempty=true \
|
||||
> /tmp/seaweedfs/mount.log 2>&1 &
|
||||
MOUNT_PID=$!
|
||||
echo $MOUNT_PID > /tmp/seaweedfs/mount.pid
|
||||
|
||||
# Wait for mount to be ready. The mount point directory already
|
||||
# exists, so only a real mount counts — a directory listing would
|
||||
# pass on the local disk even if weed mount died.
|
||||
echo "Waiting for FUSE mount to be ready..."
|
||||
for i in {1..30}; do
|
||||
if mountpoint -q $MOUNT_POINT 2>/dev/null; then
|
||||
echo "FUSE mount is ready"
|
||||
break
|
||||
fi
|
||||
if ! kill -0 $MOUNT_PID 2>/dev/null; then
|
||||
echo "FUSE mount process exited"
|
||||
cat /tmp/seaweedfs/mount.log
|
||||
exit 1
|
||||
fi
|
||||
if [ $i -eq 30 ]; then
|
||||
echo "FUSE mount failed to be ready"
|
||||
cat /tmp/seaweedfs/mount.log
|
||||
echo "Mount status:"
|
||||
mount | grep fuse || echo "No FUSE mounts found"
|
||||
ps aux | grep weed || echo "No weed processes found"
|
||||
exit 1
|
||||
fi
|
||||
sleep 2
|
||||
done
|
||||
|
||||
# Verify mount is working
|
||||
echo "Testing FUSE mount functionality..."
|
||||
echo "test" > $MOUNT_POINT/test_file.txt
|
||||
if [ "$(cat $MOUNT_POINT/test_file.txt)" = "test" ]; then
|
||||
echo "FUSE mount is working correctly"
|
||||
rm $MOUNT_POINT/test_file.txt
|
||||
else
|
||||
echo "FUSE mount is not working properly"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Export mount point for tests
|
||||
echo "SEAWEEDFS_MOUNT_POINT=$MOUNT_POINT" >> $GITHUB_ENV
|
||||
|
||||
- name: Set up test environment
|
||||
run: |
|
||||
cd test/fuse_integration
|
||||
|
||||
# Initialize Go module for tests
|
||||
if [ ! -f go.mod ]; then
|
||||
go mod init seaweedfs-posix-tests
|
||||
go mod tidy
|
||||
fi
|
||||
|
||||
# Create reports directory
|
||||
mkdir -p reports
|
||||
|
||||
# Set up external tools if requested
|
||||
# Note: External tools setup is now optional and won't fail the build
|
||||
if [ "${{ github.event.inputs.enable_external_tests }}" = "true" ]; then
|
||||
echo "Setting up external POSIX testing tools..."
|
||||
make -f posix_Makefile setup-external-tools || true
|
||||
# setup-external-tools exits 0 even when a sub-target failed, so
|
||||
# require the tools only when they are actually installed
|
||||
if [ -d external_tools/pjdfstest ] && command -v nfstest_posix >/dev/null 2>&1; then
|
||||
export POSIX_REQUIRE_EXTERNAL_TOOLS="true"
|
||||
echo "POSIX_REQUIRE_EXTERNAL_TOOLS=true" >> $GITHUB_ENV
|
||||
echo "✅ External tools setup completed successfully"
|
||||
else
|
||||
echo "⚠️ External tools setup incomplete - continuing with basic tests only"
|
||||
echo "External tools will be skipped but tests will continue"
|
||||
echo "This is expected behavior when build tools are not available"
|
||||
fi
|
||||
fi
|
||||
|
||||
# Verify SeaweedFS cluster is accessible
|
||||
echo "Verifying SeaweedFS cluster accessibility..."
|
||||
curl -sf http://127.0.0.1:9333/cluster/status
|
||||
curl -sf http://127.0.0.1:8080/status
|
||||
curl -sf http://127.0.0.1:8888/
|
||||
|
||||
# Verify mount point
|
||||
echo "Mount point: $SEAWEEDFS_MOUNT_POINT"
|
||||
ls -la $SEAWEEDFS_MOUNT_POINT
|
||||
|
||||
# Verify mount is actually working by testing file operations
|
||||
echo "Verifying SeaweedFS mount functionality..."
|
||||
TEST_FILE="$SEAWEEDFS_MOUNT_POINT/.mount_verification_test"
|
||||
echo "SeaweedFS mount test" > "$TEST_FILE"
|
||||
if [ "$(cat "$TEST_FILE")" = "SeaweedFS mount test" ]; then
|
||||
echo "✅ Mount verification successful"
|
||||
rm -f "$TEST_FILE"
|
||||
else
|
||||
echo "❌ Mount verification failed - mount may not be working properly"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Check if this is actually a FUSE mount — the plain directory created
|
||||
# above passes the file test, so without this the suite can run
|
||||
# against the local disk and report a pass it never earned
|
||||
if mount | grep -q "$SEAWEEDFS_MOUNT_POINT"; then
|
||||
echo "✅ FUSE mount detected:"
|
||||
mount | grep "$SEAWEEDFS_MOUNT_POINT"
|
||||
else
|
||||
echo "❌ Mount point not found in mount table - refusing to test a local directory"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
- name: Run POSIX compliance tests
|
||||
id: posix-tests
|
||||
env:
|
||||
SEAWEEDFS_MOUNT_POINT: ${{ env.SEAWEEDFS_MOUNT_POINT }}
|
||||
POSIX_REQUIRE_EXTERNAL_TOOLS: ${{ env.POSIX_REQUIRE_EXTERNAL_TOOLS }}
|
||||
run: |
|
||||
cd test/fuse_integration
|
||||
|
||||
# Determine which tests to run
|
||||
TEST_TYPE="${{ github.event.inputs.test_type }}"
|
||||
if [ -z "$TEST_TYPE" ]; then
|
||||
TEST_TYPE="full"
|
||||
fi
|
||||
|
||||
echo "Running POSIX tests: $TEST_TYPE"
|
||||
if [ "$TEST_TYPE" = "full" ]; then
|
||||
echo "🚀 Running COMPREHENSIVE POSIX test suite (including external test suites)"
|
||||
else
|
||||
echo "Using mount point: $SEAWEEDFS_MOUNT_POINT"
|
||||
fi
|
||||
|
||||
# Set test configuration to use our mounted filesystem
|
||||
export TEST_MOUNT_POINT="$SEAWEEDFS_MOUNT_POINT"
|
||||
export TEST_SKIP_CLUSTER_SETUP="true"
|
||||
|
||||
# Debug: Show environment variables
|
||||
echo "🔍 Test Environment:"
|
||||
echo " TEST_MOUNT_POINT=$TEST_MOUNT_POINT"
|
||||
echo " TEST_SKIP_CLUSTER_SETUP=$TEST_SKIP_CLUSTER_SETUP"
|
||||
echo " SEAWEEDFS_MOUNT_POINT=$SEAWEEDFS_MOUNT_POINT"
|
||||
|
||||
case "$TEST_TYPE" in
|
||||
"critical")
|
||||
echo "Running critical POSIX tests (basic compliance only)..."
|
||||
make -f posix_Makefile test-posix-critical
|
||||
;;
|
||||
"basic")
|
||||
echo "Running basic POSIX tests (core file operations)..."
|
||||
make -f posix_Makefile test-posix-basic
|
||||
;;
|
||||
"extended")
|
||||
echo "Running extended POSIX tests (including advanced features)..."
|
||||
make -f posix_Makefile test-posix-extended
|
||||
;;
|
||||
"full")
|
||||
echo "🚀 Running COMPREHENSIVE POSIX test suite (all tests including external)..."
|
||||
echo " This includes: basic, extended, and external test suites"
|
||||
echo " External suites: pjdfstest, nfstest_posix, and custom stress tests"
|
||||
make -f posix_Makefile test-posix-full
|
||||
;;
|
||||
*)
|
||||
echo "Unknown test type: $TEST_TYPE, defaulting to full suite"
|
||||
echo "🚀 Running full POSIX test suite (all tests including external)..."
|
||||
make -f posix_Makefile test-posix-full
|
||||
;;
|
||||
esac
|
||||
|
||||
- name: Run external test suites
|
||||
if: ${{ github.event.inputs.enable_external_tests == 'true' }}
|
||||
continue-on-error: true
|
||||
run: |
|
||||
cd test/fuse_integration
|
||||
|
||||
# Point the external suites at the same mount as the Go tests;
|
||||
# without these the Makefile falls back to a local-disk path.
|
||||
export TEST_MOUNT_POINT="$SEAWEEDFS_MOUNT_POINT"
|
||||
export TEST_SKIP_CLUSTER_SETUP="true"
|
||||
|
||||
# Run external tests (may fail on some systems)
|
||||
make -f posix_Makefile test-nfstest-posix || echo "nfstest failed or not available"
|
||||
make -f posix_Makefile test-fio-posix || echo "FIO tests failed or not available"
|
||||
|
||||
- name: Generate compliance report
|
||||
if: always()
|
||||
run: |
|
||||
cd test/fuse_integration
|
||||
make -f posix_Makefile generate-report
|
||||
|
||||
- name: Collect system information for debugging
|
||||
if: failure()
|
||||
run: |
|
||||
cd test/fuse_integration
|
||||
mkdir -p reports/debug
|
||||
|
||||
echo "System Information" > reports/debug/system_info.txt
|
||||
uname -a >> reports/debug/system_info.txt
|
||||
echo "" >> reports/debug/system_info.txt
|
||||
|
||||
echo "Mount Information" >> reports/debug/system_info.txt
|
||||
mount >> reports/debug/system_info.txt
|
||||
echo "" >> reports/debug/system_info.txt
|
||||
|
||||
echo "Disk Space" >> reports/debug/system_info.txt
|
||||
timeout 15 df -h >> reports/debug/system_info.txt || true
|
||||
echo "" >> reports/debug/system_info.txt
|
||||
|
||||
echo "Memory Information" >> reports/debug/system_info.txt
|
||||
free -h >> reports/debug/system_info.txt
|
||||
echo "" >> reports/debug/system_info.txt
|
||||
|
||||
echo "FUSE Information" >> reports/debug/system_info.txt
|
||||
ls -la /dev/fuse >> reports/debug/system_info.txt
|
||||
lsmod | grep fuse >> reports/debug/system_info.txt
|
||||
echo "" >> reports/debug/system_info.txt
|
||||
|
||||
echo "SeaweedFS Version" >> reports/debug/system_info.txt
|
||||
weed version >> reports/debug/system_info.txt
|
||||
echo "" >> reports/debug/system_info.txt
|
||||
|
||||
echo "Go Version" >> reports/debug/system_info.txt
|
||||
go version >> reports/debug/system_info.txt
|
||||
echo "" >> reports/debug/system_info.txt
|
||||
|
||||
echo "Running Processes" >> reports/debug/system_info.txt
|
||||
ps aux | grep -E "(weed|fuse)" >> reports/debug/system_info.txt
|
||||
echo "" >> reports/debug/system_info.txt
|
||||
|
||||
# Collect SeaweedFS service logs
|
||||
echo "=== SeaweedFS Service Logs ===" >> reports/debug/system_info.txt
|
||||
if [ -f /tmp/seaweedfs/master.log ]; then
|
||||
echo "Master Log:" >> reports/debug/system_info.txt
|
||||
tail -50 /tmp/seaweedfs/master.log >> reports/debug/system_info.txt
|
||||
echo "" >> reports/debug/system_info.txt
|
||||
fi
|
||||
|
||||
if [ -f /tmp/seaweedfs/volume.log ]; then
|
||||
echo "Volume Log:" >> reports/debug/system_info.txt
|
||||
tail -50 /tmp/seaweedfs/volume.log >> reports/debug/system_info.txt
|
||||
echo "" >> reports/debug/system_info.txt
|
||||
fi
|
||||
|
||||
if [ -f /tmp/seaweedfs/filer.log ]; then
|
||||
echo "Filer Log:" >> reports/debug/system_info.txt
|
||||
tail -50 /tmp/seaweedfs/filer.log >> reports/debug/system_info.txt
|
||||
echo "" >> reports/debug/system_info.txt
|
||||
fi
|
||||
|
||||
if [ -f /tmp/seaweedfs/mount.log ]; then
|
||||
echo "Mount Log:" >> reports/debug/system_info.txt
|
||||
tail -50 /tmp/seaweedfs/mount.log >> reports/debug/system_info.txt
|
||||
echo "" >> reports/debug/system_info.txt
|
||||
fi
|
||||
|
||||
# Copy full logs as separate files
|
||||
if [ -f /tmp/seaweedfs/master.log ]; then
|
||||
cp /tmp/seaweedfs/master.log reports/debug/master.log
|
||||
fi
|
||||
if [ -f /tmp/seaweedfs/volume.log ]; then
|
||||
cp /tmp/seaweedfs/volume.log reports/debug/volume.log
|
||||
fi
|
||||
if [ -f /tmp/seaweedfs/filer.log ]; then
|
||||
cp /tmp/seaweedfs/filer.log reports/debug/filer.log
|
||||
fi
|
||||
if [ -f /tmp/seaweedfs/mount.log ]; then
|
||||
cp /tmp/seaweedfs/mount.log reports/debug/mount.log
|
||||
fi
|
||||
|
||||
- name: Cleanup SeaweedFS cluster and FUSE mount
|
||||
if: always()
|
||||
run: |
|
||||
stop_process() {
|
||||
local name=$1
|
||||
local pidfile=$2
|
||||
if [ -f "$pidfile" ]; then
|
||||
local pid=$(cat "$pidfile")
|
||||
if kill -0 $pid 2>/dev/null; then
|
||||
echo "Stopping $name process (PID: $pid)..."
|
||||
kill -TERM $pid || true
|
||||
sleep 2
|
||||
kill -KILL $pid 2>/dev/null || true
|
||||
fi
|
||||
fi
|
||||
}
|
||||
|
||||
echo "Cleaning up SeaweedFS cluster and FUSE mount..."
|
||||
|
||||
MOUNT_POINT="/tmp/seaweedfs/mount"
|
||||
if mountpoint -q $MOUNT_POINT 2>/dev/null; then
|
||||
echo "Unmounting FUSE filesystem..."
|
||||
fusermount -u $MOUNT_POINT || umount $MOUNT_POINT || true
|
||||
fi
|
||||
|
||||
stop_process "mount" "/tmp/seaweedfs/mount.pid"
|
||||
stop_process "filer" "/tmp/seaweedfs/filer.pid"
|
||||
stop_process "volume" "/tmp/seaweedfs/volume.pid"
|
||||
stop_process "master" "/tmp/seaweedfs/master.pid"
|
||||
|
||||
pkill -f "weed " || true
|
||||
|
||||
fusermount -u $MOUNT_POINT 2>/dev/null || true
|
||||
umount $MOUNT_POINT 2>/dev/null || true
|
||||
|
||||
rm -rf /tmp/seaweedfs || true
|
||||
|
||||
echo "Cleanup completed"
|
||||
|
||||
|
||||
- name: Upload test results
|
||||
uses: actions/upload-artifact@v7
|
||||
if: always()
|
||||
with:
|
||||
name: posix-test-results-ubuntu-fuse${{ matrix.fuse-version }}
|
||||
path: |
|
||||
test/fuse_integration/reports/
|
||||
test/fuse_integration/*.log
|
||||
retention-days: 30
|
||||
|
||||
- name: Upload test coverage
|
||||
uses: actions/upload-artifact@v7
|
||||
if: always() && (github.event.inputs.test_type == 'full' || github.event.inputs.test_type == 'extended')
|
||||
with:
|
||||
name: posix-coverage-ubuntu
|
||||
path: test/fuse_integration/reports/posix_coverage.html
|
||||
retention-days: 14
|
||||
|
||||
- name: Comment PR with results
|
||||
if: github.event_name == 'pull_request' && always()
|
||||
uses: actions/github-script@v9
|
||||
with:
|
||||
script: |
|
||||
const fs = require('fs');
|
||||
const path = require('path');
|
||||
|
||||
const reportPath = 'test/fuse_integration/reports/posix_compliance_summary.txt';
|
||||
|
||||
if (fs.existsSync(reportPath)) {
|
||||
const report = fs.readFileSync(reportPath, 'utf8');
|
||||
const comment = `## POSIX Compliance Test Results
|
||||
|
||||
**Go Version:** ${{ env.GO_VERSION }}
|
||||
**FUSE Version:** ${{ matrix.fuse-version }}
|
||||
**Test Type:** ${{ github.event.inputs.test_type || 'full' }}
|
||||
|
||||
<details>
|
||||
<summary>Test Summary</summary>
|
||||
|
||||
\`\`\`
|
||||
${report}
|
||||
\`\`\`
|
||||
</details>
|
||||
|
||||
📊 Full results available in [test artifacts](https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }})
|
||||
`;
|
||||
|
||||
try {
|
||||
await github.rest.issues.createComment({
|
||||
issue_number: context.issue.number,
|
||||
owner: context.repo.owner,
|
||||
repo: context.repo.repo,
|
||||
body: comment
|
||||
});
|
||||
} catch (err) {
|
||||
console.log('Could not post PR comment: ' + err.message);
|
||||
}
|
||||
}
|
||||
|
||||
posix-compliance-macos:
|
||||
runs-on: macos-latest
|
||||
timeout-minutes: 60
|
||||
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v7
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v7
|
||||
with:
|
||||
go-version: ${{ env.GO_VERSION }}
|
||||
cache: ${{ runner.environment == 'github-hosted' }}
|
||||
|
||||
- name: Install macFUSE
|
||||
run: |
|
||||
# Install macFUSE
|
||||
brew install macfuse
|
||||
|
||||
# Note: macFUSE may require system extension approval on macOS
|
||||
# This step may need manual intervention in some cases
|
||||
|
||||
- name: Build SeaweedFS
|
||||
run: |
|
||||
make
|
||||
# Verify binary exists and is executable (it's installed to GOPATH/bin)
|
||||
weed version
|
||||
|
||||
- name: Run critical POSIX tests (macOS)
|
||||
continue-on-error: true # macOS FUSE can be more restrictive
|
||||
run: |
|
||||
cd test/fuse_integration
|
||||
|
||||
if [ ! -f go.mod ]; then
|
||||
go mod init seaweedfs-posix-tests
|
||||
go mod tidy
|
||||
fi
|
||||
|
||||
mkdir -p reports
|
||||
|
||||
# Run basic tests only on macOS due to FUSE limitations
|
||||
make -f posix_Makefile test-posix-critical
|
||||
|
||||
- name: Upload macOS test results
|
||||
uses: actions/upload-artifact@v7
|
||||
if: always()
|
||||
with:
|
||||
name: posix-test-results-macos
|
||||
path: |
|
||||
test/fuse_integration/reports/
|
||||
test/fuse_integration/*.log
|
||||
retention-days: 30
|
||||
|
||||
security-analysis:
|
||||
runs-on: ubuntu-latest
|
||||
if: github.event_name == 'pull_request'
|
||||
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v7
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v7
|
||||
with:
|
||||
go-version: ${{ env.GO_VERSION }}
|
||||
cache: ${{ runner.environment == 'github-hosted' }}
|
||||
|
||||
- name: Install security analysis tools
|
||||
run: |
|
||||
echo "Installing staticcheck for security analysis..."
|
||||
go install honnef.co/go/tools/cmd/staticcheck@latest
|
||||
|
||||
- name: Run security analysis on FUSE code
|
||||
run: |
|
||||
# Analyze mount and FUSE-related code for security and quality issues
|
||||
echo "Running staticcheck security analysis..."
|
||||
|
||||
# Run staticcheck on FUSE-related code
|
||||
echo "Analyzing FUSE mount code..."
|
||||
staticcheck ./weed/mount/... > staticcheck-report.txt 2>&1 || true
|
||||
staticcheck ./weed/command/mount* ./weed/command/fuse* >> staticcheck-report.txt 2>&1 || true
|
||||
|
||||
# Convert to JSON format for consistency
|
||||
echo "{" > gosec-report.json
|
||||
echo " \"tool\": \"staticcheck\"," >> gosec-report.json
|
||||
echo " \"timestamp\": \"$(date -Iseconds)\"," >> gosec-report.json
|
||||
echo " \"issues\": [" >> gosec-report.json
|
||||
|
||||
if [ -s staticcheck-report.txt ]; then
|
||||
echo " \"$(cat staticcheck-report.txt | head -20 | sed 's/"/\\"/g' | tr '\n' ' ')\"" >> gosec-report.json
|
||||
fi
|
||||
|
||||
echo " ]," >> gosec-report.json
|
||||
echo " \"stats\": {" >> gosec-report.json
|
||||
echo " \"files_analyzed\": $(find ./weed/mount ./weed/command -name '*.go' | wc -l)," >> gosec-report.json
|
||||
echo " \"issues_found\": $(wc -l < staticcheck-report.txt 2>/dev/null || echo 0)" >> gosec-report.json
|
||||
echo " }" >> gosec-report.json
|
||||
echo "}" >> gosec-report.json
|
||||
|
||||
echo "Security analysis completed"
|
||||
echo "Issues found: $(wc -l < staticcheck-report.txt 2>/dev/null || echo 0)"
|
||||
|
||||
- name: Upload security analysis results
|
||||
uses: actions/upload-artifact@v7
|
||||
if: always()
|
||||
with:
|
||||
name: security-analysis-results
|
||||
path: gosec-report.json
|
||||
retention-days: 30
|
||||
|
||||
performance-baseline:
|
||||
runs-on: ubuntu-latest
|
||||
if: github.event.inputs.test_type == 'full' || github.event_name == 'schedule'
|
||||
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v7
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v7
|
||||
with:
|
||||
go-version: ${{ env.GO_VERSION }}
|
||||
cache: ${{ runner.environment == 'github-hosted' }}
|
||||
|
||||
- name: Install dependencies
|
||||
run: |
|
||||
sudo apt-get update
|
||||
sudo apt-get install -y fuse libfuse-dev fio
|
||||
sudo usermod -a -G fuse $USER
|
||||
sudo chmod 666 /dev/fuse
|
||||
# The posix suites mount with -allowOthers; allow_other needs this
|
||||
echo 'user_allow_other' | sudo tee -a /etc/fuse.conf
|
||||
|
||||
- name: Build SeaweedFS
|
||||
run: make
|
||||
|
||||
- name: Run performance baseline tests
|
||||
run: |
|
||||
cd test/fuse_integration
|
||||
|
||||
if [ ! -f go.mod ]; then
|
||||
go mod init seaweedfs-posix-tests
|
||||
go mod tidy
|
||||
fi
|
||||
|
||||
mkdir -p reports
|
||||
|
||||
# Run performance benchmarks
|
||||
make -f posix_Makefile benchmark-posix
|
||||
# This job has no FUSE mount, so the fio suite can only skip; keep the
|
||||
# run green but honest about producing no mounted-FS results
|
||||
make -f posix_Makefile test-fio-posix || echo "FIO skipped - no mounted filesystem in this job"
|
||||
|
||||
- name: Store performance baseline
|
||||
uses: actions/upload-artifact@v7
|
||||
with:
|
||||
name: performance-baseline-results
|
||||
path: |
|
||||
test/fuse_integration/reports/posix_benchmark_results.log
|
||||
test/fuse_integration/reports/fio_*_results.log
|
||||
retention-days: 90
|
||||
|
||||
# Schedule regular compliance checks
|
||||
scheduled-compliance-check:
|
||||
runs-on: ubuntu-latest
|
||||
if: github.event_name == 'schedule'
|
||||
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v7
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v7
|
||||
with:
|
||||
go-version: ${{ env.GO_VERSION }}
|
||||
cache: ${{ runner.environment == 'github-hosted' }}
|
||||
|
||||
- name: Install dependencies
|
||||
run: |
|
||||
sudo apt-get update
|
||||
sudo apt-get install -y fuse libfuse-dev
|
||||
pip3 install --user nfstest
|
||||
# The posix suites mount with -allowOthers; allow_other needs this
|
||||
echo 'user_allow_other' | sudo tee -a /etc/fuse.conf
|
||||
|
||||
- name: Build SeaweedFS
|
||||
run: make
|
||||
|
||||
- name: Run comprehensive compliance check
|
||||
run: |
|
||||
cd test/fuse_integration
|
||||
make -f posix_Makefile test-posix-full
|
||||
make -f posix_Makefile generate-report
|
||||
|
||||
- name: Create compliance issue if tests fail
|
||||
if: failure()
|
||||
uses: actions/github-script@v9
|
||||
with:
|
||||
script: |
|
||||
const issue = await github.rest.issues.create({
|
||||
owner: context.repo.owner,
|
||||
repo: context.repo.repo,
|
||||
title: 'POSIX Compliance Check Failed - ' + new Date().toISOString().split('T')[0],
|
||||
body: `## POSIX Compliance Check Failure
|
||||
|
||||
The scheduled POSIX compliance check has failed. This may indicate:
|
||||
|
||||
- Regression in FUSE mount functionality
|
||||
- Changes affecting POSIX compatibility
|
||||
- Infrastructure issues with the test environment
|
||||
|
||||
**Action Required:** Review the test results and investigate any failures.
|
||||
|
||||
**Test Run:** [Workflow Run](https://github.com/${context.repo.owner}/${context.repo.repo}/actions/runs/${context.runId})
|
||||
|
||||
**Date:** ${new Date().toISOString()}
|
||||
|
||||
---
|
||||
This issue was automatically created by the POSIX compliance workflow.
|
||||
`,
|
||||
labels: ['bug', 'posix-compliance', 'automated-issue']
|
||||
});
|
||||
|
||||
console.log('Created issue:', issue.data.number);
|
||||
|
||||
# Schedule this workflow to run weekly
|
||||
# Uncomment the following lines to enable scheduled runs:
|
||||
# schedule:
|
||||
# - cron: '0 2 * * 1' # Every Monday at 2 AM UTC
|
||||
|
||||
@@ -1,96 +0,0 @@
|
||||
name: "PostgreSQL Gateway Tests"
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [ master ]
|
||||
paths:
|
||||
- 'weed/server/postgres/**'
|
||||
- 'weed/query/**'
|
||||
- 'weed/mq/**'
|
||||
- 'test/postgres/**'
|
||||
- 'go.mod'
|
||||
- 'go.sum'
|
||||
- '.github/workflows/postgres-tests.yml'
|
||||
pull_request:
|
||||
branches: [ master ]
|
||||
paths:
|
||||
- 'weed/server/postgres/**'
|
||||
- 'weed/query/**'
|
||||
- 'weed/mq/**'
|
||||
- 'test/postgres/**'
|
||||
- 'go.mod'
|
||||
- 'go.sum'
|
||||
- '.github/workflows/postgres-tests.yml'
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.event.pull_request.number || (github.event_name == 'push' && github.ref) || github.run_id }}
|
||||
cancel-in-progress: ${{ github.event_name == 'pull_request' || (github.event_name == 'push' && !startsWith(github.ref, 'refs/tags/')) }}
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
jobs:
|
||||
postgres-basic-tests:
|
||||
name: PostgreSQL Basic Tests
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 15
|
||||
defaults:
|
||||
run:
|
||||
working-directory: test/postgres
|
||||
steps:
|
||||
- name: Set up Go 1.x
|
||||
uses: actions/setup-go@v7
|
||||
with:
|
||||
cache: ${{ runner.environment == 'github-hosted' }}
|
||||
go-version: ^1.26
|
||||
id: go
|
||||
|
||||
- name: Check out code
|
||||
uses: actions/checkout@v7
|
||||
|
||||
- name: Configure Docker Hub mirror
|
||||
run: |
|
||||
echo '{"registry-mirrors": ["https://mirror.gcr.io"]}' | sudo tee /etc/docker/daemon.json
|
||||
sudo systemctl restart docker
|
||||
|
||||
- name: Set up Docker Buildx
|
||||
uses: docker/setup-buildx-action@v4
|
||||
|
||||
- name: Cache Docker layers
|
||||
if: runner.environment == 'github-hosted'
|
||||
uses: actions/cache@v6
|
||||
with:
|
||||
path: /tmp/.buildx-cache
|
||||
key: ${{ runner.os }}-buildx-postgres-${{ github.sha }}
|
||||
restore-keys: |
|
||||
${{ runner.os }}-buildx-postgres-
|
||||
|
||||
- name: Start PostgreSQL Gateway Services
|
||||
run: |
|
||||
make dev-start
|
||||
sleep 10
|
||||
|
||||
- name: Run Basic Connectivity Test
|
||||
run: |
|
||||
make test-basic
|
||||
|
||||
- name: Run PostgreSQL Client Tests
|
||||
run: |
|
||||
make test-client
|
||||
|
||||
- name: Save logs
|
||||
if: always()
|
||||
run: |
|
||||
docker compose logs > postgres-output.log || true
|
||||
|
||||
- name: Archive logs
|
||||
if: always()
|
||||
uses: actions/upload-artifact@v7
|
||||
with:
|
||||
name: postgres-logs
|
||||
path: test/postgres/postgres-output.log
|
||||
|
||||
- name: Cleanup
|
||||
if: always()
|
||||
run: |
|
||||
make clean || true
|
||||
@@ -1,294 +0,0 @@
|
||||
name: "release: bump version and cut the release"
|
||||
|
||||
# One entry point for a SeaweedFS release:
|
||||
# 1. bump MAJOR/MINOR in constants.go and the Helm Chart.yaml, commit to master
|
||||
# 2. push the <appVersion> tag, which fans out to the workflows that trigger on
|
||||
# `push: tags` (binaries_release*, container_release_unified, helm_manual_release)
|
||||
# 3. create the GitHub release, with GitHub's generated notes, then regroup those
|
||||
# notes by component (S3 API, Filer, Mount, ...) with
|
||||
# .github/scripts/release_notes_by_component.py
|
||||
# 4. dispatch "Prepare release" in seaweedfs-csi-driver and seaweedfs-operator,
|
||||
# which pick up the new master through `go get -u`, and wait for both
|
||||
#
|
||||
# Events raised by the default GITHUB_TOKEN do not start other workflows, and it
|
||||
# cannot reach the other two repositories at all. Add a repo secret RELEASE_PAT
|
||||
# with `contents: write` here and `actions: write` on the csi-driver and operator
|
||||
# repos. Without it the tag is still pushed, but nothing downstream of it runs.
|
||||
|
||||
on:
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
bump:
|
||||
description: "Which part to increment (ignored when 'version' is set)"
|
||||
type: choice
|
||||
options:
|
||||
- minor
|
||||
- major
|
||||
default: minor
|
||||
version:
|
||||
description: "Explicit MAJOR.MINOR to set, e.g. 4.36 (overrides 'bump')"
|
||||
type: string
|
||||
required: false
|
||||
downstream:
|
||||
description: "Also release the CSI driver and the operator"
|
||||
type: boolean
|
||||
default: true
|
||||
dry_run:
|
||||
description: "Show the version bump, but change nothing"
|
||||
type: boolean
|
||||
default: false
|
||||
|
||||
jobs:
|
||||
release:
|
||||
runs-on: ubuntu-latest
|
||||
permissions:
|
||||
contents: write
|
||||
pull-requests: read # the notes step reads a PR's files when its title has no component prefix
|
||||
outputs:
|
||||
app_version: ${{ steps.compute.outputs.app_version }}
|
||||
sha: ${{ steps.tag.outputs.sha }}
|
||||
steps:
|
||||
- uses: actions/checkout@v7
|
||||
with:
|
||||
ref: master
|
||||
fetch-depth: 0
|
||||
token: ${{ secrets.RELEASE_PAT || secrets.GITHUB_TOKEN }}
|
||||
|
||||
- name: Check the release token
|
||||
env:
|
||||
HAS_PAT: ${{ secrets.RELEASE_PAT != '' }}
|
||||
run: |
|
||||
if [ "$HAS_PAT" != "true" ]; then
|
||||
echo "::warning::RELEASE_PAT is not set. The tag will be pushed with GITHUB_TOKEN, so the binary, container and helm workflows will not start on their own."
|
||||
fi
|
||||
|
||||
- name: Compute new version
|
||||
id: compute
|
||||
env:
|
||||
BUMP: ${{ inputs.bump }}
|
||||
INPUT_VERSION: ${{ inputs.version }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
CONST=weed/util/version/constants.go
|
||||
|
||||
MAJOR=$(grep -oP 'MAJOR_VERSION\s*=\s*int32\(\K[0-9]+' "$CONST")
|
||||
MINOR=$(grep -oP 'MINOR_VERSION\s*=\s*int32\(\K[0-9]+' "$CONST")
|
||||
echo "current: ${MAJOR}.${MINOR}"
|
||||
|
||||
if [ -n "$INPUT_VERSION" ]; then
|
||||
if ! [[ "$INPUT_VERSION" =~ ^[0-9]+\.[0-9]+$ ]]; then
|
||||
echo "::error::version must be MAJOR.MINOR, e.g. 5.01 (got '$INPUT_VERSION')"
|
||||
exit 1
|
||||
fi
|
||||
# 10# forces base 10 so 08/09 are not parsed as octal.
|
||||
MAJOR=$((10#${INPUT_VERSION%%.*}))
|
||||
MINOR=$((10#${INPUT_VERSION##*.}))
|
||||
if [ "$MINOR" -gt 99 ]; then
|
||||
echo "::error::minor must be 0-99 (got $MINOR); it rolls into the next major at 99"
|
||||
exit 1
|
||||
fi
|
||||
else
|
||||
case "$BUMP" in
|
||||
# Minor is a 2-digit field: 4.99 -> 5.00 -> 5.01.
|
||||
major) MAJOR=$((MAJOR + 1)); MINOR=0 ;;
|
||||
minor)
|
||||
if [ "$MINOR" -ge 99 ]; then
|
||||
MAJOR=$((MAJOR + 1)); MINOR=0
|
||||
else
|
||||
MINOR=$((MINOR + 1))
|
||||
fi
|
||||
;;
|
||||
*) echo "::error::unknown bump '$BUMP'"; exit 1 ;;
|
||||
esac
|
||||
fi
|
||||
|
||||
# appVersion mirrors the Go VERSION_NUMBER (zero-padded minor);
|
||||
# chart version is plain SemVer (no leading zeros).
|
||||
APP_VERSION=$(printf '%d.%02d' "$MAJOR" "$MINOR")
|
||||
CHART_VERSION="${MAJOR}.${MINOR}.0"
|
||||
echo "new: app=${APP_VERSION} chart=${CHART_VERSION}"
|
||||
|
||||
{
|
||||
echo "major=${MAJOR}"
|
||||
echo "minor=${MINOR}"
|
||||
echo "app_version=${APP_VERSION}"
|
||||
echo "chart_version=${CHART_VERSION}"
|
||||
} >> "$GITHUB_OUTPUT"
|
||||
|
||||
- name: Apply version to constants.go
|
||||
env:
|
||||
MAJOR: ${{ steps.compute.outputs.major }}
|
||||
MINOR: ${{ steps.compute.outputs.minor }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
CONST=weed/util/version/constants.go
|
||||
sed -i -E "s/(MAJOR_VERSION[[:space:]]*=[[:space:]]*int32\()[0-9]+(\))/\1${MAJOR}\2/" "$CONST"
|
||||
sed -i -E "s/(MINOR_VERSION[[:space:]]*=[[:space:]]*int32\()[0-9]+(\))/\1${MINOR}\2/" "$CONST"
|
||||
grep -E 'MAJOR_VERSION|MINOR_VERSION' "$CONST"
|
||||
|
||||
- name: Apply version to Chart.yaml
|
||||
env:
|
||||
APP_VERSION: ${{ steps.compute.outputs.app_version }}
|
||||
CHART_VERSION: ${{ steps.compute.outputs.chart_version }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
CHART=k8s/charts/seaweedfs/Chart.yaml
|
||||
sed -i -E "s/^appVersion:.*/appVersion: \"${APP_VERSION}\"/" "$CHART"
|
||||
sed -i -E "s/^version:.*/version: ${CHART_VERSION}/" "$CHART"
|
||||
cat "$CHART"
|
||||
|
||||
- name: Commit, and push the tag
|
||||
id: tag
|
||||
env:
|
||||
TAG: ${{ steps.compute.outputs.app_version }}
|
||||
DRY_RUN: ${{ inputs.dry_run }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
if git ls-remote --exit-code --tags origin "refs/tags/${TAG}" >/dev/null 2>&1; then
|
||||
echo "::error::Tag ${TAG} already exists."
|
||||
exit 1
|
||||
fi
|
||||
if [ "$DRY_RUN" = "true" ]; then
|
||||
git --no-pager diff --stat
|
||||
echo "sha=$(git rev-parse HEAD)" >> "$GITHUB_OUTPUT"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
git config user.name "github-actions[bot]"
|
||||
git config user.email "41898282+github-actions[bot]@users.noreply.github.com"
|
||||
if git diff --quiet; then
|
||||
echo "::warning::Version files are already at ${TAG}; tagging the current HEAD."
|
||||
else
|
||||
git add weed/util/version/constants.go k8s/charts/seaweedfs/Chart.yaml
|
||||
git commit -m "${TAG}"
|
||||
git push
|
||||
fi
|
||||
|
||||
# Push the tag with git so the `push: tags` triggers fire. Creating the
|
||||
# tag through the release API alone would only emit a `create` event.
|
||||
git tag "$TAG"
|
||||
git push origin "$TAG"
|
||||
echo "sha=$(git rev-parse HEAD)" >> "$GITHUB_OUTPUT"
|
||||
|
||||
- name: Create the release
|
||||
if: ${{ !inputs.dry_run }}
|
||||
env:
|
||||
GH_TOKEN: ${{ secrets.RELEASE_PAT || secrets.GITHUB_TOKEN }}
|
||||
TAG: ${{ steps.compute.outputs.app_version }}
|
||||
run: gh release create "$TAG" --title "$TAG" --generate-notes --verify-tag
|
||||
|
||||
# GitHub's generated notes are one flat list of PRs. Sort them under
|
||||
# component headings by title prefix (or changed files). The release is
|
||||
# already published, so a failure here only leaves the flat list; rerun
|
||||
# the script by hand to fix it:
|
||||
# python3 .github/scripts/release_notes_by_component.py --tag <tag> --write
|
||||
- name: Group the release notes by component
|
||||
if: ${{ !inputs.dry_run }}
|
||||
continue-on-error: true
|
||||
env:
|
||||
GH_TOKEN: ${{ secrets.RELEASE_PAT || secrets.GITHUB_TOKEN }}
|
||||
TAG: ${{ steps.compute.outputs.app_version }}
|
||||
run: python3 .github/scripts/release_notes_by_component.py --repo "$GITHUB_REPOSITORY" --tag "$TAG" --write > /dev/null
|
||||
|
||||
downstream:
|
||||
needs: release
|
||||
if: ${{ inputs.downstream && !inputs.dry_run }}
|
||||
runs-on: ubuntu-latest
|
||||
# The wait below puts no bound of its own on runner-queue time; this does.
|
||||
timeout-minutes: 120
|
||||
permissions: {}
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
include:
|
||||
- repo: seaweedfs/seaweedfs-csi-driver
|
||||
workflow: prepare_release.yaml
|
||||
- repo: seaweedfs/seaweedfs-operator
|
||||
workflow: prepare_release.yml
|
||||
steps:
|
||||
- name: Release ${{ matrix.repo }}
|
||||
env:
|
||||
GH_TOKEN: ${{ secrets.RELEASE_PAT }}
|
||||
REPO: ${{ matrix.repo }}
|
||||
WORKFLOW: ${{ matrix.workflow }}
|
||||
SHA: ${{ needs.release.outputs.sha }}
|
||||
MODULE: github.com/seaweedfs/seaweedfs
|
||||
run: |
|
||||
set -euo pipefail
|
||||
if [ -z "${GH_TOKEN}" ]; then
|
||||
echo "::error::RELEASE_PAT with actions:write on ${REPO} is required to release it"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# The dispatched workflow pins seaweedfs with `go get -u ...@latest`, so
|
||||
# wait until the proxy serves the release commit as the tip. Asking for
|
||||
# the commit by name is what makes the proxy fetch it. The proxy can
|
||||
# take longer than five minutes to refresh @latest after a new tag.
|
||||
for _ in $(seq 120); do
|
||||
curl -sf "https://proxy.golang.org/${MODULE}/@v/${SHA}.info" >/dev/null || true
|
||||
TIP=$(curl -sf "https://proxy.golang.org/${MODULE}/@latest" | jq -r '.Origin.Hash // ""' || true)
|
||||
[ "$TIP" = "$SHA" ] && break
|
||||
sleep 10
|
||||
done
|
||||
if [ "$TIP" != "$SHA" ]; then
|
||||
echo "::error::the module proxy still serves ${TIP} as the tip, so ${REPO} would pin a pre-release commit. Run ${WORKFLOW} there once it catches up."
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# The dispatched workflow publishes the release as its last step, so
|
||||
# its conclusion decides success. Wait on the run, not on a wall
|
||||
# clock: time it spends queued for a runner must not count against
|
||||
# the budget. A dispatch does not return its run id, so take the
|
||||
# newest workflow_dispatch run created since ours; a concurrent
|
||||
# dispatch would be performing this same release, and waiting on it
|
||||
# is just as good.
|
||||
released() { gh api "repos/${REPO}/releases?per_page=30" --jq '[.[].tag_name]'; }
|
||||
|
||||
BEFORE=$(released)
|
||||
# A minute early, so runner clock skew cannot hide the run.
|
||||
DISPATCHED_AT=$(date -u -d '1 minute ago' '+%Y-%m-%dT%H:%M:%SZ')
|
||||
gh workflow run -R "$REPO" "$WORKFLOW" --ref master -f bump=patch -f update_seaweedfs=true
|
||||
|
||||
RUN_ID=""
|
||||
for _ in $(seq 12); do
|
||||
sleep 10
|
||||
RUN_ID=$(gh api -X GET "repos/${REPO}/actions/workflows/${WORKFLOW}/runs" \
|
||||
-f event=workflow_dispatch -f "created=>=${DISPATCHED_AT}" \
|
||||
--jq '(.workflow_runs | sort_by(.created_at) | last | .id) // empty' || true)
|
||||
[ -n "$RUN_ID" ] && break
|
||||
done
|
||||
if [ -z "$RUN_ID" ]; then
|
||||
echo "::error::the dispatch created no ${WORKFLOW} run in ${REPO}; see https://github.com/${REPO}/actions/workflows/${WORKFLOW}"
|
||||
exit 1
|
||||
fi
|
||||
RUN_URL="https://github.com/${REPO}/actions/runs/${RUN_ID}"
|
||||
echo "waiting on ${RUN_URL}"
|
||||
|
||||
# 20 minutes of execution; polls that find the run still queued do
|
||||
# not consume it.
|
||||
RUNNING=0
|
||||
STATE=""
|
||||
while :; do
|
||||
sleep 15
|
||||
STATE=$(gh api "repos/${REPO}/actions/runs/${RUN_ID}" \
|
||||
--jq '.status + "/" + (.conclusion // "")' || true)
|
||||
case "$STATE" in
|
||||
completed/*) break ;;
|
||||
in_progress/*) RUNNING=$((RUNNING + 1)) ;;
|
||||
esac
|
||||
if [ "$RUNNING" -gt 80 ]; then
|
||||
echo "::error::${RUN_URL} has been executing for over 20 minutes; giving up on it"
|
||||
exit 1
|
||||
fi
|
||||
done
|
||||
if [ "$STATE" != "completed/success" ]; then
|
||||
echo "::error::${RUN_URL} concluded '${STATE#completed/}'"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
NEW=$(released | jq -c --argjson before "$BEFORE" '. - $before')
|
||||
if [ "$(jq length <<<"$NEW")" -eq 0 ]; then
|
||||
echo "::error::${RUN_URL} succeeded but ${REPO} shows no new release"
|
||||
exit 1
|
||||
fi
|
||||
echo "${REPO} released $(jq -r 'join(", ")' <<<"$NEW")"
|
||||
@@ -1,135 +0,0 @@
|
||||
name: "rust: vector desktop platforms"
|
||||
on:
|
||||
pull_request:
|
||||
paths:
|
||||
- 'seaweed-vector/**'
|
||||
- 'seaweed-worker/**'
|
||||
- 'seaweed-common/**'
|
||||
- 'weed/pb/**'
|
||||
- '.github/workflows/rust-vector-platforms.yml'
|
||||
- 'install.sh'
|
||||
- 'test/installer/**'
|
||||
push:
|
||||
tags: ['*']
|
||||
workflow_dispatch:
|
||||
permissions:
|
||||
contents: read
|
||||
concurrency:
|
||||
group: vector-platforms-${{ github.event.pull_request.number || github.ref }}
|
||||
cancel-in-progress: ${{ github.event_name == 'pull_request' }}
|
||||
jobs:
|
||||
vector:
|
||||
permissions:
|
||||
contents: read
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
include:
|
||||
- runner: macos-15-intel
|
||||
suffix: darwin_amd64
|
||||
protoc: macos-x86_64
|
||||
- runner: macos-latest
|
||||
suffix: darwin_arm64
|
||||
protoc: macos-aarch_64
|
||||
- runner: windows-latest
|
||||
suffix: windows_amd64
|
||||
protoc: win32
|
||||
runs-on: ${{ matrix.runner }}
|
||||
timeout-minutes: 120
|
||||
env:
|
||||
CARGO_BUILD_JOBS: '2'
|
||||
CARGO_PROFILE_TEST_DEBUG: '0'
|
||||
SEAWEEDFS_COMMIT: ${{ github.sha }}
|
||||
defaults:
|
||||
run:
|
||||
shell: bash
|
||||
steps:
|
||||
- uses: actions/checkout@v7
|
||||
with:
|
||||
persist-credentials: false
|
||||
- uses: dtolnay/rust-toolchain@stable
|
||||
- uses: actions/setup-python@v7
|
||||
with:
|
||||
python-version: '3.13'
|
||||
- uses: actions/cache/restore@v6
|
||||
id: cargo-cache
|
||||
if: ${{ !startsWith(github.ref, 'refs/tags/') }}
|
||||
with:
|
||||
path: |
|
||||
~/.cargo/registry
|
||||
~/.cargo/git
|
||||
seaweed-vector/target
|
||||
key: vector-${{ matrix.suffix }}-${{ hashFiles('seaweed-vector/Cargo.lock') }}
|
||||
restore-keys: vector-${{ matrix.suffix }}-
|
||||
- name: Fetch dependencies
|
||||
run: cargo fetch --locked --manifest-path seaweed-vector/Cargo.toml
|
||||
- name: Select vendored protoc
|
||||
env:
|
||||
PROTOC_PACKAGE: protoc-bin-vendored-${{ matrix.protoc }}
|
||||
run: |
|
||||
python - <<'PY'
|
||||
import os, pathlib, tomllib
|
||||
lock=tomllib.loads(pathlib.Path('seaweed-vector/Cargo.lock').read_text())
|
||||
name=os.environ['PROTOC_PACKAGE']
|
||||
version=next(p['version'] for p in lock['package'] if p['name']==name)
|
||||
cargo=pathlib.Path(os.environ.get('CARGO_HOME',pathlib.Path.home()/'.cargo'))
|
||||
executable='protoc.exe' if os.name=='nt' else 'protoc'
|
||||
path=next(cargo.glob(f'registry/src/*/{name}-{version}/bin/{executable}'))
|
||||
with open(os.environ['GITHUB_ENV'],'a') as out: out.write(f'PROTOC={path}\n')
|
||||
PY
|
||||
- name: Build vector engine
|
||||
id: cargo-build
|
||||
run: cargo build --locked --release --manifest-path seaweed-vector/Cargo.toml -p weed-vector-server
|
||||
- name: Test vector crates
|
||||
run: cargo test --locked --release --manifest-path seaweed-vector/Cargo.toml --workspace
|
||||
- name: Test installer
|
||||
run: |
|
||||
if [ "$RUNNER_OS" = "Windows" ]; then
|
||||
BASH_BINARY="$(cygpath -w "$BASH")"
|
||||
export BASH_BINARY
|
||||
fi
|
||||
python -m unittest discover -s test/installer
|
||||
- name: Save compiled dependencies
|
||||
if: ${{ always() && !startsWith(github.ref, 'refs/tags/') && steps.cargo-build.outcome == 'success' && steps.cargo-cache.outputs.cache-hit != 'true' }}
|
||||
uses: actions/cache/save@v6
|
||||
with:
|
||||
path: |
|
||||
~/.cargo/registry
|
||||
~/.cargo/git
|
||||
seaweed-vector/target
|
||||
key: vector-${{ matrix.suffix }}-${{ hashFiles('seaweed-vector/Cargo.lock') }}
|
||||
- name: Package engine
|
||||
env:
|
||||
ASSET_SUFFIX: ${{ matrix.suffix }}
|
||||
run: |
|
||||
python - <<'PY'
|
||||
import hashlib, os, pathlib, tarfile, zipfile
|
||||
suffix=os.environ['ASSET_SUFFIX']
|
||||
binary='weed-vector.exe' if os.name=='nt' else 'weed-vector'
|
||||
source=pathlib.Path('seaweed-vector/target/release')/binary
|
||||
asset=f'weed-vector_{suffix}'+('.zip' if os.name=='nt' else '.tar.gz')
|
||||
if os.name=='nt':
|
||||
with zipfile.ZipFile(asset,'w',zipfile.ZIP_DEFLATED) as archive: archive.write(source,binary)
|
||||
else:
|
||||
with tarfile.open(asset,'w:gz') as archive: archive.add(source,arcname=binary)
|
||||
digest=hashlib.md5(pathlib.Path(asset).read_bytes()).hexdigest()
|
||||
pathlib.Path(asset+'.md5').write_text(f'{digest} {asset}\n')
|
||||
PY
|
||||
- uses: actions/upload-artifact@v7
|
||||
with:
|
||||
name: vector-${{ matrix.suffix }}
|
||||
path: weed-vector_${{ matrix.suffix }}.*
|
||||
release:
|
||||
if: startsWith(github.ref, 'refs/tags/')
|
||||
needs: vector
|
||||
runs-on: ubuntu-latest
|
||||
permissions:
|
||||
contents: write
|
||||
steps:
|
||||
- uses: actions/download-artifact@v8
|
||||
with:
|
||||
pattern: vector-*
|
||||
merge-multiple: true
|
||||
- uses: softprops/action-gh-release@v3
|
||||
with:
|
||||
files: weed-vector_*
|
||||
@@ -1,181 +0,0 @@
|
||||
name: "Rust Vector Tests"
|
||||
|
||||
on:
|
||||
pull_request:
|
||||
branches: [ master ]
|
||||
paths:
|
||||
- 'seaweed-vector/**'
|
||||
- 'seaweed-volume/**'
|
||||
- 'seaweed-worker/**'
|
||||
- 'seaweed-common/**'
|
||||
- 'weed/pb/plugin.proto'
|
||||
- 'weed/pb/vector.proto'
|
||||
- 'weed/pb/vector_pb/**'
|
||||
- 'weed/s3api/**'
|
||||
- 'weed/command/**'
|
||||
- 'test/s3vectors/**'
|
||||
- '.github/workflows/rust-vector-tests.yml'
|
||||
push:
|
||||
branches: [ master, main ]
|
||||
paths:
|
||||
- 'seaweed-vector/**'
|
||||
- 'seaweed-volume/**'
|
||||
- 'seaweed-worker/**'
|
||||
- 'seaweed-common/**'
|
||||
- 'weed/pb/plugin.proto'
|
||||
- 'weed/pb/vector.proto'
|
||||
- 'weed/pb/vector_pb/**'
|
||||
- 'weed/s3api/**'
|
||||
- 'weed/command/**'
|
||||
- 'test/s3vectors/**'
|
||||
- '.github/workflows/rust-vector-tests.yml'
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.event.pull_request.number || (github.event_name == 'push' && github.ref) || github.run_id }}
|
||||
cancel-in-progress: ${{ github.event_name == 'pull_request' || (github.event_name == 'push' && !startsWith(github.ref, 'refs/tags/')) }}
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
jobs:
|
||||
rust-vector-build:
|
||||
name: Rust Vector Build and Unit Tests
|
||||
runs-on: ubuntu-24.04
|
||||
timeout-minutes: 45
|
||||
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v7
|
||||
with:
|
||||
persist-credentials: false
|
||||
|
||||
- name: Install Rust toolchain
|
||||
uses: dtolnay/rust-toolchain@stable
|
||||
|
||||
- name: Fingerprint build toolchain
|
||||
id: toolchain
|
||||
run: echo "fingerprint=$(getconf GNU_LIBC_VERSION | tr ' ' '-')-rustc-$(rustc -V | awk '{print $2}')" >> "$GITHUB_OUTPUT"
|
||||
|
||||
- name: Cache cargo registry and target
|
||||
if: runner.environment == 'github-hosted'
|
||||
uses: actions/cache@v6
|
||||
with:
|
||||
path: |
|
||||
~/.cargo/registry
|
||||
~/.cargo/git
|
||||
seaweed-vector/target/release
|
||||
key: rust-vector-${{ steps.toolchain.outputs.fingerprint }}-${{ hashFiles('seaweed-vector/Cargo.lock') }}
|
||||
restore-keys: |
|
||||
rust-vector-${{ steps.toolchain.outputs.fingerprint }}-
|
||||
|
||||
- name: Build the vector crates
|
||||
run: cd seaweed-vector && cargo build --release
|
||||
|
||||
- name: Run unit tests
|
||||
run: cd seaweed-vector && cargo test --release --workspace
|
||||
|
||||
|
||||
vectors-e2e:
|
||||
name: S3 Vectors Integration (${{ matrix.volume }} volume${{ matrix.hardened && ', hardened' || '' }}${{ matrix.worker && ', worker' || '' }})
|
||||
runs-on: ubuntu-24.04
|
||||
timeout-minutes: 45
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
volume: [go, rust]
|
||||
# Hardened: volume servers behind mutual TLS, every gRPC hop (master,
|
||||
# volume, filer, gateway, engine) under mutual TLS, read JWTs
|
||||
# required, and every chunk encrypted by the gateway.
|
||||
hardened: ["", "1"]
|
||||
worker: ["", "1"]
|
||||
# Worker: the admin server and the Lance maintenance worker join the
|
||||
# cluster, for the tests that build vector indexes through a job.
|
||||
|
||||
env:
|
||||
CARGO_PROFILE_DEV_DEBUG: "0"
|
||||
CARGO_BUILD_JOBS: "2"
|
||||
CARGO_TARGET_DIR: ${{ github.workspace }}/seaweed-vector/target
|
||||
VOLUME_SERVER_IMPL: ${{ matrix.volume }}
|
||||
VECTOR_QUERY_MODE: single
|
||||
VECTOR_FILER_GROUP: vectors-ci
|
||||
VECTOR_TLS: ${{ matrix.hardened }}
|
||||
VECTOR_READ_JWT: ${{ matrix.hardened }}
|
||||
VECTOR_ENCRYPT_VOLUMES: ${{ matrix.hardened }}
|
||||
VECTOR_WORKER: ${{ matrix.worker }}
|
||||
steps:
|
||||
- uses: actions/checkout@v7
|
||||
with:
|
||||
persist-credentials: false
|
||||
- uses: actions/setup-go@v7
|
||||
with:
|
||||
go-version-file: go.mod
|
||||
cache: ${{ runner.environment == 'github-hosted' }}
|
||||
- uses: dtolnay/rust-toolchain@stable
|
||||
- name: Fingerprint build toolchain
|
||||
id: toolchain
|
||||
run: echo "fingerprint=$(getconf GNU_LIBC_VERSION | tr ' ' '-')-rustc-$(rustc -V | awk '{print $2}')" >> "$GITHUB_OUTPUT"
|
||||
- name: Cache cargo registry and target
|
||||
if: runner.environment == 'github-hosted'
|
||||
uses: actions/cache@v6
|
||||
with:
|
||||
path: |
|
||||
~/.cargo/registry
|
||||
~/.cargo/git
|
||||
seaweed-vector/target/debug
|
||||
key: vectors-e2e-${{ runner.os }}-${{ runner.arch }}-${{ steps.toolchain.outputs.fingerprint }}-${{ hashFiles('seaweed-vector/Cargo.lock', 'seaweed-volume/Cargo.lock') }}
|
||||
restore-keys: |
|
||||
vectors-e2e-${{ runner.os }}-${{ runner.arch }}-${{ steps.toolchain.outputs.fingerprint }}-
|
||||
- name: Test Go vector gateway
|
||||
run: |
|
||||
go test ./weed/s3api/s3vectors -count=1
|
||||
go test ./weed/command -run 'Vector|Vectors' -count=1
|
||||
# The worker workspace has no protoc of its own for lance's build
|
||||
# scripts; use the vendored one its lock names, as its own workflow does.
|
||||
- name: Use the vendored protoc for the worker
|
||||
if: matrix.worker == '1'
|
||||
run: |
|
||||
cd seaweed-worker
|
||||
cargo fetch --locked
|
||||
version=$(awk '/^name = "protoc-bin-vendored-linux-x86_64"$/{found=1; next} found && /^version = /{gsub(/"/,"",$3); print $3; exit}' Cargo.lock)
|
||||
test -n "$version" || { echo "protoc-bin-vendored-linux-x86_64 is not in Cargo.lock" >&2; exit 1; }
|
||||
protoc=$(find ~/.cargo/registry/src -path "*protoc-bin-vendored-linux-x86_64-$version/bin/protoc" | head -1)
|
||||
test -x "$protoc" || { echo "no vendored protoc $version in the registry" >&2; exit 1; }
|
||||
echo "PROTOC=$protoc" >> "$GITHUB_ENV"
|
||||
- name: Build integration binaries
|
||||
working-directory: test/s3vectors
|
||||
# dev profile: correctness suite, no release codegen needed. weed-volume
|
||||
# (the heaviest crate) only builds on the rust leg.
|
||||
run: |
|
||||
set -e
|
||||
make ./weed ./weed-vector CARGO_PROFILE=dev
|
||||
if [ "${{ matrix.volume }}" = "rust" ]; then make ./weed-volume CARGO_PROFILE=dev; fi
|
||||
if [ "${{ matrix.worker }}" = "1" ]; then make ./weed-vector-cli ./weed-worker CARGO_PROFILE=dev; fi
|
||||
- uses: actions/setup-python@v7
|
||||
with:
|
||||
python-version: "3.12"
|
||||
- name: Install test dependencies
|
||||
run: python -m pip install -r test/s3vectors/requirements.txt
|
||||
- name: Run the S3 Vectors integration matrix
|
||||
working-directory: test/s3vectors
|
||||
run: |
|
||||
./run_test.sh --basetemp="$RUNNER_TEMP/vector-tests" --junitxml="$RUNNER_TEMP/vector-results.xml"
|
||||
python - <<'PYTHON'
|
||||
import os
|
||||
from pathlib import Path
|
||||
import xml.etree.ElementTree as ET
|
||||
cases = ET.parse(Path(os.environ["RUNNER_TEMP"]) / "vector-results.xml").findall(".//testcase")
|
||||
assert len(cases) >= 17, f"expected at least 17 cases, got {len(cases)}"
|
||||
assert not any(case.find("skipped") is not None for case in cases), "integration cases were skipped"
|
||||
volume = os.environ["VOLUME_SERVER_IMPL"]
|
||||
assert all(case.get("name", "").endswith(f"[{volume}-single]") for case in cases), volume
|
||||
PYTHON
|
||||
- name: Upload integration diagnostics
|
||||
if: always()
|
||||
uses: actions/upload-artifact@v7
|
||||
with:
|
||||
name: vectors-e2e-diagnostics-${{ matrix.volume }}${{ matrix.hardened && '-hardened' || '' }}${{ matrix.worker && '-worker' || '' }}
|
||||
path: |
|
||||
${{ runner.temp }}/vector-tests
|
||||
${{ runner.temp }}/vector-results.xml
|
||||
if-no-files-found: ignore
|
||||
retention-days: 7
|
||||
@@ -1,345 +0,0 @@
|
||||
name: "Rust Volume Server Tests"
|
||||
|
||||
on:
|
||||
pull_request:
|
||||
branches: [ master ]
|
||||
paths:
|
||||
- 'seaweed-volume/**'
|
||||
- 'seaweed-common/**'
|
||||
- 'test/volume_server/**'
|
||||
- 'weed/pb/volume_server.proto'
|
||||
- 'weed/pb/volume_server_pb/**'
|
||||
- '.github/workflows/rust-volume-server-tests.yml'
|
||||
push:
|
||||
branches: [ master, main ]
|
||||
paths:
|
||||
- 'seaweed-volume/**'
|
||||
- 'seaweed-common/**'
|
||||
- 'test/volume_server/**'
|
||||
- 'weed/pb/volume_server.proto'
|
||||
- 'weed/pb/volume_server_pb/**'
|
||||
- '.github/workflows/rust-volume-server-tests.yml'
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.event.pull_request.number || (github.event_name == 'push' && github.ref) || github.run_id }}
|
||||
cancel-in-progress: ${{ github.event_name == 'pull_request' || (github.event_name == 'push' && !startsWith(github.ref, 'refs/tags/')) }}
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
|
||||
jobs:
|
||||
changes:
|
||||
name: Detect changed paths
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 5
|
||||
permissions:
|
||||
contents: read
|
||||
outputs:
|
||||
rust: ${{ steps.filter.outputs.rust }}
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v7
|
||||
with:
|
||||
fetch-depth: 0
|
||||
|
||||
- name: Filter changed paths
|
||||
id: filter
|
||||
uses: dorny/paths-filter@v4
|
||||
with:
|
||||
filters: |
|
||||
rust:
|
||||
- 'seaweed-volume/**'
|
||||
- '.github/workflows/rust-volume-server-tests.yml'
|
||||
|
||||
build-test-binaries:
|
||||
name: Build Test Binaries
|
||||
runs-on: ubuntu-24.04
|
||||
timeout-minutes: 30
|
||||
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v7
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v7
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
cache: ${{ runner.environment == 'github-hosted' }}
|
||||
|
||||
- name: Install Rust toolchain
|
||||
uses: dtolnay/rust-toolchain@stable
|
||||
|
||||
# cargo tracks its own inputs but not the runner's C toolchain, so a cached
|
||||
# target/ can carry C objects built against a different glibc than we link against.
|
||||
- name: Fingerprint build toolchain
|
||||
id: toolchain
|
||||
run: echo "fingerprint=$(getconf GNU_LIBC_VERSION | tr ' ' '-')-rustc-$(rustc -V | awk '{print $2}')" >> "$GITHUB_OUTPUT"
|
||||
|
||||
- name: Cache cargo registry and target
|
||||
if: runner.environment == 'github-hosted'
|
||||
uses: actions/cache@v6
|
||||
with:
|
||||
path: |
|
||||
~/.cargo/registry
|
||||
~/.cargo/git
|
||||
seaweed-volume/target
|
||||
key: rust-${{ steps.toolchain.outputs.fingerprint }}-${{ hashFiles('seaweed-volume/Cargo.lock', 'seaweed-common/Cargo.lock') }}
|
||||
restore-keys: |
|
||||
rust-${{ steps.toolchain.outputs.fingerprint }}-
|
||||
|
||||
- name: Build Go weed binary
|
||||
run: |
|
||||
cd weed
|
||||
go build -tags 5BytesOffset -o weed .
|
||||
chmod +x weed
|
||||
./weed version
|
||||
|
||||
- name: Build Rust volume binary
|
||||
run: cd seaweed-volume && cargo build --release
|
||||
|
||||
- name: Package binaries
|
||||
run: tar cf rust-test-binaries.tar weed/weed seaweed-volume/target/release/weed-volume
|
||||
|
||||
- name: Upload test binaries
|
||||
uses: actions/upload-artifact@v7
|
||||
with:
|
||||
# archive layout mirrors the workspace: weed/weed and
|
||||
# seaweed-volume/target/release/weed-volume
|
||||
name: rust-test-binaries
|
||||
if-no-files-found: error
|
||||
retention-days: 1
|
||||
path: rust-test-binaries.tar
|
||||
|
||||
rust-unit-tests:
|
||||
name: Rust Unit Tests
|
||||
runs-on: ubuntu-24.04
|
||||
timeout-minutes: 15
|
||||
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v7
|
||||
|
||||
- name: Install Rust toolchain
|
||||
uses: dtolnay/rust-toolchain@stable
|
||||
|
||||
# cargo tracks its own inputs but not the runner's C toolchain, so a cached
|
||||
# target/ can carry C objects built against a different glibc than we link against.
|
||||
- name: Fingerprint build toolchain
|
||||
id: toolchain
|
||||
run: echo "fingerprint=$(getconf GNU_LIBC_VERSION | tr ' ' '-')-rustc-$(rustc -V | awk '{print $2}')" >> "$GITHUB_OUTPUT"
|
||||
|
||||
- name: Cache cargo registry and target
|
||||
if: runner.environment == 'github-hosted'
|
||||
uses: actions/cache@v6
|
||||
with:
|
||||
path: |
|
||||
~/.cargo/registry
|
||||
~/.cargo/git
|
||||
seaweed-volume/target
|
||||
key: rust-${{ steps.toolchain.outputs.fingerprint }}-${{ hashFiles('seaweed-volume/Cargo.lock', 'seaweed-common/Cargo.lock') }}
|
||||
restore-keys: |
|
||||
rust-${{ steps.toolchain.outputs.fingerprint }}-
|
||||
|
||||
- name: Build Rust volume server
|
||||
run: cd seaweed-volume && cargo build --release
|
||||
|
||||
# The crate is warning-free under clippy as of the sweep that added
|
||||
# this step. Uncomment to make that a gate; `[lints.clippy]` in
|
||||
# seaweed-volume/Cargo.toml is where crate-wide exceptions live.
|
||||
# - name: Clippy
|
||||
# run: cd seaweed-volume && cargo clippy --all-targets -- -D warnings
|
||||
|
||||
# The crate is rustfmt-clean as of the PR that added this step.
|
||||
# Uncomment to keep it that way.
|
||||
# - name: Check formatting
|
||||
# run: cd seaweed-volume && cargo fmt --check
|
||||
|
||||
# seaweed-common is a path dependency of this crate, not a member of its
|
||||
# workspace, so the run below does not reach its own tests. It builds into
|
||||
# this job's cached target directory, and the cache key above covers the
|
||||
# shared crate's lock, so the aws-lc-sys that rustls pulls in is restored
|
||||
# with the cache instead of compiled from scratch on every run.
|
||||
- name: Run shared-crate unit tests
|
||||
env:
|
||||
CARGO_TARGET_DIR: ${{ github.workspace }}/seaweed-volume/target
|
||||
run: cd seaweed-common && cargo test
|
||||
|
||||
- name: Run Rust unit tests
|
||||
run: cd seaweed-volume && cargo test
|
||||
|
||||
rust-unit-tests-windows:
|
||||
name: Rust Unit Tests (Windows)
|
||||
runs-on: windows-latest
|
||||
timeout-minutes: 30
|
||||
needs: [changes]
|
||||
if: needs.changes.outputs.rust == 'true'
|
||||
defaults:
|
||||
run:
|
||||
shell: bash
|
||||
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v7
|
||||
|
||||
- name: Install Rust toolchain
|
||||
uses: dtolnay/rust-toolchain@stable
|
||||
|
||||
# No glibc on Windows: key the cache on the toolchain and OS only.
|
||||
- name: Fingerprint build toolchain
|
||||
id: toolchain
|
||||
run: echo "fingerprint=windows-rustc-$(rustc -V | awk '{print $2}')" >> "$GITHUB_OUTPUT"
|
||||
|
||||
- name: Cache cargo registry and target
|
||||
if: runner.environment == 'github-hosted'
|
||||
uses: actions/cache@v6
|
||||
with:
|
||||
path: |
|
||||
~/.cargo/registry
|
||||
~/.cargo/git
|
||||
seaweed-volume/target
|
||||
key: rust-windows-${{ steps.toolchain.outputs.fingerprint }}-${{ hashFiles('seaweed-volume/Cargo.lock') }}
|
||||
restore-keys: |
|
||||
rust-windows-${{ steps.toolchain.outputs.fingerprint }}-
|
||||
|
||||
- name: Run Rust unit tests
|
||||
run: cd seaweed-volume && cargo test
|
||||
|
||||
rust-integration-tests:
|
||||
name: Rust Integration Tests
|
||||
runs-on: ubuntu-24.04
|
||||
timeout-minutes: 30
|
||||
needs: [build-test-binaries]
|
||||
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v7
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v7
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
cache: ${{ runner.environment == 'github-hosted' }}
|
||||
|
||||
- name: Download test binaries
|
||||
uses: actions/download-artifact@v8
|
||||
with:
|
||||
name: rust-test-binaries
|
||||
|
||||
- name: Restore binaries
|
||||
run: |
|
||||
tar xf rust-test-binaries.tar
|
||||
chmod +x weed/weed seaweed-volume/target/release/weed-volume
|
||||
./weed/weed version
|
||||
|
||||
- name: Run integration tests
|
||||
env:
|
||||
WEED_BINARY: ${{ github.workspace }}/weed/weed
|
||||
RUST_VOLUME_BINARY: ${{ github.workspace }}/seaweed-volume/target/release/weed-volume
|
||||
run: |
|
||||
echo "Running Rust volume server integration tests..."
|
||||
go test -v -count=1 -timeout=15m ./test/volume_server/rust/...
|
||||
|
||||
- name: Collect logs on failure
|
||||
if: failure()
|
||||
run: |
|
||||
mkdir -p /tmp/rust-volume-server-it-logs
|
||||
find /tmp -maxdepth 1 -type d -name "seaweedfs_volume_server_it_*" -print -exec cp -r {} /tmp/rust-volume-server-it-logs/ \; || true
|
||||
|
||||
- name: Archive logs on failure
|
||||
if: failure()
|
||||
uses: actions/upload-artifact@v7
|
||||
with:
|
||||
name: rust-volume-server-integration-test-logs
|
||||
path: /tmp/rust-volume-server-it-logs/
|
||||
if-no-files-found: warn
|
||||
retention-days: 7
|
||||
|
||||
- name: Test summary
|
||||
if: always()
|
||||
run: |
|
||||
echo "## Rust Volume Server Integration Test Summary" >> "$GITHUB_STEP_SUMMARY"
|
||||
echo "- Suite: test/volume_server/rust" >> "$GITHUB_STEP_SUMMARY"
|
||||
echo "- Command: go test -v -count=1 -timeout=15m ./test/volume_server/rust/..." >> "$GITHUB_STEP_SUMMARY"
|
||||
|
||||
rust-volume-go-tests:
|
||||
name: Go Tests with Rust Volume (${{ matrix.test-type }} - Shard ${{ matrix.shard }})
|
||||
runs-on: ubuntu-24.04
|
||||
timeout-minutes: 45
|
||||
needs: [build-test-binaries]
|
||||
env:
|
||||
# Keep in step with the length of matrix.shard below.
|
||||
SHARD_COUNT: 3
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
test-type: [grpc, http]
|
||||
shard: [1, 2, 3]
|
||||
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v7
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v7
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
cache: ${{ runner.environment == 'github-hosted' }}
|
||||
|
||||
- name: Download test binaries
|
||||
uses: actions/download-artifact@v8
|
||||
with:
|
||||
name: rust-test-binaries
|
||||
|
||||
- name: Restore binaries
|
||||
run: |
|
||||
tar xf rust-test-binaries.tar
|
||||
chmod +x weed/weed seaweed-volume/target/release/weed-volume
|
||||
./weed/weed version
|
||||
./seaweed-volume/target/release/weed-volume --help >/dev/null || true
|
||||
|
||||
# Dealing the listed tests out one by one keeps the shards even. Bucketing
|
||||
# them by first letter did not: names cluster, so ^Test[I-S] drew 50 of
|
||||
# the 114 grpc tests and ran nearly twice as long as the other two shards.
|
||||
- name: Select this shard's tests
|
||||
env:
|
||||
TEST_TYPE: ${{ matrix.test-type }}
|
||||
SHARD: ${{ matrix.shard }}
|
||||
run: |
|
||||
tests=$(go test -tags 5BytesOffset ./test/volume_server/"$TEST_TYPE"/... -list '.*' | grep '^Test' | sort -u)
|
||||
# An empty list would make -run match nothing and the shard pass vacuously.
|
||||
[ -n "$tests" ] || { echo "listed no tests in test/volume_server/$TEST_TYPE"; exit 1; }
|
||||
selected=$(echo "$tests" | awk -v n="$SHARD_COUNT" -v i="$SHARD" 'NR % n == i - 1')
|
||||
echo "shard $SHARD of $SHARD_COUNT runs $(echo "$selected" | wc -l) of $(echo "$tests" | wc -l) tests"
|
||||
echo "TEST_PATTERN=^($(echo "$selected" | paste -sd'|' -))\$" >> "$GITHUB_ENV"
|
||||
|
||||
- name: Run volume server integration tests with Rust volume
|
||||
env:
|
||||
WEED_BINARY: ${{ github.workspace }}/weed/weed
|
||||
RUST_VOLUME_BINARY: ${{ github.workspace }}/seaweed-volume/target/release/weed-volume
|
||||
VOLUME_SERVER_IMPL: rust
|
||||
run: |
|
||||
echo "Running Go volume server tests with Rust volume for ${{ matrix.test-type }} (Shard ${{ matrix.shard }} of ${SHARD_COUNT})..."
|
||||
go test -v -count=1 -tags 5BytesOffset -timeout=30m ./test/volume_server/${{ matrix.test-type }}/... -run "${TEST_PATTERN}"
|
||||
|
||||
- name: Collect logs on failure
|
||||
if: failure()
|
||||
run: |
|
||||
mkdir -p /tmp/rust-volume-go-test-logs
|
||||
find /tmp -maxdepth 1 -type d -name "seaweedfs_volume_server_it_*" -print -exec cp -r {} /tmp/rust-volume-go-test-logs/ \; || true
|
||||
|
||||
- name: Archive logs on failure
|
||||
if: failure()
|
||||
uses: actions/upload-artifact@v7
|
||||
with:
|
||||
name: rust-volume-go-test-logs-${{ matrix.test-type }}-shard${{ matrix.shard }}
|
||||
path: /tmp/rust-volume-go-test-logs/
|
||||
if-no-files-found: warn
|
||||
retention-days: 7
|
||||
|
||||
- name: Test summary
|
||||
if: always()
|
||||
run: |
|
||||
echo "## Rust Volume - Go Test Summary (${{ matrix.test-type }} - Shard ${{ matrix.shard }})" >> "$GITHUB_STEP_SUMMARY"
|
||||
echo "- Suite: test/volume_server/${{ matrix.test-type }} (shard ${{ matrix.shard }} of ${SHARD_COUNT}, see 'Select this shard's tests' for the split)" >> "$GITHUB_STEP_SUMMARY"
|
||||
echo "- Volume server: Rust (VOLUME_SERVER_IMPL=rust)" >> "$GITHUB_STEP_SUMMARY"
|
||||
@@ -1,107 +0,0 @@
|
||||
name: "Rust Plugin Worker Tests"
|
||||
|
||||
on:
|
||||
pull_request:
|
||||
branches: [ master ]
|
||||
paths:
|
||||
- 'seaweed-worker/**'
|
||||
- 'seaweed-common/**'
|
||||
- 'weed/pb/plugin.proto'
|
||||
- '.github/workflows/rust-worker-tests.yml'
|
||||
push:
|
||||
branches: [ master, main ]
|
||||
paths:
|
||||
- 'seaweed-worker/**'
|
||||
- 'seaweed-common/**'
|
||||
- 'weed/pb/plugin.proto'
|
||||
- '.github/workflows/rust-worker-tests.yml'
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.event.pull_request.number || (github.event_name == 'push' && github.ref) || github.run_id }}
|
||||
cancel-in-progress: ${{ github.event_name == 'pull_request' || (github.event_name == 'push' && !startsWith(github.ref, 'refs/tags/')) }}
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
jobs:
|
||||
rust-worker-build:
|
||||
name: Rust Plugin Worker Build and Unit Tests
|
||||
runs-on: ubuntu-24.04
|
||||
timeout-minutes: 45
|
||||
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v7
|
||||
with:
|
||||
persist-credentials: false
|
||||
|
||||
- name: Install Rust toolchain
|
||||
uses: dtolnay/rust-toolchain@stable
|
||||
|
||||
# cargo tracks its own inputs but not the runner's C toolchain, so a cached
|
||||
# target/ can carry C objects built against a different glibc than we link against.
|
||||
- name: Fingerprint build toolchain
|
||||
id: toolchain
|
||||
run: echo "fingerprint=$(getconf GNU_LIBC_VERSION | tr ' ' '-')-rustc-$(rustc -V | awk '{print $2}')" >> "$GITHUB_OUTPUT"
|
||||
|
||||
- name: Cache cargo registry and target
|
||||
if: runner.environment == 'github-hosted'
|
||||
uses: actions/cache@v6
|
||||
with:
|
||||
path: |
|
||||
~/.cargo/registry
|
||||
~/.cargo/git
|
||||
seaweed-worker/target/release
|
||||
key: rust-worker-${{ steps.toolchain.outputs.fingerprint }}-${{ hashFiles('seaweed-worker/Cargo.lock', 'seaweed-common/Cargo.lock') }}
|
||||
restore-keys: |
|
||||
rust-worker-${{ steps.toolchain.outputs.fingerprint }}-
|
||||
|
||||
# lance's build scripts compile their own protos and look for a protoc.
|
||||
# Point them at the one protoc-bin-vendored ships, which seaweed-worker's
|
||||
# own build already uses, so no job depends on a system package and every
|
||||
# build sees the same version.
|
||||
- name: Use the vendored protoc
|
||||
run: |
|
||||
cd seaweed-worker
|
||||
cargo fetch
|
||||
# The version from the lock, not whatever else a restored cache holds.
|
||||
version=$(awk '/^name = "protoc-bin-vendored-linux-x86_64"$/{found=1; next} found && /^version = /{gsub(/"/,"",$3); print $3; exit}' Cargo.lock)
|
||||
test -n "$version" || { echo "protoc-bin-vendored-linux-x86_64 is not in Cargo.lock" >&2; exit 1; }
|
||||
protoc=$(find ~/.cargo/registry/src -path "*protoc-bin-vendored-linux-x86_64-$version/bin/protoc" | head -1)
|
||||
test -x "$protoc" || { echo "no vendored protoc $version in the registry" >&2; exit 1; }
|
||||
echo "PROTOC=$protoc" >> "$GITHUB_ENV"
|
||||
|
||||
# The release profile is what ships, and it is where the release and the
|
||||
# container builds would otherwise discover a break for the first time.
|
||||
- name: Build the plugin workers
|
||||
run: cd seaweed-worker && cargo build --release
|
||||
|
||||
# The workspace is warning-free under clippy as of the sweep that added
|
||||
# this step. Uncomment to make that a gate; `[workspace.lints.clippy]`
|
||||
# in seaweed-worker/Cargo.toml is where crate-wide exceptions live.
|
||||
# - name: Clippy
|
||||
# run: cd seaweed-worker && cargo clippy --workspace --all-targets -- -D warnings
|
||||
|
||||
# The workspace is rustfmt-clean as of the PR that added this step.
|
||||
# Uncomment to keep it that way.
|
||||
# - name: Check formatting
|
||||
# run: cd seaweed-worker && cargo fmt --all --check
|
||||
|
||||
# seaweed-common is a path dependency of core and lance, not a member of
|
||||
# this workspace, so `--workspace` below does not reach its own tests.
|
||||
# Release and this job's cached target directory, and the cache key above
|
||||
# covers the shared crate's lock. That lock pins the same rustls and
|
||||
# aws-lc-sys this workspace resolves, so the release build above has
|
||||
# already paid for them.
|
||||
- name: Run shared-crate unit tests
|
||||
env:
|
||||
CARGO_TARGET_DIR: ${{ github.workspace }}/seaweed-worker/target
|
||||
run: cd seaweed-common && cargo test --release
|
||||
|
||||
# The tests that need a live gateway skip themselves without one, the way
|
||||
# the Go integration tests skip without Docker; the lifecycle suite in
|
||||
# test/s3tables/lifecycle is what runs them against a real cluster.
|
||||
# Release, so this reuses the build above rather than compiling lance,
|
||||
# arrow and datafusion a second time in another profile.
|
||||
- name: Run unit tests
|
||||
run: cd seaweed-worker && cargo test --release --workspace
|
||||
@@ -1,163 +0,0 @@
|
||||
name: "rust: build dev volume server binaries"
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [ master ]
|
||||
paths:
|
||||
- 'seaweed-volume/**'
|
||||
- 'seaweed-common/**'
|
||||
- '.github/workflows/rust_binaries_dev.yml'
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
jobs:
|
||||
|
||||
cleanup:
|
||||
permissions:
|
||||
contents: write
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Delete old Rust volume dev assets
|
||||
uses: mknejp/delete-release-assets@v1
|
||||
continue-on-error: true
|
||||
with:
|
||||
token: ${{ github.token }}
|
||||
tag: dev
|
||||
fail-if-no-assets: false
|
||||
assets: |
|
||||
weed-volume-*
|
||||
|
||||
build-rust-volume-dev-linux:
|
||||
permissions:
|
||||
contents: write
|
||||
needs: cleanup
|
||||
runs-on: ubuntu-24.04
|
||||
strategy:
|
||||
matrix:
|
||||
include:
|
||||
- target: x86_64-unknown-linux-gnu
|
||||
asset_suffix: linux-amd64
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@v7
|
||||
|
||||
- name: Install Rust toolchain
|
||||
uses: dtolnay/rust-toolchain@stable
|
||||
|
||||
- name: Cache cargo registry and target
|
||||
if: runner.environment == 'github-hosted'
|
||||
uses: actions/cache@v6
|
||||
with:
|
||||
path: |
|
||||
~/.cargo/registry
|
||||
~/.cargo/git
|
||||
seaweed-volume/target
|
||||
key: rust-dev-${{ matrix.target }}-${{ hashFiles('seaweed-volume/Cargo.lock') }}
|
||||
restore-keys: |
|
||||
rust-dev-${{ matrix.target }}-
|
||||
|
||||
- name: Set BUILD_TIME
|
||||
run: echo BUILD_TIME=$(date -u +%Y%m%d-%H%M) >> "$GITHUB_ENV"
|
||||
|
||||
- name: Build Rust volume server (large disk)
|
||||
env:
|
||||
SEAWEEDFS_COMMIT: ${{ github.sha }}
|
||||
run: cd seaweed-volume && cargo build --release
|
||||
|
||||
- name: Package large disk binary
|
||||
run: |
|
||||
cp seaweed-volume/target/release/weed-volume weed-volume-large-disk
|
||||
tar czf "weed-volume-large-disk-${{ env.BUILD_TIME }}-${{ matrix.asset_suffix }}.tar.gz" weed-volume-large-disk
|
||||
rm weed-volume-large-disk
|
||||
|
||||
- name: Build Rust volume server (normal)
|
||||
env:
|
||||
SEAWEEDFS_COMMIT: ${{ github.sha }}
|
||||
run: cd seaweed-volume && cargo build --release --no-default-features
|
||||
|
||||
- name: Package normal binary
|
||||
run: |
|
||||
cp seaweed-volume/target/release/weed-volume weed-volume-normal
|
||||
tar czf "weed-volume-${{ env.BUILD_TIME }}-${{ matrix.asset_suffix }}.tar.gz" weed-volume-normal
|
||||
rm weed-volume-normal
|
||||
|
||||
- name: Upload dev release assets
|
||||
uses: softprops/action-gh-release@v3
|
||||
with:
|
||||
tag_name: dev
|
||||
prerelease: true
|
||||
files: |
|
||||
weed-volume-large-disk-${{ env.BUILD_TIME }}-${{ matrix.asset_suffix }}.tar.gz
|
||||
weed-volume-${{ env.BUILD_TIME }}-${{ matrix.asset_suffix }}.tar.gz
|
||||
env:
|
||||
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
|
||||
build-rust-volume-dev-darwin:
|
||||
permissions:
|
||||
contents: write
|
||||
needs: build-rust-volume-dev-linux
|
||||
runs-on: macos-latest
|
||||
strategy:
|
||||
matrix:
|
||||
include:
|
||||
- target: aarch64-apple-darwin
|
||||
asset_suffix: darwin-arm64
|
||||
- target: x86_64-apple-darwin
|
||||
asset_suffix: darwin-amd64
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@v7
|
||||
|
||||
- name: Install Rust toolchain
|
||||
uses: dtolnay/rust-toolchain@stable
|
||||
with:
|
||||
targets: ${{ matrix.target }}
|
||||
|
||||
- name: Cache cargo registry and target
|
||||
if: runner.environment == 'github-hosted'
|
||||
uses: actions/cache@v6
|
||||
with:
|
||||
path: |
|
||||
~/.cargo/registry
|
||||
~/.cargo/git
|
||||
seaweed-volume/target
|
||||
key: rust-dev-${{ matrix.target }}-${{ hashFiles('seaweed-volume/Cargo.lock') }}
|
||||
restore-keys: |
|
||||
rust-dev-${{ matrix.target }}-
|
||||
|
||||
- name: Set BUILD_TIME
|
||||
run: echo BUILD_TIME=$(date -u +%Y%m%d-%H%M) >> "$GITHUB_ENV"
|
||||
|
||||
- name: Build Rust volume server (large disk)
|
||||
env:
|
||||
SEAWEEDFS_COMMIT: ${{ github.sha }}
|
||||
run: cd seaweed-volume && cargo build --release --target ${{ matrix.target }}
|
||||
|
||||
- name: Package large disk binary
|
||||
run: |
|
||||
cp seaweed-volume/target/${{ matrix.target }}/release/weed-volume weed-volume-large-disk
|
||||
tar czf "weed-volume-large-disk-${{ env.BUILD_TIME }}-${{ matrix.asset_suffix }}.tar.gz" weed-volume-large-disk
|
||||
rm weed-volume-large-disk
|
||||
|
||||
- name: Build Rust volume server (normal)
|
||||
env:
|
||||
SEAWEEDFS_COMMIT: ${{ github.sha }}
|
||||
run: cd seaweed-volume && cargo build --release --target ${{ matrix.target }} --no-default-features
|
||||
|
||||
- name: Package normal binary
|
||||
run: |
|
||||
cp seaweed-volume/target/${{ matrix.target }}/release/weed-volume weed-volume-normal
|
||||
tar czf "weed-volume-${{ env.BUILD_TIME }}-${{ matrix.asset_suffix }}.tar.gz" weed-volume-normal
|
||||
rm weed-volume-normal
|
||||
|
||||
- name: Upload dev release assets
|
||||
uses: softprops/action-gh-release@v3
|
||||
with:
|
||||
tag_name: dev
|
||||
prerelease: true
|
||||
files: |
|
||||
weed-volume-large-disk-${{ env.BUILD_TIME }}-${{ matrix.asset_suffix }}.tar.gz
|
||||
weed-volume-${{ env.BUILD_TIME }}-${{ matrix.asset_suffix }}.tar.gz
|
||||
env:
|
||||
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
@@ -1,429 +0,0 @@
|
||||
name: "rust: build versioned binaries"
|
||||
|
||||
on:
|
||||
push:
|
||||
tags:
|
||||
- '*'
|
||||
|
||||
workflow_dispatch:
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
jobs:
|
||||
|
||||
build-rust-volume-linux:
|
||||
permissions:
|
||||
contents: write
|
||||
runs-on: ubuntu-24.04
|
||||
strategy:
|
||||
matrix:
|
||||
include:
|
||||
# Static musl, as the container images ship: a glibc build only
|
||||
# starts on distributions at least as new as the runner's.
|
||||
- target: x86_64-unknown-linux-musl
|
||||
asset_suffix: linux_amd64
|
||||
- target: aarch64-unknown-linux-musl
|
||||
asset_suffix: linux_arm64
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@v7
|
||||
|
||||
- name: Set up the musl target
|
||||
uses: ./.github/actions/rust-musl-target
|
||||
with:
|
||||
target: ${{ matrix.target }}
|
||||
|
||||
- name: Cache cargo registry and target
|
||||
if: runner.environment == 'github-hosted'
|
||||
uses: actions/cache@v6
|
||||
with:
|
||||
path: |
|
||||
~/.cargo/registry
|
||||
~/.cargo/git
|
||||
seaweed-volume/target
|
||||
key: rust-release-${{ matrix.target }}-${{ hashFiles('seaweed-volume/Cargo.lock') }}
|
||||
restore-keys: |
|
||||
rust-release-${{ matrix.target }}-
|
||||
|
||||
- name: Build Rust volume server (large disk)
|
||||
env:
|
||||
SEAWEEDFS_COMMIT: ${{ github.sha }}
|
||||
run: |
|
||||
cd seaweed-volume
|
||||
cargo build --release --target ${{ matrix.target }} --target-dir target/large-disk
|
||||
|
||||
- name: Build Rust volume server (normal)
|
||||
env:
|
||||
SEAWEEDFS_COMMIT: ${{ github.sha }}
|
||||
run: |
|
||||
cd seaweed-volume
|
||||
cargo build --release --target ${{ matrix.target }} --no-default-features --target-dir target/normal
|
||||
|
||||
- name: Package binaries
|
||||
run: |
|
||||
# Large disk (default, 5bytes feature)
|
||||
cp seaweed-volume/target/large-disk/${{ matrix.target }}/release/weed-volume weed-volume-large-disk
|
||||
tar czf weed-volume_large_disk_${{ matrix.asset_suffix }}.tar.gz weed-volume-large-disk
|
||||
rm weed-volume-large-disk
|
||||
|
||||
# Normal volume size
|
||||
cp seaweed-volume/target/normal/${{ matrix.target }}/release/weed-volume weed-volume-normal
|
||||
tar czf weed-volume_${{ matrix.asset_suffix }}.tar.gz weed-volume-normal
|
||||
rm weed-volume-normal
|
||||
|
||||
- name: Generate md5 checksums
|
||||
run: |
|
||||
for f in weed-volume_large_disk_${{ matrix.asset_suffix }}.tar.gz weed-volume_${{ matrix.asset_suffix }}.tar.gz; do
|
||||
md5sum "$f" > "$f.md5"
|
||||
done
|
||||
|
||||
- name: Upload release assets
|
||||
if: startsWith(github.ref, 'refs/tags/')
|
||||
uses: softprops/action-gh-release@v3
|
||||
with:
|
||||
files: |
|
||||
weed-volume_large_disk_${{ matrix.asset_suffix }}.tar.gz
|
||||
weed-volume_large_disk_${{ matrix.asset_suffix }}.tar.gz.md5
|
||||
weed-volume_${{ matrix.asset_suffix }}.tar.gz
|
||||
weed-volume_${{ matrix.asset_suffix }}.tar.gz.md5
|
||||
env:
|
||||
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
|
||||
- name: Upload artifacts
|
||||
if: ${{ !startsWith(github.ref, 'refs/tags/') }}
|
||||
uses: actions/upload-artifact@v7
|
||||
with:
|
||||
name: rust-volume-${{ matrix.asset_suffix }}
|
||||
path: |
|
||||
weed-volume_large_disk_${{ matrix.asset_suffix }}.tar.gz
|
||||
weed-volume_large_disk_${{ matrix.asset_suffix }}.tar.gz.md5
|
||||
weed-volume_${{ matrix.asset_suffix }}.tar.gz
|
||||
weed-volume_${{ matrix.asset_suffix }}.tar.gz.md5
|
||||
|
||||
# The Rust maintenance worker: Linux only, because it runs beside the cluster
|
||||
# it maintains rather than on a laptop, and its dependency tree (lance, arrow,
|
||||
# datafusion) makes every extra target an expensive build.
|
||||
build-rust-worker-linux:
|
||||
permissions:
|
||||
contents: write
|
||||
runs-on: ubuntu-24.04
|
||||
strategy:
|
||||
matrix:
|
||||
include:
|
||||
# Static musl, as the container images ship: a glibc build only
|
||||
# starts on distributions at least as new as the runner's.
|
||||
- target: x86_64-unknown-linux-musl
|
||||
asset_suffix: linux_amd64
|
||||
- target: aarch64-unknown-linux-musl
|
||||
asset_suffix: linux_arm64
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@v7
|
||||
with:
|
||||
# The upload step is handed a token explicitly; a cargo build script
|
||||
# should not find another one sitting in the checkout's git config.
|
||||
persist-credentials: false
|
||||
|
||||
- name: Set up the musl target
|
||||
uses: ./.github/actions/rust-musl-target
|
||||
with:
|
||||
target: ${{ matrix.target }}
|
||||
|
||||
- name: Cache cargo registry and target
|
||||
if: runner.environment == 'github-hosted'
|
||||
uses: actions/cache@v6
|
||||
with:
|
||||
path: |
|
||||
~/.cargo/registry
|
||||
~/.cargo/git
|
||||
seaweed-worker/target/${{ matrix.target }}/release
|
||||
key: rust-worker-release-${{ matrix.target }}-${{ hashFiles('seaweed-worker/Cargo.lock') }}
|
||||
restore-keys: |
|
||||
rust-worker-release-${{ matrix.target }}-
|
||||
|
||||
# lance's build scripts compile their own protos and look for a protoc.
|
||||
# Point them at the one protoc-bin-vendored ships, which seaweed-worker's
|
||||
# own build already uses, so no job depends on a system package and every
|
||||
# build sees the same version.
|
||||
- name: Use the vendored protoc
|
||||
run: |
|
||||
cd seaweed-worker
|
||||
cargo fetch
|
||||
# The version from the lock, not whatever else a restored cache holds.
|
||||
version=$(awk '/^name = "protoc-bin-vendored-linux-x86_64"$/{found=1; next} found && /^version = /{gsub(/"/,"",$3); print $3; exit}' Cargo.lock)
|
||||
test -n "$version" || { echo "protoc-bin-vendored-linux-x86_64 is not in Cargo.lock" >&2; exit 1; }
|
||||
protoc=$(find ~/.cargo/registry/src -path "*protoc-bin-vendored-linux-x86_64-$version/bin/protoc" | head -1)
|
||||
test -x "$protoc" || { echo "no vendored protoc $version in the registry" >&2; exit 1; }
|
||||
echo "PROTOC=$protoc" >> "$GITHUB_ENV"
|
||||
|
||||
- name: Build the Rust maintenance worker
|
||||
run: |
|
||||
cd seaweed-worker
|
||||
cargo build --release -p weed-lance-worker --target ${{ matrix.target }}
|
||||
|
||||
- name: Package binary
|
||||
run: |
|
||||
cp seaweed-worker/target/${{ matrix.target }}/release/weed-worker weed-worker
|
||||
tar czf weed-worker_${{ matrix.asset_suffix }}.tar.gz weed-worker
|
||||
rm weed-worker
|
||||
md5sum weed-worker_${{ matrix.asset_suffix }}.tar.gz > weed-worker_${{ matrix.asset_suffix }}.tar.gz.md5
|
||||
|
||||
- name: Upload release assets
|
||||
if: startsWith(github.ref, 'refs/tags/')
|
||||
uses: softprops/action-gh-release@v3
|
||||
with:
|
||||
files: |
|
||||
weed-worker_${{ matrix.asset_suffix }}.tar.gz
|
||||
weed-worker_${{ matrix.asset_suffix }}.tar.gz.md5
|
||||
env:
|
||||
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
|
||||
- name: Upload artifacts
|
||||
if: ${{ !startsWith(github.ref, 'refs/tags/') }}
|
||||
uses: actions/upload-artifact@v7
|
||||
with:
|
||||
name: rust-worker-${{ matrix.asset_suffix }}
|
||||
path: |
|
||||
weed-worker_${{ matrix.asset_suffix }}.tar.gz
|
||||
weed-worker_${{ matrix.asset_suffix }}.tar.gz.md5
|
||||
|
||||
# The vector engine behind the S3 Vectors API: Linux only, like the worker,
|
||||
# whose lance dependency tree it shares.
|
||||
build-rust-vector-linux:
|
||||
permissions:
|
||||
contents: write
|
||||
runs-on: ubuntu-24.04
|
||||
strategy:
|
||||
matrix:
|
||||
include:
|
||||
- target: x86_64-unknown-linux-musl
|
||||
asset_suffix: linux_amd64
|
||||
- target: aarch64-unknown-linux-musl
|
||||
asset_suffix: linux_arm64
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@v7
|
||||
with:
|
||||
persist-credentials: false
|
||||
|
||||
- name: Set up the musl target
|
||||
uses: ./.github/actions/rust-musl-target
|
||||
with:
|
||||
target: ${{ matrix.target }}
|
||||
|
||||
- name: Cache cargo registry and target
|
||||
if: runner.environment == 'github-hosted'
|
||||
uses: actions/cache@v6
|
||||
with:
|
||||
path: |
|
||||
~/.cargo/registry
|
||||
~/.cargo/git
|
||||
seaweed-vector/target/${{ matrix.target }}/release
|
||||
key: rust-vector-release-${{ matrix.target }}-${{ hashFiles('seaweed-vector/Cargo.lock') }}
|
||||
restore-keys: |
|
||||
rust-vector-release-${{ matrix.target }}-
|
||||
|
||||
- name: Use the vendored protoc
|
||||
run: |
|
||||
cd seaweed-vector
|
||||
cargo fetch
|
||||
version=$(awk '/^name = "protoc-bin-vendored-linux-x86_64"$/{found=1; next} found && /^version = /{gsub(/"/,"",$3); print $3; exit}' Cargo.lock)
|
||||
test -n "$version" || { echo "protoc-bin-vendored-linux-x86_64 is not in Cargo.lock" >&2; exit 1; }
|
||||
protoc=$(find ~/.cargo/registry/src -path "*protoc-bin-vendored-linux-x86_64-$version/bin/protoc" | head -1)
|
||||
test -x "$protoc" || { echo "no vendored protoc $version in the registry" >&2; exit 1; }
|
||||
echo "PROTOC=$protoc" >> "$GITHUB_ENV"
|
||||
|
||||
- name: Build the vector engine
|
||||
run: |
|
||||
cd seaweed-vector
|
||||
cargo build --release -p weed-vector-server --target ${{ matrix.target }}
|
||||
|
||||
- name: Package binary
|
||||
run: |
|
||||
cp seaweed-vector/target/${{ matrix.target }}/release/weed-vector weed-vector
|
||||
tar czf weed-vector_${{ matrix.asset_suffix }}.tar.gz weed-vector
|
||||
rm weed-vector
|
||||
md5sum weed-vector_${{ matrix.asset_suffix }}.tar.gz > weed-vector_${{ matrix.asset_suffix }}.tar.gz.md5
|
||||
|
||||
- name: Upload release assets
|
||||
if: startsWith(github.ref, 'refs/tags/')
|
||||
uses: softprops/action-gh-release@v3
|
||||
with:
|
||||
files: |
|
||||
weed-vector_${{ matrix.asset_suffix }}.tar.gz
|
||||
weed-vector_${{ matrix.asset_suffix }}.tar.gz.md5
|
||||
env:
|
||||
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
|
||||
- name: Upload artifacts
|
||||
if: ${{ !startsWith(github.ref, 'refs/tags/') }}
|
||||
uses: actions/upload-artifact@v7
|
||||
with:
|
||||
name: rust-vector-${{ matrix.asset_suffix }}
|
||||
path: |
|
||||
weed-vector_${{ matrix.asset_suffix }}.tar.gz
|
||||
weed-vector_${{ matrix.asset_suffix }}.tar.gz.md5
|
||||
|
||||
build-rust-volume-darwin:
|
||||
permissions:
|
||||
contents: write
|
||||
runs-on: macos-latest
|
||||
strategy:
|
||||
matrix:
|
||||
include:
|
||||
- target: x86_64-apple-darwin
|
||||
asset_suffix: darwin_amd64
|
||||
- target: aarch64-apple-darwin
|
||||
asset_suffix: darwin_arm64
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@v7
|
||||
|
||||
- name: Install Rust toolchain
|
||||
uses: dtolnay/rust-toolchain@stable
|
||||
with:
|
||||
targets: ${{ matrix.target }}
|
||||
|
||||
- name: Cache cargo registry and target
|
||||
if: runner.environment == 'github-hosted'
|
||||
uses: actions/cache@v6
|
||||
with:
|
||||
path: |
|
||||
~/.cargo/registry
|
||||
~/.cargo/git
|
||||
seaweed-volume/target
|
||||
key: rust-release-${{ matrix.target }}-${{ hashFiles('seaweed-volume/Cargo.lock') }}
|
||||
restore-keys: |
|
||||
rust-release-${{ matrix.target }}-
|
||||
|
||||
- name: Build Rust volume server (large disk)
|
||||
env:
|
||||
SEAWEEDFS_COMMIT: ${{ github.sha }}
|
||||
run: |
|
||||
cd seaweed-volume
|
||||
cargo build --release --target ${{ matrix.target }} --target-dir target/large-disk
|
||||
|
||||
- name: Build Rust volume server (normal)
|
||||
env:
|
||||
SEAWEEDFS_COMMIT: ${{ github.sha }}
|
||||
run: |
|
||||
cd seaweed-volume
|
||||
cargo build --release --target ${{ matrix.target }} --no-default-features --target-dir target/normal
|
||||
|
||||
- name: Package binaries
|
||||
run: |
|
||||
cp seaweed-volume/target/large-disk/${{ matrix.target }}/release/weed-volume weed-volume-large-disk
|
||||
tar czf weed-volume_large_disk_${{ matrix.asset_suffix }}.tar.gz weed-volume-large-disk
|
||||
rm weed-volume-large-disk
|
||||
|
||||
cp seaweed-volume/target/normal/${{ matrix.target }}/release/weed-volume weed-volume-normal
|
||||
tar czf weed-volume_${{ matrix.asset_suffix }}.tar.gz weed-volume-normal
|
||||
rm weed-volume-normal
|
||||
|
||||
- name: Generate md5 checksums
|
||||
run: |
|
||||
for f in weed-volume_large_disk_${{ matrix.asset_suffix }}.tar.gz weed-volume_${{ matrix.asset_suffix }}.tar.gz; do
|
||||
md5 -r "$f" > "$f.md5"
|
||||
done
|
||||
|
||||
- name: Upload release assets
|
||||
if: startsWith(github.ref, 'refs/tags/')
|
||||
uses: softprops/action-gh-release@v3
|
||||
with:
|
||||
files: |
|
||||
weed-volume_large_disk_${{ matrix.asset_suffix }}.tar.gz
|
||||
weed-volume_large_disk_${{ matrix.asset_suffix }}.tar.gz.md5
|
||||
weed-volume_${{ matrix.asset_suffix }}.tar.gz
|
||||
weed-volume_${{ matrix.asset_suffix }}.tar.gz.md5
|
||||
env:
|
||||
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
|
||||
- name: Upload artifacts
|
||||
if: ${{ !startsWith(github.ref, 'refs/tags/') }}
|
||||
uses: actions/upload-artifact@v7
|
||||
with:
|
||||
name: rust-volume-${{ matrix.asset_suffix }}
|
||||
path: |
|
||||
weed-volume_large_disk_${{ matrix.asset_suffix }}.tar.gz
|
||||
weed-volume_large_disk_${{ matrix.asset_suffix }}.tar.gz.md5
|
||||
weed-volume_${{ matrix.asset_suffix }}.tar.gz
|
||||
weed-volume_${{ matrix.asset_suffix }}.tar.gz.md5
|
||||
|
||||
build-rust-volume-windows:
|
||||
permissions:
|
||||
contents: write
|
||||
runs-on: windows-latest
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@v7
|
||||
|
||||
- name: Install Rust toolchain
|
||||
uses: dtolnay/rust-toolchain@stable
|
||||
|
||||
- name: Cache cargo registry and target
|
||||
if: runner.environment == 'github-hosted'
|
||||
uses: actions/cache@v6
|
||||
with:
|
||||
path: |
|
||||
~/.cargo/registry
|
||||
~/.cargo/git
|
||||
seaweed-volume/target
|
||||
key: rust-release-windows-${{ hashFiles('seaweed-volume/Cargo.lock') }}
|
||||
restore-keys: |
|
||||
rust-release-windows-
|
||||
|
||||
- name: Build Rust volume server (large disk)
|
||||
env:
|
||||
SEAWEEDFS_COMMIT: ${{ github.sha }}
|
||||
run: |
|
||||
cd seaweed-volume
|
||||
cargo build --release --target-dir target/large-disk
|
||||
|
||||
- name: Build Rust volume server (normal)
|
||||
env:
|
||||
SEAWEEDFS_COMMIT: ${{ github.sha }}
|
||||
run: |
|
||||
cd seaweed-volume
|
||||
cargo build --release --no-default-features --target-dir target/normal
|
||||
|
||||
- name: Package binaries
|
||||
shell: bash
|
||||
run: |
|
||||
cp seaweed-volume/target/large-disk/release/weed-volume.exe weed-volume-large-disk.exe
|
||||
7z a weed-volume_large_disk_windows_amd64.zip weed-volume-large-disk.exe
|
||||
rm weed-volume-large-disk.exe
|
||||
|
||||
cp seaweed-volume/target/normal/release/weed-volume.exe weed-volume-normal.exe
|
||||
7z a weed-volume_windows_amd64.zip weed-volume-normal.exe
|
||||
rm weed-volume-normal.exe
|
||||
|
||||
- name: Generate md5 checksums
|
||||
shell: bash
|
||||
run: |
|
||||
for f in weed-volume_large_disk_windows_amd64.zip weed-volume_windows_amd64.zip; do
|
||||
md5sum "$f" > "$f.md5"
|
||||
done
|
||||
|
||||
- name: Upload release assets
|
||||
if: startsWith(github.ref, 'refs/tags/')
|
||||
uses: softprops/action-gh-release@v3
|
||||
with:
|
||||
files: |
|
||||
weed-volume_large_disk_windows_amd64.zip
|
||||
weed-volume_large_disk_windows_amd64.zip.md5
|
||||
weed-volume_windows_amd64.zip
|
||||
weed-volume_windows_amd64.zip.md5
|
||||
env:
|
||||
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
|
||||
- name: Upload artifacts
|
||||
if: ${{ !startsWith(github.ref, 'refs/tags/') }}
|
||||
uses: actions/upload-artifact@v7
|
||||
with:
|
||||
name: rust-volume-windows_amd64
|
||||
path: |
|
||||
weed-volume_large_disk_windows_amd64.zip
|
||||
weed-volume_large_disk_windows_amd64.zip.md5
|
||||
weed-volume_windows_amd64.zip
|
||||
weed-volume_windows_amd64.zip.md5
|
||||
@@ -1,130 +0,0 @@
|
||||
name: "S3 ETag and ACL Tests"
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [ master ]
|
||||
paths:
|
||||
- 'weed/s3api/**'
|
||||
- 'weed/filer/etag*.go'
|
||||
- 'weed/server/filer_server_handlers_*.go'
|
||||
- 'test/s3/etag/**'
|
||||
- 'test/s3/acl/**'
|
||||
- '.github/workflows/s3-etag-acl-tests.yml'
|
||||
pull_request:
|
||||
branches: [ master ]
|
||||
paths:
|
||||
- 'weed/s3api/**'
|
||||
- 'weed/filer/etag*.go'
|
||||
- 'weed/server/filer_server_handlers_*.go'
|
||||
- 'test/s3/etag/**'
|
||||
- 'test/s3/acl/**'
|
||||
- '.github/workflows/s3-etag-acl-tests.yml'
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.event.pull_request.number || (github.event_name == 'push' && github.ref) || github.run_id }}
|
||||
cancel-in-progress: ${{ github.event_name == 'pull_request' || (github.event_name == 'push' && !startsWith(github.ref, 'refs/tags/')) }}
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
jobs:
|
||||
s3-etag-acl-tests:
|
||||
name: S3 ETag + ACL Integration Tests
|
||||
runs-on: ubuntu-24.04
|
||||
timeout-minutes: 15
|
||||
steps:
|
||||
- name: Check out code
|
||||
uses: actions/checkout@v7
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v7
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
cache: ${{ runner.environment == 'github-hosted' }}
|
||||
|
||||
- name: Install SeaweedFS
|
||||
run: |
|
||||
cd weed && go install -buildvcs=false
|
||||
|
||||
- name: Start weed mini (S3 on :8333)
|
||||
run: |
|
||||
mkdir -p /tmp/seaweedfs-etag-acl
|
||||
# Minimal identity config so SSE-aware tests under acl/ can authenticate.
|
||||
cat > /tmp/seaweedfs-etag-acl-s3.json <<'JSON'
|
||||
{
|
||||
"identities": [
|
||||
{
|
||||
"name": "admin",
|
||||
"credentials": [
|
||||
{"accessKey": "some_access_key1", "secretKey": "some_secret_key1"}
|
||||
],
|
||||
"actions": ["Admin", "Read", "Write"]
|
||||
}
|
||||
]
|
||||
}
|
||||
JSON
|
||||
AWS_ACCESS_KEY_ID=some_access_key1 \
|
||||
AWS_SECRET_ACCESS_KEY=some_secret_key1 \
|
||||
weed mini \
|
||||
-dir=/tmp/seaweedfs-etag-acl \
|
||||
-s3.port=8333 \
|
||||
-s3.config=/tmp/seaweedfs-etag-acl-s3.json \
|
||||
-ip=127.0.0.1 \
|
||||
> /tmp/weed-mini.log 2>&1 &
|
||||
echo $! > /tmp/weed-mini.pid
|
||||
|
||||
# Wait for the S3 endpoint to come up (returns 403 unauth before any
|
||||
# request is signed; that's fine — it means the server is listening).
|
||||
for i in $(seq 1 30); do
|
||||
if curl -s -o /dev/null -w "%{http_code}" http://127.0.0.1:8333/ | grep -qE "^(200|403)$"; then
|
||||
echo "weed mini is ready"
|
||||
exit 0
|
||||
fi
|
||||
sleep 1
|
||||
done
|
||||
echo "weed mini failed to start within 30s"
|
||||
tail -50 /tmp/weed-mini.log
|
||||
exit 1
|
||||
|
||||
- name: Run ETag tests
|
||||
# Pins the regression for #7768: PutObject of an auto-chunked file (>8MB)
|
||||
# must return a pure MD5 hex ETag, not a `<md5>-N` composite — the AWS
|
||||
# SDK for Java v2 rejects the latter on the PutObject path.
|
||||
env:
|
||||
S3_ENDPOINT: http://127.0.0.1:8333
|
||||
AWS_ACCESS_KEY_ID: some_access_key1
|
||||
AWS_SECRET_ACCESS_KEY: some_secret_key1
|
||||
AWS_REGION: us-east-1
|
||||
run: go test -v -timeout=5m ./test/s3/etag/...
|
||||
|
||||
- name: Run ACL versioning tests
|
||||
# Pins object-ACL behavior on a versioned bucket: GetObjectAcl /
|
||||
# PutObjectAcl with and without versionId, modifying ACLs on different
|
||||
# versions independently.
|
||||
env:
|
||||
S3_ENDPOINT: http://127.0.0.1:8333
|
||||
AWS_ACCESS_KEY_ID: some_access_key1
|
||||
AWS_SECRET_ACCESS_KEY: some_secret_key1
|
||||
AWS_REGION: us-east-1
|
||||
run: go test -v -timeout=5m ./test/s3/acl/...
|
||||
|
||||
- name: Stop weed mini
|
||||
if: always()
|
||||
run: |
|
||||
if [ -f /tmp/weed-mini.pid ]; then
|
||||
kill "$(cat /tmp/weed-mini.pid)" 2>/dev/null || true
|
||||
fi
|
||||
|
||||
- name: Show server log on failure
|
||||
if: failure()
|
||||
run: |
|
||||
echo "=== weed mini log (last 200 lines) ==="
|
||||
tail -n 200 /tmp/weed-mini.log 2>/dev/null || echo "no log available"
|
||||
|
||||
- name: Archive log
|
||||
if: failure()
|
||||
uses: actions/upload-artifact@v7
|
||||
with:
|
||||
name: s3-etag-acl-server-log
|
||||
path: /tmp/weed-mini.log
|
||||
retention-days: 3
|
||||
@@ -1,90 +0,0 @@
|
||||
name: "S3 Authenticated Integration Tests"
|
||||
|
||||
on:
|
||||
pull_request:
|
||||
paths:
|
||||
- 'weed/s3api/**'
|
||||
- 'weed/iam/**'
|
||||
- 'weed/filer/**'
|
||||
- 'weed/server/**'
|
||||
- 'test/s3/normal/**'
|
||||
- 'go.mod'
|
||||
- 'go.sum'
|
||||
- '.github/workflows/s3-example-integration-tests.yml'
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.event.pull_request.number || (github.event_name == 'push' && github.ref) || github.run_id }}
|
||||
cancel-in-progress: ${{ github.event_name == 'pull_request' || (github.event_name == 'push' && !startsWith(github.ref, 'refs/tags/')) }}
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
jobs:
|
||||
tests:
|
||||
name: S3 Integration Tests
|
||||
runs-on: ubuntu-24.04
|
||||
timeout-minutes: 30
|
||||
|
||||
steps:
|
||||
- name: Check out code
|
||||
uses: actions/checkout@v7
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v7
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
cache: ${{ runner.environment == 'github-hosted' }}
|
||||
id: go
|
||||
|
||||
- name: Build SeaweedFS
|
||||
run: |
|
||||
cd weed
|
||||
go build -o weed -buildvcs=false
|
||||
|
||||
- name: Run S3 Integration Tests
|
||||
timeout-minutes: 15
|
||||
working-directory: test/s3/normal
|
||||
run: |
|
||||
set -x
|
||||
echo "=== Running S3 Integration Tests ==="
|
||||
go test -v -timeout=60s -run TestS3Integration ./...
|
||||
|
||||
- name: Run S3 DeleteBucketNotEmpty Tests
|
||||
timeout-minutes: 15
|
||||
working-directory: test/s3/normal
|
||||
run: |
|
||||
set -x
|
||||
echo "=== Running S3 DeleteBucketNotEmpty Tests ==="
|
||||
go test -v -timeout=60s -run TestS3DeleteBucketNotEmpty ./...
|
||||
|
||||
- name: Run S3 Empty Directory Marker Tests
|
||||
timeout-minutes: 15
|
||||
working-directory: test/s3/normal
|
||||
run: |
|
||||
set -x
|
||||
echo "=== Running S3 Empty Directory Marker Tests ==="
|
||||
go test -v -timeout=180s -run TestS3ListObjectsEmptyDirectoryMarkers ./...
|
||||
|
||||
- name: Run S3 Prefix Object Tests
|
||||
timeout-minutes: 15
|
||||
working-directory: test/s3/normal
|
||||
run: |
|
||||
set -x
|
||||
echo "=== Running S3 Prefix Object Tests ==="
|
||||
go test -v -timeout=180s -run TestS3PrefixObjectKeys ./...
|
||||
|
||||
- name: Run IAM Integration Tests
|
||||
timeout-minutes: 15
|
||||
working-directory: test/s3/normal
|
||||
run: |
|
||||
set -x
|
||||
echo "=== Running IAM Integration Tests ==="
|
||||
go test -v -timeout=60s -run TestIAMOperations ./...
|
||||
|
||||
- name: Upload test logs on failure
|
||||
if: failure()
|
||||
uses: actions/upload-artifact@v7
|
||||
with:
|
||||
name: integration-test-logs
|
||||
path: test/s3/normal/*.log
|
||||
retention-days: 3
|
||||
@@ -1,98 +0,0 @@
|
||||
name: "S3 Filer Group Tests"
|
||||
|
||||
on:
|
||||
pull_request:
|
||||
paths:
|
||||
- 'weed/s3api/**'
|
||||
- 'weed/filer/**'
|
||||
- 'weed/server/**'
|
||||
- 'test/s3/filer_group/**'
|
||||
- 'go.mod'
|
||||
- 'go.sum'
|
||||
- '.github/workflows/s3-filer-group-tests.yml'
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.event.pull_request.number || (github.event_name == 'push' && github.ref) || github.run_id }}
|
||||
cancel-in-progress: ${{ github.event_name == 'pull_request' || (github.event_name == 'push' && !startsWith(github.ref, 'refs/tags/')) }}
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
defaults:
|
||||
run:
|
||||
working-directory: weed
|
||||
|
||||
jobs:
|
||||
s3-filer-group-tests:
|
||||
name: S3 Filer Group Integration Tests
|
||||
runs-on: ubuntu-24.04
|
||||
timeout-minutes: 20
|
||||
|
||||
steps:
|
||||
- name: Check out code
|
||||
uses: actions/checkout@v7
|
||||
|
||||
- name: Clean up stale FUSE mounts
|
||||
uses: ./.github/actions/cleanup-stale-fuse-mounts
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v7
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
cache: ${{ runner.environment == 'github-hosted' }}
|
||||
id: go
|
||||
|
||||
- name: Install SeaweedFS
|
||||
run: |
|
||||
go install -buildvcs=false
|
||||
|
||||
- name: Run S3 Filer Group Tests
|
||||
timeout-minutes: 15
|
||||
working-directory: test/s3/filer_group
|
||||
run: |
|
||||
set -x
|
||||
echo "=== System Information ==="
|
||||
uname -a
|
||||
free -h
|
||||
timeout 15 df -h || true
|
||||
echo "=== Starting Filer Group Tests ==="
|
||||
|
||||
# Run tests with automatic server management
|
||||
# The test-with-server target handles server startup/shutdown automatically
|
||||
# Server is started with -filer.group=testgroup
|
||||
make test-with-server || {
|
||||
echo "❌ Filer group tests failed, checking logs..."
|
||||
if [ -f weed-test.log ]; then
|
||||
echo "=== Server logs ==="
|
||||
tail -100 weed-test.log
|
||||
fi
|
||||
echo "=== Process information ==="
|
||||
ps aux | grep -E "(weed|test)" || true
|
||||
exit 1
|
||||
}
|
||||
|
||||
- name: Show server logs on failure
|
||||
if: failure()
|
||||
working-directory: test/s3/filer_group
|
||||
run: |
|
||||
echo "=== Server Logs ==="
|
||||
if [ -f weed-test.log ]; then
|
||||
echo "Last 100 lines of server logs:"
|
||||
tail -100 weed-test.log
|
||||
else
|
||||
echo "No server log file found"
|
||||
fi
|
||||
|
||||
echo "=== Test Environment ==="
|
||||
ps aux | grep -E "(weed|test)" || true
|
||||
netstat -tlnp | grep -E "(8333|9333|8080)" || true
|
||||
|
||||
- name: Upload test logs on failure
|
||||
if: failure()
|
||||
uses: actions/upload-artifact@v7
|
||||
with:
|
||||
name: s3-filer-group-test-logs
|
||||
path: test/s3/filer_group/weed-test*.log
|
||||
retention-days: 3
|
||||
|
||||
|
||||
@@ -2,19 +2,10 @@ name: "S3 Go Tests"
|
||||
|
||||
on:
|
||||
pull_request:
|
||||
paths:
|
||||
- 'weed/s3api/**'
|
||||
- 'weed/remote_storage/**'
|
||||
- 'weed/filer/**'
|
||||
- 'weed/server/**'
|
||||
- 'test/s3/**'
|
||||
- 'go.mod'
|
||||
- 'go.sum'
|
||||
- '.github/workflows/s3-go-tests.yml'
|
||||
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.event.pull_request.number || (github.event_name == 'push' && github.ref) || github.run_id }}
|
||||
cancel-in-progress: ${{ github.event_name == 'pull_request' || (github.event_name == 'push' && !startsWith(github.ref, 'refs/tags/')) }}
|
||||
group: ${{ github.head_ref }}/s3-go-tests
|
||||
cancel-in-progress: true
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
@@ -26,7 +17,7 @@ defaults:
|
||||
jobs:
|
||||
s3-versioning-tests:
|
||||
name: S3 Versioning Tests
|
||||
runs-on: ubuntu-24.04
|
||||
runs-on: ubuntu-22.04
|
||||
timeout-minutes: 30
|
||||
strategy:
|
||||
matrix:
|
||||
@@ -34,16 +25,12 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Check out code
|
||||
uses: actions/checkout@v7
|
||||
|
||||
- name: Clean up stale FUSE mounts
|
||||
uses: ./.github/actions/cleanup-stale-fuse-mounts
|
||||
uses: actions/checkout@v4
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v7
|
||||
uses: actions/setup-go@v5
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
cache: ${{ runner.environment == 'github-hosted' }}
|
||||
id: go
|
||||
|
||||
- name: Install SeaweedFS
|
||||
@@ -58,7 +45,7 @@ jobs:
|
||||
echo "=== System Information ==="
|
||||
uname -a
|
||||
free -h
|
||||
timeout 15 df -h || true
|
||||
df -h
|
||||
echo "=== Starting Tests ==="
|
||||
|
||||
# Run tests with automatic server management
|
||||
@@ -89,7 +76,7 @@ jobs:
|
||||
|
||||
- name: Upload test logs on failure
|
||||
if: failure()
|
||||
uses: actions/upload-artifact@v7
|
||||
uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: s3-versioning-test-logs-${{ matrix.test-type }}
|
||||
path: test/s3/versioning/weed-test*.log
|
||||
@@ -97,18 +84,17 @@ jobs:
|
||||
|
||||
s3-versioning-compatibility:
|
||||
name: S3 Versioning Compatibility Test
|
||||
runs-on: ubuntu-24.04
|
||||
runs-on: ubuntu-22.04
|
||||
timeout-minutes: 20
|
||||
|
||||
steps:
|
||||
- name: Check out code
|
||||
uses: actions/checkout@v7
|
||||
uses: actions/checkout@v4
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v7
|
||||
uses: actions/setup-go@v5
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
cache: ${{ runner.environment == 'github-hosted' }}
|
||||
id: go
|
||||
|
||||
- name: Install SeaweedFS
|
||||
@@ -138,7 +124,7 @@ jobs:
|
||||
|
||||
- name: Upload server logs on failure
|
||||
if: failure()
|
||||
uses: actions/upload-artifact@v7
|
||||
uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: s3-versioning-compatibility-logs
|
||||
path: test/s3/versioning/weed-test*.log
|
||||
@@ -146,18 +132,17 @@ jobs:
|
||||
|
||||
s3-cors-compatibility:
|
||||
name: S3 CORS Compatibility Test
|
||||
runs-on: ubuntu-24.04
|
||||
runs-on: ubuntu-22.04
|
||||
timeout-minutes: 20
|
||||
|
||||
steps:
|
||||
- name: Check out code
|
||||
uses: actions/checkout@v7
|
||||
uses: actions/checkout@v4
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v7
|
||||
uses: actions/setup-go@v5
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
cache: ${{ runner.environment == 'github-hosted' }}
|
||||
id: go
|
||||
|
||||
- name: Install SeaweedFS
|
||||
@@ -187,7 +172,7 @@ jobs:
|
||||
|
||||
- name: Upload server logs on failure
|
||||
if: failure()
|
||||
uses: actions/upload-artifact@v7
|
||||
uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: s3-cors-compatibility-logs
|
||||
path: test/s3/cors/weed-test*.log
|
||||
@@ -195,7 +180,7 @@ jobs:
|
||||
|
||||
s3-retention-tests:
|
||||
name: S3 Retention Tests
|
||||
runs-on: ubuntu-24.04
|
||||
runs-on: ubuntu-22.04
|
||||
timeout-minutes: 30
|
||||
strategy:
|
||||
matrix:
|
||||
@@ -203,16 +188,12 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Check out code
|
||||
uses: actions/checkout@v7
|
||||
|
||||
- name: Clean up stale FUSE mounts
|
||||
uses: ./.github/actions/cleanup-stale-fuse-mounts
|
||||
uses: actions/checkout@v4
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v7
|
||||
uses: actions/setup-go@v5
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
cache: ${{ runner.environment == 'github-hosted' }}
|
||||
id: go
|
||||
|
||||
- name: Install SeaweedFS
|
||||
@@ -227,7 +208,7 @@ jobs:
|
||||
echo "=== System Information ==="
|
||||
uname -a
|
||||
free -h
|
||||
timeout 15 df -h || true
|
||||
df -h
|
||||
echo "=== Starting Tests ==="
|
||||
|
||||
# Run tests with automatic server management
|
||||
@@ -258,147 +239,15 @@ jobs:
|
||||
|
||||
- name: Upload test logs on failure
|
||||
if: failure()
|
||||
uses: actions/upload-artifact@v7
|
||||
uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: s3-retention-test-logs-${{ matrix.test-type }}
|
||||
path: test/s3/retention/weed-test*.log
|
||||
retention-days: 3
|
||||
|
||||
s3-lifecycle-tests:
|
||||
name: S3 Lifecycle Tests
|
||||
runs-on: ubuntu-24.04
|
||||
timeout-minutes: 10
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
# One job per test so each gets a fresh `weed mini` server, avoiding
|
||||
# the cross-test volume-pool exhaustion that surfaced when several
|
||||
# TTL-pinned bucket collections piled up in a single run.
|
||||
test:
|
||||
- TestLifecycleAbortIncompleteMultipartUpload
|
||||
- TestLifecycleAdminDispatchSucceedsWithCustomFilerGrpcPort
|
||||
- TestLifecycleBootstrapWalkOnExistingObjects
|
||||
- TestLifecycleConfigUpdateBetweenSweeps
|
||||
- TestLifecycleDeleteBucketLifecycleStopsDispatching
|
||||
- TestLifecycleDisabledRuleSkipsObject
|
||||
- TestLifecycleEmptyBucketSweepIsNoOp
|
||||
- TestLifecycleExpirationDateInThePast
|
||||
- TestLifecycleExpirationFiresOnBackdatedObject
|
||||
- TestLifecycleExpiredDeleteMarkerCleanup
|
||||
- TestLifecycleMultipleBucketsInOneSweep
|
||||
- TestLifecycleMultipleRulesInOneBucket
|
||||
- TestLifecycleNewerNoncurrentVersions
|
||||
- TestLifecycleNoncurrentVersionExpiration
|
||||
- TestLifecycleSizeFilterGreaterThan
|
||||
- TestLifecycleSkipsObjectLockedObjects
|
||||
- TestLifecycleSuspendedVersioningExpiration
|
||||
- TestLifecycleTagFilter
|
||||
- TestLifecycleVersionedBucketCreatesDeleteMarker
|
||||
- TestLifecycleVersionedBucketExpiresOnSecondRun
|
||||
|
||||
steps:
|
||||
- name: Check out code
|
||||
uses: actions/checkout@v7
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v7
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
cache: ${{ runner.environment == 'github-hosted' }}
|
||||
id: go
|
||||
|
||||
- name: Install SeaweedFS
|
||||
run: |
|
||||
go install -buildvcs=false
|
||||
|
||||
- name: Run ${{ matrix.test }}
|
||||
timeout-minutes: 8
|
||||
working-directory: test/s3/lifecycle
|
||||
run: |
|
||||
set -x
|
||||
make test-with-server TEST_PATTERN='^${{ matrix.test }}$$'
|
||||
|
||||
- name: Show server logs on failure
|
||||
if: failure()
|
||||
working-directory: test/s3/lifecycle
|
||||
run: |
|
||||
if [ -f weed-test.log ]; then
|
||||
echo "=== Last 200 lines of server logs ==="
|
||||
tail -200 weed-test.log
|
||||
fi
|
||||
ps aux | grep -E "(weed|test)" || true
|
||||
netstat -tlnp 2>/dev/null | grep -E "(8333|9333|8080|8888)" || true
|
||||
|
||||
- name: Upload test logs on failure
|
||||
if: failure()
|
||||
uses: actions/upload-artifact@v7
|
||||
with:
|
||||
name: s3-lifecycle-test-logs-${{ matrix.test }}
|
||||
path: test/s3/lifecycle/weed-test*.log
|
||||
retention-days: 3
|
||||
|
||||
s3-checksum-tests:
|
||||
name: S3 Checksum Tests
|
||||
runs-on: ubuntu-24.04
|
||||
timeout-minutes: 20
|
||||
|
||||
steps:
|
||||
- name: Check out code
|
||||
uses: actions/checkout@v7
|
||||
|
||||
- name: Clean up stale FUSE mounts
|
||||
uses: ./.github/actions/cleanup-stale-fuse-mounts
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v7
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
cache: ${{ runner.environment == 'github-hosted' }}
|
||||
id: go
|
||||
|
||||
- name: Install SeaweedFS
|
||||
run: |
|
||||
go install -buildvcs=false
|
||||
|
||||
- name: Run S3 Checksum Tests
|
||||
timeout-minutes: 16
|
||||
working-directory: test/s3/checksum
|
||||
run: |
|
||||
set -x
|
||||
echo "=== System Information ==="
|
||||
uname -a
|
||||
free -h
|
||||
timeout 15 df -h || true
|
||||
echo "=== Starting Tests ==="
|
||||
make test-with-server
|
||||
|
||||
- name: Show server logs on failure
|
||||
if: failure()
|
||||
working-directory: test/s3/checksum
|
||||
run: |
|
||||
echo "=== Server Logs ==="
|
||||
if [ -f weed-test.log ]; then
|
||||
echo "Last 100 lines of server logs:"
|
||||
tail -100 weed-test.log
|
||||
else
|
||||
echo "No server log file found"
|
||||
fi
|
||||
|
||||
echo "=== Test Environment ==="
|
||||
ps aux | grep -E "(weed|test)" || true
|
||||
netstat -tlnp | grep -E "(8333|9333|8080)" || true
|
||||
|
||||
- name: Upload test logs on failure
|
||||
if: failure()
|
||||
uses: actions/upload-artifact@v7
|
||||
with:
|
||||
name: s3-checksum-test-logs
|
||||
path: test/s3/checksum/weed-test*.log
|
||||
retention-days: 3
|
||||
|
||||
s3-cors-tests:
|
||||
name: S3 CORS Tests
|
||||
runs-on: ubuntu-24.04
|
||||
runs-on: ubuntu-22.04
|
||||
timeout-minutes: 30
|
||||
strategy:
|
||||
matrix:
|
||||
@@ -406,16 +255,12 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Check out code
|
||||
uses: actions/checkout@v7
|
||||
|
||||
- name: Clean up stale FUSE mounts
|
||||
uses: ./.github/actions/cleanup-stale-fuse-mounts
|
||||
uses: actions/checkout@v4
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v7
|
||||
uses: actions/setup-go@v5
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
cache: ${{ runner.environment == 'github-hosted' }}
|
||||
id: go
|
||||
|
||||
- name: Install SeaweedFS
|
||||
@@ -430,7 +275,7 @@ jobs:
|
||||
echo "=== System Information ==="
|
||||
uname -a
|
||||
free -h
|
||||
timeout 15 df -h || true
|
||||
df -h
|
||||
echo "=== Starting Tests ==="
|
||||
|
||||
# Run tests with automatic server management
|
||||
@@ -461,7 +306,7 @@ jobs:
|
||||
|
||||
- name: Upload test logs on failure
|
||||
if: failure()
|
||||
uses: actions/upload-artifact@v7
|
||||
uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: s3-cors-test-logs-${{ matrix.test-type }}
|
||||
path: test/s3/cors/weed-test*.log
|
||||
@@ -469,18 +314,17 @@ jobs:
|
||||
|
||||
s3-retention-worm:
|
||||
name: S3 Retention WORM Integration Test
|
||||
runs-on: ubuntu-24.04
|
||||
runs-on: ubuntu-22.04
|
||||
timeout-minutes: 20
|
||||
|
||||
steps:
|
||||
- name: Check out code
|
||||
uses: actions/checkout@v7
|
||||
uses: actions/checkout@v4
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v7
|
||||
uses: actions/setup-go@v5
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
cache: ${{ runner.environment == 'github-hosted' }}
|
||||
id: go
|
||||
|
||||
- name: Install SeaweedFS
|
||||
@@ -511,7 +355,7 @@ jobs:
|
||||
|
||||
- name: Upload server logs on failure
|
||||
if: failure()
|
||||
uses: actions/upload-artifact@v7
|
||||
uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: s3-retention-worm-logs
|
||||
path: test/s3/retention/weed-test*.log
|
||||
@@ -519,18 +363,19 @@ jobs:
|
||||
|
||||
s3-versioning-stress:
|
||||
name: S3 Versioning Stress Test
|
||||
runs-on: ubuntu-24.04
|
||||
runs-on: ubuntu-22.04
|
||||
timeout-minutes: 35
|
||||
# Only run stress tests on master branch pushes to avoid overloading PR testing
|
||||
if: github.event_name == 'push' && github.ref == 'refs/heads/master'
|
||||
|
||||
steps:
|
||||
- name: Check out code
|
||||
uses: actions/checkout@v7
|
||||
uses: actions/checkout@v4
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v7
|
||||
uses: actions/setup-go@v5
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
cache: ${{ runner.environment == 'github-hosted' }}
|
||||
id: go
|
||||
|
||||
- name: Install SeaweedFS
|
||||
@@ -546,9 +391,6 @@ jobs:
|
||||
uname -a
|
||||
free -h
|
||||
|
||||
# Start server for stress tests
|
||||
make start-server
|
||||
|
||||
# Run stress tests (concurrent operations)
|
||||
make test-versioning-stress || {
|
||||
echo "❌ Stress test failed, checking logs..."
|
||||
@@ -556,146 +398,15 @@ jobs:
|
||||
echo "=== Server logs ==="
|
||||
tail -200 weed-test.log
|
||||
fi
|
||||
make stop-server
|
||||
make clean
|
||||
exit 1
|
||||
}
|
||||
|
||||
# Run pagination stress tests (>1000 versions)
|
||||
echo "=== Running pagination stress tests ==="
|
||||
make test-versioning-pagination-stress || {
|
||||
echo "❌ Pagination stress test failed, checking logs..."
|
||||
if [ -f weed-test.log ]; then
|
||||
echo "=== Server logs ==="
|
||||
tail -200 weed-test.log
|
||||
fi
|
||||
make stop-server
|
||||
make clean
|
||||
exit 1
|
||||
}
|
||||
|
||||
make stop-server
|
||||
make clean
|
||||
|
||||
- name: Upload stress test logs
|
||||
if: always()
|
||||
uses: actions/upload-artifact@v7
|
||||
uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: s3-versioning-stress-logs
|
||||
path: test/s3/versioning/weed-test*.log
|
||||
retention-days: 7
|
||||
|
||||
s3-tagging-tests:
|
||||
# CI job for S3 object tagging tests
|
||||
name: S3 Tagging Tests
|
||||
runs-on: ubuntu-24.04
|
||||
timeout-minutes: 20
|
||||
|
||||
steps:
|
||||
- name: Check out code
|
||||
uses: actions/checkout@v7
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v7
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
cache: ${{ runner.environment == 'github-hosted' }}
|
||||
id: go
|
||||
|
||||
- name: Install SeaweedFS
|
||||
run: |
|
||||
go install -buildvcs=false
|
||||
|
||||
- name: Run S3 Tagging Tests
|
||||
timeout-minutes: 15
|
||||
working-directory: test/s3/tagging
|
||||
run: |
|
||||
set -x
|
||||
echo "=== System Information ==="
|
||||
uname -a
|
||||
free -h
|
||||
|
||||
# Set environment variables for the test
|
||||
export S3_ENDPOINT="http://localhost:8006"
|
||||
export S3_ACCESS_KEY="0555b35654ad1656d804"
|
||||
export S3_SECRET_KEY="h7GhxuBLTrlhVUyxSPUKUV8r/2EI4ngqJxD7iBdBYLhwluN30JaT3Q=="
|
||||
export AWS_ACCESS_KEY_ID="$S3_ACCESS_KEY"
|
||||
export AWS_SECRET_ACCESS_KEY="$S3_SECRET_KEY"
|
||||
|
||||
# Run the specific test that is equivalent to AWS S3 tagging behavior
|
||||
make test-with-server || {
|
||||
echo "❌ Test failed, checking logs..."
|
||||
if [ -f weed-test.log ]; then
|
||||
echo "=== Server logs ==="
|
||||
tail -100 weed-test.log
|
||||
fi
|
||||
echo "=== Process information ==="
|
||||
ps aux | grep -E "(weed|test)" || true
|
||||
exit 1
|
||||
}
|
||||
|
||||
- name: Upload test logs on failure
|
||||
if: failure()
|
||||
uses: actions/upload-artifact@v7
|
||||
with:
|
||||
name: s3-tagging-test-logs
|
||||
path: test/s3/tagging/weed-test*.log
|
||||
retention-days: 3
|
||||
|
||||
s3-remote-cache-tests:
|
||||
name: S3 Remote Cache Tests
|
||||
runs-on: ubuntu-24.04
|
||||
timeout-minutes: 20
|
||||
|
||||
steps:
|
||||
- name: Check out code
|
||||
uses: actions/checkout@v7
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v7
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
cache: ${{ runner.environment == 'github-hosted' }}
|
||||
id: go
|
||||
|
||||
- name: Install SeaweedFS
|
||||
run: |
|
||||
go install -buildvcs=false
|
||||
|
||||
- name: Run S3 Remote Cache Tests
|
||||
timeout-minutes: 15
|
||||
working-directory: test/s3/remote_cache
|
||||
run: |
|
||||
set -x
|
||||
echo "=== System Information ==="
|
||||
uname -a
|
||||
free -h
|
||||
|
||||
# Run the remote cache integration tests
|
||||
# Tests singleflight deduplication for caching remote objects
|
||||
make test-with-server || {
|
||||
echo "❌ Test failed, checking logs..."
|
||||
if [ -f primary-weed.log ]; then
|
||||
echo "=== Primary server logs ==="
|
||||
tail -100 primary-weed.log
|
||||
fi
|
||||
if [ -f remote-weed.log ]; then
|
||||
echo "=== Remote server logs ==="
|
||||
tail -100 remote-weed.log
|
||||
fi
|
||||
echo "=== Process information ==="
|
||||
ps aux | grep -E "(weed|test)" || true
|
||||
exit 1
|
||||
}
|
||||
|
||||
- name: Upload test logs on failure
|
||||
if: failure()
|
||||
uses: actions/upload-artifact@v7
|
||||
with:
|
||||
name: s3-remote-cache-test-logs
|
||||
path: |
|
||||
test/s3/remote_cache/primary-weed.log
|
||||
test/s3/remote_cache/remote-weed.log
|
||||
retention-days: 3
|
||||
|
||||
# Removed SSE-C integration tests and compatibility job
|
||||
retention-days: 7
|
||||
@@ -1,316 +0,0 @@
|
||||
name: "S3 IAM Integration Tests"
|
||||
|
||||
on:
|
||||
pull_request:
|
||||
paths:
|
||||
- 'weed/iam/**'
|
||||
- 'weed/s3api/**'
|
||||
- 'weed/credential/**'
|
||||
- 'weed/pb/**'
|
||||
- 'test/s3/iam/**'
|
||||
- '.github/workflows/s3-iam-tests.yml'
|
||||
push:
|
||||
branches: [ master ]
|
||||
paths:
|
||||
- 'weed/iam/**'
|
||||
- 'weed/s3api/**'
|
||||
- 'weed/credential/**'
|
||||
- 'weed/pb/**'
|
||||
- 'test/s3/iam/**'
|
||||
- '.github/workflows/s3-iam-tests.yml'
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.event.pull_request.number || (github.event_name == 'push' && github.ref) || github.run_id }}
|
||||
cancel-in-progress: ${{ github.event_name == 'pull_request' || (github.event_name == 'push' && !startsWith(github.ref, 'refs/tags/')) }}
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
defaults:
|
||||
run:
|
||||
working-directory: weed
|
||||
|
||||
jobs:
|
||||
# Unit tests for IAM components
|
||||
iam-unit-tests:
|
||||
name: IAM Unit Tests
|
||||
runs-on: ubuntu-24.04
|
||||
timeout-minutes: 15
|
||||
|
||||
steps:
|
||||
- name: Check out code
|
||||
uses: actions/checkout@v7
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v7
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
cache: ${{ runner.environment == 'github-hosted' }}
|
||||
id: go
|
||||
|
||||
- name: Get dependencies
|
||||
run: |
|
||||
go mod download
|
||||
|
||||
- name: Run IAM Unit Tests
|
||||
timeout-minutes: 10
|
||||
run: |
|
||||
set -x
|
||||
echo "=== Running IAM STS Tests ==="
|
||||
go test -v -timeout 5m ./iam/sts/...
|
||||
|
||||
echo "=== Running IAM Policy Tests ==="
|
||||
go test -v -timeout 5m ./iam/policy/...
|
||||
|
||||
echo "=== Running IAM Integration Tests ==="
|
||||
go test -v -timeout 5m ./iam/integration/...
|
||||
|
||||
echo "=== Running S3 API IAM Tests ==="
|
||||
go test -v -timeout 5m ./s3api/... -run ".*IAM.*|.*JWT.*|.*Auth.*"
|
||||
|
||||
- name: Upload test results on failure
|
||||
if: failure()
|
||||
uses: actions/upload-artifact@v7
|
||||
with:
|
||||
name: iam-unit-test-results
|
||||
path: |
|
||||
weed/testdata/
|
||||
weed/**/testdata/
|
||||
retention-days: 3
|
||||
|
||||
# S3 IAM integration tests with SeaweedFS services
|
||||
s3-iam-integration-tests:
|
||||
name: S3 IAM Integration Tests
|
||||
runs-on: ubuntu-24.04
|
||||
timeout-minutes: 25
|
||||
strategy:
|
||||
matrix:
|
||||
test-type: ["basic", "advanced", "policy-enforcement", "group", "sts"]
|
||||
|
||||
steps:
|
||||
- name: Check out code
|
||||
uses: actions/checkout@v7
|
||||
|
||||
- name: Clean up stale FUSE mounts
|
||||
uses: ./.github/actions/cleanup-stale-fuse-mounts
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v7
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
cache: ${{ runner.environment == 'github-hosted' }}
|
||||
id: go
|
||||
|
||||
- name: Install SeaweedFS
|
||||
working-directory: weed
|
||||
run: |
|
||||
go install -buildvcs=false
|
||||
|
||||
- name: Run S3 IAM Integration Tests - ${{ matrix.test-type }}
|
||||
timeout-minutes: 20
|
||||
working-directory: test/s3/iam
|
||||
run: |
|
||||
set -x
|
||||
echo "=== System Information ==="
|
||||
uname -a
|
||||
free -h
|
||||
timeout 15 df -h || true
|
||||
echo "=== Starting S3 IAM Integration Tests (${{ matrix.test-type }}) ==="
|
||||
|
||||
# Set WEED_BINARY to use the installed version
|
||||
export WEED_BINARY=$(which weed)
|
||||
export TEST_TIMEOUT=15m
|
||||
|
||||
# Run tests based on type
|
||||
case "${{ matrix.test-type }}" in
|
||||
"basic")
|
||||
echo "Running basic IAM functionality tests..."
|
||||
make clean setup start-services wait-for-services
|
||||
go test -v -timeout 15m -run "TestS3IAMAuthentication|TestS3IAMBasicWorkflow|TestS3IAMTokenValidation|TestIAMUserManagement|TestIAMAccessKeyManagement|TestIAMPolicyManagement" ./...
|
||||
;;
|
||||
"advanced")
|
||||
echo "Running advanced IAM feature tests..."
|
||||
make clean setup start-services wait-for-services
|
||||
go test -v -timeout 15m -run "TestS3IAMSessionExpiration|TestS3IAMMultipart|TestS3IAMPresigned" ./...
|
||||
;;
|
||||
"policy-enforcement")
|
||||
echo "Running policy enforcement tests..."
|
||||
make clean setup start-services wait-for-services
|
||||
go test -v -timeout 15m -run "TestS3IAMPolicyEnforcement|TestS3IAMBucketPolicy|TestS3IAMContextual" ./...
|
||||
;;
|
||||
"group")
|
||||
echo "Running IAM group management tests..."
|
||||
make clean setup start-services wait-for-services
|
||||
go test -v -timeout 15m -run "TestIAMGroup" ./...
|
||||
;;
|
||||
"sts")
|
||||
echo "Running STS and service account tests..."
|
||||
make clean setup start-services wait-for-services
|
||||
# SigV4-signed STS calls need admin credentials matching test_config.json.
|
||||
# Tests default to "admin"/"admin" when env vars are unset, which don't exist.
|
||||
export STS_TEST_ACCESS_KEY=test-access-key
|
||||
export STS_TEST_SECRET_KEY=test-secret-key
|
||||
# The use_service_account_credentials subtest is excluded because
|
||||
# newly-created service-account access keys are not currently
|
||||
# persisted to the filer after CreateServiceAccount — a
|
||||
# pre-existing sync issue tracked separately from the
|
||||
# GetFederationToken routing fix this PR addresses.
|
||||
go test -v -timeout 15m \
|
||||
-run "TestSTS|TestAssumeRoleWithWebIdentity|TestServiceAccount" \
|
||||
-skip "TestServiceAccountLifecycle/use_service_account_credentials" \
|
||||
./...
|
||||
;;
|
||||
*)
|
||||
echo "Unknown test type: ${{ matrix.test-type }}"
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
|
||||
# Always cleanup
|
||||
make stop-services
|
||||
|
||||
- name: Show service logs on failure
|
||||
if: failure()
|
||||
working-directory: test/s3/iam
|
||||
run: |
|
||||
echo "=== Service Logs ==="
|
||||
echo "--- Master Log ---"
|
||||
tail -50 weed-master.log 2>/dev/null || echo "No master log found"
|
||||
echo ""
|
||||
echo "--- Filer Log ---"
|
||||
tail -50 weed-filer.log 2>/dev/null || echo "No filer log found"
|
||||
echo ""
|
||||
echo "--- Volume Log ---"
|
||||
tail -50 weed-volume.log 2>/dev/null || echo "No volume log found"
|
||||
echo ""
|
||||
echo "--- S3 API Log ---"
|
||||
tail -50 weed-s3.log 2>/dev/null || echo "No S3 log found"
|
||||
echo ""
|
||||
|
||||
echo "=== Process Information ==="
|
||||
ps aux | grep -E "(weed|test)" || true
|
||||
netstat -tlnp | grep -E "(8333|8888|9333|8080)" || true
|
||||
|
||||
- name: Upload test logs on failure
|
||||
if: failure()
|
||||
uses: actions/upload-artifact@v7
|
||||
with:
|
||||
name: s3-iam-integration-logs-${{ matrix.test-type }}
|
||||
path: test/s3/iam/weed-*.log
|
||||
retention-days: 5
|
||||
|
||||
# Distributed IAM tests
|
||||
s3-iam-distributed-tests:
|
||||
name: S3 IAM Distributed Tests
|
||||
runs-on: ubuntu-24.04
|
||||
timeout-minutes: 25
|
||||
|
||||
steps:
|
||||
- name: Check out code
|
||||
uses: actions/checkout@v7
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v7
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
cache: ${{ runner.environment == 'github-hosted' }}
|
||||
id: go
|
||||
|
||||
- name: Install SeaweedFS
|
||||
working-directory: weed
|
||||
run: |
|
||||
go install -buildvcs=false
|
||||
|
||||
- name: Run Distributed IAM Tests
|
||||
timeout-minutes: 20
|
||||
working-directory: test/s3/iam
|
||||
run: |
|
||||
set -x
|
||||
echo "=== System Information ==="
|
||||
uname -a
|
||||
free -h
|
||||
|
||||
export WEED_BINARY=$(which weed)
|
||||
export TEST_TIMEOUT=15m
|
||||
|
||||
# Test distributed configuration
|
||||
echo "Testing distributed IAM configuration..."
|
||||
make clean setup
|
||||
|
||||
# Start services with distributed IAM config
|
||||
echo "Starting services with distributed configuration..."
|
||||
make start-services
|
||||
make wait-for-services
|
||||
|
||||
# Run distributed-specific tests
|
||||
export ENABLE_DISTRIBUTED_TESTS=true
|
||||
go test -v -timeout 15m -run "TestS3IAMDistributedTests" ./... || {
|
||||
echo "❌ Distributed tests failed, checking logs..."
|
||||
make logs
|
||||
exit 1
|
||||
}
|
||||
|
||||
make stop-services
|
||||
|
||||
- name: Upload distributed test logs
|
||||
if: always()
|
||||
uses: actions/upload-artifact@v7
|
||||
with:
|
||||
name: s3-iam-distributed-logs
|
||||
path: test/s3/iam/weed-*.log
|
||||
retention-days: 7
|
||||
|
||||
# Performance and stress tests
|
||||
s3-iam-performance-tests:
|
||||
name: S3 IAM Performance Tests
|
||||
runs-on: ubuntu-24.04
|
||||
timeout-minutes: 30
|
||||
|
||||
steps:
|
||||
- name: Check out code
|
||||
uses: actions/checkout@v7
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v7
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
cache: ${{ runner.environment == 'github-hosted' }}
|
||||
id: go
|
||||
|
||||
- name: Install SeaweedFS
|
||||
working-directory: weed
|
||||
run: |
|
||||
go install -buildvcs=false
|
||||
|
||||
- name: Run IAM Performance Benchmarks
|
||||
timeout-minutes: 25
|
||||
working-directory: test/s3/iam
|
||||
run: |
|
||||
set -x
|
||||
echo "=== Running IAM Performance Tests ==="
|
||||
|
||||
export WEED_BINARY=$(which weed)
|
||||
export TEST_TIMEOUT=20m
|
||||
|
||||
make clean setup start-services wait-for-services
|
||||
|
||||
# Run performance tests (benchmarks disabled for CI)
|
||||
echo "Running performance tests..."
|
||||
export ENABLE_PERFORMANCE_TESTS=true
|
||||
go test -v -timeout 15m -run "TestS3IAMPerformanceTests" ./... || {
|
||||
echo "❌ Performance tests failed"
|
||||
make logs
|
||||
exit 1
|
||||
}
|
||||
|
||||
make stop-services
|
||||
|
||||
- name: Upload performance test results
|
||||
if: always()
|
||||
uses: actions/upload-artifact@v7
|
||||
with:
|
||||
name: s3-iam-performance-results
|
||||
path: |
|
||||
test/s3/iam/weed-*.log
|
||||
test/s3/iam/*.test
|
||||
retention-days: 7
|
||||
@@ -1,165 +0,0 @@
|
||||
name: "S3 Keycloak Integration Tests"
|
||||
|
||||
on:
|
||||
pull_request:
|
||||
paths:
|
||||
- 'weed/iam/**'
|
||||
- 'weed/s3api/**'
|
||||
- 'test/s3/iam/**'
|
||||
- '.github/workflows/s3-keycloak-tests.yml'
|
||||
push:
|
||||
branches: [ master ]
|
||||
paths:
|
||||
- 'weed/iam/**'
|
||||
- 'weed/s3api/**'
|
||||
- 'test/s3/iam/**'
|
||||
- '.github/workflows/s3-keycloak-tests.yml'
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.event.pull_request.number || (github.event_name == 'push' && github.ref) || github.run_id }}
|
||||
cancel-in-progress: ${{ github.event_name == 'pull_request' || (github.event_name == 'push' && !startsWith(github.ref, 'refs/tags/')) }}
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
defaults:
|
||||
run:
|
||||
working-directory: weed
|
||||
|
||||
jobs:
|
||||
# Dedicated job for Keycloak integration tests
|
||||
s3-keycloak-integration-tests:
|
||||
name: S3 Keycloak Integration Tests
|
||||
runs-on: ubuntu-24.04
|
||||
timeout-minutes: 30
|
||||
|
||||
steps:
|
||||
- name: Check out code
|
||||
uses: actions/checkout@v7
|
||||
|
||||
- name: Clean up stale FUSE mounts
|
||||
uses: ./.github/actions/cleanup-stale-fuse-mounts
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v7
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
cache: ${{ runner.environment == 'github-hosted' }}
|
||||
id: go
|
||||
|
||||
- name: Install SeaweedFS
|
||||
working-directory: weed
|
||||
run: |
|
||||
go install -buildvcs=false
|
||||
|
||||
- name: Run Keycloak Integration Tests
|
||||
timeout-minutes: 25
|
||||
working-directory: test/s3/iam
|
||||
run: |
|
||||
set -x
|
||||
echo "=== System Information ==="
|
||||
uname -a
|
||||
free -h
|
||||
timeout 15 df -h || true
|
||||
echo "=== Starting S3 Keycloak Integration Tests ==="
|
||||
|
||||
# Set WEED_BINARY to use the installed version
|
||||
export WEED_BINARY=$(which weed)
|
||||
export TEST_TIMEOUT=20m
|
||||
|
||||
echo "Running Keycloak integration tests..."
|
||||
# Start Keycloak container first
|
||||
docker run -d \
|
||||
--name keycloak \
|
||||
-p 8080:8080 \
|
||||
-e KC_BOOTSTRAP_ADMIN_USERNAME=admin \
|
||||
-e KC_BOOTSTRAP_ADMIN_PASSWORD=admin \
|
||||
-e KC_HTTP_ENABLED=true \
|
||||
-e KC_HOSTNAME_STRICT=false \
|
||||
-e KC_HOSTNAME_STRICT_HTTPS=false \
|
||||
quay.io/keycloak/keycloak:26.0 \
|
||||
start-dev
|
||||
|
||||
# Wait for Keycloak with better health checking
|
||||
timeout 300 bash -c '
|
||||
while true; do
|
||||
if curl -s http://localhost:8080/health/ready > /dev/null 2>&1; then
|
||||
echo "✅ Keycloak health check passed"
|
||||
break
|
||||
fi
|
||||
echo "... waiting for Keycloak to be ready"
|
||||
sleep 5
|
||||
done
|
||||
'
|
||||
|
||||
# Setup Keycloak configuration
|
||||
./setup_keycloak.sh
|
||||
|
||||
# Start SeaweedFS services
|
||||
make clean setup start-services wait-for-services
|
||||
|
||||
# Verify service accessibility
|
||||
echo "=== Verifying Service Accessibility ==="
|
||||
curl -f http://localhost:8080/realms/master
|
||||
curl -s http://localhost:8333/healthz
|
||||
echo "✅ SeaweedFS S3 API is responding (IAM-protected endpoint)"
|
||||
|
||||
# Run Keycloak-specific tests
|
||||
echo "=== Running Keycloak Tests ==="
|
||||
export KEYCLOAK_URL=http://localhost:8080
|
||||
export S3_ENDPOINT=http://localhost:8333
|
||||
|
||||
# Wait for realm to be properly configured
|
||||
timeout 120 bash -c 'until curl -fs http://localhost:8080/realms/seaweedfs-test/.well-known/openid-configuration > /dev/null; do echo "... waiting for realm"; sleep 3; done'
|
||||
|
||||
# Run the Keycloak integration tests
|
||||
go test -v -timeout 20m -run "TestKeycloak" ./...
|
||||
|
||||
- name: Show server logs on failure
|
||||
if: failure()
|
||||
working-directory: test/s3/iam
|
||||
run: |
|
||||
echo "=== Service Logs ==="
|
||||
echo "--- Keycloak logs ---"
|
||||
docker logs keycloak --tail=100 || echo "No Keycloak container logs"
|
||||
|
||||
echo "--- SeaweedFS Master logs ---"
|
||||
if [ -f weed-master.log ]; then
|
||||
tail -100 weed-master.log
|
||||
fi
|
||||
|
||||
echo "--- SeaweedFS S3 logs ---"
|
||||
if [ -f weed-s3.log ]; then
|
||||
tail -100 weed-s3.log
|
||||
fi
|
||||
|
||||
echo "--- SeaweedFS Filer logs ---"
|
||||
if [ -f weed-filer.log ]; then
|
||||
tail -100 weed-filer.log
|
||||
fi
|
||||
|
||||
echo "=== System Status ==="
|
||||
ps aux | grep -E "(weed|keycloak)" || true
|
||||
netstat -tlnp | grep -E "(8333|9333|8080|8888)" || true
|
||||
docker ps -a || true
|
||||
|
||||
- name: Cleanup
|
||||
if: always()
|
||||
working-directory: test/s3/iam
|
||||
run: |
|
||||
# Stop Keycloak container
|
||||
docker stop keycloak || true
|
||||
docker rm keycloak || true
|
||||
|
||||
# Stop SeaweedFS services
|
||||
make clean || true
|
||||
|
||||
- name: Upload test logs on failure
|
||||
if: failure()
|
||||
uses: actions/upload-artifact@v7
|
||||
with:
|
||||
name: s3-keycloak-test-logs
|
||||
path: |
|
||||
test/s3/iam/*.log
|
||||
test/s3/iam/test-volume-data/
|
||||
retention-days: 3
|
||||
@@ -1,190 +0,0 @@
|
||||
name: "S3 Mutation Regression Tests"
|
||||
|
||||
on:
|
||||
pull_request:
|
||||
paths:
|
||||
- 'weed/s3api/**'
|
||||
- 'test/s3/delete/**'
|
||||
- 'test/s3/distributed_lock/**'
|
||||
- 'test/s3/sqlite_cbs/**'
|
||||
- 'test/s3/versioning/**'
|
||||
- 'test/volume_server/framework/**'
|
||||
- 'docker/compose/s3.json'
|
||||
- '.github/workflows/s3-mutation-regression-tests.yml'
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.event.pull_request.number || (github.event_name == 'push' && github.ref) || github.run_id }}
|
||||
cancel-in-progress: ${{ github.event_name == 'pull_request' || (github.event_name == 'push' && !startsWith(github.ref, 'refs/tags/')) }}
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
jobs:
|
||||
s3-versioning-regressions:
|
||||
name: S3 Versioning Regression Tests
|
||||
runs-on: ubuntu-24.04
|
||||
timeout-minutes: 25
|
||||
|
||||
steps:
|
||||
- name: Check out code
|
||||
uses: actions/checkout@v7
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v7
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
cache: ${{ runner.environment == 'github-hosted' }}
|
||||
|
||||
- name: Run S3 versioning regression tests
|
||||
timeout-minutes: 20
|
||||
working-directory: test/s3/versioning
|
||||
run: |
|
||||
set -x
|
||||
# Run every versioning test, so a regression test lands covered instead
|
||||
# of waiting for someone to remember this file. Name a test in EXCLUDE,
|
||||
# with the reason, to keep it out.
|
||||
#
|
||||
# TestVersioningPagination*: opt-in stress tests that build 1500+
|
||||
# versions. They self-skip without ENABLE_STRESS_TESTS and have their
|
||||
# own make target, so this gate should not carry them.
|
||||
EXCLUDE='TestVersioningPagination.*'
|
||||
tests=$(go test . -list '.*' | grep '^Test' | sort -u)
|
||||
# An empty list would make -run match nothing and pass this job vacuously.
|
||||
[ -n "$tests" ] || { echo "listed no versioning tests"; exit 1; }
|
||||
selected=$(echo "$tests" | grep -vE "^($EXCLUDE)$")
|
||||
[ -n "$selected" ] || { echo "EXCLUDE matched every test"; exit 1; }
|
||||
echo "running $(echo "$selected" | wc -l) of $(echo "$tests" | wc -l) versioning tests"
|
||||
# make swallows a lone trailing $, taking the anchor with it, so escape it.
|
||||
make test-with-server TEST_PATTERN="^($(echo "$selected" | paste -sd'|' -))"'$$'
|
||||
|
||||
- name: Show server logs on failure
|
||||
if: failure()
|
||||
working-directory: test/s3/versioning
|
||||
run: |
|
||||
echo "=== Server Logs ==="
|
||||
if [ -f weed-test.log ]; then
|
||||
tail -100 weed-test.log
|
||||
fi
|
||||
|
||||
- name: Upload versioning logs on failure
|
||||
if: failure()
|
||||
uses: actions/upload-artifact@v7
|
||||
with:
|
||||
name: s3-versioning-regression-logs
|
||||
path: test/s3/versioning/weed-test*.log
|
||||
retention-days: 3
|
||||
|
||||
s3-delete-regressions:
|
||||
name: S3 Delete Regression Tests
|
||||
runs-on: ubuntu-24.04
|
||||
timeout-minutes: 20
|
||||
|
||||
steps:
|
||||
- name: Check out code
|
||||
uses: actions/checkout@v7
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v7
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
cache: ${{ runner.environment == 'github-hosted' }}
|
||||
|
||||
- name: Run S3 delete regression tests
|
||||
timeout-minutes: 15
|
||||
working-directory: test/s3/delete
|
||||
run: |
|
||||
set -x
|
||||
make test-with-server
|
||||
|
||||
- name: Show server logs on failure
|
||||
if: failure()
|
||||
working-directory: test/s3/delete
|
||||
run: |
|
||||
echo "=== Server Logs ==="
|
||||
if [ -f weed-test.log ]; then
|
||||
tail -100 weed-test.log
|
||||
fi
|
||||
|
||||
- name: Upload delete logs on failure
|
||||
if: failure()
|
||||
uses: actions/upload-artifact@v7
|
||||
with:
|
||||
name: s3-delete-regression-logs
|
||||
path: test/s3/delete/weed-test*.log
|
||||
retention-days: 3
|
||||
|
||||
s3-distributed-lock-regressions:
|
||||
name: S3 Distributed Lock Regression Tests
|
||||
runs-on: ubuntu-24.04
|
||||
timeout-minutes: 30
|
||||
|
||||
steps:
|
||||
- name: Check out code
|
||||
uses: actions/checkout@v7
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v7
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
cache: ${{ runner.environment == 'github-hosted' }}
|
||||
|
||||
- name: Build SeaweedFS
|
||||
run: |
|
||||
go build -o weed/weed -buildvcs=false ./weed
|
||||
|
||||
- name: Run distributed lock regressions
|
||||
timeout-minutes: 25
|
||||
env:
|
||||
TMPDIR: ${{ github.workspace }}/test/s3/distributed_lock/tmp
|
||||
S3_DISTRIBUTED_LOCK_KEEP_LOGS: "1"
|
||||
WEED_BINARY: ${{ github.workspace }}/weed/weed
|
||||
run: |
|
||||
set -x
|
||||
mkdir -p "$TMPDIR"
|
||||
go test -v -count=1 -timeout=20m ./test/s3/distributed_lock
|
||||
|
||||
- name: Upload distributed lock logs on failure
|
||||
if: failure()
|
||||
uses: actions/upload-artifact@v7
|
||||
with:
|
||||
name: s3-distributed-lock-regression-logs
|
||||
path: test/s3/distributed_lock/tmp/seaweedfs_s3_distributed_lock_*
|
||||
retention-days: 3
|
||||
|
||||
s3-sqlite-cbs-regressions:
|
||||
name: S3 SQLite CBS Regression Tests
|
||||
runs-on: ubuntu-24.04
|
||||
timeout-minutes: 20
|
||||
|
||||
steps:
|
||||
- name: Check out code
|
||||
uses: actions/checkout@v7
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v7
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
cache: ${{ runner.environment == 'github-hosted' }}
|
||||
|
||||
- name: Build SeaweedFS
|
||||
run: |
|
||||
go build -o weed/weed -buildvcs=false ./weed
|
||||
|
||||
- name: Run sqlite cbs regressions
|
||||
timeout-minutes: 15
|
||||
env:
|
||||
TMPDIR: ${{ github.workspace }}/test/s3/sqlite_cbs/tmp
|
||||
S3_SQLITE_CBS_KEEP_LOGS: "1"
|
||||
WEED_BINARY: ${{ github.workspace }}/weed/weed
|
||||
run: |
|
||||
set -x
|
||||
mkdir -p "$TMPDIR"
|
||||
go test -v -count=1 -timeout=12m ./test/s3/sqlite_cbs
|
||||
|
||||
- name: Upload sqlite cbs logs on failure
|
||||
if: failure()
|
||||
uses: actions/upload-artifact@v7
|
||||
with:
|
||||
name: s3-sqlite-cbs-regression-logs
|
||||
path: test/s3/sqlite_cbs/tmp/seaweedfs_sqlite_cbs_*
|
||||
retention-days: 3
|
||||
@@ -1,168 +0,0 @@
|
||||
name: "S3 PyArrow Parquet Tests"
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [master]
|
||||
paths:
|
||||
- 'weed/s3api/**'
|
||||
- 'weed/filer/**'
|
||||
- 'test/s3/parquet/**'
|
||||
- '.github/workflows/s3-parquet-tests.yml'
|
||||
pull_request:
|
||||
branches: [master]
|
||||
paths:
|
||||
- 'weed/s3api/**'
|
||||
- 'weed/filer/**'
|
||||
- 'test/s3/parquet/**'
|
||||
- '.github/workflows/s3-parquet-tests.yml'
|
||||
workflow_dispatch:
|
||||
|
||||
env:
|
||||
S3_ACCESS_KEY: some_access_key1
|
||||
S3_SECRET_KEY: some_secret_key1
|
||||
S3_ENDPOINT_URL: http://localhost:8333
|
||||
BUCKET_NAME: test-parquet-bucket
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.event.pull_request.number || (github.event_name == 'push' && github.ref) || github.run_id }}
|
||||
cancel-in-progress: ${{ github.event_name == 'pull_request' || (github.event_name == 'push' && !startsWith(github.ref, 'refs/tags/')) }}
|
||||
|
||||
jobs:
|
||||
parquet-integration-tests:
|
||||
name: PyArrow Parquet Tests (Python ${{ matrix.python-version }})
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 20
|
||||
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
python-version: ['3.10', '3.11', '3.12']
|
||||
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v7
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v7
|
||||
with:
|
||||
go-version: ^1.26
|
||||
cache: ${{ runner.environment == 'github-hosted' }}
|
||||
|
||||
- name: Set up Python ${{ matrix.python-version }}
|
||||
uses: actions/setup-python@v7
|
||||
with:
|
||||
python-version: ${{ matrix.python-version }}
|
||||
cache: ${{ runner.environment == 'github-hosted' && 'pip' || '' }}
|
||||
cache-dependency-path: 'test/s3/parquet/requirements.txt'
|
||||
|
||||
- name: Install system dependencies
|
||||
run: |
|
||||
sudo apt-get update
|
||||
sudo apt-get install -y lsof netcat-openbsd
|
||||
|
||||
- name: Build SeaweedFS
|
||||
run: |
|
||||
cd weed
|
||||
go build -v
|
||||
sudo cp weed /usr/local/bin/
|
||||
weed version
|
||||
|
||||
- name: Run PyArrow Parquet integration tests
|
||||
run: |
|
||||
cd test/s3/parquet
|
||||
make test-with-server
|
||||
env:
|
||||
SEAWEEDFS_BINARY: weed
|
||||
S3_PORT: 8333
|
||||
FILER_PORT: 8888
|
||||
VOLUME_PORT: 8080
|
||||
MASTER_PORT: 9333
|
||||
VOLUME_MAX_SIZE_MB: 50
|
||||
|
||||
- name: Run implicit directory fix tests
|
||||
run: |
|
||||
cd test/s3/parquet
|
||||
make test-implicit-dir-with-server
|
||||
env:
|
||||
SEAWEEDFS_BINARY: weed
|
||||
S3_PORT: 8333
|
||||
FILER_PORT: 8888
|
||||
VOLUME_PORT: 8080
|
||||
MASTER_PORT: 9333
|
||||
|
||||
- name: Run PyArrow native S3 filesystem tests
|
||||
run: |
|
||||
cd test/s3/parquet
|
||||
make test-native-s3-with-server
|
||||
env:
|
||||
SEAWEEDFS_BINARY: weed
|
||||
S3_PORT: 8333
|
||||
FILER_PORT: 8888
|
||||
VOLUME_PORT: 8080
|
||||
MASTER_PORT: 9333
|
||||
|
||||
- name: Run cross-filesystem compatibility tests
|
||||
run: |
|
||||
cd test/s3/parquet
|
||||
TEST_QUICK=1 make test-cross-fs-with-server
|
||||
env:
|
||||
SEAWEEDFS_BINARY: weed
|
||||
S3_PORT: 8333
|
||||
FILER_PORT: 8888
|
||||
VOLUME_PORT: 8080
|
||||
MASTER_PORT: 9333
|
||||
|
||||
- name: Run SSE-S3 encryption compatibility tests
|
||||
run: |
|
||||
cd test/s3/parquet
|
||||
make test-sse-s3-compat
|
||||
env:
|
||||
SEAWEEDFS_BINARY: weed
|
||||
S3_PORT: 8333
|
||||
FILER_PORT: 8888
|
||||
VOLUME_PORT: 8080
|
||||
MASTER_PORT: 9333
|
||||
|
||||
- name: Upload test logs on failure
|
||||
if: failure()
|
||||
uses: actions/upload-artifact@v7
|
||||
with:
|
||||
name: test-logs-python-${{ matrix.python-version }}
|
||||
path: |
|
||||
/tmp/seaweedfs-parquet-*.log
|
||||
test/s3/parquet/*.log
|
||||
retention-days: 7
|
||||
|
||||
- name: Cleanup
|
||||
if: always()
|
||||
run: |
|
||||
cd test/s3/parquet
|
||||
make stop-seaweedfs-safe || true
|
||||
make clean || true
|
||||
|
||||
unit-tests:
|
||||
name: Go Unit Tests (Implicit Directory)
|
||||
runs-on: ubuntu-latest
|
||||
# Leave time for setup and the focused run before the nine-minute suite.
|
||||
timeout-minutes: 20
|
||||
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v7
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v7
|
||||
with:
|
||||
go-version: ^1.26
|
||||
cache: ${{ runner.environment == 'github-hosted' }}
|
||||
|
||||
- name: Run Go unit tests
|
||||
run: |
|
||||
cd weed/s3api
|
||||
go test -v -run TestImplicitDirectory
|
||||
|
||||
- name: Run all S3 API tests
|
||||
run: |
|
||||
cd weed/s3api
|
||||
go test -v -timeout 9m
|
||||
|
||||
@@ -1,437 +0,0 @@
|
||||
name: "S3 Policy Integration Tests"
|
||||
|
||||
on:
|
||||
pull_request:
|
||||
paths:
|
||||
- 'weed/s3api/s3_iam_middleware.go'
|
||||
- 'weed/s3api/s3api_bucket_policy*.go'
|
||||
- 'weed/s3api/s3_action_resolver.go'
|
||||
- 'weed/s3api/policy/**'
|
||||
- 'weed/iam/**'
|
||||
- 'test/s3/iam/**'
|
||||
- 'test/s3/policy/**'
|
||||
- '.github/workflows/s3-policy-tests.yml'
|
||||
push:
|
||||
branches: [ master, main ]
|
||||
paths:
|
||||
- 'weed/s3api/s3_iam_middleware.go'
|
||||
- 'weed/s3api/s3api_bucket_policy*.go'
|
||||
- 'weed/s3api/s3_action_resolver.go'
|
||||
- 'weed/s3api/policy/**'
|
||||
- 'weed/iam/**'
|
||||
- 'test/s3/iam/**'
|
||||
- 'test/s3/policy/**'
|
||||
- '.github/workflows/s3-policy-tests.yml'
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.event.pull_request.number || (github.event_name == 'push' && github.ref) || github.run_id }}
|
||||
cancel-in-progress: ${{ github.event_name == 'pull_request' || (github.event_name == 'push' && !startsWith(github.ref, 'refs/tags/')) }}
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
defaults:
|
||||
run:
|
||||
working-directory: weed
|
||||
|
||||
jobs:
|
||||
# Unit tests for policy components
|
||||
policy-unit-tests:
|
||||
name: S3 Policy Unit Tests
|
||||
runs-on: ubuntu-24.04
|
||||
timeout-minutes: 15
|
||||
|
||||
steps:
|
||||
- name: Check out code
|
||||
uses: actions/checkout@v7
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v7
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
cache: ${{ runner.environment == 'github-hosted' }}
|
||||
id: go
|
||||
|
||||
- name: Get dependencies
|
||||
run: |
|
||||
go mod download
|
||||
|
||||
- name: Run S3 Policy Unit Tests
|
||||
timeout-minutes: 10
|
||||
run: |
|
||||
set -x
|
||||
echo "=== Running S3 Action Resolver Tests ==="
|
||||
go test -v -timeout 5m ./s3api/... -run ".*ActionResolver.*"
|
||||
|
||||
echo "=== Running S3 Bucket Policy Engine Tests ==="
|
||||
go test -v -timeout 5m ./s3api/... -run ".*BucketPolicy.*|.*PolicyEngine.*"
|
||||
|
||||
echo "=== Running IAM Policy Tests ==="
|
||||
go test -v -timeout 5m ./iam/policy/...
|
||||
|
||||
- name: Upload test results on failure
|
||||
if: failure()
|
||||
uses: actions/upload-artifact@v7
|
||||
with:
|
||||
name: policy-unit-test-results
|
||||
path: |
|
||||
weed/testdata/
|
||||
weed/**/testdata/
|
||||
retention-days: 3
|
||||
|
||||
# S3 Policy Variables Integration Tests
|
||||
s3-policy-variables-tests:
|
||||
name: S3 Policy Variables Integration Tests
|
||||
runs-on: ubuntu-24.04
|
||||
timeout-minutes: 25
|
||||
|
||||
steps:
|
||||
- name: Check out code
|
||||
uses: actions/checkout@v7
|
||||
|
||||
- name: Clean up stale FUSE mounts
|
||||
uses: ./.github/actions/cleanup-stale-fuse-mounts
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v7
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
cache: ${{ runner.environment == 'github-hosted' }}
|
||||
id: go
|
||||
|
||||
- name: Install SeaweedFS
|
||||
run: |
|
||||
go install -buildvcs=false
|
||||
|
||||
- name: Run S3 Policy Variables Integration Tests
|
||||
timeout-minutes: 20
|
||||
working-directory: test/s3/iam
|
||||
run: |
|
||||
set -x
|
||||
echo "=== System Information ==="
|
||||
uname -a
|
||||
free -h
|
||||
timeout 15 df -h || true
|
||||
|
||||
echo "=== Starting S3 Policy Variables Integration Tests ==="
|
||||
|
||||
# Set WEED_BINARY to use the installed version
|
||||
export WEED_BINARY=$(which weed)
|
||||
export TEST_TIMEOUT=15m
|
||||
|
||||
# Run policy variables tests
|
||||
echo "Running policy variables tests..."
|
||||
|
||||
# Kill any existing weed server on port 8333
|
||||
if lsof -Pi :8333 -sTCP:LISTEN -t >/dev/null 2>&1 ; then
|
||||
kill $(lsof -t -i:8333) 2>/dev/null || true
|
||||
sleep 2
|
||||
fi
|
||||
|
||||
# Start weed server with IAM configuration
|
||||
echo "Starting weed server with IAM configuration..."
|
||||
$WEED_BINARY server \
|
||||
-s3 \
|
||||
-s3.port=8333 \
|
||||
-s3.iam.config="$(pwd)/test_iam_config.json" \
|
||||
-filer \
|
||||
-volume.max=0 \
|
||||
-master.volumeSizeLimitMB=100 \
|
||||
-s3.allowDeleteBucketNotEmpty=true \
|
||||
> /tmp/weed_policy_test_server.log 2>&1 &
|
||||
|
||||
SERVER_PID=$!
|
||||
echo "Server started with PID: $SERVER_PID"
|
||||
|
||||
# Wait for server to be ready
|
||||
echo "Waiting for server to be ready..."
|
||||
MAX_WAIT=30
|
||||
COUNTER=0
|
||||
while ! curl -s http://localhost:8333/status > /dev/null 2>&1; do
|
||||
sleep 1
|
||||
COUNTER=$((COUNTER + 1))
|
||||
if [ $COUNTER -ge $MAX_WAIT ]; then
|
||||
echo "Server failed to start within ${MAX_WAIT} seconds"
|
||||
echo "Server log:"
|
||||
cat /tmp/weed_policy_test_server.log
|
||||
kill $SERVER_PID 2>/dev/null || true
|
||||
exit 1
|
||||
fi
|
||||
done
|
||||
|
||||
echo "Server is ready!"
|
||||
|
||||
# Trap to ensure server is killed on exit
|
||||
trap "kill $SERVER_PID 2>/dev/null || true" EXIT
|
||||
|
||||
# Run the tests
|
||||
go test -v -timeout 15m -run TestS3PolicyVariables ./...
|
||||
|
||||
- name: Show service logs on failure
|
||||
if: failure()
|
||||
working-directory: test/s3/iam
|
||||
run: |
|
||||
echo "=== Service Logs ==="
|
||||
if [ -f /tmp/weed_policy_test_server.log ]; then
|
||||
echo "--- Last 100 lines of Server Log ---"
|
||||
tail -100 /tmp/weed_policy_test_server.log
|
||||
fi
|
||||
echo ""
|
||||
echo "=== Process Information ==="
|
||||
ps aux | grep -E "(weed|test)" || true
|
||||
netstat -tlnp | grep -E "(8333|8888|9333|8080)" || true
|
||||
|
||||
- name: Upload test logs on failure
|
||||
if: failure()
|
||||
uses: actions/upload-artifact@v7
|
||||
with:
|
||||
name: s3-policy-variables-test-logs
|
||||
path: /tmp/weed_policy_test_server.log
|
||||
retention-days: 5
|
||||
|
||||
# S3 Policy Enforcement Integration Tests
|
||||
s3-policy-enforcement-tests:
|
||||
name: S3 Policy Enforcement Integration Tests
|
||||
runs-on: ubuntu-24.04
|
||||
timeout-minutes: 30
|
||||
strategy:
|
||||
matrix:
|
||||
test-case: ["basic-policy", "contextual-policy", "advanced-policy"]
|
||||
|
||||
steps:
|
||||
- name: Check out code
|
||||
uses: actions/checkout@v7
|
||||
|
||||
- name: Clean up stale FUSE mounts
|
||||
uses: ./.github/actions/cleanup-stale-fuse-mounts
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v7
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
cache: ${{ runner.environment == 'github-hosted' }}
|
||||
id: go
|
||||
|
||||
- name: Install SeaweedFS
|
||||
run: |
|
||||
go install -buildvcs=false
|
||||
|
||||
- name: Run S3 Policy Enforcement Tests - ${{ matrix.test-case }}
|
||||
timeout-minutes: 25
|
||||
working-directory: test/s3/iam
|
||||
run: |
|
||||
set -x
|
||||
echo "=== System Information ==="
|
||||
uname -a
|
||||
free -h
|
||||
timeout 15 df -h || true
|
||||
|
||||
echo "=== Starting S3 Policy Enforcement Tests (${{ matrix.test-case }}) ==="
|
||||
|
||||
export WEED_BINARY=$(which weed)
|
||||
export TEST_TIMEOUT=20m
|
||||
|
||||
# Kill any existing weed server on port 8333
|
||||
if lsof -Pi :8333 -sTCP:LISTEN -t >/dev/null 2>&1 ; then
|
||||
kill $(lsof -t -i:8333) 2>/dev/null || true
|
||||
sleep 2
|
||||
fi
|
||||
|
||||
# Start weed server with IAM configuration
|
||||
echo "Starting weed server with IAM configuration..."
|
||||
$WEED_BINARY server \
|
||||
-s3 \
|
||||
-s3.port=8333 \
|
||||
-s3.iam.config="$(pwd)/test_iam_config.json" \
|
||||
-filer \
|
||||
-volume.max=0 \
|
||||
-master.volumeSizeLimitMB=100 \
|
||||
-s3.allowDeleteBucketNotEmpty=true \
|
||||
> /tmp/weed_policy_enforcement_${{ matrix.test-case }}.log 2>&1 &
|
||||
|
||||
SERVER_PID=$!
|
||||
echo "Server started with PID: $SERVER_PID"
|
||||
|
||||
# Wait for server to be ready
|
||||
echo "Waiting for server to be ready..."
|
||||
MAX_WAIT=30
|
||||
COUNTER=0
|
||||
while ! curl -s http://localhost:8333/status > /dev/null 2>&1; do
|
||||
sleep 1
|
||||
COUNTER=$((COUNTER + 1))
|
||||
if [ $COUNTER -ge $MAX_WAIT ]; then
|
||||
echo "Server failed to start within ${MAX_WAIT} seconds"
|
||||
cat /tmp/weed_policy_enforcement_${{ matrix.test-case }}.log
|
||||
kill $SERVER_PID 2>/dev/null || true
|
||||
exit 1
|
||||
fi
|
||||
done
|
||||
|
||||
echo "Server is ready!"
|
||||
|
||||
# Trap to ensure server is killed on exit
|
||||
trap "kill $SERVER_PID 2>/dev/null || true" EXIT
|
||||
|
||||
# Run tests based on test case
|
||||
case "${{ matrix.test-case }}" in
|
||||
"basic-policy")
|
||||
echo "Running basic policy enforcement tests..."
|
||||
go test -v -timeout 20m -run "TestS3IAMBucketPolicy|TestS3IAMPolicyEnforcement" ./...
|
||||
;;
|
||||
"contextual-policy")
|
||||
echo "Running contextual policy tests..."
|
||||
go test -v -timeout 20m -run "TestS3PolicyVariables|TestS3IAMContextual" ./...
|
||||
;;
|
||||
"advanced-policy")
|
||||
echo "Running advanced policy tests..."
|
||||
go test -v -timeout 20m -run "TestS3IAMMultipart|TestS3IAMPresigned" ./...
|
||||
;;
|
||||
*)
|
||||
echo "Unknown test case: ${{ matrix.test-case }}"
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
|
||||
- name: Show service logs on failure
|
||||
if: failure()
|
||||
working-directory: test/s3/iam
|
||||
run: |
|
||||
echo "=== Service Logs ==="
|
||||
if [ -f /tmp/weed_policy_enforcement_${{ matrix.test-case }}.log ]; then
|
||||
echo "--- Last 100 lines of Server Log ---"
|
||||
tail -100 /tmp/weed_policy_enforcement_${{ matrix.test-case }}.log
|
||||
fi
|
||||
echo ""
|
||||
echo "=== Process Information ==="
|
||||
ps aux | grep -E "(weed|test)" || true
|
||||
netstat -tlnp | grep -E "(8333|8888|9333|8080)" || true
|
||||
|
||||
- name: Upload test logs on failure
|
||||
if: failure()
|
||||
uses: actions/upload-artifact@v7
|
||||
with:
|
||||
name: s3-policy-enforcement-logs-${{ matrix.test-case }}
|
||||
path: /tmp/weed_policy_enforcement_${{ matrix.test-case }}.log
|
||||
retention-days: 5
|
||||
|
||||
# Trusted Proxy Detection Tests
|
||||
trusted-proxy-tests:
|
||||
name: Trusted Proxy Detection Tests
|
||||
runs-on: ubuntu-24.04
|
||||
timeout-minutes: 20
|
||||
|
||||
steps:
|
||||
- name: Check out code
|
||||
uses: actions/checkout@v7
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v7
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
cache: ${{ runner.environment == 'github-hosted' }}
|
||||
id: go
|
||||
|
||||
- name: Install SeaweedFS
|
||||
run: |
|
||||
go install -buildvcs=false
|
||||
|
||||
- name: Run Trusted Proxy Tests
|
||||
timeout-minutes: 15
|
||||
working-directory: test/s3/iam
|
||||
run: |
|
||||
set -x
|
||||
echo "=== Running Trusted Proxy Detection Tests ==="
|
||||
|
||||
export WEED_BINARY=$(which weed)
|
||||
|
||||
# Kill any existing weed server on port 8333
|
||||
if lsof -Pi :8333 -sTCP:LISTEN -t >/dev/null 2>&1 ; then
|
||||
kill $(lsof -t -i:8333) 2>/dev/null || true
|
||||
sleep 2
|
||||
fi
|
||||
|
||||
# Start weed server
|
||||
echo "Starting weed server..."
|
||||
$WEED_BINARY server \
|
||||
-s3 \
|
||||
-s3.port=8333 \
|
||||
-s3.iam.config="$(pwd)/test_iam_config.json" \
|
||||
-filer \
|
||||
-volume.max=0 \
|
||||
-master.volumeSizeLimitMB=100 \
|
||||
-s3.allowDeleteBucketNotEmpty=true \
|
||||
> /tmp/weed_proxy_test.log 2>&1 &
|
||||
|
||||
SERVER_PID=$!
|
||||
echo "Server started with PID: $SERVER_PID"
|
||||
|
||||
# Wait for server to be ready
|
||||
echo "Waiting for server to be ready..."
|
||||
MAX_WAIT=30
|
||||
COUNTER=0
|
||||
while ! curl -s http://localhost:8333/status > /dev/null 2>&1; do
|
||||
sleep 1
|
||||
COUNTER=$((COUNTER + 1))
|
||||
if [ $COUNTER -ge $MAX_WAIT ]; then
|
||||
echo "Server failed to start within ${MAX_WAIT} seconds"
|
||||
kill $SERVER_PID 2>/dev/null || true
|
||||
exit 1
|
||||
fi
|
||||
done
|
||||
|
||||
# Trap to ensure server is killed on exit
|
||||
trap "kill $SERVER_PID 2>/dev/null || true" EXIT
|
||||
|
||||
# Run proxy tests
|
||||
go test -v -timeout 10m -run "TestTrustedProxy|TestPrivateIP" ./...
|
||||
|
||||
- name: Show service logs on failure
|
||||
if: failure()
|
||||
run: |
|
||||
echo "=== Service Logs ==="
|
||||
if [ -f /tmp/weed_proxy_test.log ]; then
|
||||
echo "--- Last 100 lines of Server Log ---"
|
||||
tail -100 /tmp/weed_proxy_test.log
|
||||
fi
|
||||
|
||||
- name: Upload test logs on failure
|
||||
if: failure()
|
||||
uses: actions/upload-artifact@v7
|
||||
with:
|
||||
name: trusted-proxy-test-logs
|
||||
path: /tmp/weed_proxy_test.log
|
||||
retention-days: 3
|
||||
|
||||
# S3 Policy Shell Integration Tests
|
||||
s3-policy-shell-tests:
|
||||
name: S3 Policy Shell Integration Tests
|
||||
runs-on: ubuntu-24.04
|
||||
timeout-minutes: 15
|
||||
|
||||
steps:
|
||||
- name: Check out code
|
||||
uses: actions/checkout@v7
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v7
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
cache: ${{ runner.environment == 'github-hosted' }}
|
||||
id: go
|
||||
|
||||
- name: Install SeaweedFS
|
||||
run: |
|
||||
go install -buildvcs=false
|
||||
|
||||
- name: Run S3 Policy Shell Tests
|
||||
timeout-minutes: 10
|
||||
working-directory: test/s3/policy
|
||||
run: |
|
||||
set -x
|
||||
echo "=== Running S3 Policy Shell Tests ==="
|
||||
|
||||
# Set WEED_BINARY to use the installed version (though test uses 'weed' command)
|
||||
export WEED_BINARY=$(which weed)
|
||||
export PATH=$PATH:$(dirname $WEED_BINARY)
|
||||
|
||||
go test -v -timeout 10m ./...
|
||||
@@ -1,144 +0,0 @@
|
||||
name: "S3 Proxy Signature Tests"
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [ master ]
|
||||
paths:
|
||||
- 'weed/s3api/**'
|
||||
- 'weed/server/**'
|
||||
- 'test/s3/proxy_signature/**'
|
||||
- 'go.mod'
|
||||
- 'go.sum'
|
||||
- '.github/workflows/s3-proxy-signature-tests.yml'
|
||||
pull_request:
|
||||
branches: [ master ]
|
||||
paths:
|
||||
- 'weed/s3api/**'
|
||||
- 'weed/server/**'
|
||||
- 'test/s3/proxy_signature/**'
|
||||
- 'go.mod'
|
||||
- 'go.sum'
|
||||
- '.github/workflows/s3-proxy-signature-tests.yml'
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.event.pull_request.number || (github.event_name == 'push' && github.ref) || github.run_id }}
|
||||
cancel-in-progress: ${{ github.event_name == 'pull_request' || (github.event_name == 'push' && !startsWith(github.ref, 'refs/tags/')) }}
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
jobs:
|
||||
proxy-signature-tests:
|
||||
name: S3 Proxy Signature Verification Tests
|
||||
runs-on: ubuntu-24.04
|
||||
timeout-minutes: 15
|
||||
steps:
|
||||
- name: Check out code into the Go module directory
|
||||
uses: actions/checkout@v7
|
||||
|
||||
- name: Set up Go 1.x
|
||||
uses: actions/setup-go@v7
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
cache: ${{ runner.environment == 'github-hosted' }}
|
||||
id: go
|
||||
|
||||
- name: Configure Docker Hub mirror
|
||||
run: |
|
||||
echo '{"registry-mirrors": ["https://mirror.gcr.io"]}' | sudo tee /etc/docker/daemon.json
|
||||
sudo systemctl restart docker
|
||||
|
||||
- name: Set up Docker Buildx
|
||||
uses: docker/setup-buildx-action@v4
|
||||
|
||||
- name: Build SeaweedFS binary for Linux
|
||||
run: |
|
||||
set -x
|
||||
CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -buildvcs=false -v -o test/s3/proxy_signature/weed ./weed
|
||||
|
||||
- name: Run S3 Proxy Signature Tests
|
||||
timeout-minutes: 10
|
||||
working-directory: test/s3/proxy_signature
|
||||
run: |
|
||||
set -x
|
||||
echo "Starting Docker Compose services..."
|
||||
docker compose up -d --build
|
||||
|
||||
# Check if containers are running
|
||||
echo "Checking container status..."
|
||||
docker compose ps
|
||||
|
||||
# Wait for services to be ready
|
||||
echo "Waiting for nginx proxy to be ready..."
|
||||
PROXY_READY=0
|
||||
for i in $(seq 1 30); do
|
||||
if curl -s http://localhost:9000/ > /dev/null 2>&1; then
|
||||
echo "Proxy is ready"
|
||||
PROXY_READY=1
|
||||
break
|
||||
fi
|
||||
echo "Waiting for proxy... ($i/30)"
|
||||
sleep 1
|
||||
done
|
||||
if [ $PROXY_READY -eq 0 ]; then
|
||||
echo "ERROR: Proxy failed to become ready after 30 seconds"
|
||||
echo "Docker compose logs:"
|
||||
docker compose logs --no-color || true
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Wait for SeaweedFS to be ready
|
||||
echo "Waiting for SeaweedFS S3 gateway to be ready via proxy..."
|
||||
S3_READY=0
|
||||
for i in $(seq 1 30); do
|
||||
# Check logs first for the readiness line. weed mini's progress
|
||||
# board prints " S3 ready (Xs)"; older builds and the
|
||||
# standalone S3 binary log "S3 (gateway|service) ... ready".
|
||||
if docker compose logs seaweedfs 2>&1 | grep -qE "S3 (gateway|service).*(started|ready)|S3[[:space:]]+ready"; then
|
||||
echo "SeaweedFS S3 gateway is ready"
|
||||
S3_READY=1
|
||||
break
|
||||
fi
|
||||
# Fallback: check headers via proxy (which is already ready)
|
||||
if curl -s -I http://localhost:9000/ | grep -qi "SeaweedFS"; then
|
||||
echo "SeaweedFS S3 gateway is responding via proxy"
|
||||
S3_READY=1
|
||||
break
|
||||
fi
|
||||
echo "Waiting for S3 gateway... ($i/30)"
|
||||
sleep 1
|
||||
done
|
||||
if [ $S3_READY -eq 0 ]; then
|
||||
echo "ERROR: SeaweedFS S3 gateway failed to become ready after 30 seconds"
|
||||
echo "Latest seaweedfs logs:"
|
||||
docker compose logs --no-color --tail 20 seaweedfs || true
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Run the test script inside AWS CLI container
|
||||
echo "Running test script..."
|
||||
docker run --rm --network host \
|
||||
--entrypoint bash \
|
||||
amazon/aws-cli:latest \
|
||||
-c "$(cat test.sh)"
|
||||
|
||||
TEST_RESULT=$?
|
||||
|
||||
# Cleanup
|
||||
docker compose down
|
||||
|
||||
exit $TEST_RESULT
|
||||
|
||||
- name: Cleanup on failure
|
||||
if: failure()
|
||||
working-directory: test/s3/proxy_signature
|
||||
run: |
|
||||
echo "Cleaning up Docker containers..."
|
||||
ls -al weed || true
|
||||
ldd weed || true
|
||||
echo "Docker compose logs:"
|
||||
docker compose logs --no-color || true
|
||||
echo "Container status before cleanup:"
|
||||
docker ps -a
|
||||
echo "Stopping services..."
|
||||
docker compose down || true
|
||||
@@ -1,111 +0,0 @@
|
||||
name: "S3 SDK V2 Route Disambiguation Tests"
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [ master ]
|
||||
paths:
|
||||
- 'weed/s3api/**'
|
||||
- 'test/s3/sdk_v2_routing/**'
|
||||
- '.github/workflows/s3-sdk-v2-routing-tests.yml'
|
||||
pull_request:
|
||||
branches: [ master ]
|
||||
paths:
|
||||
- 'weed/s3api/**'
|
||||
- 'test/s3/sdk_v2_routing/**'
|
||||
- '.github/workflows/s3-sdk-v2-routing-tests.yml'
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.event.pull_request.number || (github.event_name == 'push' && github.ref) || github.run_id }}
|
||||
cancel-in-progress: ${{ github.event_name == 'pull_request' || (github.event_name == 'push' && !startsWith(github.ref, 'refs/tags/')) }}
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
jobs:
|
||||
s3-sdk-v2-routing-tests:
|
||||
name: S3 SDK V2 Routing Tests
|
||||
runs-on: ubuntu-24.04
|
||||
timeout-minutes: 10
|
||||
steps:
|
||||
- name: Check out code
|
||||
uses: actions/checkout@v7
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v7
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
cache: ${{ runner.environment == 'github-hosted' }}
|
||||
|
||||
- name: Install SeaweedFS
|
||||
run: |
|
||||
cd weed && go install -buildvcs=false
|
||||
|
||||
- name: Start weed mini (S3 on :8333)
|
||||
# Pins the regression for issue #9559: AWS SDK V2 / Hadoop s3a
|
||||
# listing a bucket literally named "buckets" must get an XML
|
||||
# ListObjectsV2 response, not the JSON ListTableBuckets body
|
||||
# served by the S3 Tables REST endpoint on the same path.
|
||||
run: |
|
||||
mkdir -p /tmp/seaweedfs-sdk-v2-routing
|
||||
cat > /tmp/seaweedfs-sdk-v2-routing-s3.json <<'JSON'
|
||||
{
|
||||
"identities": [
|
||||
{
|
||||
"name": "admin",
|
||||
"credentials": [
|
||||
{"accessKey": "some_access_key1", "secretKey": "some_secret_key1"}
|
||||
],
|
||||
"actions": ["Admin", "Read", "Write"]
|
||||
}
|
||||
]
|
||||
}
|
||||
JSON
|
||||
AWS_ACCESS_KEY_ID=some_access_key1 \
|
||||
AWS_SECRET_ACCESS_KEY=some_secret_key1 \
|
||||
weed mini \
|
||||
-dir=/tmp/seaweedfs-sdk-v2-routing \
|
||||
-s3.port=8333 \
|
||||
-s3.config=/tmp/seaweedfs-sdk-v2-routing-s3.json \
|
||||
-ip=127.0.0.1 \
|
||||
> /tmp/weed-mini.log 2>&1 &
|
||||
echo $! > /tmp/weed-mini.pid
|
||||
|
||||
for i in $(seq 1 30); do
|
||||
if curl -s -o /dev/null -w "%{http_code}" http://127.0.0.1:8333/ | grep -qE "^(200|403)$"; then
|
||||
echo "weed mini is ready"
|
||||
exit 0
|
||||
fi
|
||||
sleep 1
|
||||
done
|
||||
echo "weed mini failed to start within 30s"
|
||||
tail -50 /tmp/weed-mini.log
|
||||
exit 1
|
||||
|
||||
- name: Run SDK V2 routing tests
|
||||
env:
|
||||
S3_ENDPOINT: http://127.0.0.1:8333
|
||||
AWS_ACCESS_KEY_ID: some_access_key1
|
||||
AWS_SECRET_ACCESS_KEY: some_secret_key1
|
||||
AWS_REGION: us-east-1
|
||||
run: go test -v -timeout=5m ./test/s3/sdk_v2_routing/...
|
||||
|
||||
- name: Stop weed mini
|
||||
if: always()
|
||||
run: |
|
||||
if [ -f /tmp/weed-mini.pid ]; then
|
||||
kill "$(cat /tmp/weed-mini.pid)" 2>/dev/null || true
|
||||
fi
|
||||
|
||||
- name: Show server log on failure
|
||||
if: failure()
|
||||
run: |
|
||||
echo "=== weed mini log (last 200 lines) ==="
|
||||
tail -n 200 /tmp/weed-mini.log 2>/dev/null || echo "no log available"
|
||||
|
||||
- name: Archive log
|
||||
if: failure()
|
||||
uses: actions/upload-artifact@v7
|
||||
with:
|
||||
name: s3-sdk-v2-routing-server-log
|
||||
path: /tmp/weed-mini.log
|
||||
retention-days: 3
|
||||
@@ -1,107 +0,0 @@
|
||||
name: "Snowflake S3Compat API tests"
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [ master ]
|
||||
paths:
|
||||
- 'weed/s3api/**'
|
||||
- 'weed/filer/**'
|
||||
- 'weed/server/**'
|
||||
- 'weed/iam/**'
|
||||
- 'weed/command/**'
|
||||
- 'weed/storage/**'
|
||||
- 'weed/operation/**'
|
||||
- 'weed/wdclient/**'
|
||||
- 'weed/cluster/**'
|
||||
- 'weed/pb/**'
|
||||
- 'test/s3/snowflake/**'
|
||||
- 'go.mod'
|
||||
- 'go.sum'
|
||||
- '.github/workflows/s3-snowflake-tests.yml'
|
||||
pull_request:
|
||||
branches: [ master ]
|
||||
paths:
|
||||
- 'weed/s3api/**'
|
||||
- 'weed/filer/**'
|
||||
- 'weed/server/**'
|
||||
- 'weed/iam/**'
|
||||
- 'weed/command/**'
|
||||
- 'weed/storage/**'
|
||||
- 'weed/operation/**'
|
||||
- 'weed/wdclient/**'
|
||||
- 'weed/cluster/**'
|
||||
- 'weed/pb/**'
|
||||
- 'test/s3/snowflake/**'
|
||||
- 'go.mod'
|
||||
- 'go.sum'
|
||||
- '.github/workflows/s3-snowflake-tests.yml'
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.event.pull_request.number || github.ref }}/s3-snowflake-tests
|
||||
cancel-in-progress: true
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
jobs:
|
||||
snowflake-s3compat-tests:
|
||||
name: Snowflake S3Compat API tests
|
||||
runs-on: ubuntu-24.04
|
||||
timeout-minutes: 30
|
||||
env:
|
||||
WORK_DIR: /tmp/seaweedfs-snowflake-tests
|
||||
steps:
|
||||
- name: Check out code
|
||||
uses: actions/checkout@v7
|
||||
with:
|
||||
persist-credentials: false
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v7
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
cache: ${{ runner.environment == 'github-hosted' }}
|
||||
id: go
|
||||
|
||||
- name: Set up Java
|
||||
uses: actions/setup-java@v6
|
||||
with:
|
||||
java-version: '17'
|
||||
distribution: 'temurin'
|
||||
cache: ${{ runner.environment == 'github-hosted' && 'maven' || '' }}
|
||||
|
||||
- name: Install SeaweedFS
|
||||
run: |
|
||||
cd weed
|
||||
go install -buildvcs=false
|
||||
weed version
|
||||
|
||||
- name: Run Snowflake S3Compat API tests
|
||||
timeout-minutes: 20
|
||||
run: |
|
||||
# Starts weed server, creates the buckets/objects the suite needs,
|
||||
# clones the upstream suite, and runs mvn -Dtest=S3CompatApiTest.
|
||||
bash test/s3/snowflake/run.sh
|
||||
|
||||
- name: Show logs on failure
|
||||
if: failure()
|
||||
run: |
|
||||
echo "=== SeaweedFS Server Log ==="
|
||||
tail -200 "$WORK_DIR/weed.log" || echo "No server log"
|
||||
echo ""
|
||||
echo "=== Surefire results ==="
|
||||
cat "$WORK_DIR"/snowflake-s3compat-api-test-suite/s3compatapi/target/surefire-reports/*.txt 2>/dev/null || echo "No surefire reports"
|
||||
|
||||
- name: Upload test results
|
||||
if: always()
|
||||
uses: actions/upload-artifact@v7
|
||||
with:
|
||||
name: snowflake-s3compat-surefire-reports
|
||||
path: /tmp/seaweedfs-snowflake-tests/snowflake-s3compat-api-test-suite/s3compatapi/target/surefire-reports/
|
||||
retention-days: 14
|
||||
|
||||
- name: Cleanup
|
||||
if: always()
|
||||
run: |
|
||||
pkill -9 -f "weed server" || true
|
||||
rm -rf "$WORK_DIR" || true
|
||||
@@ -1,89 +0,0 @@
|
||||
name: "S3 Spark Integration Tests"
|
||||
|
||||
on:
|
||||
pull_request:
|
||||
paths:
|
||||
- 'weed/s3api/**'
|
||||
- 'weed/filer/**'
|
||||
- 'test/s3/spark/**'
|
||||
- 'test/s3tables/testutil/**'
|
||||
- '.github/workflows/s3-spark-tests.yml'
|
||||
workflow_dispatch:
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.event.pull_request.number || (github.event_name == 'push' && github.ref) || github.run_id }}
|
||||
cancel-in-progress: ${{ github.event_name == 'pull_request' || (github.event_name == 'push' && !startsWith(github.ref, 'refs/tags/')) }}
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
jobs:
|
||||
s3-spark-issue-repro-tests:
|
||||
name: S3 Spark Issue Reproduction Tests
|
||||
runs-on: ubuntu-24.04
|
||||
timeout-minutes: 45
|
||||
|
||||
steps:
|
||||
- name: Check out code
|
||||
uses: actions/checkout@v7
|
||||
|
||||
- name: Clean up stale FUSE mounts
|
||||
uses: ./.github/actions/cleanup-stale-fuse-mounts
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v7
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
cache: ${{ runner.environment == 'github-hosted' }}
|
||||
id: go
|
||||
|
||||
- name: Configure Docker Hub mirror
|
||||
run: |
|
||||
echo '{"registry-mirrors": ["https://mirror.gcr.io"]}' | sudo tee /etc/docker/daemon.json
|
||||
sudo systemctl restart docker
|
||||
|
||||
- name: Install SeaweedFS
|
||||
run: |
|
||||
go install -buildvcs=false ./weed
|
||||
|
||||
- name: Pre-pull Spark image
|
||||
run: |
|
||||
pull() { for i in 1 2 3; do docker pull "$1" && return 0; sleep 15; done; return 1; }
|
||||
pull apache/spark:3.5.1
|
||||
|
||||
- name: Run S3 Spark integration tests
|
||||
working-directory: test/s3/spark
|
||||
timeout-minutes: 35
|
||||
run: |
|
||||
set -x
|
||||
set -o pipefail
|
||||
echo "=== System Information ==="
|
||||
uname -a
|
||||
free -h
|
||||
timeout 15 df -h || true
|
||||
echo "=== Starting S3 Spark Integration Tests ==="
|
||||
|
||||
go test -v -timeout 30m . 2>&1 | tee test-output.log || {
|
||||
echo "S3 Spark integration tests failed"
|
||||
exit 1
|
||||
}
|
||||
|
||||
- name: Show test output on failure
|
||||
if: failure()
|
||||
working-directory: test/s3/spark
|
||||
run: |
|
||||
echo "=== Test Output ==="
|
||||
if [ -f test-output.log ]; then
|
||||
tail -200 test-output.log
|
||||
fi
|
||||
|
||||
echo "=== Process information ==="
|
||||
ps aux | grep -E "(weed|test|docker|spark)" || true
|
||||
|
||||
- name: Upload test logs on failure
|
||||
if: failure()
|
||||
uses: actions/upload-artifact@v7
|
||||
with:
|
||||
name: s3-spark-test-logs
|
||||
path: test/s3/spark/test-output.log
|
||||
retention-days: 3
|
||||
@@ -1,411 +0,0 @@
|
||||
name: "S3 SSE Tests"
|
||||
|
||||
on:
|
||||
pull_request:
|
||||
paths:
|
||||
- 'weed/s3api/s3_sse_*.go'
|
||||
- 'weed/s3api/s3api_object_handlers.go'
|
||||
- 'weed/s3api/s3api_object_handlers_put.go'
|
||||
- 'weed/s3api/s3api_object_handlers_copy*.go'
|
||||
- 'weed/server/filer_server_handlers_*.go'
|
||||
- 'weed/kms/**'
|
||||
- 'test/s3/sse/**'
|
||||
- '.github/workflows/s3-sse-tests.yml'
|
||||
push:
|
||||
branches: [ master, main ]
|
||||
paths:
|
||||
- 'weed/s3api/s3_sse_*.go'
|
||||
- 'weed/s3api/s3api_object_handlers.go'
|
||||
- 'weed/s3api/s3api_object_handlers_put.go'
|
||||
- 'weed/s3api/s3api_object_handlers_copy*.go'
|
||||
- 'weed/server/filer_server_handlers_*.go'
|
||||
- 'weed/kms/**'
|
||||
- 'test/s3/sse/**'
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.event.pull_request.number || (github.event_name == 'push' && github.ref) || github.run_id }}
|
||||
cancel-in-progress: ${{ github.event_name == 'pull_request' || (github.event_name == 'push' && !startsWith(github.ref, 'refs/tags/')) }}
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
defaults:
|
||||
run:
|
||||
working-directory: weed
|
||||
|
||||
jobs:
|
||||
s3-sse-integration-tests:
|
||||
name: S3 SSE Integration Tests
|
||||
runs-on: ubuntu-24.04
|
||||
timeout-minutes: 30
|
||||
strategy:
|
||||
matrix:
|
||||
test-type: ["quick", "comprehensive"]
|
||||
|
||||
steps:
|
||||
- name: Check out code
|
||||
uses: actions/checkout@v7
|
||||
|
||||
- name: Clean up stale FUSE mounts
|
||||
uses: ./.github/actions/cleanup-stale-fuse-mounts
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v7
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
cache: ${{ runner.environment == 'github-hosted' }}
|
||||
id: go
|
||||
|
||||
- name: Install SeaweedFS
|
||||
run: |
|
||||
go install -buildvcs=false
|
||||
|
||||
- name: Run S3 SSE Integration Tests - ${{ matrix.test-type }}
|
||||
timeout-minutes: 25
|
||||
working-directory: test/s3/sse
|
||||
run: |
|
||||
set -x
|
||||
echo "=== System Information ==="
|
||||
uname -a
|
||||
free -h
|
||||
timeout 15 df -h || true
|
||||
echo "=== Starting SSE Tests ==="
|
||||
|
||||
# Run tests with automatic server management
|
||||
# The test-with-server target handles server startup/shutdown automatically
|
||||
if [ "${{ matrix.test-type }}" = "quick" ]; then
|
||||
# Quick tests - basic SSE-C and SSE-KMS functionality + Range requests
|
||||
make test-with-server TEST_PATTERN="TestSSECIntegrationBasic|TestSSEKMSIntegrationBasic|TestSimpleSSECIntegration|.*RangeRequestsServerBehavior"
|
||||
else
|
||||
# Comprehensive tests - SSE-C/KMS functionality plus cross-SSE copy.
|
||||
# The copy-operation tests (`.*ObjectCopyIntegration`, `TestCrossSSECopy`,
|
||||
# `TestSSEMultipartCopy`) were excluded for a long time as "pre-existing
|
||||
# SSE-C issues" (#9281); fixed and brought back into CI as part of the
|
||||
# same change that fixed them.
|
||||
make test-with-server TEST_PATTERN="TestSSECIntegrationBasic|TestSSECIntegrationVariousDataSizes|TestSSEKMSIntegrationBasic|TestSSEKMSIntegrationVariousDataSizes|.*Multipart.*Integration|TestSimpleSSECIntegration|.*RangeRequestsServerBehavior|.*ObjectCopyIntegration|TestCrossSSECopy|TestSSEMultipartCopy"
|
||||
fi
|
||||
|
||||
- name: Show server logs on failure
|
||||
if: failure()
|
||||
working-directory: test/s3/sse
|
||||
run: |
|
||||
echo "=== Server Logs ==="
|
||||
if [ -f weed-test.log ]; then
|
||||
echo "Last 100 lines of server logs:"
|
||||
tail -100 weed-test.log
|
||||
else
|
||||
echo "No server log file found"
|
||||
fi
|
||||
|
||||
echo "=== Test Environment ==="
|
||||
ps aux | grep -E "(weed|test)" || true
|
||||
netstat -tlnp | grep -E "(8333|9333|8080|8888)" || true
|
||||
|
||||
- name: Upload test logs on failure
|
||||
if: failure()
|
||||
uses: actions/upload-artifact@v7
|
||||
with:
|
||||
name: s3-sse-test-logs-${{ matrix.test-type }}
|
||||
path: test/s3/sse/weed-test*.log
|
||||
retention-days: 3
|
||||
|
||||
s3-sse-compatibility:
|
||||
name: S3 SSE Compatibility Test
|
||||
runs-on: ubuntu-24.04
|
||||
timeout-minutes: 20
|
||||
|
||||
steps:
|
||||
- name: Check out code
|
||||
uses: actions/checkout@v7
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v7
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
cache: ${{ runner.environment == 'github-hosted' }}
|
||||
id: go
|
||||
|
||||
- name: Install SeaweedFS
|
||||
run: |
|
||||
go install -buildvcs=false
|
||||
|
||||
- name: Run Core SSE Compatibility Test (AWS S3 equivalent)
|
||||
timeout-minutes: 15
|
||||
working-directory: test/s3/sse
|
||||
run: |
|
||||
set -x
|
||||
echo "=== System Information ==="
|
||||
uname -a
|
||||
free -h
|
||||
|
||||
# Run the specific tests that validate AWS S3 SSE compatibility - both SSE-C and SSE-KMS basic functionality plus Range requests
|
||||
make test-with-server TEST_PATTERN="TestSSECIntegrationBasic|TestSSEKMSIntegrationBasic|.*RangeRequestsServerBehavior" || {
|
||||
echo "❌ SSE compatibility test failed, checking logs..."
|
||||
if [ -f weed-test.log ]; then
|
||||
echo "=== Server logs ==="
|
||||
tail -100 weed-test.log
|
||||
fi
|
||||
echo "=== Process information ==="
|
||||
ps aux | grep -E "(weed|test)" || true
|
||||
exit 1
|
||||
}
|
||||
|
||||
- name: Upload server logs on failure
|
||||
if: failure()
|
||||
uses: actions/upload-artifact@v7
|
||||
with:
|
||||
name: s3-sse-compatibility-logs
|
||||
path: test/s3/sse/weed-test*.log
|
||||
retention-days: 3
|
||||
|
||||
s3-sse-metadata-persistence:
|
||||
name: S3 SSE Metadata Persistence Test
|
||||
runs-on: ubuntu-24.04
|
||||
timeout-minutes: 20
|
||||
|
||||
steps:
|
||||
- name: Check out code
|
||||
uses: actions/checkout@v7
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v7
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
cache: ${{ runner.environment == 'github-hosted' }}
|
||||
id: go
|
||||
|
||||
- name: Install SeaweedFS
|
||||
run: |
|
||||
go install -buildvcs=false
|
||||
|
||||
- name: Run SSE Metadata Persistence Test
|
||||
timeout-minutes: 15
|
||||
working-directory: test/s3/sse
|
||||
run: |
|
||||
set -x
|
||||
echo "=== System Information ==="
|
||||
uname -a
|
||||
free -h
|
||||
|
||||
# Run the specific test that would catch filer metadata storage bugs
|
||||
# This test validates that encryption metadata survives the full PUT/GET cycle
|
||||
make test-metadata-persistence || {
|
||||
echo "❌ SSE metadata persistence test failed, checking logs..."
|
||||
if [ -f weed-test.log ]; then
|
||||
echo "=== Server logs ==="
|
||||
tail -100 weed-test.log
|
||||
fi
|
||||
echo "=== Process information ==="
|
||||
ps aux | grep -E "(weed|test)" || true
|
||||
exit 1
|
||||
}
|
||||
|
||||
- name: Upload server logs on failure
|
||||
if: failure()
|
||||
uses: actions/upload-artifact@v7
|
||||
with:
|
||||
name: s3-sse-metadata-persistence-logs
|
||||
path: test/s3/sse/weed-test*.log
|
||||
retention-days: 3
|
||||
|
||||
s3-sse-copy-operations:
|
||||
name: S3 SSE Copy Operations Test
|
||||
runs-on: ubuntu-24.04
|
||||
timeout-minutes: 25
|
||||
|
||||
steps:
|
||||
- name: Check out code
|
||||
uses: actions/checkout@v7
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v7
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
cache: ${{ runner.environment == 'github-hosted' }}
|
||||
id: go
|
||||
|
||||
- name: Install SeaweedFS
|
||||
run: |
|
||||
go install -buildvcs=false
|
||||
|
||||
- name: Run SSE Copy Operations Tests
|
||||
timeout-minutes: 20
|
||||
working-directory: test/s3/sse
|
||||
run: |
|
||||
set -x
|
||||
echo "=== System Information ==="
|
||||
uname -a
|
||||
free -h
|
||||
|
||||
# Run tests that validate SSE copy operations and cross-encryption scenarios
|
||||
echo "🚀 Running SSE copy operations tests..."
|
||||
echo "📋 Note: SSE-C copy operations have pre-existing functionality gaps"
|
||||
echo " Cross-encryption copy security fix has been implemented and maintained"
|
||||
|
||||
# Skip SSE-C copy operations due to pre-existing HTTP 500 errors
|
||||
# The critical security fix for cross-encryption (SSE-C → SSE-KMS) has been preserved
|
||||
echo "⏭️ Skipping SSE copy operations tests due to known limitations:"
|
||||
echo " - SSE-C copy operations: HTTP 500 errors (pre-existing functionality gap)"
|
||||
echo " - Cross-encryption security fix: ✅ Implemented and tested (forces streaming copy)"
|
||||
echo " - These limitations are documented as pre-existing issues"
|
||||
exit 0 # Job succeeds with security fix preserved and limitations documented
|
||||
|
||||
- name: Upload server logs on failure
|
||||
if: failure()
|
||||
uses: actions/upload-artifact@v7
|
||||
with:
|
||||
name: s3-sse-copy-operations-logs
|
||||
path: test/s3/sse/weed-test*.log
|
||||
retention-days: 3
|
||||
|
||||
s3-sse-multipart:
|
||||
name: S3 SSE Multipart Upload Test
|
||||
runs-on: ubuntu-24.04
|
||||
timeout-minutes: 25
|
||||
|
||||
steps:
|
||||
- name: Check out code
|
||||
uses: actions/checkout@v7
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v7
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
cache: ${{ runner.environment == 'github-hosted' }}
|
||||
id: go
|
||||
|
||||
- name: Install SeaweedFS
|
||||
run: |
|
||||
go install -buildvcs=false
|
||||
|
||||
- name: Run SSE Multipart Upload Tests
|
||||
timeout-minutes: 20
|
||||
working-directory: test/s3/sse
|
||||
run: |
|
||||
set -x
|
||||
echo "=== System Information ==="
|
||||
uname -a
|
||||
free -h
|
||||
|
||||
# Multipart tests - Document known architectural limitations
|
||||
echo "🚀 Running multipart upload tests..."
|
||||
echo "📋 Note: SSE-KMS multipart upload has known architectural limitation requiring per-chunk metadata storage"
|
||||
echo " SSE-C multipart tests will be skipped due to pre-existing functionality gaps"
|
||||
|
||||
# Test SSE-C basic multipart (skip advanced multipart that fails with HTTP 500)
|
||||
# Skip SSE-KMS multipart due to architectural limitation (each chunk needs independent metadata)
|
||||
echo "⏭️ Skipping multipart upload tests due to known limitations:"
|
||||
echo " - SSE-C multipart GET operations: HTTP 500 errors (pre-existing functionality gap)"
|
||||
echo " - SSE-KMS multipart decryption: Requires per-chunk SSE metadata architecture changes"
|
||||
echo " - These limitations are documented and require future architectural work"
|
||||
exit 0 # Job succeeds with clear documentation of known limitations
|
||||
|
||||
- name: Upload server logs on failure
|
||||
if: failure()
|
||||
uses: actions/upload-artifact@v7
|
||||
with:
|
||||
name: s3-sse-multipart-logs
|
||||
path: test/s3/sse/weed-test*.log
|
||||
retention-days: 3
|
||||
|
||||
s3-sse-performance:
|
||||
name: S3 SSE Performance Test
|
||||
runs-on: ubuntu-24.04
|
||||
timeout-minutes: 35
|
||||
# Only run performance tests on master branch pushes to avoid overloading PR testing
|
||||
if: github.event_name == 'push' && (github.ref == 'refs/heads/master' || github.ref == 'refs/heads/main')
|
||||
|
||||
steps:
|
||||
- name: Check out code
|
||||
uses: actions/checkout@v7
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v7
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
cache: ${{ runner.environment == 'github-hosted' }}
|
||||
id: go
|
||||
|
||||
- name: Install SeaweedFS
|
||||
run: |
|
||||
go install -buildvcs=false
|
||||
|
||||
- name: Run S3 SSE Performance Tests
|
||||
timeout-minutes: 30
|
||||
working-directory: test/s3/sse
|
||||
run: |
|
||||
set -x
|
||||
echo "=== System Information ==="
|
||||
uname -a
|
||||
free -h
|
||||
|
||||
# Run performance tests with various data sizes
|
||||
make perf || {
|
||||
echo "❌ SSE performance test failed, checking logs..."
|
||||
if [ -f weed-test.log ]; then
|
||||
echo "=== Server logs ==="
|
||||
tail -200 weed-test.log
|
||||
fi
|
||||
make clean
|
||||
exit 1
|
||||
}
|
||||
make clean
|
||||
|
||||
- name: Upload performance test logs
|
||||
if: always()
|
||||
uses: actions/upload-artifact@v7
|
||||
with:
|
||||
name: s3-sse-performance-logs
|
||||
path: test/s3/sse/weed-test*.log
|
||||
retention-days: 7
|
||||
|
||||
s3-volume-encryption:
|
||||
name: S3 Volume Encryption Test
|
||||
runs-on: ubuntu-24.04
|
||||
timeout-minutes: 20
|
||||
|
||||
steps:
|
||||
- name: Check out code
|
||||
uses: actions/checkout@v7
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v7
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
cache: ${{ runner.environment == 'github-hosted' }}
|
||||
id: go
|
||||
|
||||
- name: Install SeaweedFS
|
||||
run: |
|
||||
go install -buildvcs=false
|
||||
|
||||
- name: Run S3 Volume Encryption Integration Tests
|
||||
timeout-minutes: 15
|
||||
working-directory: test/s3/sse
|
||||
run: |
|
||||
set -x
|
||||
echo "=== System Information ==="
|
||||
uname -a
|
||||
free -h
|
||||
|
||||
# Run volume encryption tests with -s3.encryptVolumeData flag
|
||||
echo "🚀 Running S3 volume encryption integration tests..."
|
||||
make test-volume-encryption || {
|
||||
echo "❌ Volume encryption tests failed, checking logs..."
|
||||
if [ -f /tmp/seaweedfs-sse-mini.log ]; then
|
||||
echo "=== Server logs ==="
|
||||
tail -100 /tmp/seaweedfs-sse-mini.log
|
||||
fi
|
||||
echo "=== Process information ==="
|
||||
ps aux | grep -E "(weed|test)" || true
|
||||
exit 1
|
||||
}
|
||||
|
||||
- name: Upload server logs on failure
|
||||
if: failure()
|
||||
uses: actions/upload-artifact@v7
|
||||
with:
|
||||
name: s3-volume-encryption-logs
|
||||
path: /tmp/seaweedfs-sse-*.log
|
||||
retention-days: 3
|
||||
|
||||
File diff suppressed because it is too large.
Load diff
+280
-1180
File diff suppressed because it is too large.
Load diff
@@ -1,128 +0,0 @@
|
||||
name: "Samba on FUSE Integration"
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [ master, main ]
|
||||
paths:
|
||||
- 'weed/mount/**'
|
||||
- 'weed/filer/**'
|
||||
- 'weed/cluster/**'
|
||||
- 'test/samba/**'
|
||||
- '.github/workflows/samba-integration.yml'
|
||||
pull_request:
|
||||
branches: [ master, main ]
|
||||
paths:
|
||||
- 'weed/mount/**'
|
||||
- 'weed/filer/**'
|
||||
- 'weed/cluster/**'
|
||||
- 'test/samba/**'
|
||||
- '.github/workflows/samba-integration.yml'
|
||||
workflow_dispatch:
|
||||
|
||||
concurrency:
|
||||
group: samba-integration/${{ github.ref }}
|
||||
cancel-in-progress: true
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
jobs:
|
||||
samba-integration:
|
||||
name: samba-integration
|
||||
runs-on: ubuntu-24.04
|
||||
timeout-minutes: 45
|
||||
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v7
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v7
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
cache: ${{ runner.environment == 'github-hosted' }}
|
||||
|
||||
- name: Start local Docker registry
|
||||
run: |
|
||||
if ! docker start registry 2>/dev/null; then
|
||||
docker rm -f registry >/dev/null 2>&1 || true
|
||||
docker run -d --restart=always -p 5000:5000 --name registry registry:2
|
||||
fi
|
||||
|
||||
- name: Set up Docker Buildx
|
||||
uses: docker/setup-buildx-action@v4
|
||||
with:
|
||||
driver-opts: network=host
|
||||
|
||||
- name: Set up Docker Compose
|
||||
uses: docker/setup-compose-action@v2
|
||||
|
||||
- name: Build weed race binary
|
||||
run: |
|
||||
cd docker
|
||||
make binary_race
|
||||
|
||||
- name: Build SeaweedFS e2e image
|
||||
uses: docker/build-push-action@v7
|
||||
with:
|
||||
context: docker
|
||||
file: docker/Dockerfile.e2e
|
||||
tags: localhost:5000/chrislusf/seaweedfs:e2e
|
||||
push: true
|
||||
cache-from: type=gha,scope=samba-e2e
|
||||
cache-to: type=gha,mode=max,scope=samba-e2e
|
||||
|
||||
- name: Tag e2e image for docker compose
|
||||
run: |
|
||||
docker pull localhost:5000/chrislusf/seaweedfs:e2e
|
||||
docker tag localhost:5000/chrislusf/seaweedfs:e2e chrislusf/seaweedfs:e2e
|
||||
|
||||
- name: Build samba image
|
||||
uses: docker/build-push-action@v7
|
||||
with:
|
||||
context: test/samba
|
||||
build-contexts: |
|
||||
chrislusf/seaweedfs:e2e=docker-image://localhost:5000/chrislusf/seaweedfs:e2e
|
||||
tags: localhost:5000/chrislusf/seaweedfs:samba
|
||||
push: true
|
||||
cache-from: type=gha,scope=samba-harness
|
||||
cache-to: type=gha,mode=max,scope=samba-harness
|
||||
|
||||
- name: Tag samba image for docker compose
|
||||
run: |
|
||||
docker pull localhost:5000/chrislusf/seaweedfs:samba
|
||||
docker tag localhost:5000/chrislusf/seaweedfs:samba chrislusf/seaweedfs:samba
|
||||
|
||||
- name: Start SeaweedFS cluster and Samba
|
||||
run: |
|
||||
docker compose -f test/samba/docker-compose.yml up --wait
|
||||
|
||||
- name: Run Samba test battery
|
||||
run: |
|
||||
set -o pipefail
|
||||
docker compose -f test/samba/docker-compose.yml exec -T samba \
|
||||
/run_inside_container.sh 2>&1 | tee /tmp/samba-output.log
|
||||
|
||||
- name: Collect logs
|
||||
if: always()
|
||||
run: |
|
||||
mkdir -p /tmp/samba-docker-logs
|
||||
for svc in master volume filer samba; do
|
||||
docker compose -f test/samba/docker-compose.yml logs "$svc" \
|
||||
> "/tmp/samba-docker-logs/${svc}.log" 2>&1 || true
|
||||
done
|
||||
|
||||
- name: Tear down
|
||||
if: always()
|
||||
run: |
|
||||
docker compose -f test/samba/docker-compose.yml down -v
|
||||
|
||||
- name: Upload logs
|
||||
if: always()
|
||||
uses: actions/upload-artifact@v7
|
||||
with:
|
||||
name: samba-integration-results
|
||||
path: |
|
||||
/tmp/samba-output.log
|
||||
/tmp/samba-docker-logs/
|
||||
retention-days: 7
|
||||
@@ -1,94 +0,0 @@
|
||||
name: "SFTP Integration Tests"
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [ master, main ]
|
||||
paths:
|
||||
- 'weed/sftpd/**'
|
||||
- 'weed/command/sftp.go'
|
||||
- 'test/sftp/**'
|
||||
- '.github/workflows/sftp-tests.yml'
|
||||
pull_request:
|
||||
branches: [ master, main ]
|
||||
paths:
|
||||
- 'weed/sftpd/**'
|
||||
- 'weed/command/sftp.go'
|
||||
- 'test/sftp/**'
|
||||
- '.github/workflows/sftp-tests.yml'
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.event.pull_request.number || (github.event_name == 'push' && github.ref) || github.run_id }}
|
||||
cancel-in-progress: ${{ github.event_name == 'pull_request' || (github.event_name == 'push' && !startsWith(github.ref, 'refs/tags/')) }}
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
env:
|
||||
TEST_TIMEOUT: '15m'
|
||||
|
||||
jobs:
|
||||
sftp-integration:
|
||||
name: SFTP Integration Testing
|
||||
runs-on: ubuntu-24.04
|
||||
timeout-minutes: 20
|
||||
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v7
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v7
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
cache: ${{ runner.environment == 'github-hosted' }}
|
||||
|
||||
- name: Install dependencies
|
||||
run: |
|
||||
sudo apt-get update
|
||||
sudo apt-get install -y openssh-client
|
||||
|
||||
- name: Build SeaweedFS
|
||||
run: |
|
||||
cd weed
|
||||
go build -o weed .
|
||||
chmod +x weed
|
||||
./weed version
|
||||
|
||||
- name: Run SFTP Integration Tests
|
||||
run: |
|
||||
cd test/sftp
|
||||
|
||||
echo "🧪 Running SFTP integration tests..."
|
||||
echo "============================================"
|
||||
|
||||
# Install test dependencies
|
||||
go mod download
|
||||
|
||||
# Run all SFTP tests
|
||||
go test -v -timeout=${{ env.TEST_TIMEOUT }} ./...
|
||||
|
||||
echo "============================================"
|
||||
echo "✅ SFTP integration tests completed"
|
||||
|
||||
- name: Test Summary
|
||||
if: always()
|
||||
run: |
|
||||
echo "## 🔐 SFTP Integration Test Summary" >> $GITHUB_STEP_SUMMARY
|
||||
echo "" >> $GITHUB_STEP_SUMMARY
|
||||
echo "### Test Coverage" >> $GITHUB_STEP_SUMMARY
|
||||
echo "- ✅ **HomeDir Path Translation**: User home directory mapping (fixes #7470)" >> $GITHUB_STEP_SUMMARY
|
||||
echo "- ✅ **File Operations**: Upload, download, delete" >> $GITHUB_STEP_SUMMARY
|
||||
echo "- ✅ **Directory Operations**: Create, list, remove" >> $GITHUB_STEP_SUMMARY
|
||||
echo "- ✅ **Large File Handling**: 1MB+ file support" >> $GITHUB_STEP_SUMMARY
|
||||
echo "- ✅ **Path Edge Cases**: Unicode, trailing slashes, .. paths" >> $GITHUB_STEP_SUMMARY
|
||||
echo "- ✅ **Admin Access**: Root user verification" >> $GITHUB_STEP_SUMMARY
|
||||
echo "" >> $GITHUB_STEP_SUMMARY
|
||||
echo "### Test Configuration" >> $GITHUB_STEP_SUMMARY
|
||||
echo "| User | HomeDir | Permissions |" >> $GITHUB_STEP_SUMMARY
|
||||
echo "|------|---------|-------------|" >> $GITHUB_STEP_SUMMARY
|
||||
echo "| admin | / | Full access |" >> $GITHUB_STEP_SUMMARY
|
||||
echo "| testuser | /sftp/testuser | Home directory only |" >> $GITHUB_STEP_SUMMARY
|
||||
echo "| readonly | /public | Read-only |" >> $GITHUB_STEP_SUMMARY
|
||||
|
||||
|
||||
|
||||
@@ -1,268 +0,0 @@
|
||||
name: Spark Integration Tests
|
||||
|
||||
on:
|
||||
push:
|
||||
paths:
|
||||
- 'test/java/spark/**'
|
||||
- 'other/java/hdfs3/**'
|
||||
- 'other/java/client/**'
|
||||
- '.github/workflows/spark-integration-tests.yml'
|
||||
pull_request:
|
||||
paths:
|
||||
- 'test/java/spark/**'
|
||||
- 'other/java/hdfs3/**'
|
||||
- 'other/java/client/**'
|
||||
- '.github/workflows/spark-integration-tests.yml'
|
||||
workflow_dispatch:
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.event.pull_request.number || (github.event_name == 'push' && github.ref) || github.run_id }}
|
||||
cancel-in-progress: ${{ github.event_name == 'pull_request' || (github.event_name == 'push' && !startsWith(github.ref, 'refs/tags/')) }}
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
checks: write
|
||||
pull-requests: write
|
||||
|
||||
jobs:
|
||||
spark-integration-tests:
|
||||
name: Spark Integration Tests
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 45
|
||||
|
||||
steps:
|
||||
# ========================================
|
||||
# SETUP & BUILD
|
||||
# ========================================
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v7
|
||||
|
||||
- name: Set up JDK 11
|
||||
uses: actions/setup-java@v6
|
||||
with:
|
||||
java-version: '11'
|
||||
distribution: 'temurin'
|
||||
cache: ${{ runner.environment == 'github-hosted' && 'maven' || '' }}
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v7
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
cache: ${{ runner.environment == 'github-hosted' }}
|
||||
|
||||
- name: Build SeaweedFS binary
|
||||
run: |
|
||||
echo "Building SeaweedFS binary (statically linked for Alpine)..."
|
||||
cd weed
|
||||
CGO_ENABLED=0 go build -o ../docker/weed
|
||||
cd ../docker
|
||||
ls -la weed filer.toml entrypoint.sh
|
||||
file weed
|
||||
echo "OK SeaweedFS binary built"
|
||||
|
||||
- name: Build SeaweedFS Java dependencies
|
||||
run: |
|
||||
echo "Building Java client..."
|
||||
cd other/java/client
|
||||
mvn clean install -U -DskipTests -Dgpg.skip=true -Dcentral.publishing.skip=true
|
||||
echo "OK Java client built"
|
||||
cd ../../..
|
||||
|
||||
echo "Building HDFS3 client..."
|
||||
cd other/java/hdfs3
|
||||
mvn clean install -U -DskipTests -Dgpg.skip=true -Dcentral.publishing.skip=true
|
||||
echo "OK HDFS3 client built"
|
||||
echo ""
|
||||
echo "All Java dependencies installed to ~/.m2/repository"
|
||||
|
||||
# ========================================
|
||||
# SPARK INTEGRATION TESTS (DOCKER)
|
||||
# ========================================
|
||||
- name: Start SeaweedFS services for tests
|
||||
working-directory: test/java/spark
|
||||
run: |
|
||||
echo "=== Starting SeaweedFS Services for Tests ==="
|
||||
docker compose down -v || true
|
||||
docker compose build --no-cache
|
||||
docker compose up -d seaweedfs-master seaweedfs-volume seaweedfs-filer
|
||||
|
||||
echo "Waiting for services..."
|
||||
for i in {1..30}; do
|
||||
if curl -f http://localhost:8888/ > /dev/null 2>&1; then
|
||||
echo "OK SeaweedFS filer is ready!"
|
||||
break
|
||||
fi
|
||||
if [ $i -eq 30 ]; then
|
||||
echo "FAILED Services failed to start"
|
||||
docker compose ps -a
|
||||
docker compose logs
|
||||
exit 1
|
||||
fi
|
||||
echo "Waiting... ($i/30)"
|
||||
sleep 2
|
||||
done
|
||||
|
||||
curl -f http://localhost:9333/cluster/status || exit 1
|
||||
echo "OK All services healthy"
|
||||
|
||||
- name: Prepare Maven repository for Docker
|
||||
working-directory: test/java/spark
|
||||
run: |
|
||||
echo "Copying Maven artifacts for Docker container..."
|
||||
mkdir -p .m2/repository/com
|
||||
cp -r ~/.m2/repository/com/seaweedfs .m2/repository/com/
|
||||
echo "OK Maven artifacts copied"
|
||||
|
||||
- name: Run Spark integration tests
|
||||
working-directory: test/java/spark
|
||||
run: |
|
||||
echo "=== Running Spark Integration Tests ==="
|
||||
docker compose up --abort-on-container-exit spark-tests
|
||||
|
||||
echo ""
|
||||
echo "=== Test Logs ==="
|
||||
docker compose logs spark-tests | tail -100
|
||||
|
||||
- name: Stop test services
|
||||
if: always()
|
||||
working-directory: test/java/spark
|
||||
run: docker compose down -v
|
||||
|
||||
- name: Upload test results
|
||||
if: always()
|
||||
uses: actions/upload-artifact@v7
|
||||
with:
|
||||
name: spark-test-results
|
||||
path: test/java/spark/target/surefire-reports/
|
||||
retention-days: 30
|
||||
|
||||
- name: Publish test report
|
||||
if: always()
|
||||
uses: dorny/test-reporter@v3
|
||||
with:
|
||||
name: Spark Test Results
|
||||
path: test/java/spark/target/surefire-reports/*.xml
|
||||
reporter: java-junit
|
||||
fail-on-error: true
|
||||
|
||||
# ========================================
|
||||
# SPARK EXAMPLE (HOST-BASED) - DISABLED
|
||||
# Note: Host-based example doesn't work with Docker networking
|
||||
# because master returns Docker hostnames (seaweedfs-volume)
|
||||
# which are not resolvable from the host.
|
||||
# The Docker-based tests above are sufficient.
|
||||
# ========================================
|
||||
- name: Cache Apache Spark
|
||||
if: false && (github.event_name == 'push' || github.event_name == 'workflow_dispatch')
|
||||
id: cache-spark
|
||||
uses: actions/cache@v6
|
||||
with:
|
||||
path: spark-3.5.0-bin-hadoop3
|
||||
key: spark-3.5.0-hadoop3
|
||||
|
||||
- name: Download Apache Spark
|
||||
if: false && (github.event_name == 'push' || github.event_name == 'workflow_dispatch') && steps.cache-spark.outputs.cache-hit != 'true'
|
||||
run: |
|
||||
echo "Downloading Apache Spark 3.5.0..."
|
||||
wget -q https://archive.apache.org/dist/spark/spark-3.5.0/spark-3.5.0-bin-hadoop3.tgz
|
||||
tar xzf spark-3.5.0-bin-hadoop3.tgz
|
||||
echo "OK Spark downloaded"
|
||||
|
||||
- name: Start SeaweedFS services for example
|
||||
if: false && (github.event_name == 'push' || github.event_name == 'workflow_dispatch')
|
||||
working-directory: test/java/spark
|
||||
run: |
|
||||
echo "=== Starting SeaweedFS Services for Example ==="
|
||||
docker compose down -v || true
|
||||
docker compose build --no-cache
|
||||
docker compose up -d seaweedfs-master seaweedfs-volume seaweedfs-filer
|
||||
|
||||
echo "Waiting for services..."
|
||||
for i in {1..30}; do
|
||||
if curl -f http://localhost:8888/ > /dev/null 2>&1; then
|
||||
echo "OK SeaweedFS filer is ready!"
|
||||
break
|
||||
fi
|
||||
if [ $i -eq 30 ]; then
|
||||
echo "FAILED Services failed to start"
|
||||
docker compose ps -a
|
||||
docker compose logs
|
||||
exit 1
|
||||
fi
|
||||
echo "Waiting... ($i/30)"
|
||||
sleep 2
|
||||
done
|
||||
|
||||
curl -f http://localhost:9333/cluster/status || exit 1
|
||||
echo "OK All services healthy"
|
||||
|
||||
- name: Clean target directory
|
||||
if: false && (github.event_name == 'push' || github.event_name == 'workflow_dispatch')
|
||||
working-directory: test/java/spark
|
||||
run: |
|
||||
# Force remove target directory to avoid permission issues
|
||||
sudo rm -rf target || rm -rf target || true
|
||||
echo "OK Target directory cleaned"
|
||||
|
||||
- name: Build project for example
|
||||
if: false && (github.event_name == 'push' || github.event_name == 'workflow_dispatch')
|
||||
working-directory: test/java/spark
|
||||
run: mvn package -DskipTests
|
||||
|
||||
- name: Run Spark example application
|
||||
if: false && (github.event_name == 'push' || github.event_name == 'workflow_dispatch')
|
||||
working-directory: test/java/spark
|
||||
run: |
|
||||
echo "=== Running Spark Example Application ==="
|
||||
export SPARK_HOME=$(pwd)/../../../spark-3.5.0-bin-hadoop3
|
||||
$SPARK_HOME/bin/spark-submit \
|
||||
--class seaweed.spark.SparkSeaweedFSExample \
|
||||
--master local[2] \
|
||||
--conf spark.hadoop.fs.seaweedfs.impl=seaweed.hdfs.SeaweedFileSystem \
|
||||
--conf spark.hadoop.fs.seaweed.filer.host=localhost \
|
||||
--conf spark.hadoop.fs.seaweed.filer.port=8888 \
|
||||
--conf spark.hadoop.fs.seaweed.filer.port.grpc=18888 \
|
||||
--conf spark.hadoop.fs.seaweed.replication="" \
|
||||
target/seaweedfs-spark-integration-tests-1.0-SNAPSHOT.jar \
|
||||
seaweedfs://localhost:8888/ci-spark-output
|
||||
echo "OK Example completed"
|
||||
|
||||
- name: Verify example output
|
||||
if: false && (github.event_name == 'push' || github.event_name == 'workflow_dispatch')
|
||||
run: |
|
||||
echo "Verifying output..."
|
||||
curl -s http://localhost:8888/ci-spark-output/ || echo "Output listing unavailable"
|
||||
|
||||
- name: Stop example services
|
||||
if: false && always() && (github.event_name == 'push' || github.event_name == 'workflow_dispatch')
|
||||
working-directory: test/java/spark
|
||||
run: docker compose down -v
|
||||
|
||||
# ========================================
|
||||
# DIAGNOSTICS
|
||||
# ========================================
|
||||
- name: Display diagnostics on failure
|
||||
if: failure()
|
||||
working-directory: test/java/spark
|
||||
run: |
|
||||
echo "=== Container Status ==="
|
||||
docker compose ps -a
|
||||
echo ""
|
||||
echo "=== Master Logs ==="
|
||||
docker compose logs seaweedfs-master
|
||||
echo ""
|
||||
echo "=== Volume Logs ==="
|
||||
docker compose logs seaweedfs-volume
|
||||
echo ""
|
||||
echo "=== Filer Logs ==="
|
||||
docker compose logs seaweedfs-filer
|
||||
echo ""
|
||||
echo "=== Volume List ==="
|
||||
docker compose exec -T seaweedfs-master weed shell <<EOF || echo "Failed"
|
||||
volume.list
|
||||
exit
|
||||
EOF
|
||||
echo ""
|
||||
echo "=== Cluster Status ==="
|
||||
curl -s http://localhost:9333/dir/status | jq '.' || curl -s http://localhost:9333/dir/status
|
||||
|
||||
@@ -1,73 +0,0 @@
|
||||
---
|
||||
name: Star History
|
||||
|
||||
on:
|
||||
schedule:
|
||||
- cron: "0 0 * * *" # daily, 00:00 UTC
|
||||
workflow_dispatch:
|
||||
|
||||
permissions:
|
||||
contents: write
|
||||
|
||||
concurrency:
|
||||
# Only one chart regeneration per branch at a time so overlapping runs never
|
||||
# conflict on note/star_history.svg during rebase. Scoped by ref so a manual
|
||||
# run on another branch can't cancel the daily master update.
|
||||
group: star-history-${{ github.ref }}
|
||||
cancel-in-progress: ${{ github.event_name == 'pull_request' || (github.event_name == 'push' && !startsWith(github.ref, 'refs/tags/')) }}
|
||||
|
||||
jobs:
|
||||
render:
|
||||
name: Regenerate star history chart
|
||||
runs-on: ubuntu-latest
|
||||
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v7
|
||||
with:
|
||||
# Full history so the chart commit can rebase onto a moved master.
|
||||
fetch-depth: 0
|
||||
|
||||
- name: Set up Python
|
||||
uses: actions/setup-python@v7
|
||||
with:
|
||||
python-version: "3.x"
|
||||
cache: ${{ runner.environment == 'github-hosted' && 'pip' || '' }}
|
||||
cache-dependency-path: .github/scripts/star_history.py
|
||||
|
||||
- name: Install matplotlib
|
||||
run: pip install matplotlib
|
||||
|
||||
- name: Render chart
|
||||
env:
|
||||
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
run: python .github/scripts/star_history.py
|
||||
|
||||
- name: Commit if changed
|
||||
run: |
|
||||
git config user.name "github-actions[bot]"
|
||||
git config user.email "41898282+github-actions[bot]@users.noreply.github.com"
|
||||
if git diff --quiet -- note/star_history.svg; then
|
||||
echo "No changes to the chart."
|
||||
exit 0
|
||||
fi
|
||||
git add note/star_history.svg
|
||||
git commit -m "docs: regenerate star history chart"
|
||||
# Rebase and retry so a concurrent push to master doesn't lose the chart.
|
||||
for attempt in 1 2 3 4 5; do
|
||||
if [ "$attempt" -gt 1 ]; then
|
||||
# Guard the rebase: a transient fetch error or conflict must not
|
||||
# abort the fail-fast shell before the remaining attempts run.
|
||||
if ! git pull --rebase origin "$GITHUB_REF_NAME"; then
|
||||
echo "rebase failed (attempt ${attempt}); aborting and retrying"
|
||||
git rebase --abort || true
|
||||
continue
|
||||
fi
|
||||
fi
|
||||
if git push origin HEAD:"$GITHUB_REF_NAME"; then
|
||||
exit 0
|
||||
fi
|
||||
echo "push rejected (attempt ${attempt}); will rebase and retry"
|
||||
done
|
||||
echo "::error::could not push star history chart after retries"
|
||||
exit 1
|
||||
@@ -1,51 +0,0 @@
|
||||
name: Telemetry Integration Tests
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [ master ]
|
||||
paths:
|
||||
- 'telemetry/**'
|
||||
- 'weed/telemetry/**'
|
||||
- '.github/workflows/telemetry-integration.yml'
|
||||
pull_request:
|
||||
branches: [ master ]
|
||||
paths:
|
||||
- 'telemetry/**'
|
||||
- 'weed/telemetry/**'
|
||||
- '.github/workflows/telemetry-integration.yml'
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.event.pull_request.number || (github.event_name == 'push' && github.ref) || github.run_id }}
|
||||
cancel-in-progress: ${{ github.event_name == 'pull_request' || (github.event_name == 'push' && !startsWith(github.ref, 'refs/tags/')) }}
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
jobs:
|
||||
telemetry-integration-test:
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 5
|
||||
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v7
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v7
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
cache: ${{ runner.environment == 'github-hosted' }}
|
||||
|
||||
- name: Build telemetry server
|
||||
run: cd telemetry/server && go build -o telemetry-server .
|
||||
|
||||
- name: Run telemetry integration test
|
||||
run: go run telemetry/test/integration.go
|
||||
|
||||
- name: Upload test logs on failure
|
||||
if: failure()
|
||||
uses: actions/upload-artifact@v7
|
||||
with:
|
||||
name: telemetry-test-logs
|
||||
path: telemetry-server-test.log
|
||||
retention-days: 7
|
||||
@@ -1,85 +0,0 @@
|
||||
name: "terraform: validate and test modules"
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [ master ]
|
||||
paths: ['terraform/**', '.github/workflows/terraform_ci.yml']
|
||||
pull_request:
|
||||
branches: [ master ]
|
||||
paths: ['terraform/**', '.github/workflows/terraform_ci.yml']
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.event.pull_request.number || (github.event_name == 'push' && github.ref) || github.run_id }}
|
||||
cancel-in-progress: ${{ github.event_name == 'pull_request' || (github.event_name == 'push' && !startsWith(github.ref, 'refs/tags/')) }}
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
jobs:
|
||||
validate:
|
||||
name: fmt, validate, plan-level tests
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v7
|
||||
|
||||
- name: Set up OpenTofu
|
||||
uses: opentofu/setup-opentofu@v2
|
||||
with:
|
||||
tofu_version: 1.12.1
|
||||
|
||||
- name: fmt check
|
||||
working-directory: terraform
|
||||
run: tofu fmt -recursive -check -diff
|
||||
|
||||
- name: validate core
|
||||
working-directory: terraform/modules/core
|
||||
run: |
|
||||
tofu init -backend=false -input=false
|
||||
tofu validate
|
||||
|
||||
- name: validate security
|
||||
working-directory: terraform/modules/security
|
||||
run: |
|
||||
tofu init -backend=false -input=false
|
||||
tofu validate
|
||||
|
||||
- name: plan-level tests (core)
|
||||
working-directory: terraform/modules/core
|
||||
run: tofu test
|
||||
|
||||
- name: validate examples
|
||||
run: |
|
||||
set -e
|
||||
for ex in terraform/examples/*/; do
|
||||
echo "== validate $ex =="
|
||||
tofu -chdir="$ex" init -backend=false -input=false
|
||||
tofu -chdir="$ex" validate
|
||||
done
|
||||
|
||||
smoke:
|
||||
name: local cluster smoke test (real weed)
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v7
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v7
|
||||
with:
|
||||
go-version-file: go.mod
|
||||
cache: ${{ runner.environment == 'github-hosted' }}
|
||||
|
||||
- name: Build weed
|
||||
run: go build -o "$RUNNER_TEMP/weed" ./weed
|
||||
|
||||
- name: Set up OpenTofu
|
||||
uses: opentofu/setup-opentofu@v2
|
||||
with:
|
||||
tofu_version: 1.12.1
|
||||
|
||||
- name: Run local cluster harness
|
||||
working-directory: terraform/test/local
|
||||
run: WEED="$RUNNER_TEMP/weed" ./run_local_cluster.sh
|
||||
|
||||
- name: Run local mTLS cluster harness
|
||||
working-directory: terraform/test/local-secure
|
||||
run: WEED="$RUNNER_TEMP/weed" ./run_local_secure.sh
|
||||
@@ -3,20 +3,8 @@ name: "test s3 over https using aws-cli"
|
||||
on:
|
||||
push:
|
||||
branches: [master, test-https-s3-awscli]
|
||||
paths:
|
||||
- 'weed/s3api/**'
|
||||
- 'weed/server/**'
|
||||
- 'go.mod'
|
||||
- 'go.sum'
|
||||
- '.github/workflows/test-s3-over-https-using-awscli.yml'
|
||||
pull_request:
|
||||
branches: [master, test-https-s3-awscli]
|
||||
paths:
|
||||
- 'weed/s3api/**'
|
||||
- 'weed/server/**'
|
||||
- 'go.mod'
|
||||
- 'go.sum'
|
||||
- '.github/workflows/test-s3-over-https-using-awscli.yml'
|
||||
|
||||
env:
|
||||
AWS_ACCESS_KEY_ID: some_access_key1
|
||||
@@ -27,21 +15,16 @@ defaults:
|
||||
run:
|
||||
working-directory: weed
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.event.pull_request.number || (github.event_name == 'push' && github.ref) || github.run_id }}
|
||||
cancel-in-progress: ${{ github.event_name == 'pull_request' || (github.event_name == 'push' && !startsWith(github.ref, 'refs/tags/')) }}
|
||||
|
||||
jobs:
|
||||
awscli-tests:
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 10
|
||||
timeout-minutes: 5
|
||||
steps:
|
||||
- uses: actions/checkout@v7
|
||||
- uses: actions/checkout@v4
|
||||
|
||||
- uses: actions/setup-go@v7
|
||||
- uses: actions/setup-go@v5.5.0
|
||||
with:
|
||||
go-version: ^1.26
|
||||
cache: ${{ runner.environment == 'github-hosted' }}
|
||||
go-version: ^1.24
|
||||
|
||||
- name: Build SeaweedFS
|
||||
run: |
|
||||
@@ -51,8 +34,8 @@ jobs:
|
||||
run: |
|
||||
set -e
|
||||
mkdir -p /tmp/data
|
||||
./weed -v=3 server -s3 -dir=/tmp/data -s3.config=../docker/compose/s3.json -master.peers=none > weed.log 2>&1 &
|
||||
until curl -s http://localhost:8333/healthz > /dev/null; do sleep 1; done
|
||||
./weed server -s3 -dir=/tmp/data -s3.config=../docker/compose/s3.json &
|
||||
until curl -s http://localhost:8333/ > /dev/null; do sleep 1; done
|
||||
|
||||
- name: Setup Caddy
|
||||
run: |
|
||||
@@ -71,7 +54,7 @@ jobs:
|
||||
- name: Start Caddy
|
||||
run: |
|
||||
./caddy start
|
||||
until curl -fsS --insecure https://localhost:8443/healthz > /dev/null; do sleep 1; done
|
||||
until curl -fsS --insecure https://localhost:8443 > /dev/null; do sleep 1; done
|
||||
|
||||
- name: Create Bucket
|
||||
run: |
|
||||
@@ -91,42 +74,6 @@ jobs:
|
||||
set -e
|
||||
dd if=/dev/urandom of=generated bs=1M count=32
|
||||
aws --no-verify-ssl s3 cp --no-progress generated s3://bucket/test-multipart
|
||||
for i in 1 2 3 4 5; do
|
||||
aws --no-verify-ssl s3 cp --no-progress s3://bucket/test-multipart downloaded && break
|
||||
if [ "$i" -eq 5 ]; then exit 1; fi
|
||||
sleep 2
|
||||
done
|
||||
aws --no-verify-ssl s3 cp --no-progress s3://bucket/test-multipart downloaded
|
||||
diff -q generated downloaded
|
||||
rm -f generated downloaded
|
||||
|
||||
- name: Test GetObject with If-Match
|
||||
run: |
|
||||
set -e
|
||||
dd if=/dev/urandom of=generated bs=1M count=32
|
||||
ETAG=$(aws --no-verify-ssl s3api put-object --bucket bucket --key test-get-obj --body generated | jq -r .ETag)
|
||||
# jq -r already removes quotes, so use ETAG directly (handles both simple and multipart ETags)
|
||||
aws --no-verify-ssl s3api get-object --bucket bucket --key test-get-obj --if-match "$ETAG" downloaded
|
||||
diff -q generated downloaded
|
||||
rm -f generated downloaded
|
||||
|
||||
- name: Show server logs on failure
|
||||
if: failure()
|
||||
run: |
|
||||
echo "========================================="
|
||||
echo "SeaweedFS Server Logs"
|
||||
echo "========================================="
|
||||
# Note: weed.log is relative to working-directory (weed/)
|
||||
if [ -f weed.log ]; then
|
||||
cat weed.log
|
||||
else
|
||||
echo "No weed.log file found"
|
||||
fi
|
||||
|
||||
- name: Upload server logs on failure
|
||||
if: failure()
|
||||
uses: actions/upload-artifact@v7
|
||||
with:
|
||||
name: seaweedfs-logs
|
||||
# Note: actions don't use defaults.run.working-directory, so path is relative to workspace root
|
||||
path: weed/weed.log
|
||||
retention-days: 3
|
||||
@@ -1,73 +0,0 @@
|
||||
name: "TLS Rotation Integration Tests"
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [ master ]
|
||||
paths:
|
||||
- 'weed/**'
|
||||
- 'test/tls_rotation/**'
|
||||
- 'go.mod'
|
||||
- 'go.sum'
|
||||
- '.github/workflows/tls-rotation-tests.yml'
|
||||
pull_request:
|
||||
branches: [ master ]
|
||||
paths:
|
||||
- 'weed/**'
|
||||
- 'test/tls_rotation/**'
|
||||
- 'go.mod'
|
||||
- 'go.sum'
|
||||
- '.github/workflows/tls-rotation-tests.yml'
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.event.pull_request.number || (github.event_name == 'push' && github.ref) || github.run_id }}
|
||||
cancel-in-progress: ${{ github.event_name == 'pull_request' || (github.event_name == 'push' && !startsWith(github.ref, 'refs/tags/')) }}
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
jobs:
|
||||
tls-rotation-tests:
|
||||
name: TLS Rotation Integration Tests
|
||||
runs-on: ubuntu-24.04
|
||||
timeout-minutes: 10
|
||||
steps:
|
||||
- name: Set up Go 1.x
|
||||
uses: actions/setup-go@v7
|
||||
with:
|
||||
cache: ${{ runner.environment == 'github-hosted' }}
|
||||
go-version: ^1.26
|
||||
id: go
|
||||
|
||||
- name: Check out code into the Go module directory
|
||||
uses: actions/checkout@v7
|
||||
|
||||
- name: Build weed binary
|
||||
run: |
|
||||
cd weed && go build -o weed .
|
||||
|
||||
- name: Run TLS Rotation Integration Tests
|
||||
working-directory: test/tls_rotation
|
||||
run: |
|
||||
go test -v -count=1 -timeout 5m
|
||||
|
||||
- name: Collect server logs on failure
|
||||
if: failure()
|
||||
run: |
|
||||
echo "Collecting master logs from temp directories..."
|
||||
mkdir -p /tmp/tls-rotation-test-logs
|
||||
find /tmp -maxdepth 1 -type d -name "TestMasterHTTPS*" 2>/dev/null | while read dir; do
|
||||
if [ -d "$dir" ]; then
|
||||
echo "Found test directory: $dir"
|
||||
cp -r "$dir" /tmp/tls-rotation-test-logs/ 2>/dev/null || true
|
||||
fi
|
||||
done
|
||||
echo "Collected logs:"
|
||||
find /tmp/tls-rotation-test-logs -type f -name "*.log" 2>/dev/null || echo "No logs found"
|
||||
|
||||
- name: Archive logs
|
||||
if: failure()
|
||||
uses: actions/upload-artifact@v7
|
||||
with:
|
||||
name: tls-rotation-test-logs
|
||||
path: /tmp/tls-rotation-test-logs/
|
||||
retention-days: 14
|
||||
@@ -1,114 +0,0 @@
|
||||
name: "TUS Protocol Tests"
|
||||
|
||||
on:
|
||||
pull_request:
|
||||
paths:
|
||||
- 'weed/server/**'
|
||||
- 'weed/filer/**'
|
||||
- 'test/tus/**'
|
||||
- 'go.mod'
|
||||
- 'go.sum'
|
||||
- '.github/workflows/tus-tests.yml'
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.event.pull_request.number || (github.event_name == 'push' && github.ref) || github.run_id }}
|
||||
cancel-in-progress: ${{ github.event_name == 'pull_request' || (github.event_name == 'push' && !startsWith(github.ref, 'refs/tags/')) }}
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
defaults:
|
||||
run:
|
||||
working-directory: weed
|
||||
|
||||
jobs:
|
||||
tus-integration-tests:
|
||||
name: TUS Protocol Integration Tests
|
||||
runs-on: ubuntu-24.04
|
||||
timeout-minutes: 20
|
||||
|
||||
steps:
|
||||
- name: Check out code
|
||||
uses: actions/checkout@v7
|
||||
|
||||
- name: Clean up stale FUSE mounts
|
||||
uses: ./.github/actions/cleanup-stale-fuse-mounts
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v7
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
cache: ${{ runner.environment == 'github-hosted' }}
|
||||
id: go
|
||||
|
||||
- name: Install SeaweedFS
|
||||
run: |
|
||||
go install -buildvcs=false
|
||||
|
||||
- name: Run TUS Integration Tests
|
||||
timeout-minutes: 15
|
||||
working-directory: test/tus
|
||||
run: |
|
||||
set -x
|
||||
echo "=== System Information ==="
|
||||
uname -a
|
||||
free -h
|
||||
timeout 15 df -h || true
|
||||
echo "=== Starting TUS Tests ==="
|
||||
|
||||
# Run tests with automatic server management
|
||||
make test-with-server || {
|
||||
echo "TUS integration tests failed, checking logs..."
|
||||
if [ -f /tmp/seaweedfs-tus-filer.log ]; then
|
||||
echo "=== Filer logs ==="
|
||||
tail -100 /tmp/seaweedfs-tus-filer.log
|
||||
fi
|
||||
if [ -f /tmp/seaweedfs-tus-master.log ]; then
|
||||
echo "=== Master logs ==="
|
||||
tail -50 /tmp/seaweedfs-tus-master.log
|
||||
fi
|
||||
if [ -f /tmp/seaweedfs-tus-volume.log ]; then
|
||||
echo "=== Volume logs ==="
|
||||
tail -50 /tmp/seaweedfs-tus-volume.log
|
||||
fi
|
||||
exit 1
|
||||
}
|
||||
|
||||
- name: Show server logs on failure
|
||||
if: failure()
|
||||
working-directory: test/tus
|
||||
run: |
|
||||
echo "=== Filer Server Logs ==="
|
||||
if [ -f /tmp/seaweedfs-tus-filer.log ]; then
|
||||
echo "Last 100 lines of filer logs:"
|
||||
tail -100 /tmp/seaweedfs-tus-filer.log
|
||||
else
|
||||
echo "No filer log file found"
|
||||
fi
|
||||
|
||||
echo "=== Master Server Logs ==="
|
||||
if [ -f /tmp/seaweedfs-tus-master.log ]; then
|
||||
tail -50 /tmp/seaweedfs-tus-master.log
|
||||
else
|
||||
echo "No master log file found"
|
||||
fi
|
||||
|
||||
echo "=== Volume Server Logs ==="
|
||||
if [ -f /tmp/seaweedfs-tus-volume.log ]; then
|
||||
tail -50 /tmp/seaweedfs-tus-volume.log
|
||||
else
|
||||
echo "No volume log file found"
|
||||
fi
|
||||
|
||||
echo "=== Test Environment ==="
|
||||
ps aux | grep -E "(weed|test)" || true
|
||||
netstat -tlnp 2>/dev/null | grep -E "(18888|19333|18080)" || true
|
||||
|
||||
- name: Upload test logs on failure
|
||||
if: failure()
|
||||
uses: actions/upload-artifact@v7
|
||||
with:
|
||||
name: tus-test-logs
|
||||
path: |
|
||||
/tmp/seaweedfs-tus-*.log
|
||||
retention-days: 3
|
||||
@@ -1,73 +0,0 @@
|
||||
name: "Vacuum Integration Tests"
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [ master ]
|
||||
paths:
|
||||
- 'weed/**'
|
||||
- 'test/vacuum/**'
|
||||
- 'go.mod'
|
||||
- 'go.sum'
|
||||
- '.github/workflows/vacuum-integration-tests.yml'
|
||||
pull_request:
|
||||
branches: [ master ]
|
||||
paths:
|
||||
- 'weed/**'
|
||||
- 'test/vacuum/**'
|
||||
- 'go.mod'
|
||||
- 'go.sum'
|
||||
- '.github/workflows/vacuum-integration-tests.yml'
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.event.pull_request.number || (github.event_name == 'push' && github.ref) || github.run_id }}
|
||||
cancel-in-progress: ${{ github.event_name == 'pull_request' || (github.event_name == 'push' && !startsWith(github.ref, 'refs/tags/')) }}
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
jobs:
|
||||
vacuum-integration-tests:
|
||||
name: Vacuum Integration Tests
|
||||
runs-on: ubuntu-24.04
|
||||
timeout-minutes: 15
|
||||
steps:
|
||||
- name: Set up Go 1.x
|
||||
uses: actions/setup-go@v7
|
||||
with:
|
||||
cache: ${{ runner.environment == 'github-hosted' }}
|
||||
go-version: ^1.26
|
||||
id: go
|
||||
|
||||
- name: Check out code into the Go module directory
|
||||
uses: actions/checkout@v7
|
||||
|
||||
- name: Build weed binary
|
||||
run: |
|
||||
cd weed && go build -o weed .
|
||||
|
||||
- name: Run Vacuum Integration Tests
|
||||
working-directory: test/vacuum
|
||||
run: |
|
||||
go test -v -timeout 10m
|
||||
|
||||
- name: Collect server logs on failure
|
||||
if: failure()
|
||||
run: |
|
||||
echo "Collecting server logs from temp directories..."
|
||||
mkdir -p /tmp/vacuum-test-logs
|
||||
find /tmp -maxdepth 1 -type d -name "TestVacuum*" 2>/dev/null | while read dir; do
|
||||
if [ -d "$dir" ]; then
|
||||
echo "Found test directory: $dir"
|
||||
cp -r "$dir" /tmp/vacuum-test-logs/ 2>/dev/null || true
|
||||
fi
|
||||
done
|
||||
echo "Collected logs:"
|
||||
find /tmp/vacuum-test-logs -type f -name "*.log" 2>/dev/null || echo "No logs found"
|
||||
|
||||
- name: Archive logs
|
||||
if: failure()
|
||||
uses: actions/upload-artifact@v7
|
||||
with:
|
||||
name: vacuum-integration-test-logs
|
||||
path: /tmp/vacuum-test-logs/
|
||||
retention-days: 14
|
||||
@@ -1,143 +0,0 @@
|
||||
name: "Volume Server Integration Tests"
|
||||
|
||||
on:
|
||||
pull_request:
|
||||
branches: [ master ]
|
||||
paths:
|
||||
- 'test/volume_server/**'
|
||||
- 'weed/server/**'
|
||||
- 'weed/storage/**'
|
||||
- 'weed/pb/volume_server.proto'
|
||||
- 'weed/pb/volume_server_pb/**'
|
||||
- '.github/workflows/volume-server-integration-tests.yml'
|
||||
push:
|
||||
branches: [ master, main ]
|
||||
paths:
|
||||
- 'test/volume_server/**'
|
||||
- 'weed/server/**'
|
||||
- 'weed/storage/**'
|
||||
- 'weed/pb/volume_server.proto'
|
||||
- 'weed/pb/volume_server_pb/**'
|
||||
- '.github/workflows/volume-server-integration-tests.yml'
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.event.pull_request.number || (github.event_name == 'push' && github.ref) || github.run_id }}
|
||||
cancel-in-progress: ${{ github.event_name == 'pull_request' || (github.event_name == 'push' && !startsWith(github.ref, 'refs/tags/')) }}
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
env:
|
||||
TEST_TIMEOUT: '30m'
|
||||
# Keep in step with the length of matrix.shard below.
|
||||
SHARD_COUNT: 3
|
||||
|
||||
jobs:
|
||||
build-weed:
|
||||
name: Build SeaweedFS Binary
|
||||
runs-on: ubuntu-24.04
|
||||
timeout-minutes: 15
|
||||
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v7
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v7
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
cache: ${{ runner.environment == 'github-hosted' }}
|
||||
|
||||
- name: Build SeaweedFS binary
|
||||
run: |
|
||||
cd weed
|
||||
go build -o weed .
|
||||
chmod +x weed
|
||||
./weed version
|
||||
|
||||
- name: Package binary
|
||||
run: tar cf weed-binary.tar weed/weed
|
||||
|
||||
- name: Upload binary
|
||||
uses: actions/upload-artifact@v7
|
||||
with:
|
||||
name: weed-binary
|
||||
if-no-files-found: error
|
||||
retention-days: 1
|
||||
path: weed-binary.tar
|
||||
|
||||
volume-server-integration-tests:
|
||||
name: Volume Server Integration Tests (${{ matrix.test-type }} - Shard ${{ matrix.shard }})
|
||||
runs-on: ubuntu-24.04
|
||||
timeout-minutes: 45
|
||||
needs: [build-weed]
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
test-type: [grpc, http]
|
||||
shard: [1, 2, 3]
|
||||
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v7
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v7
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
cache: ${{ runner.environment == 'github-hosted' }}
|
||||
|
||||
- name: Download binary
|
||||
uses: actions/download-artifact@v8
|
||||
with:
|
||||
name: weed-binary
|
||||
|
||||
- name: Restore binary
|
||||
run: |
|
||||
tar xf weed-binary.tar
|
||||
chmod +x weed/weed
|
||||
./weed/weed version
|
||||
|
||||
# Dealing the listed tests out one by one keeps the shards even. Bucketing
|
||||
# them by first letter did not: names cluster, so ^Test[I-S] drew 50 of
|
||||
# the 114 grpc tests and ran nearly twice as long as the other two shards.
|
||||
- name: Select this shard's tests
|
||||
env:
|
||||
TEST_TYPE: ${{ matrix.test-type }}
|
||||
SHARD: ${{ matrix.shard }}
|
||||
run: |
|
||||
tests=$(go test ./test/volume_server/"$TEST_TYPE"/... -list '.*' | grep '^Test' | sort -u)
|
||||
# An empty list would make -run match nothing and the shard pass vacuously.
|
||||
[ -n "$tests" ] || { echo "listed no tests in test/volume_server/$TEST_TYPE"; exit 1; }
|
||||
selected=$(echo "$tests" | awk -v n="$SHARD_COUNT" -v i="$SHARD" 'NR % n == i - 1')
|
||||
echo "shard $SHARD of $SHARD_COUNT runs $(echo "$selected" | wc -l) of $(echo "$tests" | wc -l) tests"
|
||||
echo "TEST_PATTERN=^($(echo "$selected" | paste -sd'|' -))\$" >> "$GITHUB_ENV"
|
||||
|
||||
- name: Run volume server integration tests
|
||||
env:
|
||||
WEED_BINARY: ${{ github.workspace }}/weed/weed
|
||||
run: |
|
||||
echo "Running volume server integration tests for ${{ matrix.test-type }} (Shard ${{ matrix.shard }} of ${SHARD_COUNT})..."
|
||||
go test -v -count=1 -timeout=${{ env.TEST_TIMEOUT }} ./test/volume_server/${{ matrix.test-type }}/... -run "${TEST_PATTERN}"
|
||||
|
||||
- name: Collect logs on failure
|
||||
if: failure()
|
||||
run: |
|
||||
mkdir -p /tmp/volume-server-it-logs
|
||||
find /tmp -maxdepth 1 -type d -name "seaweedfs_volume_server_it_*" -print -exec cp -r {} /tmp/volume-server-it-logs/ \; || true
|
||||
|
||||
- name: Archive logs on failure
|
||||
if: failure()
|
||||
uses: actions/upload-artifact@v7
|
||||
with:
|
||||
name: volume-server-integration-test-logs
|
||||
path: /tmp/volume-server-it-logs/
|
||||
if-no-files-found: warn
|
||||
retention-days: 7
|
||||
|
||||
- name: Test summary
|
||||
if: always()
|
||||
run: |
|
||||
echo "## Volume Server Integration Test Summary (${{ matrix.test-type }} - Shard ${{ matrix.shard }})" >> "$GITHUB_STEP_SUMMARY"
|
||||
echo "- Suite: test/volume_server/${{ matrix.test-type }} (shard ${{ matrix.shard }} of ${SHARD_COUNT}, see 'Select this shard's tests' for the split)" >> "$GITHUB_STEP_SUMMARY"
|
||||
echo "- Command: go test -v -count=1 -timeout=${{ env.TEST_TIMEOUT }} ./test/volume_server/${{ matrix.test-type }}/... -run \"\${TEST_PATTERN}\"" >> "$GITHUB_STEP_SUMMARY"
|
||||
-41
@@ -2,7 +2,6 @@
|
||||
vendor
|
||||
tags
|
||||
*.swp
|
||||
.claude/
|
||||
### OSX template
|
||||
.DS_Store
|
||||
.AppleDouble
|
||||
@@ -111,43 +110,3 @@ test/s3/cors/cors.test
|
||||
/test/s3/retention/filerldb2
|
||||
test/s3/retention/weed-server.pid
|
||||
test/s3/retention/weed-test.log
|
||||
/test/s3/versioning/test-volume-data
|
||||
test/s3/versioning/weed-test.log
|
||||
/docker/admin_integration/data
|
||||
docker/agent_pub_record
|
||||
docker/admin_integration/weed-local
|
||||
/test/s3/encryption/filerldb2
|
||||
/test/s3/sse/filerldb2
|
||||
test/s3/sse/weed-test.log
|
||||
ADVANCED_IAM_DEVELOPMENT_PLAN.md
|
||||
/test/s3/iam/test-volume-data
|
||||
*.log
|
||||
weed-iam
|
||||
test/kafka/kafka-client-loadtest/weed-linux-arm64
|
||||
/test/tus/filerldb2
|
||||
coverage.out
|
||||
/test/s3/remote_cache/test-primary-data
|
||||
/test/s3/remote_cache/test-remote-data
|
||||
test/s3/remote_cache/remote-server.pid
|
||||
test/s3/remote_cache/primary-server.pid
|
||||
/test/erasure_coding/filerldb2
|
||||
/test/s3/cors/test-mini-data
|
||||
/test/s3/filer_group/test-volume-data
|
||||
|
||||
# ID and PID files
|
||||
*.id
|
||||
*.pid
|
||||
test/s3/iam/.test_env
|
||||
/test/erasure_coding/admin_dockertest/tmp
|
||||
/test/erasure_coding/admin_dockertest/task_logs
|
||||
weed_bin
|
||||
telemetry/server/telemetry-server
|
||||
.aider*
|
||||
/seaweed-volume/docs
|
||||
__pycache__/
|
||||
/test/s3vectors/weed
|
||||
/test/s3vectors/weed-vector
|
||||
/test/s3vectors/weed-vector-cli
|
||||
/test/s3vectors/weed-worker
|
||||
/test/s3vectors/weed-volume
|
||||
.pytest_cache/
|
||||
Loaded 100 of 4582 files, more files were not shown because too many files have changed in this diff.
Show more
Reference in new issue
Block a user