mirror of
https://github.com/seaweedfs/seaweedfs.git
synced 2026-10-30 08:50:48 +01:00
Merge remote-tracking branch 'origin/master' into codex-rust-volume-server-bootstrap
# Conflicts: # .github/workflows/volume-server-integration-tests.yml # test/volume_server/framework/cluster.go # test/volume_server/framework/cluster_dual.go # test/volume_server/framework/volume_fixture.go # test/volume_server/grpc/data_stream_success_test.go # test/volume_server/http/public_cors_methods_test.go # test/volume_server/matrix/config_profiles.go
This commit is contained in:
user.email committed
commit
9436099d55
3609 files changed
+709333
-121315
No files matched your search
@@ -0,0 +1,11 @@
|
||||
name: Clean up stale SeaweedFS test mounts
|
||||
description: Detach disconnected FUSE test mounts left by earlier jobs on a shared runner.
|
||||
runs:
|
||||
using: composite
|
||||
steps:
|
||||
- name: Unmount disconnected SeaweedFS test mounts
|
||||
if: runner.os == 'Linux' && runner.environment == 'self-hosted'
|
||||
shell: bash
|
||||
env:
|
||||
CLEANUP_SCRIPT: ${{ github.action_path }}/cleanup.py
|
||||
run: python3 "$CLEANUP_SCRIPT"
|
||||
@@ -0,0 +1,89 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Detach disconnected SeaweedFS integration-test mounts without touching live ones."""
|
||||
|
||||
import errno
|
||||
import os
|
||||
from pathlib import Path
|
||||
import re
|
||||
import shutil
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
|
||||
|
||||
def test_mounts(mountinfo):
|
||||
"""Read the kernel table directly: traversing a stale mount may block."""
|
||||
roots = {tempfile.gettempdir(), "/tmp", "/var/tmp"}
|
||||
for line in mountinfo.splitlines():
|
||||
fields, filesystem = line.split(" - ", 1)
|
||||
if filesystem.split()[0].split(".")[0] != "fuse":
|
||||
continue
|
||||
fields = fields.split()
|
||||
# mountinfo escapes whitespace and backslashes with octal sequences.
|
||||
target = re.sub(r"\\([0-7]{3})", lambda m: chr(int(m[1], 8)), fields[4])
|
||||
path = Path(target)
|
||||
# Test mounts are a mount* leaf under a seaweedfs_fuse_* scratch dir
|
||||
# (seaweedfs_fuse_test_*/mount, seaweedfs_fuse_failover_*/mount0, ...)
|
||||
# or the posix-compliance workflow's fixed /tmp/seaweedfs/mount.
|
||||
if (not path.name.startswith("mount")
|
||||
or str(path.parent.parent) not in roots
|
||||
or not (path.parent.name.startswith("seaweedfs_fuse")
|
||||
or path.parent.name == "seaweedfs")):
|
||||
continue
|
||||
yield fields[0], target
|
||||
|
||||
|
||||
def mounted(mount_id, target):
|
||||
return (mount_id, target) in test_mounts(Path("/proc/self/mountinfo").read_text())
|
||||
|
||||
|
||||
def cleanup():
|
||||
failed = False
|
||||
for mount_id, target in test_mounts(Path("/proc/self/mountinfo").read_text()):
|
||||
try:
|
||||
# A separate, bounded probe prevents a wedged FUSE daemon from
|
||||
# hanging cleanup. Only ENOTCONN proves the daemon disconnected;
|
||||
# a timeout or another error is not enough to detach a live mount.
|
||||
probe = subprocess.run(
|
||||
[sys.executable, __file__, "--probe", target], timeout=5,
|
||||
capture_output=True, check=False,
|
||||
)
|
||||
except subprocess.TimeoutExpired:
|
||||
print(f"::warning::FUSE probe timed out; leaving {target} mounted")
|
||||
continue
|
||||
if probe.returncode != errno.ENOTCONN or not mounted(mount_id, target):
|
||||
continue
|
||||
|
||||
print(f"Unmounting disconnected FUSE test mount: {target}", flush=True)
|
||||
# Lazy unmount detaches a dead mount even when an old process still
|
||||
# holds it open. Prefer the unprivileged FUSE helper; sudo is fallback.
|
||||
for command in (
|
||||
["fusermount3", "-uz", "--", target],
|
||||
["fusermount", "-uz", "--", target],
|
||||
["sudo", "-n", "umount", "-l", "--", target],
|
||||
):
|
||||
try:
|
||||
subprocess.run(command, timeout=10, check=False)
|
||||
except (OSError, subprocess.TimeoutExpired) as error:
|
||||
print(f"{command[0]}: {error}", file=sys.stderr)
|
||||
if not mounted(mount_id, target):
|
||||
break
|
||||
else:
|
||||
print(f"::error::Could not unmount disconnected FUSE test mount: {target}")
|
||||
failed = True
|
||||
continue
|
||||
# The detached mount's parent still holds the crashed run's data and
|
||||
# logs; test_mounts only yields targets under a SeaweedFS test dir in
|
||||
# a temp root, so the parent is safe to reclaim here.
|
||||
shutil.rmtree(Path(target).parent, ignore_errors=True)
|
||||
return int(failed)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
if len(sys.argv) == 3 and sys.argv[1] == "--probe":
|
||||
try:
|
||||
os.stat(sys.argv[2])
|
||||
except OSError as error:
|
||||
sys.exit(error.errno)
|
||||
sys.exit(0)
|
||||
sys.exit(cleanup())
|
||||
@@ -0,0 +1,137 @@
|
||||
import errno
|
||||
import importlib.util
|
||||
from pathlib import Path
|
||||
import subprocess
|
||||
import unittest
|
||||
from unittest.mock import patch
|
||||
|
||||
spec = importlib.util.spec_from_file_location("cleanup", Path(__file__).with_name("cleanup.py"))
|
||||
cleanup = importlib.util.module_from_spec(spec)
|
||||
spec.loader.exec_module(cleanup)
|
||||
|
||||
TARGET = "/tmp/seaweedfs_fuse_test_123/mount"
|
||||
MOUNTINFO = f"42 1 0:9 / {TARGET} rw - fuse.seaweedfs seaweedfs rw\n"
|
||||
|
||||
|
||||
class CleanupTests(unittest.TestCase):
|
||||
def run_cleanup(self, results, mounted=True):
|
||||
with patch.object(cleanup.Path, "read_text", return_value=MOUNTINFO), \
|
||||
patch.object(cleanup, "mounted", side_effect=mounted if isinstance(mounted, list) else None,
|
||||
return_value=mounted) as check, \
|
||||
patch.object(cleanup.shutil, "rmtree") as rmtree, \
|
||||
patch.object(cleanup.subprocess, "run", side_effect=results) as run:
|
||||
status = cleanup.cleanup()
|
||||
return status, run, check, rmtree
|
||||
|
||||
def result(self, status):
|
||||
return subprocess.CompletedProcess([], status)
|
||||
|
||||
def test_only_fuse_integration_test_mounts_are_candidates(self):
|
||||
rows = MOUNTINFO
|
||||
rows += MOUNTINFO.replace("fuse.seaweedfs", "ext4")
|
||||
rows += MOUNTINFO.replace(TARGET, "/mnt/production")
|
||||
rows += MOUNTINFO.replace(TARGET, "/tmp/seaweedfs_fuse_test_123/data")
|
||||
rows += MOUNTINFO.replace(TARGET, "/home/user/seaweedfs_fuse_test_123/mount")
|
||||
rows += MOUNTINFO.replace(TARGET, "/tmp/seaweedfs/data")
|
||||
self.assertEqual(list(cleanup.test_mounts(rows)), [("42", TARGET)])
|
||||
|
||||
def test_all_test_mount_layouts_are_candidates(self):
|
||||
targets = (
|
||||
"/tmp/seaweedfs_fuse_test_123/mount",
|
||||
"/tmp/seaweedfs_fuse_failover_456/mount0",
|
||||
"/tmp/seaweedfs_fuse_failover_456/mount1",
|
||||
"/var/tmp/seaweedfs_fuse_p2p_test_789/mount2",
|
||||
"/tmp/seaweedfs/mount",
|
||||
)
|
||||
rows = "".join(
|
||||
MOUNTINFO.replace("42", str(40 + i)).replace(TARGET, target)
|
||||
for i, target in enumerate(targets)
|
||||
)
|
||||
self.assertEqual(
|
||||
list(cleanup.test_mounts(rows)),
|
||||
[(str(40 + i), target) for i, target in enumerate(targets)],
|
||||
)
|
||||
|
||||
def test_mountinfo_paths_are_unescaped_once(self):
|
||||
target = "/tmp/seaweedfs_fuse_test_space and\\slash/mount"
|
||||
encoded = target.replace("\\", r"\134").replace(" ", r"\040")
|
||||
rows = MOUNTINFO.replace(TARGET, encoded)
|
||||
self.assertEqual(list(cleanup.test_mounts(rows)), [("42", target)])
|
||||
|
||||
def test_no_mounts_is_a_noop(self):
|
||||
with patch.object(cleanup.Path, "read_text", return_value=""), \
|
||||
patch.object(cleanup.subprocess, "run") as run:
|
||||
self.assertEqual(cleanup.cleanup(), 0)
|
||||
run.assert_not_called()
|
||||
|
||||
def test_healthy_mount_is_untouched(self):
|
||||
status, run, _, _ = self.run_cleanup([self.result(0)])
|
||||
self.assertEqual(status, 0)
|
||||
self.assertEqual(run.call_count, 1)
|
||||
|
||||
def test_other_probe_errors_are_untouched(self):
|
||||
for error in (errno.EACCES, errno.ENOENT, errno.EIO):
|
||||
with self.subTest(error=error):
|
||||
status, run, _, _ = self.run_cleanup([self.result(error)])
|
||||
self.assertEqual(status, 0)
|
||||
self.assertEqual(run.call_count, 1)
|
||||
|
||||
def test_probe_timeout_does_not_detach_a_possibly_live_mount(self):
|
||||
status, run, _, _ = self.run_cleanup([subprocess.TimeoutExpired("probe", 5)])
|
||||
self.assertEqual(status, 0)
|
||||
self.assertEqual(run.call_count, 1)
|
||||
|
||||
def test_disconnected_mount_is_lazily_unmounted(self):
|
||||
status, run, _, _ = self.run_cleanup([self.result(errno.ENOTCONN), self.result(0)], [True, False])
|
||||
self.assertEqual(status, 0)
|
||||
self.assertEqual(run.call_args_list[1].args[0], ["fusermount3", "-uz", "--", TARGET])
|
||||
self.assertEqual(run.call_args_list[0].kwargs["timeout"], 5)
|
||||
self.assertEqual(run.call_args_list[1].kwargs["timeout"], 10)
|
||||
|
||||
def test_replaced_or_removed_mount_is_untouched(self):
|
||||
status, run, _, _ = self.run_cleanup([self.result(errno.ENOTCONN)], False)
|
||||
self.assertEqual(status, 0)
|
||||
self.assertEqual(run.call_count, 1)
|
||||
|
||||
def test_mount_identity_is_checked_without_stat(self):
|
||||
with patch.object(cleanup.Path, "read_text", return_value=MOUNTINFO):
|
||||
self.assertTrue(cleanup.mounted("42", TARGET))
|
||||
self.assertFalse(cleanup.mounted("43", TARGET))
|
||||
|
||||
def test_missing_or_timed_out_helper_uses_fallback(self):
|
||||
for error in (FileNotFoundError(), subprocess.TimeoutExpired("fusermount3", 10)):
|
||||
with self.subTest(error=error):
|
||||
status, run, _, _ = self.run_cleanup(
|
||||
[self.result(errno.ENOTCONN), error, self.result(0)], [True, True, False])
|
||||
self.assertEqual(status, 0)
|
||||
self.assertEqual(run.call_args.args[0], ["fusermount", "-uz", "--", TARGET])
|
||||
|
||||
def test_failed_helpers_fall_back_to_sudo(self):
|
||||
status, run, _, _ = self.run_cleanup(
|
||||
[self.result(errno.ENOTCONN), self.result(1), self.result(1), self.result(0)],
|
||||
[True, True, True, False])
|
||||
self.assertEqual(status, 0)
|
||||
self.assertEqual(run.call_args.args[0], ["sudo", "-n", "umount", "-l", "--", TARGET])
|
||||
|
||||
def test_cleanup_fails_when_stale_mount_remains(self):
|
||||
status, run, _, rmtree = self.run_cleanup(
|
||||
[self.result(errno.ENOTCONN), self.result(1), self.result(1), self.result(1)])
|
||||
self.assertEqual(status, 1)
|
||||
self.assertEqual(run.call_count, 4)
|
||||
rmtree.assert_not_called()
|
||||
|
||||
def test_unmounted_test_dir_is_reclaimed(self):
|
||||
status, _, _, rmtree = self.run_cleanup(
|
||||
[self.result(errno.ENOTCONN), self.result(0)], [True, False])
|
||||
self.assertEqual(status, 0)
|
||||
rmtree.assert_called_once_with(Path(TARGET).parent, ignore_errors=True)
|
||||
|
||||
def test_probe_process_reports_errno(self):
|
||||
for target, expected in ((str(Path(__file__)), 0), (str(Path(__file__) / "absent"), errno.ENOTDIR)):
|
||||
with self.subTest(target=target):
|
||||
result = subprocess.run([cleanup.sys.executable, cleanup.__file__, "--probe", target], check=False)
|
||||
self.assertEqual(result.returncode, expected)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -0,0 +1,72 @@
|
||||
name: Fix fusermount3 setuid
|
||||
description: >
|
||||
Make sure the fusermount3 an unprivileged mount will find can actually mount.
|
||||
Some runner images carry a second, source-built fusermount3 in /usr/local/bin
|
||||
that shadows the distro one in PATH; it is neither setuid nor root-owned, so
|
||||
every mount fails with "mount failed: Operation not permitted". Both the Go
|
||||
mount and sw-fuse resolve the helper through PATH.
|
||||
Run it after the step that apt-installs fuse3.
|
||||
|
||||
runs:
|
||||
using: composite
|
||||
steps:
|
||||
- name: Point PATH at a fusermount3 that can mount
|
||||
shell: bash
|
||||
run: |
|
||||
set -euo pipefail
|
||||
|
||||
# setuid only grants root when root owns the file, and exec follows
|
||||
# symlinks, so judge the target.
|
||||
can_mount() {
|
||||
[ -u "$1" ] && [ "$(stat -Lc %u "$1")" = 0 ]
|
||||
}
|
||||
|
||||
bin=$(command -v fusermount3 || true)
|
||||
if [ -z "$bin" ]; then
|
||||
# Self-hosted runners are not guaranteed to carry fuse3.
|
||||
sudo apt-get update
|
||||
sudo apt-get install -y fuse3
|
||||
bin=$(command -v fusermount3 || true)
|
||||
if [ -z "$bin" ]; then
|
||||
echo "no fusermount3 in PATH after installing fuse3" >&2
|
||||
exit 1
|
||||
fi
|
||||
fi
|
||||
if can_mount "$bin"; then
|
||||
ls -l "$bin"
|
||||
exit 0
|
||||
fi
|
||||
echo "$bin cannot mount unprivileged:"
|
||||
ls -l "$bin"
|
||||
|
||||
# The distro fusermount3 is setuid root and is the one meant to be used.
|
||||
# Reach it through a symlink earlier in PATH: exec resolves the link, so
|
||||
# the target keeps its setuid bit, and nothing on the image is modified.
|
||||
for distro in /usr/bin/fusermount3 /bin/fusermount3; do
|
||||
if [ "$distro" != "$bin" ] && can_mount "$distro"; then
|
||||
mkdir -p "$RUNNER_TEMP/fuse-bin"
|
||||
ln -sf "$distro" "$RUNNER_TEMP/fuse-bin/fusermount3"
|
||||
echo "$RUNNER_TEMP/fuse-bin" >> "$GITHUB_PATH"
|
||||
echo "using $distro instead"
|
||||
ls -l "$distro"
|
||||
exit 0
|
||||
fi
|
||||
done
|
||||
|
||||
# No usable distro binary, so setting the bit is the only way out - but
|
||||
# the target came from PATH: do that only for a root-owned system binary,
|
||||
# never for anything else that happens to sit there.
|
||||
case "$bin" in
|
||||
/bin/*|/sbin/*|/usr/bin/*|/usr/sbin/*|/usr/local/bin/*|/usr/local/sbin/*) ;;
|
||||
*) echo "refusing to setuid $bin: outside the system bin paths" >&2; exit 1 ;;
|
||||
esac
|
||||
if [ -L "$bin" ] || [ ! -f "$bin" ] || [ ! -x "$bin" ]; then
|
||||
echo "refusing to setuid $bin: not a regular executable file" >&2
|
||||
exit 1
|
||||
fi
|
||||
if [ "$(stat -c %u "$bin")" != 0 ] || [ "$(stat -c %g "$bin")" != 0 ]; then
|
||||
echo "refusing to setuid $bin: not owned by root:root" >&2
|
||||
exit 1
|
||||
fi
|
||||
sudo chmod u+s "$bin"
|
||||
ls -l "$bin"
|
||||
@@ -0,0 +1,44 @@
|
||||
name: Sign container images
|
||||
description: >
|
||||
Keyless cosign signature on each image, then a verification pass against the
|
||||
identity the signature should carry, so a misconfigured job fails here and not
|
||||
on someone's cluster. That identity is the calling workflow's own,
|
||||
https://github.com/<owner>/<repo>/.github/workflows/<file>@<ref>.
|
||||
The calling job needs `id-token: write` and a registry login for every image.
|
||||
|
||||
inputs:
|
||||
images:
|
||||
description: Image references by digest (name@sha256:...), whitespace separated.
|
||||
required: true
|
||||
|
||||
runs:
|
||||
using: composite
|
||||
steps:
|
||||
- name: Install cosign
|
||||
uses: sigstore/cosign-installer@v4.1.2
|
||||
|
||||
- name: Sign
|
||||
shell: bash
|
||||
env:
|
||||
IMAGES: ${{ inputs.images }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
# The .sig tag layout: the OCI-referrer bundle cosign 3 writes by default
|
||||
# is not read by the Kyverno and policy-controller releases in use today.
|
||||
# Cosign 3 also defaults to --use-signing-config, which insists on a
|
||||
# bundle for its output; turning it off falls back to the default
|
||||
# Fulcio and Rekor URLs, which is all the .sig layout ever used.
|
||||
cosign sign --yes --recursive \
|
||||
--new-bundle-format=false --use-signing-config=false \
|
||||
$IMAGES
|
||||
|
||||
- name: Verify
|
||||
shell: bash
|
||||
env:
|
||||
IMAGES: ${{ inputs.images }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
cosign verify \
|
||||
--certificate-oidc-issuer https://token.actions.githubusercontent.com \
|
||||
--certificate-identity "https://github.com/$GITHUB_WORKFLOW_REF" \
|
||||
$IMAGES
|
||||
@@ -1,10 +1,14 @@
|
||||
version: 2
|
||||
updates:
|
||||
- package-ecosystem: "github-actions"
|
||||
directory: "/"
|
||||
directories:
|
||||
- "/"
|
||||
- "/.github/actions/sign-image"
|
||||
schedule:
|
||||
interval: "weekly"
|
||||
- package-ecosystem: gomod
|
||||
directory: "/"
|
||||
schedule:
|
||||
interval: weekly
|
||||
ignore:
|
||||
- dependency-name: "github.com/seaweedfs/goexif"
|
||||
Executable
+240
@@ -0,0 +1,240 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Check JWT extraction and, with --context, real Helm upgrade persistence."""
|
||||
|
||||
import argparse
|
||||
import base64
|
||||
import json
|
||||
from pathlib import Path
|
||||
import shutil
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
import uuid
|
||||
|
||||
try:
|
||||
import tomllib
|
||||
except ModuleNotFoundError: # CI also exercises Python 3.10.
|
||||
import tomli as tomllib
|
||||
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[2]
|
||||
CHART = ROOT / "k8s/charts/seaweedfs"
|
||||
SECTIONS = ("jwt.signing", "jwt.signing.read", "jwt.filer_signing", "jwt.filer_signing.read")
|
||||
KEYS = {section: f"active-{index}" for index, section in enumerate(SECTIONS)}
|
||||
ESCAPED_HEADER = '["jw\\u0074".signing]\nkey = "existing"'
|
||||
|
||||
|
||||
def run(*args):
|
||||
return subprocess.run(args, check=True, text=True, capture_output=True).stdout
|
||||
|
||||
|
||||
def keys(raw):
|
||||
document = tomllib.loads(raw)
|
||||
result = {}
|
||||
for section in SECTIONS:
|
||||
table = document
|
||||
for part in section.split("."):
|
||||
table = table.get(part, {})
|
||||
if "key" in table:
|
||||
result[section] = table["key"]
|
||||
return result
|
||||
|
||||
|
||||
def fixtures():
|
||||
canonical = "\n".join(f'[{section}]\nkey = "{value}"' for section, value in KEYS.items())
|
||||
yield "canonical / final line without newline", canonical
|
||||
yield "commented stale keys", canonical.replace("key =", '# key = "stale"\nkey =')
|
||||
yield "commented headers / mixed line endings", "\n".join(
|
||||
f'# [{section}]\r\n# key = "stale"\r\n[{section}]\nkey = "{value}"'
|
||||
for section, value in KEYS.items()
|
||||
)
|
||||
yield "indented headers and keys / trailing comments", "\n".join(
|
||||
f' \t[ {section} ] \t# header [note]\n \tkey \t= \t"{value}" # key comment'
|
||||
for section, value in KEYS.items()
|
||||
)
|
||||
yield "CRLF", canonical.replace("\n", "\r\n")
|
||||
yield "quoted and spaced section names", "\n".join(
|
||||
f'["{section.split(".")[0]}" . \'{section.split(".")[1]}\''
|
||||
+ (f' . "{section.split(".")[2]}"' if section.count(".") == 2 else "")
|
||||
+ f'] # original table\nkey = "{value}"'
|
||||
for section, value in KEYS.items()
|
||||
)
|
||||
yield "unrelated quoted header before JWT keys", '["custom section"]\nkey = "other"\n' + canonical
|
||||
yield "brackets in comments", canonical.replace("key =", "# consult [notes]\nkey =")
|
||||
yield "quoted values and quoted key names", "\n".join((
|
||||
'[jwt.signing]\n"key" = "brackets[inside]#value"',
|
||||
"[jwt.signing.read]\n'key' = 'literal\\path[#value]'",
|
||||
r'[jwt.filer_signing]' + '\n' + r'key = "escaped\"quote\\slash\u0041"',
|
||||
'[jwt.filer_signing.read]\nkey = ""',
|
||||
))
|
||||
yield "absent keys and sections / unrelated key", '\n'.join((
|
||||
'[jwt.signing]\nexpires_after_seconds = 10',
|
||||
'[jwt.signing.read]\nkey = "read-only"',
|
||||
'[unrelated]\nkey = "not-a-jwt-key"',
|
||||
'# [jwt.filer_signing]\n# key = "not-active"',
|
||||
))
|
||||
yield "no existing security.toml", ""
|
||||
|
||||
|
||||
def check_generated(value):
|
||||
decoded = base64.b64decode(value, validate=True).decode("ascii")
|
||||
assert len(decoded) == 10 and decoded.isascii() and decoded.isalnum(), "invalid generated JWT key"
|
||||
|
||||
|
||||
def check_helpers(helm, reference_helm):
|
||||
# Exercise the real helper, not a second implementation of its matching rules.
|
||||
with tempfile.TemporaryDirectory(prefix="helm-jwt-helper-") as directory:
|
||||
chart = Path(directory)
|
||||
(chart / "templates").mkdir()
|
||||
(chart / "Chart.yaml").write_text("apiVersion: v2\nname: jwt-regression\nversion: 0.0.0\n")
|
||||
shutil.copyfile(CHART / "templates/shared/_helpers.tpl", chart / "templates/_helpers.tpl")
|
||||
entries = [
|
||||
json.dumps(section) + ': {{ include "seaweedfs.existingTomlKey" (list '
|
||||
+ json.dumps(section) + ' .Values.raw) | toJson }}'
|
||||
for section in SECTIONS
|
||||
]
|
||||
template = chart / "templates/keys.yaml"
|
||||
prefix = '{"apiVersion":"v1","kind":"ConfigMap","metadata":{"name":"keys"},"data":{'
|
||||
helper_template = prefix + ",".join(entries) + "}}"
|
||||
reference_entries = [
|
||||
json.dumps(section) + ': {{ dig '
|
||||
+ " ".join(json.dumps(part) for part in section.split("."))
|
||||
+ ' "key" "__ABSENT__" (fromToml .Values.raw) | toJson }}'
|
||||
for section in SECTIONS
|
||||
]
|
||||
reference_template = prefix + ",".join(reference_entries) + "}}"
|
||||
|
||||
def render(binary, source, raw):
|
||||
template.write_text(source)
|
||||
values = chart / "input.json"
|
||||
values.write_text(json.dumps({"raw": raw}))
|
||||
output = run(binary, "template", "keys", str(chart), "-f", str(values))
|
||||
return json.loads(output[output.index("{"):])["data"]
|
||||
|
||||
for name, raw in fixtures():
|
||||
expected = keys(raw)
|
||||
tokens = render(helm, helper_template, raw)
|
||||
actual = {section: tomllib.loads("key = " + token)["key"]
|
||||
for section, token in tokens.items() if token != ""}
|
||||
assert actual == expected, f"{name}: extracted keys differ from stored TOML"
|
||||
if reference_helm:
|
||||
reference = render(reference_helm, reference_template, raw)
|
||||
reference = {section: value for section, value in reference.items() if value != "__ABSENT__"}
|
||||
assert actual == reference, f"{name}: keys differ from fromToml/dig"
|
||||
print(f"PASS helper: {name}")
|
||||
|
||||
# A present but unsupported value must not silently become a fresh key.
|
||||
try:
|
||||
render(helm, helper_template, '[jwt.signing]\nkey = """multi\nline"""')
|
||||
except subprocess.CalledProcessError as error:
|
||||
assert "refusing to replace an existing key" in error.stderr, error.stderr
|
||||
else:
|
||||
raise AssertionError("multiline existing key was silently accepted or replaced")
|
||||
print("PASS helper: unsupported existing value fails without rotation")
|
||||
|
||||
try:
|
||||
render(helm, helper_template, ESCAPED_HEADER)
|
||||
except subprocess.CalledProcessError as error:
|
||||
assert "unsupported quoted section header" in error.stderr, error.stderr
|
||||
else:
|
||||
raise AssertionError("unsupported quoted header silently rotated its key")
|
||||
print("PASS helper: unsupported quoted header fails without rotation")
|
||||
|
||||
|
||||
def check_upgrades(helm, context):
|
||||
namespace = "jwt-key-persist-" + uuid.uuid4().hex[:8]
|
||||
current = "jk-seaweedfs-security-config"
|
||||
legacy = "seaweedfs-security-config"
|
||||
kubectl = ["kubectl", "--context", context, "-n", namespace]
|
||||
release_args = ["jk", str(CHART), "--kube-context", context, "-n", namespace]
|
||||
# No workload is needed to exercise Helm's real ConfigMap lookup and update.
|
||||
for setting in (
|
||||
"master.enabled=false", "volume.enabled=false", "filer.enabled=false",
|
||||
"global.seaweedfs.createClusterRole=false",
|
||||
"global.seaweedfs.securityConfig.jwtSigning.volumeWrite=true",
|
||||
"global.seaweedfs.securityConfig.jwtSigning.volumeRead=true",
|
||||
"global.seaweedfs.securityConfig.jwtSigning.filerWrite=true",
|
||||
"global.seaweedfs.securityConfig.jwtSigning.filerRead=true",
|
||||
):
|
||||
release_args += ["--set", setting]
|
||||
|
||||
def stored():
|
||||
cm = json.loads(run(*kubectl, "get", "configmap", current, "-o", "json"))
|
||||
return keys(cm["data"]["security.toml"])
|
||||
|
||||
def upgrade():
|
||||
run(helm, "upgrade", *release_args)
|
||||
return stored()
|
||||
|
||||
def patch(raw):
|
||||
# Seed previous-release content without taking Helm 4's SSA ownership.
|
||||
run(*kubectl, "patch", "configmap", current, "--type=merge", "--field-manager=helm", "-p",
|
||||
json.dumps({"data": {"security.toml": raw}}))
|
||||
|
||||
run(*kubectl, "create", "namespace", namespace)
|
||||
try:
|
||||
run(helm, "install", *release_args)
|
||||
initial = stored()
|
||||
assert set(initial) == set(SECTIONS), "install omitted a JWT section"
|
||||
for value in initial.values():
|
||||
check_generated(value)
|
||||
assert upgrade() == initial, "no-op upgrade changed an existing key"
|
||||
print("PASS upgrade: all four generated keys persist")
|
||||
|
||||
for name, raw in fixtures():
|
||||
patch(raw)
|
||||
actual = upgrade()
|
||||
expected = keys(raw)
|
||||
assert set(actual) == set(SECTIONS), f"{name}: upgrade omitted a JWT section"
|
||||
for section in SECTIONS:
|
||||
if section in expected:
|
||||
assert actual[section] == expected[section], f"{name}: changed {section}"
|
||||
else:
|
||||
check_generated(actual[section])
|
||||
assert upgrade() == actual, f"{name}: subsequent upgrade changed a key"
|
||||
print(f"PASS upgrade: {name}")
|
||||
|
||||
# Migration from the old chart name, followed by precedence of the current name.
|
||||
legacy_raw = "\n".join(f'[{section}]\nkey = "legacy-{index}"'
|
||||
for index, section in enumerate(SECTIONS))
|
||||
run(*kubectl, "create", "configmap", legacy, "--from-literal=security.toml=" + legacy_raw)
|
||||
run(*kubectl, "delete", "configmap", current)
|
||||
assert upgrade() == keys(legacy_raw), "legacy ConfigMap keys were not preserved"
|
||||
print("PASS upgrade: legacy ConfigMap migration")
|
||||
current_raw = next(fixtures())[1]
|
||||
patch(current_raw)
|
||||
assert upgrade() == keys(current_raw), "legacy ConfigMap overrode current ConfigMap"
|
||||
print("PASS upgrade: current ConfigMap takes precedence")
|
||||
|
||||
patch(ESCAPED_HEADER)
|
||||
try:
|
||||
upgrade()
|
||||
except subprocess.CalledProcessError as error:
|
||||
assert "unsupported quoted section header" in error.stderr, error.stderr
|
||||
else:
|
||||
raise AssertionError("unsupported quoted header silently rotated its key")
|
||||
assert keys(run(*kubectl, "get", "configmap", current, "-o",
|
||||
"jsonpath={.data.security\\.toml}"))["jwt.signing"] == "existing"
|
||||
print("PASS upgrade: unsupported quoted header leaves stored key untouched")
|
||||
finally:
|
||||
run(*kubectl, "delete", "namespace", namespace, "--wait=false")
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("--helm", default="helm")
|
||||
parser.add_argument("--reference-helm", help="Helm >=3.17 binary for differential checks")
|
||||
parser.add_argument("--context", help="explicit disposable Kubernetes context for live upgrade checks")
|
||||
args = parser.parse_args()
|
||||
print(run(args.helm, "version", "--short").strip())
|
||||
check_helpers(args.helm, args.reference_helm)
|
||||
if args.context:
|
||||
check_upgrades(args.helm, args.context)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
try:
|
||||
main()
|
||||
except subprocess.CalledProcessError as error:
|
||||
print(error.stderr, file=sys.stderr)
|
||||
sys.exit(error.returncode)
|
||||
@@ -0,0 +1,87 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Render the repository's star history to note/star_history.svg.
|
||||
|
||||
Uses the GitHub REST stargazers endpoint with the starred-at accept header,
|
||||
which caps at 40,000 entries (400 pages of 100). The chart is regenerated on
|
||||
a schedule; if the repo grows past that cap the script stops at 40,000 and
|
||||
logs a warning rather than under-reporting.
|
||||
"""
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
import urllib.request
|
||||
from datetime import datetime
|
||||
|
||||
import matplotlib
|
||||
|
||||
matplotlib.use("Agg")
|
||||
import matplotlib.pyplot as plt # noqa: E402
|
||||
from matplotlib.dates import AutoDateLocator, DateFormatter # noqa: E402
|
||||
|
||||
REPO = "seaweedfs/seaweedfs"
|
||||
TOKEN = os.environ["GITHUB_TOKEN"]
|
||||
OUT = os.environ.get("OUT", "note/star_history.svg")
|
||||
PAGE_CAP = 400 # GitHub's hard limit on stargazer pagination
|
||||
|
||||
|
||||
def fetch_stargazers():
|
||||
stars = []
|
||||
page = 1
|
||||
while page <= PAGE_CAP:
|
||||
url = f"https://api.github.com/repos/{REPO}/stargazers?per_page=100&page={page}"
|
||||
req = urllib.request.Request(
|
||||
url,
|
||||
headers={
|
||||
"Accept": "application/vnd.github.star+json",
|
||||
"Authorization": f"Bearer {TOKEN}",
|
||||
"X-GitHub-Api-Version": "2022-11-28",
|
||||
"User-Agent": "seaweedfs-star-history",
|
||||
},
|
||||
)
|
||||
with urllib.request.urlopen(req) as resp:
|
||||
batch = json.load(resp)
|
||||
if not batch:
|
||||
break
|
||||
for u in batch:
|
||||
sa = u.get("starred_at")
|
||||
if sa:
|
||||
stars.append(datetime.fromisoformat(sa.replace("Z", "+00:00")))
|
||||
if len(batch) < 100:
|
||||
break
|
||||
page += 1
|
||||
if page > PAGE_CAP:
|
||||
print(
|
||||
f"::warning::Hit the {PAGE_CAP}-page stargazer pagination cap; "
|
||||
"chart reflects the first 40,000 stars only."
|
||||
)
|
||||
return stars
|
||||
|
||||
|
||||
def render(stars, out):
|
||||
stars.sort()
|
||||
counts = list(range(1, len(stars) + 1))
|
||||
fig, ax = plt.subplots(figsize=(10, 6), dpi=130)
|
||||
ax.plot(stars, counts, color="#0969da", linewidth=1.6)
|
||||
ax.set_xlabel("Date")
|
||||
ax.set_ylabel("Stars")
|
||||
ax.set_title(f"{REPO} star history")
|
||||
ax.grid(True, linestyle="--", alpha=0.3)
|
||||
ax.xaxis.set_major_locator(AutoDateLocator())
|
||||
ax.xaxis.set_major_formatter(DateFormatter("%Y-%m"))
|
||||
fig.autofmt_xdate()
|
||||
fig.tight_layout()
|
||||
fig.savefig(out, format="svg", transparent=False)
|
||||
plt.close(fig)
|
||||
|
||||
|
||||
def main():
|
||||
stars = fetch_stargazers()
|
||||
if not stars:
|
||||
print("::error::No stargazers fetched; not updating the chart.")
|
||||
sys.exit(1)
|
||||
render(stars, OUT)
|
||||
print(f"Rendered {len(stars)} stars to {OUT}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,30 @@
|
||||
name: "actions: unit tests"
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [ master ]
|
||||
paths:
|
||||
- '.github/actions/**'
|
||||
- '.github/workflows/action-tests.yml'
|
||||
pull_request:
|
||||
branches: [ master ]
|
||||
paths:
|
||||
- '.github/actions/**'
|
||||
- '.github/workflows/action-tests.yml'
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.event.pull_request.number || (github.event_name == 'push' && github.ref) || github.run_id }}
|
||||
cancel-in-progress: ${{ github.event_name == 'pull_request' || (github.event_name == 'push' && !startsWith(github.ref, 'refs/tags/')) }}
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
jobs:
|
||||
cleanup-stale-fuse-mounts:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Check out code
|
||||
uses: actions/checkout@v7
|
||||
|
||||
- name: Run cleanup action unit tests
|
||||
run: python3 .github/actions/cleanup-stale-fuse-mounts/test_cleanup.py
|
||||
@@ -3,6 +3,15 @@ name: "go: build dev binaries"
|
||||
on:
|
||||
push:
|
||||
branches: [ master ]
|
||||
paths:
|
||||
- 'weed/**'
|
||||
- 'go.mod'
|
||||
- 'go.sum'
|
||||
- '.github/workflows/binaries_dev.yml'
|
||||
|
||||
concurrency:
|
||||
group: binaries-dev-${{ github.ref }}
|
||||
cancel-in-progress: true
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
@@ -18,6 +27,7 @@ jobs:
|
||||
|
||||
- name: Delete old release assets
|
||||
uses: mknejp/delete-release-assets@v1
|
||||
continue-on-error: true
|
||||
with:
|
||||
token: ${{ github.token }}
|
||||
tag: dev
|
||||
@@ -38,7 +48,7 @@ jobs:
|
||||
steps:
|
||||
|
||||
- name: Check out code into the Go module directory
|
||||
uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v2
|
||||
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v2
|
||||
|
||||
- name: Set BUILD_TIME env
|
||||
run: echo BUILD_TIME=$(date -u +%Y%m%d-%H%M) >> ${GITHUB_ENV}
|
||||
@@ -87,7 +97,7 @@ jobs:
|
||||
steps:
|
||||
|
||||
- name: Check out code into the Go module directory
|
||||
uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v2
|
||||
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v2
|
||||
|
||||
- name: Set BUILD_TIME env
|
||||
run: echo BUILD_TIME=$(date -u +%Y%m%d-%H%M) >> ${GITHUB_ENV}
|
||||
|
||||
@@ -28,7 +28,7 @@ jobs:
|
||||
# Steps represent a sequence of tasks that will be executed as part of the job
|
||||
steps:
|
||||
# Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it
|
||||
- uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v2
|
||||
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v2
|
||||
- name: Go Release Binaries Normal Volume Size
|
||||
uses: wangyoucao577/go-release-action@279495102627de7960cbc33434ab01a12bae144b # v1.22
|
||||
with:
|
||||
|
||||
@@ -28,7 +28,7 @@ jobs:
|
||||
# Steps represent a sequence of tasks that will be executed as part of the job
|
||||
steps:
|
||||
# Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it
|
||||
- uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v2
|
||||
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v2
|
||||
- name: Go Release Binaries Normal Volume Size
|
||||
uses: wangyoucao577/go-release-action@279495102627de7960cbc33434ab01a12bae144b # v1.22
|
||||
with:
|
||||
|
||||
@@ -28,7 +28,7 @@ jobs:
|
||||
# Steps represent a sequence of tasks that will be executed as part of the job
|
||||
steps:
|
||||
# Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it
|
||||
- uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v2
|
||||
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v2
|
||||
- name: Go Release Binaries Normal Volume Size
|
||||
uses: wangyoucao577/go-release-action@279495102627de7960cbc33434ab01a12bae144b # v1.22
|
||||
with:
|
||||
|
||||
@@ -28,7 +28,7 @@ jobs:
|
||||
# Steps represent a sequence of tasks that will be executed as part of the job
|
||||
steps:
|
||||
# Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it
|
||||
- uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v2
|
||||
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v2
|
||||
- name: Go Release Binaries Normal Volume Size
|
||||
uses: wangyoucao577/go-release-action@279495102627de7960cbc33434ab01a12bae144b # v1.22
|
||||
with:
|
||||
|
||||
@@ -28,7 +28,7 @@ jobs:
|
||||
# Steps represent a sequence of tasks that will be executed as part of the job
|
||||
steps:
|
||||
# Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it
|
||||
- uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v2
|
||||
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v2
|
||||
- name: Go Release Binaries Normal Volume Size
|
||||
uses: wangyoucao577/go-release-action@279495102627de7960cbc33434ab01a12bae144b # v1.22
|
||||
with:
|
||||
|
||||
@@ -28,7 +28,7 @@ jobs:
|
||||
# Steps represent a sequence of tasks that will be executed as part of the job
|
||||
steps:
|
||||
# Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it
|
||||
- uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v2
|
||||
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v2
|
||||
- name: Go Release Binaries Normal Volume Size
|
||||
uses: wangyoucao577/go-release-action@279495102627de7960cbc33434ab01a12bae144b # v1.22
|
||||
with:
|
||||
|
||||
@@ -2,10 +2,15 @@ name: "Code Scanning - Action"
|
||||
|
||||
on:
|
||||
pull_request:
|
||||
paths:
|
||||
- '**/*.go'
|
||||
- 'go.mod'
|
||||
- 'go.sum'
|
||||
- '.github/workflows/codeql.yml'
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.head_ref }}/codeql
|
||||
cancel-in-progress: true
|
||||
group: ${{ github.workflow }}-${{ github.event.pull_request.number || (github.event_name == 'push' && github.ref) || github.run_id }}
|
||||
cancel-in-progress: ${{ github.event_name == 'pull_request' || (github.event_name == 'push' && !startsWith(github.ref, 'refs/tags/')) }}
|
||||
|
||||
jobs:
|
||||
CodeQL-Build:
|
||||
@@ -18,11 +23,17 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8
|
||||
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v7
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
cache: ${{ runner.environment == 'github-hosted' }}
|
||||
|
||||
# Initializes the CodeQL tools for scanning.
|
||||
- name: Initialize CodeQL
|
||||
uses: github/codeql-action/init@v4
|
||||
uses: github/codeql-action/init@v4.39.0
|
||||
# Override language selection by uncommenting this and choosing your languages
|
||||
with:
|
||||
languages: go
|
||||
@@ -30,7 +41,7 @@ jobs:
|
||||
# Autobuild attempts to build any compiled languages (C/C++, C#, or Java).
|
||||
# If this step fails, then you should remove it and run the build manually (see below).
|
||||
- name: Autobuild
|
||||
uses: github/codeql-action/autobuild@v4
|
||||
uses: github/codeql-action/autobuild@v4.39.0
|
||||
|
||||
# ℹ️ Command-line programs to run using the OS shell.
|
||||
# 📚 See https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#jobsjob_idstepsrun
|
||||
@@ -44,4 +55,4 @@ jobs:
|
||||
# make release
|
||||
|
||||
- name: Perform CodeQL Analysis
|
||||
uses: github/codeql-action/analyze@v4
|
||||
uses: github/codeql-action/analyze@v4.39.0
|
||||
@@ -0,0 +1,27 @@
|
||||
# Codespell configuration is within .codespellrc
|
||||
---
|
||||
name: Codespell
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [master]
|
||||
pull_request:
|
||||
branches: [master]
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.event.pull_request.number || (github.event_name == 'push' && github.ref) || github.run_id }}
|
||||
cancel-in-progress: ${{ github.event_name == 'pull_request' || (github.event_name == 'push' && !startsWith(github.ref, 'refs/tags/')) }}
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
jobs:
|
||||
codespell:
|
||||
name: Check for spelling errors
|
||||
runs-on: ubuntu-latest
|
||||
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v7
|
||||
- name: Codespell
|
||||
uses: codespell-project/actions-codespell@8f01853be192eb0f849a5c7d721450e7a467c579 # v2.2
|
||||
@@ -3,23 +3,149 @@ name: "docker: build dev containers"
|
||||
on:
|
||||
push:
|
||||
branches: [ master ]
|
||||
paths:
|
||||
- 'weed/**'
|
||||
- 'seaweed-volume/**'
|
||||
- 'seaweed-common/**'
|
||||
- 'seaweed-worker/**'
|
||||
- 'docker/**'
|
||||
- 'go.mod'
|
||||
- 'go.sum'
|
||||
- '.github/workflows/container_dev.yml'
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.event.pull_request.number || (github.event_name == 'push' && github.ref) || github.run_id }}
|
||||
cancel-in-progress: ${{ github.event_name == 'pull_request' || (github.event_name == 'push' && !startsWith(github.ref, 'refs/tags/')) }}
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
jobs:
|
||||
|
||||
# ── Pre-build the Rust binaries natively ────────────────────────────
|
||||
build-rust-binaries:
|
||||
runs-on: ubuntu-22.04
|
||||
strategy:
|
||||
matrix:
|
||||
include:
|
||||
- target: x86_64-unknown-linux-musl
|
||||
arch: amd64
|
||||
- target: aarch64-unknown-linux-musl
|
||||
arch: arm64
|
||||
cross: true
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v7
|
||||
|
||||
- name: Install Rust toolchain
|
||||
uses: dtolnay/rust-toolchain@stable
|
||||
with:
|
||||
targets: ${{ matrix.target }}
|
||||
|
||||
- name: Install musl tools (amd64)
|
||||
if: ${{ !matrix.cross }}
|
||||
run: sudo apt-get install -y musl-tools
|
||||
|
||||
- name: Install cross-compilation tools (arm64)
|
||||
if: matrix.cross
|
||||
run: |
|
||||
sudo apt-get install -y gcc-aarch64-linux-gnu
|
||||
echo "CARGO_TARGET_AARCH64_UNKNOWN_LINUX_MUSL_LINKER=aarch64-linux-gnu-gcc" >> "$GITHUB_ENV"
|
||||
# Disable glibc fortify source — its __memcpy_chk etc. symbols don't exist in musl
|
||||
echo "CFLAGS_aarch64_unknown_linux_musl=-U_FORTIFY_SOURCE" >> "$GITHUB_ENV"
|
||||
|
||||
- name: Cache cargo registry and target
|
||||
if: runner.environment == 'github-hosted'
|
||||
uses: actions/cache@v6
|
||||
with:
|
||||
path: |
|
||||
~/.cargo/registry
|
||||
~/.cargo/git
|
||||
seaweed-volume/target
|
||||
seaweed-worker/target/${{ matrix.target }}/release
|
||||
key: rust-docker-dev-${{ matrix.target }}-${{ hashFiles('seaweed-volume/Cargo.lock', 'seaweed-worker/Cargo.lock') }}
|
||||
restore-keys: |
|
||||
rust-docker-dev-${{ matrix.target }}-
|
||||
|
||||
# lance's build scripts compile their own protos and look for a protoc.
|
||||
# Point them at the one protoc-bin-vendored ships, which seaweed-worker's
|
||||
# own build already uses, so no job depends on a system package and every
|
||||
# build sees the same version.
|
||||
- name: Use the vendored protoc
|
||||
run: |
|
||||
cd seaweed-worker
|
||||
cargo fetch
|
||||
# The version from the lock, not whatever else a restored cache holds.
|
||||
version=$(awk '/^name = "protoc-bin-vendored-linux-x86_64"$/{found=1; next} found && /^version = /{gsub(/"/,"",$3); print $3; exit}' Cargo.lock)
|
||||
test -n "$version" || { echo "protoc-bin-vendored-linux-x86_64 is not in Cargo.lock" >&2; exit 1; }
|
||||
protoc=$(find ~/.cargo/registry/src -path "*protoc-bin-vendored-linux-x86_64-$version/bin/protoc" | head -1)
|
||||
test -x "$protoc" || { echo "no vendored protoc $version in the registry" >&2; exit 1; }
|
||||
echo "PROTOC=$protoc" >> "$GITHUB_ENV"
|
||||
|
||||
- name: Build normal variant
|
||||
env:
|
||||
SEAWEEDFS_COMMIT: ${{ github.sha }}
|
||||
run: |
|
||||
cd seaweed-volume
|
||||
cargo build --release --target ${{ matrix.target }} --no-default-features
|
||||
cp target/${{ matrix.target }}/release/weed-volume ../weed-volume-normal-${{ matrix.arch }}
|
||||
|
||||
- name: Build the Rust maintenance worker
|
||||
run: |
|
||||
cd seaweed-worker
|
||||
cargo build --release -p weed-lance-worker --target ${{ matrix.target }}
|
||||
cp target/${{ matrix.target }}/release/weed-worker ../weed-worker-${{ matrix.arch }}
|
||||
|
||||
- name: Upload artifacts
|
||||
uses: actions/upload-artifact@v7
|
||||
with:
|
||||
name: rust-bins-${{ matrix.arch }}
|
||||
path: |
|
||||
weed-volume-normal-${{ matrix.arch }}
|
||||
weed-worker-${{ matrix.arch }}
|
||||
|
||||
build-dev-containers:
|
||||
needs: [build-rust-binaries]
|
||||
runs-on: [ubuntu-latest]
|
||||
permissions:
|
||||
contents: read
|
||||
id-token: write
|
||||
|
||||
steps:
|
||||
-
|
||||
name: Checkout
|
||||
uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v2
|
||||
-
|
||||
name: Docker meta
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v7
|
||||
|
||||
- name: Download pre-built Rust binaries
|
||||
uses: actions/download-artifact@v8
|
||||
with:
|
||||
pattern: rust-bins-*
|
||||
merge-multiple: true
|
||||
path: ./rust-bins
|
||||
|
||||
- name: Place Rust binaries in Docker context
|
||||
run: |
|
||||
mkdir -p docker/weed-volume-prebuilt
|
||||
for arch in amd64 arm64; do
|
||||
src="./rust-bins/weed-volume-normal-${arch}"
|
||||
if [ -f "$src" ]; then
|
||||
cp "$src" "docker/weed-volume-prebuilt/weed-volume-${arch}"
|
||||
echo "Placed pre-built Rust binary for ${arch}"
|
||||
fi
|
||||
done
|
||||
mkdir -p docker/weed-worker-prebuilt
|
||||
for arch in amd64 arm64; do
|
||||
src="./rust-bins/weed-worker-${arch}"
|
||||
if [ -f "$src" ]; then
|
||||
cp "$src" "docker/weed-worker-prebuilt/weed-worker-${arch}"
|
||||
echo "Placed pre-built Rust worker for ${arch}"
|
||||
fi
|
||||
done
|
||||
ls -la docker/weed-volume-prebuilt/
|
||||
ls -la docker/weed-worker-prebuilt/
|
||||
|
||||
- name: Docker meta
|
||||
id: docker_meta
|
||||
uses: docker/metadata-action@318604b99e75e41977312d83839a89be02ca4893 # v3
|
||||
uses: docker/metadata-action@v6
|
||||
with:
|
||||
images: |
|
||||
chrislusf/seaweedfs
|
||||
@@ -30,40 +156,41 @@ jobs:
|
||||
org.opencontainers.image.title=seaweedfs
|
||||
org.opencontainers.image.description=SeaweedFS is a distributed storage system for blobs, objects, files, and data lake, to store and serve billions of files fast!
|
||||
org.opencontainers.image.vendor=Chris Lu
|
||||
-
|
||||
name: Set up QEMU
|
||||
uses: docker/setup-qemu-action@c7c53464625b32c7a7e944ae62b3e17d2b600130 # v1
|
||||
-
|
||||
name: Create BuildKit config
|
||||
|
||||
- name: Set up QEMU
|
||||
uses: docker/setup-qemu-action@v4.4.0
|
||||
|
||||
- name: Create BuildKit config
|
||||
run: |
|
||||
cat > /tmp/buildkitd.toml <<EOF
|
||||
[registry."docker.io"]
|
||||
mirrors = ["https://mirror.gcr.io"]
|
||||
EOF
|
||||
-
|
||||
name: Set up Docker Buildx
|
||||
uses: docker/setup-buildx-action@e468171a9de216ec08956ac3ada2f0791b6bd435 # v1
|
||||
|
||||
- name: Set up Docker Buildx
|
||||
uses: docker/setup-buildx-action@v4
|
||||
with:
|
||||
buildkitd-flags: "--debug"
|
||||
buildkitd-config: /tmp/buildkitd.toml
|
||||
-
|
||||
name: Login to Docker Hub
|
||||
|
||||
- name: Login to Docker Hub
|
||||
if: github.event_name != 'pull_request'
|
||||
uses: docker/login-action@5e57cd118135c172c3672efd75eb46360885c0ef # v1
|
||||
uses: docker/login-action@v4.6.0
|
||||
with:
|
||||
username: ${{ secrets.DOCKER_USERNAME }}
|
||||
password: ${{ secrets.DOCKER_PASSWORD }}
|
||||
-
|
||||
name: Login to GHCR
|
||||
|
||||
- name: Login to GHCR
|
||||
if: github.event_name != 'pull_request'
|
||||
uses: docker/login-action@5e57cd118135c172c3672efd75eb46360885c0ef # v1
|
||||
uses: docker/login-action@v4.6.0
|
||||
with:
|
||||
registry: ghcr.io
|
||||
username: ${{ secrets.GHCR_USERNAME }}
|
||||
password: ${{ secrets.GHCR_TOKEN }}
|
||||
-
|
||||
name: Build
|
||||
uses: docker/build-push-action@263435318d21b8e681c14492fe198d362a7d2c83 # v2
|
||||
|
||||
- name: Build
|
||||
id: build
|
||||
uses: docker/build-push-action@v7
|
||||
with:
|
||||
context: ./docker
|
||||
push: ${{ github.event_name != 'pull_request' }}
|
||||
@@ -71,3 +198,17 @@ jobs:
|
||||
platforms: linux/amd64, linux/arm64
|
||||
tags: ${{ steps.docker_meta.outputs.tags }}
|
||||
labels: ${{ steps.docker_meta.outputs.labels }}
|
||||
cache-from: type=gha
|
||||
cache-to: type=gha,mode=max
|
||||
# Key the checkout layer on the pushed sha so cached layers can't
|
||||
# serve stale code; the Dockerfile fetch fallback picks it up.
|
||||
build-args: |
|
||||
BRANCH=${{ github.sha }}
|
||||
|
||||
- name: Sign
|
||||
if: github.event_name != 'pull_request'
|
||||
uses: ./.github/actions/sign-image
|
||||
with:
|
||||
images: >-
|
||||
chrislusf/seaweedfs@${{ steps.build.outputs.digest }}
|
||||
ghcr.io/chrislusf/seaweedfs@${{ steps.build.outputs.digest }}
|
||||
@@ -24,16 +24,23 @@ on:
|
||||
required: false
|
||||
default: ''
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.event.pull_request.number || (github.event_name == 'push' && github.ref) || github.event.inputs.image_tag || format('foundationdb_{0}_seaweedfs_{1}', github.event.inputs.fdb_version || '7.4.5', github.event.inputs.seaweedfs_ref || github.ref_name) }}
|
||||
cancel-in-progress: ${{ github.event_name == 'pull_request' || (github.event_name == 'push' && !startsWith(github.ref, 'refs/tags/')) }}
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
jobs:
|
||||
build-foundationdb-image:
|
||||
runs-on: ubuntu-latest
|
||||
permissions:
|
||||
contents: read
|
||||
id-token: write
|
||||
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v6
|
||||
uses: actions/checkout@v7
|
||||
with:
|
||||
fetch-depth: 0
|
||||
|
||||
@@ -58,9 +65,10 @@ jobs:
|
||||
sudo ldconfig
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v6
|
||||
uses: actions/setup-go@v7
|
||||
with:
|
||||
go-version-file: go.mod
|
||||
cache: ${{ runner.environment == 'github-hosted' }}
|
||||
|
||||
- name: Run FoundationDB tagged tests
|
||||
env:
|
||||
@@ -126,31 +134,36 @@ jobs:
|
||||
echo "seaweedfs_ref=$seaweed" >> "$GITHUB_OUTPUT"
|
||||
|
||||
- name: Set up QEMU
|
||||
uses: docker/setup-qemu-action@v3
|
||||
uses: docker/setup-qemu-action@v4.4.0
|
||||
|
||||
- name: Set up Docker Buildx
|
||||
uses: docker/setup-buildx-action@v3
|
||||
uses: docker/setup-buildx-action@v4
|
||||
|
||||
- name: Login to Docker Hub
|
||||
if: github.event_name != 'pull_request'
|
||||
uses: docker/login-action@v3
|
||||
uses: docker/login-action@v4.6.0
|
||||
with:
|
||||
username: ${{ secrets.DOCKER_USERNAME }}
|
||||
password: ${{ secrets.DOCKER_PASSWORD }}
|
||||
|
||||
- name: Determine branch to build
|
||||
id: branch
|
||||
env:
|
||||
INPUT_REF: ${{ inputs.seaweedfs_ref }}
|
||||
HEAD_REF: ${{ github.head_ref }}
|
||||
REF_NAME: ${{ github.ref_name }}
|
||||
run: |
|
||||
if [ -n "${{ inputs.seaweedfs_ref }}" ]; then
|
||||
echo "branch=${{ inputs.seaweedfs_ref }}" >> "$GITHUB_OUTPUT"
|
||||
if [ -n "$INPUT_REF" ]; then
|
||||
echo "branch=$INPUT_REF" >> "$GITHUB_OUTPUT"
|
||||
elif [ "${{ github.event_name }}" = "pull_request" ]; then
|
||||
echo "branch=${{ github.head_ref }}" >> "$GITHUB_OUTPUT"
|
||||
echo "branch=$HEAD_REF" >> "$GITHUB_OUTPUT"
|
||||
else
|
||||
echo "branch=${{ github.ref_name }}" >> "$GITHUB_OUTPUT"
|
||||
echo "branch=$REF_NAME" >> "$GITHUB_OUTPUT"
|
||||
fi
|
||||
|
||||
- name: Build and push image
|
||||
uses: docker/build-push-action@v6
|
||||
id: build
|
||||
uses: docker/build-push-action@v7
|
||||
with:
|
||||
context: ./docker
|
||||
push: ${{ github.event_name != 'pull_request' }}
|
||||
@@ -167,3 +180,8 @@ jobs:
|
||||
org.opencontainers.image.description=SeaweedFS is a distributed storage system for blobs, objects, files, and data lake, to store and serve billions of files fast!
|
||||
org.opencontainers.image.vendor=Chris Lu
|
||||
|
||||
- name: Sign
|
||||
if: github.event_name != 'pull_request'
|
||||
uses: ./.github/actions/sign-image
|
||||
with:
|
||||
images: chrislusf/seaweedfs@${{ steps.build.outputs.digest }}
|
||||
@@ -1,9 +1,10 @@
|
||||
name: "docker: build latest container"
|
||||
|
||||
# Manual fallback only. On tag push, container_release_unified.yml already
|
||||
# re-tags the released versioned image as `latest` / `latest_large_disk`,
|
||||
# so a full rebuild here is unnecessary. Run this manually if you need to
|
||||
# rebuild `latest` from an arbitrary ref.
|
||||
on:
|
||||
push:
|
||||
tags:
|
||||
- '*'
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
source_ref:
|
||||
@@ -23,15 +24,27 @@ on:
|
||||
- all
|
||||
- standard
|
||||
- large_disk
|
||||
publish:
|
||||
description: 'Publish images and manifests'
|
||||
required: true
|
||||
type: boolean
|
||||
default: false
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.event.pull_request.number || (github.event_name == 'push' && github.ref) || github.event.inputs.image_tag || github.run_id }}
|
||||
cancel-in-progress: false
|
||||
queue: max
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
security-events: write
|
||||
|
||||
jobs:
|
||||
setup:
|
||||
runs-on: ubuntu-latest
|
||||
outputs:
|
||||
variants: ${{ steps.set-variants.outputs.variants }}
|
||||
publish: ${{ steps.set-publish.outputs.publish }}
|
||||
steps:
|
||||
- name: Select variants for this run
|
||||
id: set-variants
|
||||
@@ -42,24 +55,128 @@ jobs:
|
||||
variants='["standard","large_disk"]'
|
||||
fi
|
||||
echo "variants=$variants" >> "$GITHUB_OUTPUT"
|
||||
- name: Select publish mode
|
||||
id: set-publish
|
||||
run: |
|
||||
if [ "${{ github.event_name }}" = "workflow_dispatch" ]; then
|
||||
echo "publish=${{ github.event.inputs.publish }}" >> "$GITHUB_OUTPUT"
|
||||
else
|
||||
echo "publish=true" >> "$GITHUB_OUTPUT"
|
||||
fi
|
||||
|
||||
# ── Pre-build the Rust binaries natively ────────────────────────────
|
||||
build-rust-binaries:
|
||||
runs-on: ubuntu-22.04
|
||||
strategy:
|
||||
matrix:
|
||||
include:
|
||||
- target: x86_64-unknown-linux-musl
|
||||
arch: amd64
|
||||
- target: aarch64-unknown-linux-musl
|
||||
arch: arm64
|
||||
cross: true
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v7
|
||||
with:
|
||||
ref: ${{ github.event_name == 'workflow_dispatch' && github.event.inputs.source_ref || github.ref }}
|
||||
|
||||
- name: Install Rust toolchain
|
||||
uses: dtolnay/rust-toolchain@stable
|
||||
with:
|
||||
targets: ${{ matrix.target }}
|
||||
|
||||
- name: Install musl tools (amd64)
|
||||
if: ${{ !matrix.cross }}
|
||||
run: sudo apt-get install -y musl-tools
|
||||
|
||||
- name: Install cross-compilation tools (arm64)
|
||||
if: matrix.cross
|
||||
run: |
|
||||
sudo apt-get install -y gcc-aarch64-linux-gnu
|
||||
echo "CARGO_TARGET_AARCH64_UNKNOWN_LINUX_MUSL_LINKER=aarch64-linux-gnu-gcc" >> "$GITHUB_ENV"
|
||||
# Disable glibc fortify source — its __memcpy_chk etc. symbols don't exist in musl
|
||||
echo "CFLAGS_aarch64_unknown_linux_musl=-U_FORTIFY_SOURCE" >> "$GITHUB_ENV"
|
||||
|
||||
- name: Cache cargo registry and target
|
||||
if: runner.environment == 'github-hosted'
|
||||
uses: actions/cache@v6
|
||||
with:
|
||||
path: |
|
||||
~/.cargo/registry
|
||||
~/.cargo/git
|
||||
seaweed-volume/target
|
||||
seaweed-worker/target/${{ matrix.target }}/release
|
||||
key: rust-docker-${{ matrix.target }}-${{ hashFiles('seaweed-volume/Cargo.lock', 'seaweed-worker/Cargo.lock') }}
|
||||
restore-keys: |
|
||||
rust-docker-${{ matrix.target }}-
|
||||
|
||||
# lance's build scripts compile their own protos and look for a protoc.
|
||||
# Point them at the one protoc-bin-vendored ships, which seaweed-worker's
|
||||
# own build already uses, so no job depends on a system package and every
|
||||
# build sees the same version.
|
||||
- name: Use the vendored protoc
|
||||
run: |
|
||||
cd seaweed-worker
|
||||
cargo fetch
|
||||
# The version from the lock, not whatever else a restored cache holds.
|
||||
version=$(awk '/^name = "protoc-bin-vendored-linux-x86_64"$/{found=1; next} found && /^version = /{gsub(/"/,"",$3); print $3; exit}' Cargo.lock)
|
||||
test -n "$version" || { echo "protoc-bin-vendored-linux-x86_64 is not in Cargo.lock" >&2; exit 1; }
|
||||
protoc=$(find ~/.cargo/registry/src -path "*protoc-bin-vendored-linux-x86_64-$version/bin/protoc" | head -1)
|
||||
test -x "$protoc" || { echo "no vendored protoc $version in the registry" >&2; exit 1; }
|
||||
echo "PROTOC=$protoc" >> "$GITHUB_ENV"
|
||||
|
||||
- name: Build large-disk variant
|
||||
env:
|
||||
SEAWEEDFS_COMMIT: ${{ github.sha }}
|
||||
run: |
|
||||
cd seaweed-volume
|
||||
cargo build --release --target ${{ matrix.target }}
|
||||
cp target/${{ matrix.target }}/release/weed-volume ../weed-volume-large-disk-${{ matrix.arch }}
|
||||
|
||||
- name: Build normal variant
|
||||
env:
|
||||
SEAWEEDFS_COMMIT: ${{ github.sha }}
|
||||
run: |
|
||||
cd seaweed-volume
|
||||
cargo build --release --target ${{ matrix.target }} --no-default-features
|
||||
cp target/${{ matrix.target }}/release/weed-volume ../weed-volume-normal-${{ matrix.arch }}
|
||||
|
||||
- name: Build the Rust maintenance worker
|
||||
run: |
|
||||
cd seaweed-worker
|
||||
cargo build --release -p weed-lance-worker --target ${{ matrix.target }}
|
||||
cp target/${{ matrix.target }}/release/weed-worker ../weed-worker-${{ matrix.arch }}
|
||||
|
||||
- name: Upload artifacts
|
||||
uses: actions/upload-artifact@v7
|
||||
with:
|
||||
name: rust-bins-${{ matrix.arch }}
|
||||
path: |
|
||||
weed-volume-large-disk-${{ matrix.arch }}
|
||||
weed-volume-normal-${{ matrix.arch }}
|
||||
weed-worker-${{ matrix.arch }}
|
||||
|
||||
build:
|
||||
needs: [setup]
|
||||
needs: [setup, build-rust-binaries]
|
||||
runs-on: ubuntu-latest
|
||||
strategy:
|
||||
matrix:
|
||||
platform: [amd64, arm64, arm, 386]
|
||||
platform: [amd64, arm64, arm, 386, ppc64le, s390x]
|
||||
variant: ${{ fromJSON(needs.setup.outputs.variants) }}
|
||||
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v6
|
||||
uses: actions/checkout@v7
|
||||
with:
|
||||
ref: ${{ github.event_name == 'workflow_dispatch' && github.event.inputs.source_ref || github.ref }}
|
||||
- name: Clean up stale FUSE mounts
|
||||
uses: ./.github/actions/cleanup-stale-fuse-mounts
|
||||
|
||||
- name: Free Disk Space
|
||||
run: |
|
||||
echo "Available disk space before cleanup:"
|
||||
df -h
|
||||
timeout 15 df -h || true
|
||||
# Remove pre-installed tools
|
||||
sudo rm -rf /usr/share/dotnet /usr/local/lib/android /opt/ghc /opt/hostedtoolcache/CodeQL
|
||||
# Clean package managers
|
||||
@@ -71,7 +188,7 @@ jobs:
|
||||
[ -d ~/.cache/go-build ] && rm -rf ~/.cache/go-build || true
|
||||
[ -d /go/pkg ] && rm -rf /go/pkg || true
|
||||
echo "Available disk space after cleanup:"
|
||||
df -h
|
||||
timeout 15 df -h || true
|
||||
|
||||
- name: Configure variant
|
||||
id: config
|
||||
@@ -79,14 +196,44 @@ jobs:
|
||||
if [ "${{ matrix.variant }}" == "large_disk" ]; then
|
||||
echo "tag_suffix=_large_disk" >> $GITHUB_OUTPUT
|
||||
echo "build_args=TAGS=5BytesOffset" >> $GITHUB_OUTPUT
|
||||
echo "rust_variant=large-disk" >> $GITHUB_OUTPUT
|
||||
else
|
||||
echo "tag_suffix=" >> $GITHUB_OUTPUT
|
||||
echo "build_args=" >> $GITHUB_OUTPUT
|
||||
echo "rust_variant=normal" >> $GITHUB_OUTPUT
|
||||
fi
|
||||
|
||||
- name: Download pre-built Rust binaries
|
||||
uses: actions/download-artifact@v8
|
||||
with:
|
||||
pattern: rust-bins-*
|
||||
merge-multiple: true
|
||||
path: ./rust-bins
|
||||
|
||||
- name: Place Rust binaries in Docker context
|
||||
run: |
|
||||
mkdir -p docker/weed-volume-prebuilt
|
||||
for arch in amd64 arm64; do
|
||||
src="./rust-bins/weed-volume-${{ steps.config.outputs.rust_variant }}-${arch}"
|
||||
if [ -f "$src" ]; then
|
||||
cp "$src" "docker/weed-volume-prebuilt/weed-volume-${arch}"
|
||||
echo "Placed pre-built Rust binary for ${arch}"
|
||||
fi
|
||||
done
|
||||
mkdir -p docker/weed-worker-prebuilt
|
||||
for arch in amd64 arm64; do
|
||||
src="./rust-bins/weed-worker-${arch}"
|
||||
if [ -f "$src" ]; then
|
||||
cp "$src" "docker/weed-worker-prebuilt/weed-worker-${arch}"
|
||||
echo "Placed pre-built Rust worker for ${arch}"
|
||||
fi
|
||||
done
|
||||
ls -la docker/weed-volume-prebuilt/
|
||||
ls -la docker/weed-worker-prebuilt/
|
||||
|
||||
- name: Docker meta
|
||||
id: docker_meta
|
||||
uses: docker/metadata-action@v5
|
||||
uses: docker/metadata-action@v6
|
||||
with:
|
||||
images: |
|
||||
chrislusf/seaweedfs
|
||||
@@ -98,7 +245,7 @@ jobs:
|
||||
org.opencontainers.image.vendor=Chris Lu
|
||||
- name: Set up QEMU
|
||||
if: matrix.platform != 'amd64'
|
||||
uses: docker/setup-qemu-action@v3
|
||||
uses: docker/setup-qemu-action@v4.4.0
|
||||
- name: Create BuildKit config
|
||||
run: |
|
||||
cat > /tmp/buildkitd.toml <<EOF
|
||||
@@ -106,30 +253,30 @@ jobs:
|
||||
mirrors = ["https://mirror.gcr.io"]
|
||||
EOF
|
||||
- name: Set up Docker Buildx
|
||||
uses: docker/setup-buildx-action@v3
|
||||
uses: docker/setup-buildx-action@v4
|
||||
with:
|
||||
buildkitd-flags: "--debug"
|
||||
buildkitd-config: /tmp/buildkitd.toml
|
||||
- name: Login to Docker Hub
|
||||
if: github.event_name != 'pull_request'
|
||||
uses: docker/login-action@v3
|
||||
if: needs.setup.outputs.publish == 'true'
|
||||
uses: docker/login-action@v4.6.0
|
||||
with:
|
||||
username: ${{ secrets.DOCKER_USERNAME }}
|
||||
password: ${{ secrets.DOCKER_PASSWORD }}
|
||||
- name: Login to GHCR
|
||||
if: github.event_name != 'pull_request'
|
||||
uses: docker/login-action@v3
|
||||
if: needs.setup.outputs.publish == 'true'
|
||||
uses: docker/login-action@v4.6.0
|
||||
with:
|
||||
registry: ghcr.io
|
||||
username: ${{ secrets.GHCR_USERNAME }}
|
||||
password: ${{ secrets.GHCR_TOKEN }}
|
||||
- name: Build ${{ matrix.platform }} ${{ matrix.variant }}
|
||||
uses: docker/build-push-action@v6
|
||||
uses: docker/build-push-action@v7
|
||||
env:
|
||||
DOCKER_BUILDKIT: 1
|
||||
with:
|
||||
context: ./docker
|
||||
push: ${{ github.event_name != 'pull_request' }}
|
||||
push: ${{ needs.setup.outputs.publish == 'true' }}
|
||||
file: ./docker/Dockerfile.go_build
|
||||
platforms: linux/${{ matrix.platform }}
|
||||
# Push to GHCR only during build to avoid Docker Hub rate limits
|
||||
@@ -149,19 +296,173 @@ jobs:
|
||||
# Remove Go build cache
|
||||
sudo rm -rf /tmp/go-build*
|
||||
|
||||
create-manifest:
|
||||
trivy-scan:
|
||||
runs-on: ubuntu-latest
|
||||
needs: [setup, build]
|
||||
if: github.event_name != 'pull_request'
|
||||
needs: [setup, build, build-rust-binaries]
|
||||
strategy:
|
||||
matrix:
|
||||
variant: ${{ fromJSON(needs.setup.outputs.variants) }}
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v6
|
||||
- name: Configure variant
|
||||
id: config
|
||||
run: |
|
||||
if [ "${{ matrix.variant }}" == "large_disk" ]; then
|
||||
echo "tag_suffix=_large_disk" >> $GITHUB_OUTPUT
|
||||
else
|
||||
echo "tag_suffix=" >> $GITHUB_OUTPUT
|
||||
fi
|
||||
- name: Login to GHCR
|
||||
if: needs.setup.outputs.publish == 'true'
|
||||
uses: docker/login-action@v4.6.0
|
||||
with:
|
||||
registry: ghcr.io
|
||||
username: ${{ secrets.GHCR_USERNAME }}
|
||||
password: ${{ secrets.GHCR_TOKEN }}
|
||||
- name: Checkout for local scan build
|
||||
if: needs.setup.outputs.publish != 'true'
|
||||
uses: actions/checkout@v7
|
||||
with:
|
||||
ref: ${{ github.event_name == 'workflow_dispatch' && github.event.inputs.source_ref || github.ref }}
|
||||
|
||||
- name: Download pre-built Rust binaries for local scan
|
||||
if: needs.setup.outputs.publish != 'true'
|
||||
uses: actions/download-artifact@v8
|
||||
with:
|
||||
pattern: rust-bins-*
|
||||
merge-multiple: true
|
||||
path: ./rust-bins
|
||||
- name: Place Rust binaries in Docker context for local scan
|
||||
if: needs.setup.outputs.publish != 'true'
|
||||
run: |
|
||||
rust_variant="normal"
|
||||
if [ "${{ matrix.variant }}" == "large_disk" ]; then
|
||||
rust_variant="large-disk"
|
||||
fi
|
||||
mkdir -p docker/weed-volume-prebuilt
|
||||
for arch in amd64 arm64; do
|
||||
src="./rust-bins/weed-volume-${rust_variant}-${arch}"
|
||||
if [ -f "$src" ]; then
|
||||
cp "$src" "docker/weed-volume-prebuilt/weed-volume-${arch}"
|
||||
echo "Placed pre-built Rust binary for ${arch}"
|
||||
fi
|
||||
done
|
||||
mkdir -p docker/weed-worker-prebuilt
|
||||
for arch in amd64 arm64; do
|
||||
src="./rust-bins/weed-worker-${arch}"
|
||||
if [ -f "$src" ]; then
|
||||
cp "$src" "docker/weed-worker-prebuilt/weed-worker-${arch}"
|
||||
echo "Placed pre-built Rust worker for ${arch}"
|
||||
fi
|
||||
done
|
||||
ls -la docker/weed-volume-prebuilt/
|
||||
ls -la docker/weed-worker-prebuilt/
|
||||
- name: Create BuildKit config for local scan build
|
||||
if: needs.setup.outputs.publish != 'true'
|
||||
run: |
|
||||
cat > /tmp/buildkitd.toml <<EOF
|
||||
[registry."docker.io"]
|
||||
mirrors = ["https://mirror.gcr.io"]
|
||||
EOF
|
||||
- name: Set up Docker Buildx for local scan build
|
||||
if: needs.setup.outputs.publish != 'true'
|
||||
uses: docker/setup-buildx-action@v4
|
||||
with:
|
||||
buildkitd-flags: "--debug"
|
||||
buildkitd-config: /tmp/buildkitd.toml
|
||||
- name: Build local scan image tarball
|
||||
if: needs.setup.outputs.publish != 'true'
|
||||
uses: docker/build-push-action@v7
|
||||
env:
|
||||
DOCKER_BUILDKIT: 1
|
||||
with:
|
||||
context: ./docker
|
||||
file: ./docker/Dockerfile.go_build
|
||||
platforms: linux/amd64
|
||||
outputs: type=docker,dest=/tmp/seaweedfs${{ steps.config.outputs.tag_suffix }}-amd64.tar
|
||||
build-args: |
|
||||
BUILDKIT_INLINE_CACHE=1
|
||||
BRANCH=${{ github.event_name == 'workflow_dispatch' && github.event.inputs.source_ref || github.sha }}
|
||||
${{ matrix.variant == 'large_disk' && 'TAGS=5BytesOffset' || '' }}
|
||||
- name: Trivy report (published image)
|
||||
if: needs.setup.outputs.publish == 'true'
|
||||
# Pin to SHA - mutable tags were compromised (GHSA-69fq-xp46-6x23)
|
||||
uses: aquasecurity/trivy-action@ed142fd0673e97e23eac54620cfb913e5ce36c25 # v0.36.0
|
||||
with:
|
||||
scan-type: image
|
||||
# Scan amd64 only - OS packages are identical across architectures
|
||||
# since they all use the same alpine base, so a single-arch scan
|
||||
# provides sufficient coverage without multiplying CI time.
|
||||
image-ref: ghcr.io/chrislusf/seaweedfs:${{ github.event_name == 'workflow_dispatch' && github.event.inputs.image_tag || 'latest' }}${{ steps.config.outputs.tag_suffix }}-amd64
|
||||
scanners: vuln
|
||||
vuln-type: os,library
|
||||
severity: HIGH,CRITICAL
|
||||
ignore-unfixed: true
|
||||
limit-severities-for-sarif: true
|
||||
format: sarif
|
||||
output: trivy-results.sarif
|
||||
exit-code: '0'
|
||||
- name: Trivy report (local tarball)
|
||||
if: needs.setup.outputs.publish != 'true'
|
||||
uses: aquasecurity/trivy-action@ed142fd0673e97e23eac54620cfb913e5ce36c25 # v0.36.0
|
||||
with:
|
||||
input: /tmp/seaweedfs${{ steps.config.outputs.tag_suffix }}-amd64.tar
|
||||
scanners: vuln
|
||||
vuln-type: os,library
|
||||
severity: HIGH,CRITICAL
|
||||
ignore-unfixed: true
|
||||
limit-severities-for-sarif: true
|
||||
format: sarif
|
||||
output: trivy-results.sarif
|
||||
exit-code: '0'
|
||||
- name: Upload Trivy scan results to GitHub Security
|
||||
uses: github/codeql-action/upload-sarif@v4.39.0
|
||||
if: always()
|
||||
with:
|
||||
sarif_file: trivy-results.sarif
|
||||
- name: Trivy gate (published image)
|
||||
if: needs.setup.outputs.publish == 'true'
|
||||
# Gate only on fixable high/critical vulnerabilities. Non-fixable
|
||||
# findings are still visible in the SARIF upload above.
|
||||
uses: aquasecurity/trivy-action@ed142fd0673e97e23eac54620cfb913e5ce36c25 # v0.36.0
|
||||
with:
|
||||
scan-type: image
|
||||
image-ref: ghcr.io/chrislusf/seaweedfs:${{ github.event_name == 'workflow_dispatch' && github.event.inputs.image_tag || 'latest' }}${{ steps.config.outputs.tag_suffix }}-amd64
|
||||
scanners: vuln
|
||||
vuln-type: os,library
|
||||
severity: HIGH,CRITICAL
|
||||
ignore-unfixed: true
|
||||
format: table
|
||||
exit-code: '1'
|
||||
skip-setup-trivy: true
|
||||
- name: Trivy gate (local tarball)
|
||||
if: needs.setup.outputs.publish != 'true'
|
||||
uses: aquasecurity/trivy-action@ed142fd0673e97e23eac54620cfb913e5ce36c25 # v0.36.0
|
||||
with:
|
||||
input: /tmp/seaweedfs${{ steps.config.outputs.tag_suffix }}-amd64.tar
|
||||
scanners: vuln
|
||||
vuln-type: os,library
|
||||
severity: HIGH,CRITICAL
|
||||
ignore-unfixed: true
|
||||
format: table
|
||||
exit-code: '1'
|
||||
skip-setup-trivy: true
|
||||
|
||||
create-manifest:
|
||||
runs-on: ubuntu-latest
|
||||
needs: [setup, build, trivy-scan]
|
||||
if: needs.setup.outputs.publish == 'true' && github.event_name != 'pull_request'
|
||||
permissions:
|
||||
contents: read
|
||||
id-token: write
|
||||
strategy:
|
||||
matrix:
|
||||
variant: ${{ fromJSON(needs.setup.outputs.variants) }}
|
||||
steps:
|
||||
- name: Checkout the signing action
|
||||
uses: actions/checkout@v7
|
||||
with:
|
||||
sparse-checkout: .github/actions
|
||||
persist-credentials: false
|
||||
|
||||
- name: Configure variant
|
||||
id: config
|
||||
run: |
|
||||
@@ -173,19 +474,19 @@ jobs:
|
||||
|
||||
- name: Docker meta
|
||||
id: docker_meta
|
||||
uses: docker/metadata-action@v5
|
||||
uses: docker/metadata-action@v6
|
||||
with:
|
||||
images: |
|
||||
chrislusf/seaweedfs
|
||||
ghcr.io/chrislusf/seaweedfs
|
||||
tags: type=raw,value=${{ github.event_name == 'workflow_dispatch' && github.event.inputs.image_tag || 'latest' }},suffix=${{ steps.config.outputs.tag_suffix }}
|
||||
- name: Login to Docker Hub
|
||||
uses: docker/login-action@v3
|
||||
uses: docker/login-action@v4.6.0
|
||||
with:
|
||||
username: ${{ secrets.DOCKER_USERNAME }}
|
||||
password: ${{ secrets.DOCKER_PASSWORD }}
|
||||
- name: Login to GHCR
|
||||
uses: docker/login-action@v3
|
||||
uses: docker/login-action@v4.6.0
|
||||
with:
|
||||
registry: ghcr.io
|
||||
username: ${{ secrets.GHCR_USERNAME }}
|
||||
@@ -194,21 +495,31 @@ jobs:
|
||||
run: |
|
||||
# Install crane for efficient multi-arch image copying
|
||||
cd $(mktemp -d)
|
||||
curl -sL "https://github.com/google/go-containerregistry/releases/latest/download/go-containerregistry_Linux_x86_64.tar.gz" | tar xz
|
||||
curl -sLO https://github.com/google/go-containerregistry/releases/download/v0.22.0/go-containerregistry_Linux_x86_64.tar.gz
|
||||
echo "edb74d53fad9a596860f59d1c5d04a43dfb5f441dc71f57060dd0bf39483c833 go-containerregistry_Linux_x86_64.tar.gz" | sha256sum -c -
|
||||
tar xzf go-containerregistry_Linux_x86_64.tar.gz crane
|
||||
sudo mv crane /usr/local/bin/
|
||||
crane version
|
||||
- name: Create and push manifest
|
||||
id: manifest
|
||||
env:
|
||||
BASE_TAG: ${{ github.event_name == 'workflow_dispatch' && github.event.inputs.image_tag || 'latest' }}
|
||||
run: |
|
||||
SUFFIX="${{ steps.config.outputs.tag_suffix }}"
|
||||
BASE_TAG="${{ github.event_name == 'workflow_dispatch' && github.event.inputs.image_tag || 'latest' }}"
|
||||
|
||||
# Create manifest on GHCR first (no rate limits)
|
||||
echo "Creating GHCR manifest (no rate limits)..."
|
||||
docker buildx imagetools create -t ghcr.io/chrislusf/seaweedfs:${BASE_TAG}${SUFFIX} \
|
||||
--metadata-file /tmp/manifest.json \
|
||||
ghcr.io/chrislusf/seaweedfs:${BASE_TAG}${SUFFIX}-amd64 \
|
||||
ghcr.io/chrislusf/seaweedfs:${BASE_TAG}${SUFFIX}-arm64 \
|
||||
ghcr.io/chrislusf/seaweedfs:${BASE_TAG}${SUFFIX}-arm \
|
||||
ghcr.io/chrislusf/seaweedfs:${BASE_TAG}${SUFFIX}-386
|
||||
ghcr.io/chrislusf/seaweedfs:${BASE_TAG}${SUFFIX}-386 \
|
||||
ghcr.io/chrislusf/seaweedfs:${BASE_TAG}${SUFFIX}-ppc64le \
|
||||
ghcr.io/chrislusf/seaweedfs:${BASE_TAG}${SUFFIX}-s390x
|
||||
# The copy and the signature below use the digest this run pushed, not whatever the tag points at by then.
|
||||
DIGEST=$(jq -er '."containerimage.descriptor".digest' /tmp/manifest.json)
|
||||
echo "digest=${DIGEST}" >> "$GITHUB_OUTPUT"
|
||||
|
||||
# Copy the complete multi-arch image from GHCR to Docker Hub
|
||||
# This only requires one pull from GHCR (no rate limit) and one push to Docker Hub
|
||||
@@ -244,16 +555,25 @@ jobs:
|
||||
# Use crane or skopeo to copy, fallback to docker if not available
|
||||
if command -v crane &> /dev/null; then
|
||||
echo "Using crane to copy..."
|
||||
retry_with_backoff crane copy ghcr.io/chrislusf/seaweedfs:${BASE_TAG}${SUFFIX} chrislusf/seaweedfs:${BASE_TAG}${SUFFIX}
|
||||
retry_with_backoff crane copy ghcr.io/chrislusf/seaweedfs@${DIGEST} chrislusf/seaweedfs:${BASE_TAG}${SUFFIX}
|
||||
elif command -v skopeo &> /dev/null; then
|
||||
echo "Using skopeo to copy..."
|
||||
retry_with_backoff skopeo copy --all docker://ghcr.io/chrislusf/seaweedfs:${BASE_TAG}${SUFFIX} docker://chrislusf/seaweedfs:${BASE_TAG}${SUFFIX}
|
||||
retry_with_backoff skopeo copy --all docker://ghcr.io/chrislusf/seaweedfs@${DIGEST} docker://chrislusf/seaweedfs:${BASE_TAG}${SUFFIX}
|
||||
else
|
||||
echo "Using docker buildx imagetools (pulling 4 images from Docker Hub)..."
|
||||
echo "Using docker buildx imagetools (pulling 6 images from Docker Hub)..."
|
||||
# Fallback: create manifest directly on Docker Hub (pulls from Docker Hub - rate limited)
|
||||
retry_with_backoff docker buildx imagetools create -t chrislusf/seaweedfs:${BASE_TAG}${SUFFIX} \
|
||||
ghcr.io/chrislusf/seaweedfs:${BASE_TAG}${SUFFIX}-amd64 \
|
||||
ghcr.io/chrislusf/seaweedfs:${BASE_TAG}${SUFFIX}-arm64 \
|
||||
ghcr.io/chrislusf/seaweedfs:${BASE_TAG}${SUFFIX}-arm \
|
||||
ghcr.io/chrislusf/seaweedfs:${BASE_TAG}${SUFFIX}-386
|
||||
ghcr.io/chrislusf/seaweedfs:${BASE_TAG}${SUFFIX}-386 \
|
||||
ghcr.io/chrislusf/seaweedfs:${BASE_TAG}${SUFFIX}-ppc64le \
|
||||
ghcr.io/chrislusf/seaweedfs:${BASE_TAG}${SUFFIX}-s390x
|
||||
fi
|
||||
|
||||
- name: Sign
|
||||
uses: ./.github/actions/sign-image
|
||||
with:
|
||||
images: >-
|
||||
ghcr.io/chrislusf/seaweedfs@${{ steps.manifest.outputs.digest }}
|
||||
chrislusf/seaweedfs@${{ steps.manifest.outputs.digest }}
|
||||
@@ -4,29 +4,45 @@ on:
|
||||
push:
|
||||
tags:
|
||||
- '*'
|
||||
workflow_dispatch: {}
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
release_tag:
|
||||
description: 'Release tag to publish (e.g. 3.93)'
|
||||
required: true
|
||||
default: ''
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.event.pull_request.number || (github.event_name == 'push' && github.ref) || format('refs/tags/{0}', github.event.inputs.release_tag) || github.run_id }}
|
||||
cancel-in-progress: false
|
||||
queue: max
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
env:
|
||||
RELEASE_TAG: ${{ github.event_name == 'workflow_dispatch' && github.event.inputs.release_tag || github.ref_name }}
|
||||
|
||||
jobs:
|
||||
|
||||
build-large-release-container_foundationdb:
|
||||
runs-on: [ubuntu-latest]
|
||||
permissions:
|
||||
contents: read
|
||||
id-token: write
|
||||
|
||||
steps:
|
||||
-
|
||||
name: Checkout
|
||||
uses: actions/checkout@v6
|
||||
uses: actions/checkout@v7
|
||||
-
|
||||
name: Docker meta
|
||||
id: docker_meta
|
||||
uses: docker/metadata-action@v5
|
||||
uses: docker/metadata-action@v6
|
||||
with:
|
||||
images: |
|
||||
chrislusf/seaweedfs
|
||||
tags: |
|
||||
type=ref,event=tag,suffix=_large_disk_foundationdb
|
||||
type=raw,value=${{ env.RELEASE_TAG }}_large_disk_foundationdb
|
||||
flavor: |
|
||||
latest=false
|
||||
labels: |
|
||||
@@ -35,14 +51,14 @@ jobs:
|
||||
org.opencontainers.image.vendor=Chris Lu
|
||||
-
|
||||
name: Set up QEMU
|
||||
uses: docker/setup-qemu-action@v3
|
||||
uses: docker/setup-qemu-action@v4.4.0
|
||||
-
|
||||
name: Set up Docker Buildx
|
||||
uses: docker/setup-buildx-action@v3
|
||||
uses: docker/setup-buildx-action@v4
|
||||
-
|
||||
name: Login to Docker Hub
|
||||
if: github.event_name != 'pull_request'
|
||||
uses: docker/login-action@v3
|
||||
uses: docker/login-action@v4.6.0
|
||||
with:
|
||||
username: ${{ secrets.DOCKER_USERNAME }}
|
||||
password: ${{ secrets.DOCKER_PASSWORD }}
|
||||
@@ -57,7 +73,8 @@ jobs:
|
||||
fi
|
||||
-
|
||||
name: Build
|
||||
uses: docker/build-push-action@v6
|
||||
id: build
|
||||
uses: docker/build-push-action@v7
|
||||
with:
|
||||
context: ./docker
|
||||
push: ${{ github.event_name != 'pull_request' }}
|
||||
@@ -68,4 +85,10 @@ jobs:
|
||||
platforms: linux/amd64
|
||||
tags: ${{ steps.docker_meta.outputs.tags }}
|
||||
labels: ${{ steps.docker_meta.outputs.labels }}
|
||||
-
|
||||
name: Sign
|
||||
if: github.event_name != 'pull_request'
|
||||
uses: ./.github/actions/sign-image
|
||||
with:
|
||||
images: chrislusf/seaweedfs@${{ steps.build.outputs.digest }}
|
||||
|
||||
@@ -25,13 +25,15 @@ on:
|
||||
rocksdb_version:
|
||||
description: 'RocksDB git tag to use when variant=rocksdb'
|
||||
required: false
|
||||
default: 'v10.10.1'
|
||||
default: 'v11.1.2'
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
security-events: write
|
||||
|
||||
env:
|
||||
RELEASE_TAG: ${{ github.event_name == 'workflow_dispatch' && github.event.inputs.release_tag || github.ref_name }}
|
||||
IMAGE: ghcr.io/chrislusf/seaweedfs
|
||||
|
||||
# Limit concurrent builds to avoid rate limits
|
||||
concurrency:
|
||||
@@ -39,86 +41,207 @@ concurrency:
|
||||
cancel-in-progress: false
|
||||
|
||||
jobs:
|
||||
build:
|
||||
runs-on: ubuntu-latest
|
||||
|
||||
# ── Pre-build the Rust binaries natively ────────────────────────────
|
||||
# The volume server and the Rust maintenance worker, cross-compiled for
|
||||
# amd64 and arm64 without QEMU, turning a 5-hour emulated cargo build into
|
||||
# ~15 minutes of native compilation.
|
||||
build-rust-binaries:
|
||||
runs-on: ubuntu-22.04
|
||||
strategy:
|
||||
# Build sequentially to avoid rate limits
|
||||
max-parallel: 2
|
||||
matrix:
|
||||
include:
|
||||
- target: x86_64-unknown-linux-musl
|
||||
arch: amd64
|
||||
- target: aarch64-unknown-linux-musl
|
||||
arch: arm64
|
||||
cross: true
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v7
|
||||
|
||||
- name: Install Rust toolchain
|
||||
uses: dtolnay/rust-toolchain@stable
|
||||
with:
|
||||
targets: ${{ matrix.target }}
|
||||
|
||||
- name: Install musl tools (amd64)
|
||||
if: ${{ !matrix.cross }}
|
||||
run: sudo apt-get install -y musl-tools
|
||||
|
||||
- name: Install cross-compilation tools (arm64)
|
||||
if: matrix.cross
|
||||
run: |
|
||||
sudo apt-get install -y gcc-aarch64-linux-gnu
|
||||
echo "CARGO_TARGET_AARCH64_UNKNOWN_LINUX_MUSL_LINKER=aarch64-linux-gnu-gcc" >> "$GITHUB_ENV"
|
||||
# Disable glibc fortify source — its __memcpy_chk etc. symbols don't exist in musl
|
||||
echo "CFLAGS_aarch64_unknown_linux_musl=-U_FORTIFY_SOURCE" >> "$GITHUB_ENV"
|
||||
|
||||
- name: Cache cargo registry and target
|
||||
if: runner.environment == 'github-hosted'
|
||||
uses: actions/cache@v6
|
||||
with:
|
||||
path: |
|
||||
~/.cargo/registry
|
||||
~/.cargo/git
|
||||
seaweed-volume/target
|
||||
seaweed-worker/target/${{ matrix.target }}/release
|
||||
key: rust-docker-${{ matrix.target }}-${{ hashFiles('seaweed-volume/Cargo.lock', 'seaweed-worker/Cargo.lock') }}
|
||||
restore-keys: |
|
||||
rust-docker-${{ matrix.target }}-
|
||||
|
||||
# lance's build scripts compile their own protos and look for a protoc.
|
||||
# Point them at the one protoc-bin-vendored ships, which seaweed-worker's
|
||||
# own build already uses, so no job depends on a system package and every
|
||||
# build sees the same version.
|
||||
- name: Use the vendored protoc
|
||||
run: |
|
||||
cd seaweed-worker
|
||||
cargo fetch
|
||||
# The version from the lock, not whatever else a restored cache holds.
|
||||
version=$(awk '/^name = "protoc-bin-vendored-linux-x86_64"$/{found=1; next} found && /^version = /{gsub(/"/,"",$3); print $3; exit}' Cargo.lock)
|
||||
test -n "$version" || { echo "protoc-bin-vendored-linux-x86_64 is not in Cargo.lock" >&2; exit 1; }
|
||||
protoc=$(find ~/.cargo/registry/src -path "*protoc-bin-vendored-linux-x86_64-$version/bin/protoc" | head -1)
|
||||
test -x "$protoc" || { echo "no vendored protoc $version in the registry" >&2; exit 1; }
|
||||
echo "PROTOC=$protoc" >> "$GITHUB_ENV"
|
||||
|
||||
- name: Build large-disk variant
|
||||
env:
|
||||
SEAWEEDFS_COMMIT: ${{ github.sha }}
|
||||
run: |
|
||||
cd seaweed-volume
|
||||
cargo build --release --target ${{ matrix.target }}
|
||||
cp target/${{ matrix.target }}/release/weed-volume ../weed-volume-large-disk-${{ matrix.arch }}
|
||||
|
||||
- name: Build normal variant
|
||||
env:
|
||||
SEAWEEDFS_COMMIT: ${{ github.sha }}
|
||||
run: |
|
||||
cd seaweed-volume
|
||||
cargo build --release --target ${{ matrix.target }} --no-default-features
|
||||
cp target/${{ matrix.target }}/release/weed-volume ../weed-volume-normal-${{ matrix.arch }}
|
||||
|
||||
- name: Build the Rust maintenance worker
|
||||
run: |
|
||||
cd seaweed-worker
|
||||
cargo build --release -p weed-lance-worker --target ${{ matrix.target }}
|
||||
cp target/${{ matrix.target }}/release/weed-worker ../weed-worker-${{ matrix.arch }}
|
||||
|
||||
- name: Upload artifacts
|
||||
uses: actions/upload-artifact@v7
|
||||
with:
|
||||
name: rust-bins-${{ matrix.arch }}
|
||||
path: |
|
||||
weed-volume-large-disk-${{ matrix.arch }}
|
||||
weed-volume-normal-${{ matrix.arch }}
|
||||
weed-worker-${{ matrix.arch }}
|
||||
|
||||
# One job per (variant, platform) on a native runner, pushed by digest;
|
||||
# the merge job stitches the digests into one multi-arch tag.
|
||||
build:
|
||||
needs: [build-rust-binaries]
|
||||
runs-on: ${{ matrix.runner }}
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
include:
|
||||
# Normal volume - multi-arch
|
||||
- variant: normal
|
||||
platforms: linux/amd64,linux/arm64,linux/arm/v7,linux/386
|
||||
dockerfile: ./docker/Dockerfile.go_build
|
||||
build_args: ""
|
||||
tag_suffix: ""
|
||||
|
||||
# Large disk - multi-arch
|
||||
- variant: large_disk
|
||||
platforms: linux/amd64,linux/arm64,linux/arm/v7,linux/386
|
||||
dockerfile: ./docker/Dockerfile.go_build
|
||||
build_args: TAGS=5BytesOffset
|
||||
tag_suffix: _large_disk
|
||||
|
||||
# Full tags - amd64 only
|
||||
- variant: full
|
||||
platforms: linux/amd64
|
||||
dockerfile: ./docker/Dockerfile.go_build
|
||||
build_args: TAGS=elastic,gocdk,rclone,sqlite,tarantool,tikv,ydb
|
||||
tag_suffix: _full
|
||||
|
||||
# Large disk + full tags - amd64 only
|
||||
- variant: large_disk_full
|
||||
platforms: linux/amd64
|
||||
dockerfile: ./docker/Dockerfile.go_build
|
||||
build_args: TAGS=5BytesOffset,elastic,gocdk,rclone,sqlite,tarantool,tikv,ydb
|
||||
tag_suffix: _large_disk_full
|
||||
|
||||
- { variant: normal, tag_suffix: "", dockerfile: ./docker/Dockerfile.go_build, build_args: "", rust_variant: normal, platform: linux/amd64, arch: amd64, runner: ubuntu-latest, qemu: false }
|
||||
- { variant: normal, tag_suffix: "", dockerfile: ./docker/Dockerfile.go_build, build_args: "", rust_variant: normal, platform: linux/arm64, arch: arm64, runner: ubuntu-24.04-arm, qemu: false }
|
||||
- { variant: normal, tag_suffix: "", dockerfile: ./docker/Dockerfile.go_build, build_args: "", rust_variant: normal, platform: linux/arm/v7, arch: armv7, runner: ubuntu-latest, qemu: true }
|
||||
- { variant: normal, tag_suffix: "", dockerfile: ./docker/Dockerfile.go_build, build_args: "", rust_variant: normal, platform: linux/386, arch: i386, runner: ubuntu-latest, qemu: false }
|
||||
- { variant: normal, tag_suffix: "", dockerfile: ./docker/Dockerfile.go_build, build_args: "", rust_variant: normal, platform: linux/ppc64le, arch: ppc64le, runner: ubuntu-latest, qemu: true }
|
||||
- { variant: normal, tag_suffix: "", dockerfile: ./docker/Dockerfile.go_build, build_args: "", rust_variant: normal, platform: linux/s390x, arch: s390x, runner: ubuntu-latest, qemu: true }
|
||||
|
||||
# Large disk - multi-arch
|
||||
- { variant: large_disk, tag_suffix: _large_disk, dockerfile: ./docker/Dockerfile.go_build, build_args: TAGS=5BytesOffset, rust_variant: large-disk, platform: linux/amd64, arch: amd64, runner: ubuntu-latest, qemu: false }
|
||||
- { variant: large_disk, tag_suffix: _large_disk, dockerfile: ./docker/Dockerfile.go_build, build_args: TAGS=5BytesOffset, rust_variant: large-disk, platform: linux/arm64, arch: arm64, runner: ubuntu-24.04-arm, qemu: false }
|
||||
- { variant: large_disk, tag_suffix: _large_disk, dockerfile: ./docker/Dockerfile.go_build, build_args: TAGS=5BytesOffset, rust_variant: large-disk, platform: linux/arm/v7, arch: armv7, runner: ubuntu-latest, qemu: true }
|
||||
- { variant: large_disk, tag_suffix: _large_disk, dockerfile: ./docker/Dockerfile.go_build, build_args: TAGS=5BytesOffset, rust_variant: large-disk, platform: linux/386, arch: i386, runner: ubuntu-latest, qemu: false }
|
||||
- { variant: large_disk, tag_suffix: _large_disk, dockerfile: ./docker/Dockerfile.go_build, build_args: TAGS=5BytesOffset, rust_variant: large-disk, platform: linux/ppc64le, arch: ppc64le, runner: ubuntu-latest, qemu: true }
|
||||
- { variant: large_disk, tag_suffix: _large_disk, dockerfile: ./docker/Dockerfile.go_build, build_args: TAGS=5BytesOffset, rust_variant: large-disk, platform: linux/s390x, arch: s390x, runner: ubuntu-latest, qemu: true }
|
||||
|
||||
# Full tags - multi-arch
|
||||
- { variant: full, tag_suffix: _full, dockerfile: ./docker/Dockerfile.go_build, build_args: "TAGS=elastic,gocdk,rclone,sqlite,tarantool,tikv,ydb", rust_variant: normal, platform: linux/amd64, arch: amd64, runner: ubuntu-latest, qemu: false }
|
||||
- { variant: full, tag_suffix: _full, dockerfile: ./docker/Dockerfile.go_build, build_args: "TAGS=elastic,gocdk,rclone,sqlite,tarantool,tikv,ydb", rust_variant: normal, platform: linux/arm64, arch: arm64, runner: ubuntu-24.04-arm, qemu: false }
|
||||
|
||||
# Large disk + full tags - multi-arch
|
||||
- { variant: large_disk_full, tag_suffix: _large_disk_full, dockerfile: ./docker/Dockerfile.go_build, build_args: "TAGS=5BytesOffset,elastic,gocdk,rclone,sqlite,tarantool,tikv,ydb", rust_variant: large-disk, platform: linux/amd64, arch: amd64, runner: ubuntu-latest, qemu: false }
|
||||
- { variant: large_disk_full, tag_suffix: _large_disk_full, dockerfile: ./docker/Dockerfile.go_build, build_args: "TAGS=5BytesOffset,elastic,gocdk,rclone,sqlite,tarantool,tikv,ydb", rust_variant: large-disk, platform: linux/arm64, arch: arm64, runner: ubuntu-24.04-arm, qemu: false }
|
||||
|
||||
# RocksDB large disk - amd64 only
|
||||
- variant: rocksdb
|
||||
platforms: linux/amd64
|
||||
dockerfile: ./docker/Dockerfile.rocksdb_large
|
||||
build_args: ""
|
||||
tag_suffix: _large_disk_rocksdb
|
||||
|
||||
- { variant: rocksdb, tag_suffix: _large_disk_rocksdb, dockerfile: ./docker/Dockerfile.rocksdb_large, build_args: "", rust_variant: large-disk, platform: linux/amd64, arch: amd64, runner: ubuntu-latest, qemu: false }
|
||||
steps:
|
||||
- name: Skip unselected variant
|
||||
if: github.event_name == 'workflow_dispatch' && github.event.inputs.variant != 'all' && github.event.inputs.variant != matrix.variant
|
||||
run: echo "Skipping ${{ matrix.variant }} (${{ matrix.platform }})" && exit 0
|
||||
|
||||
- name: Checkout
|
||||
if: github.event_name != 'workflow_dispatch' || github.event.inputs.variant == 'all' || github.event.inputs.variant == matrix.variant
|
||||
uses: actions/checkout@v6
|
||||
|
||||
uses: actions/checkout@v7
|
||||
|
||||
- name: Clean up stale FUSE mounts
|
||||
if: github.event_name != 'workflow_dispatch' || github.event.inputs.variant == 'all' || github.event.inputs.variant == matrix.variant
|
||||
uses: ./.github/actions/cleanup-stale-fuse-mounts
|
||||
|
||||
- name: Download pre-built Rust binaries
|
||||
if: github.event_name != 'workflow_dispatch' || github.event.inputs.variant == 'all' || github.event.inputs.variant == matrix.variant
|
||||
uses: actions/download-artifact@v8
|
||||
with:
|
||||
pattern: rust-bins-*
|
||||
merge-multiple: true
|
||||
path: ./rust-bins
|
||||
|
||||
- name: Place Rust binaries in Docker context
|
||||
if: github.event_name != 'workflow_dispatch' || github.event.inputs.variant == 'all' || github.event.inputs.variant == matrix.variant
|
||||
run: |
|
||||
mkdir -p docker/weed-volume-prebuilt
|
||||
for arch in amd64 arm64; do
|
||||
src="./rust-bins/weed-volume-${{ matrix.rust_variant }}-${arch}"
|
||||
if [ -f "$src" ]; then
|
||||
cp "$src" "docker/weed-volume-prebuilt/weed-volume-${arch}"
|
||||
echo "Placed pre-built Rust binary for ${arch}"
|
||||
fi
|
||||
done
|
||||
mkdir -p docker/weed-worker-prebuilt
|
||||
for arch in amd64 arm64; do
|
||||
src="./rust-bins/weed-worker-${arch}"
|
||||
if [ -f "$src" ]; then
|
||||
cp "$src" "docker/weed-worker-prebuilt/weed-worker-${arch}"
|
||||
echo "Placed pre-built Rust worker for ${arch}"
|
||||
fi
|
||||
done
|
||||
ls -la docker/weed-volume-prebuilt/
|
||||
ls -la docker/weed-worker-prebuilt/
|
||||
|
||||
- name: Free Disk Space
|
||||
if: github.event_name != 'workflow_dispatch' || github.event.inputs.variant == 'all' || github.event.inputs.variant == matrix.variant
|
||||
run: |
|
||||
echo "Available disk space before cleanup:"
|
||||
df -h
|
||||
sudo rm -rf /usr/share/dotnet /usr/local/lib/android /opt/ghc /opt/hostedtoolcache/CodeQL
|
||||
sudo apt-get clean
|
||||
sudo rm -rf /var/lib/apt/lists/*
|
||||
sudo docker system prune -af --volumes
|
||||
[ -d ~/.cache/go-build ] && rm -rf ~/.cache/go-build || true
|
||||
[ -d /go/pkg ] && rm -rf /go/pkg || true
|
||||
echo "Available disk space after cleanup:"
|
||||
df -h
|
||||
|
||||
timeout 15 df -h || true
|
||||
|
||||
- name: Docker meta
|
||||
if: github.event_name != 'workflow_dispatch' || github.event.inputs.variant == 'all' || github.event.inputs.variant == matrix.variant
|
||||
id: docker_meta
|
||||
uses: docker/metadata-action@v5
|
||||
uses: docker/metadata-action@v6
|
||||
with:
|
||||
images: |
|
||||
chrislusf/seaweedfs
|
||||
ghcr.io/chrislusf/seaweedfs
|
||||
images: ${{ env.IMAGE }}
|
||||
tags: type=raw,value=${{ env.RELEASE_TAG }}${{ matrix.tag_suffix }}
|
||||
flavor: latest=false
|
||||
labels: |
|
||||
org.opencontainers.image.title=seaweedfs
|
||||
org.opencontainers.image.description=SeaweedFS is a distributed storage system for blobs, objects, files, and data lake, to store and serve billions of files fast!
|
||||
org.opencontainers.image.vendor=Chris Lu
|
||||
|
||||
|
||||
- name: Set up QEMU
|
||||
if: (github.event_name != 'workflow_dispatch' || github.event.inputs.variant == 'all' || github.event.inputs.variant == matrix.variant) && contains(matrix.platforms, 'arm')
|
||||
uses: docker/setup-qemu-action@v3
|
||||
|
||||
if: (github.event_name != 'workflow_dispatch' || github.event.inputs.variant == 'all' || github.event.inputs.variant == matrix.variant) && matrix.qemu
|
||||
uses: docker/setup-qemu-action@v4.4.0
|
||||
|
||||
- name: Create BuildKit config
|
||||
if: github.event_name != 'workflow_dispatch' || github.event.inputs.variant == 'all' || github.event.inputs.variant == matrix.variant
|
||||
run: |
|
||||
@@ -126,150 +249,358 @@ jobs:
|
||||
[registry."docker.io"]
|
||||
mirrors = ["https://mirror.gcr.io"]
|
||||
EOF
|
||||
|
||||
|
||||
- name: Set up Docker Buildx
|
||||
if: github.event_name != 'workflow_dispatch' || github.event.inputs.variant == 'all' || github.event.inputs.variant == matrix.variant
|
||||
uses: docker/setup-buildx-action@v3
|
||||
uses: docker/setup-buildx-action@v4
|
||||
with:
|
||||
buildkitd-config: /tmp/buildkitd.toml
|
||||
|
||||
- name: Login to Docker Hub
|
||||
if: (github.event_name != 'workflow_dispatch' || github.event.inputs.variant == 'all' || github.event.inputs.variant == matrix.variant) && github.event_name != 'pull_request'
|
||||
uses: docker/login-action@v3
|
||||
with:
|
||||
username: ${{ secrets.DOCKER_USERNAME }}
|
||||
password: ${{ secrets.DOCKER_PASSWORD }}
|
||||
|
||||
|
||||
- name: Login to GHCR
|
||||
if: (github.event_name != 'workflow_dispatch' || github.event.inputs.variant == 'all' || github.event.inputs.variant == matrix.variant) && github.event_name != 'pull_request'
|
||||
uses: docker/login-action@v3
|
||||
if: github.event_name != 'workflow_dispatch' || github.event.inputs.variant == 'all' || github.event.inputs.variant == matrix.variant
|
||||
uses: docker/login-action@v4.6.0
|
||||
with:
|
||||
registry: ghcr.io
|
||||
username: ${{ secrets.GHCR_USERNAME }}
|
||||
password: ${{ secrets.GHCR_TOKEN }}
|
||||
|
||||
- name: Build and push ${{ matrix.variant }}
|
||||
|
||||
- name: Build and push ${{ matrix.variant }} (${{ matrix.platform }})
|
||||
if: github.event_name != 'workflow_dispatch' || github.event.inputs.variant == 'all' || github.event.inputs.variant == matrix.variant
|
||||
uses: docker/build-push-action@v6
|
||||
id: build
|
||||
uses: docker/build-push-action@v7
|
||||
env:
|
||||
DOCKER_BUILDKIT: 1
|
||||
with:
|
||||
context: ./docker
|
||||
push: ${{ github.event_name != 'pull_request' }}
|
||||
file: ${{ matrix.dockerfile }}
|
||||
platforms: ${{ matrix.platforms }}
|
||||
# Push to GHCR to avoid Docker Hub rate limits on pulls
|
||||
tags: |
|
||||
ghcr.io/chrislusf/seaweedfs:${{ env.RELEASE_TAG }}${{ matrix.tag_suffix }}
|
||||
platforms: ${{ matrix.platform }}
|
||||
labels: ${{ steps.docker_meta.outputs.labels }}
|
||||
cache-from: type=gha,scope=${{ matrix.variant }}
|
||||
cache-to: type=gha,mode=max,scope=${{ matrix.variant }}
|
||||
# mode=max keeps resolvedDependencies complete and records the
|
||||
# builder identity; attestations ride inside each pushed index and
|
||||
# survive the imagetools merge into the multi-arch manifest.
|
||||
provenance: mode=max
|
||||
sbom: true
|
||||
outputs: type=image,name=${{ env.IMAGE }},push-by-digest=true,name-canonical=true,push=true
|
||||
cache-from: type=gha,scope=${{ matrix.variant }}-${{ matrix.arch }}
|
||||
# max only for rocksdb: its RocksDB compile is sha-independent and worth
|
||||
# keeping; go-build layers are sha-busted every release, so min elsewhere.
|
||||
cache-to: type=gha,mode=${{ matrix.variant == 'rocksdb' && 'max' || 'min' }},scope=${{ matrix.variant }}-${{ matrix.arch }}
|
||||
build-args: |
|
||||
${{ matrix.build_args }}
|
||||
BUILDKIT_INLINE_CACHE=1
|
||||
BRANCH=${{ github.sha }}
|
||||
${{ matrix.variant == 'rocksdb' && format('ROCKSDB_VERSION={0}', github.event.inputs.rocksdb_version || 'v10.10.1') || '' }}
|
||||
|
||||
- name: Clean up build artifacts
|
||||
if: always() && (github.event_name != 'workflow_dispatch' || github.event.inputs.variant == 'all' || github.event.inputs.variant == matrix.variant)
|
||||
run: |
|
||||
sudo docker system prune -f
|
||||
sudo rm -rf /tmp/go-build*
|
||||
${{ matrix.variant == 'rocksdb' && format('ROCKSDB_VERSION={0}', github.event.inputs.rocksdb_version || 'v11.1.2') || '' }}
|
||||
|
||||
copy-to-dockerhub:
|
||||
runs-on: ubuntu-latest
|
||||
needs: [build]
|
||||
if: github.event_name != 'pull_request'
|
||||
strategy:
|
||||
matrix:
|
||||
variant: [normal, large_disk, full, large_disk_full, rocksdb]
|
||||
include:
|
||||
- variant: normal
|
||||
tag_suffix: ""
|
||||
- variant: large_disk
|
||||
tag_suffix: _large_disk
|
||||
- variant: full
|
||||
tag_suffix: _full
|
||||
- variant: large_disk_full
|
||||
tag_suffix: _large_disk_full
|
||||
- variant: rocksdb
|
||||
tag_suffix: _large_disk_rocksdb
|
||||
|
||||
steps:
|
||||
- name: Login to Docker Hub
|
||||
- name: Export digest
|
||||
if: github.event_name != 'workflow_dispatch' || github.event.inputs.variant == 'all' || github.event.inputs.variant == matrix.variant
|
||||
uses: docker/login-action@v3
|
||||
run: |
|
||||
mkdir -p /tmp/digests
|
||||
digest="${{ steps.build.outputs.digest }}"
|
||||
touch "/tmp/digests/${digest#sha256:}"
|
||||
|
||||
- name: Upload digest
|
||||
if: github.event_name != 'workflow_dispatch' || github.event.inputs.variant == 'all' || github.event.inputs.variant == matrix.variant
|
||||
uses: actions/upload-artifact@v7
|
||||
with:
|
||||
username: ${{ secrets.DOCKER_USERNAME }}
|
||||
password: ${{ secrets.DOCKER_PASSWORD }}
|
||||
|
||||
name: digest-${{ matrix.variant }}-${{ matrix.arch }}
|
||||
path: /tmp/digests/*
|
||||
if-no-files-found: error
|
||||
retention-days: 1
|
||||
|
||||
# Assemble each variant's per-platform digests into one tag, mirror it to
|
||||
# Docker Hub, and sign the result on both registries.
|
||||
merge:
|
||||
needs: [build]
|
||||
runs-on: ubuntu-latest
|
||||
permissions:
|
||||
contents: read
|
||||
id-token: write
|
||||
attestations: write
|
||||
packages: write
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
include:
|
||||
- { variant: normal, tag_suffix: "" }
|
||||
- { variant: large_disk, tag_suffix: _large_disk }
|
||||
- { variant: full, tag_suffix: _full }
|
||||
- { variant: large_disk_full, tag_suffix: _large_disk_full }
|
||||
- { variant: rocksdb, tag_suffix: _large_disk_rocksdb }
|
||||
steps:
|
||||
- name: Checkout the signing action
|
||||
if: github.event_name != 'workflow_dispatch' || github.event.inputs.variant == 'all' || github.event.inputs.variant == matrix.variant
|
||||
uses: actions/checkout@v7
|
||||
with:
|
||||
sparse-checkout: .github/actions
|
||||
persist-credentials: false
|
||||
|
||||
- name: Download digests
|
||||
if: github.event_name != 'workflow_dispatch' || github.event.inputs.variant == 'all' || github.event.inputs.variant == matrix.variant
|
||||
uses: actions/download-artifact@v8
|
||||
with:
|
||||
pattern: digest-${{ matrix.variant }}-*
|
||||
merge-multiple: true
|
||||
path: /tmp/digests
|
||||
|
||||
- name: Set up Docker Buildx
|
||||
if: github.event_name != 'workflow_dispatch' || github.event.inputs.variant == 'all' || github.event.inputs.variant == matrix.variant
|
||||
uses: docker/setup-buildx-action@v4
|
||||
|
||||
- name: Login to GHCR
|
||||
if: github.event_name != 'workflow_dispatch' || github.event.inputs.variant == 'all' || github.event.inputs.variant == matrix.variant
|
||||
uses: docker/login-action@v3
|
||||
uses: docker/login-action@v4.6.0
|
||||
with:
|
||||
registry: ghcr.io
|
||||
username: ${{ secrets.GHCR_USERNAME }}
|
||||
password: ${{ secrets.GHCR_TOKEN }}
|
||||
|
||||
|
||||
- name: Create multi-arch tag on GHCR
|
||||
if: github.event_name != 'workflow_dispatch' || github.event.inputs.variant == 'all' || github.event.inputs.variant == matrix.variant
|
||||
id: manifest
|
||||
working-directory: /tmp/digests
|
||||
run: |
|
||||
docker buildx imagetools create \
|
||||
-t ${{ env.IMAGE }}:${{ env.RELEASE_TAG }}${{ matrix.tag_suffix }} \
|
||||
--metadata-file /tmp/manifest.json \
|
||||
$(printf '${{ env.IMAGE }}@sha256:%s ' *)
|
||||
docker buildx imagetools inspect ${{ env.IMAGE }}:${{ env.RELEASE_TAG }}${{ matrix.tag_suffix }}
|
||||
# The copy and the signature below use the digest this run pushed, not whatever the tag points at by then.
|
||||
digest=$(jq -er '."containerimage.descriptor".digest' /tmp/manifest.json)
|
||||
echo "digest=${digest}" >> "$GITHUB_OUTPUT"
|
||||
|
||||
- name: Login to Docker Hub
|
||||
if: github.event_name != 'workflow_dispatch' || github.event.inputs.variant == 'all' || github.event.inputs.variant == matrix.variant
|
||||
uses: docker/login-action@v4.6.0
|
||||
with:
|
||||
username: ${{ secrets.DOCKER_USERNAME }}
|
||||
password: ${{ secrets.DOCKER_PASSWORD }}
|
||||
|
||||
- name: Install crane
|
||||
if: github.event_name != 'workflow_dispatch' || github.event.inputs.variant == 'all' || github.event.inputs.variant == matrix.variant
|
||||
run: |
|
||||
cd $(mktemp -d)
|
||||
curl -sL "https://github.com/google/go-containerregistry/releases/latest/download/go-containerregistry_Linux_x86_64.tar.gz" | tar xz
|
||||
curl -sLO https://github.com/google/go-containerregistry/releases/download/v0.22.0/go-containerregistry_Linux_x86_64.tar.gz
|
||||
echo "edb74d53fad9a596860f59d1c5d04a43dfb5f441dc71f57060dd0bf39483c833 go-containerregistry_Linux_x86_64.tar.gz" | sha256sum -c -
|
||||
tar xzf go-containerregistry_Linux_x86_64.tar.gz crane
|
||||
sudo mv crane /usr/local/bin/
|
||||
crane version
|
||||
|
||||
- name: Copy ${{ matrix.variant }} from GHCR to Docker Hub
|
||||
|
||||
- name: Copy ${{ matrix.variant }} to Docker Hub
|
||||
if: github.event_name != 'workflow_dispatch' || github.event.inputs.variant == 'all' || github.event.inputs.variant == matrix.variant
|
||||
run: |
|
||||
# Function to retry with exponential backoff
|
||||
retry_with_backoff() {
|
||||
local max_attempts=5
|
||||
local timeout=1
|
||||
local attempt=1
|
||||
local exit_code=0
|
||||
|
||||
while [ $attempt -le $max_attempts ]; do
|
||||
if "$@"; then
|
||||
return 0
|
||||
else
|
||||
exit_code=$?
|
||||
fi
|
||||
|
||||
if [ $attempt -lt $max_attempts ]; then
|
||||
echo "Attempt $attempt failed. Retrying in ${timeout}s..." >&2
|
||||
sleep $timeout
|
||||
timeout=$((timeout * 2))
|
||||
fi
|
||||
|
||||
attempt=$((attempt + 1))
|
||||
done
|
||||
|
||||
echo "Command failed after $max_attempts attempts" >&2
|
||||
return $exit_code
|
||||
}
|
||||
|
||||
# Copy multi-arch image from GHCR to Docker Hub with retry
|
||||
# This is much more efficient than pulling/pushing individual arch images
|
||||
|
||||
echo "Copying ${{ matrix.variant }} from GHCR to Docker Hub..."
|
||||
retry_with_backoff crane copy \
|
||||
ghcr.io/chrislusf/seaweedfs:${{ env.RELEASE_TAG }}${{ matrix.tag_suffix }} \
|
||||
${{ env.IMAGE }}@${{ steps.manifest.outputs.digest }} \
|
||||
chrislusf/seaweedfs:${{ env.RELEASE_TAG }}${{ matrix.tag_suffix }}
|
||||
|
||||
echo "✓ Successfully copied ${{ matrix.variant }} to Docker Hub"
|
||||
echo "Copied ${{ matrix.variant }} to Docker Hub"
|
||||
|
||||
- name: Sign ${{ matrix.variant }}
|
||||
if: github.event_name != 'workflow_dispatch' || github.event.inputs.variant == 'all' || github.event.inputs.variant == matrix.variant
|
||||
uses: ./.github/actions/sign-image
|
||||
with:
|
||||
images: >-
|
||||
${{ env.IMAGE }}@${{ steps.manifest.outputs.digest }}
|
||||
chrislusf/seaweedfs@${{ steps.manifest.outputs.digest }}
|
||||
|
||||
# Attest the merged multi-arch digest itself, so digest-pinning consumers
|
||||
# can verify provenance against the exact index they pull. Both steps
|
||||
# reuse the GHCR login above and push as OCI referrers.
|
||||
- name: Attest build provenance for ${{ matrix.variant }}
|
||||
if: github.event_name != 'workflow_dispatch' || github.event.inputs.variant == 'all' || github.event.inputs.variant == matrix.variant
|
||||
uses: actions/attest-build-provenance@v4
|
||||
with:
|
||||
subject-name: ${{ env.IMAGE }}
|
||||
subject-digest: ${{ steps.manifest.outputs.digest }}
|
||||
push-to-registry: true
|
||||
|
||||
# SBOMs are generated and attested per platform manifest: an index-wide
|
||||
# scan only describes syft's default platform, while the manifests it
|
||||
# merges differ (some ship the real binary, others placeholders), so the
|
||||
# SBOM has to be bound to the platform digest a consumer actually pulls.
|
||||
- name: Install syft
|
||||
if: github.event_name != 'workflow_dispatch' || github.event.inputs.variant == 'all' || github.event.inputs.variant == matrix.variant
|
||||
uses: anchore/sbom-action/download-syft@v0
|
||||
|
||||
- name: Attest per-platform SBOMs for ${{ matrix.variant }}
|
||||
if: github.event_name != 'workflow_dispatch' || github.event.inputs.variant == 'all' || github.event.inputs.variant == matrix.variant
|
||||
working-directory: /tmp/digests
|
||||
run: |
|
||||
set -euo pipefail
|
||||
for f in *; do
|
||||
# Each saved digest is an OCI index (image + attestation manifests);
|
||||
# scan and attest the image manifest a same-platform pull resolves.
|
||||
ref="${{ env.IMAGE }}@sha256:${f}"
|
||||
image=$(crane manifest "$ref" | jq -r '
|
||||
[.manifests[]? | select((.annotations["vnd.docker.reference.type"] // "") != "attestation-manifest"
|
||||
and (.platform.os // "unknown") != "unknown") | .digest][0] // empty')
|
||||
ref="${{ env.IMAGE }}@${image:-sha256:${f}}"
|
||||
syft "registry:${ref}" -o "spdx-json=/tmp/sbom-${f}.spdx.json"
|
||||
cosign attest --yes \
|
||||
--predicate "/tmp/sbom-${f}.spdx.json" \
|
||||
--type spdxjson \
|
||||
"$ref"
|
||||
done
|
||||
|
||||
# Report-only trivy scan: uploads fixable HIGH/CRITICAL findings to GitHub
|
||||
# Security for visibility, but never blocks the release. Releases (including
|
||||
# `latest`) ship regardless — vulnerabilities are tracked, not gated, since
|
||||
# we sometimes need to publish through known findings (e.g. unfixed upstream
|
||||
# CVE, base-image lag).
|
||||
trivy-scan:
|
||||
runs-on: ubuntu-latest
|
||||
needs: [merge]
|
||||
if: github.event_name == 'push'
|
||||
continue-on-error: true
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
include:
|
||||
- source_suffix: ""
|
||||
variant: normal
|
||||
- source_suffix: _large_disk
|
||||
variant: large_disk
|
||||
steps:
|
||||
- name: Login to GHCR
|
||||
uses: docker/login-action@v4.6.0
|
||||
with:
|
||||
registry: ghcr.io
|
||||
username: ${{ secrets.GHCR_USERNAME }}
|
||||
password: ${{ secrets.GHCR_TOKEN }}
|
||||
|
||||
- name: Trivy report (${{ matrix.variant }})
|
||||
# Pin to SHA - mutable tags were compromised (GHSA-69fq-xp46-6x23)
|
||||
uses: aquasecurity/trivy-action@ed142fd0673e97e23eac54620cfb913e5ce36c25 # v0.36.0
|
||||
with:
|
||||
scan-type: image
|
||||
# Scan the multi-arch tag on GHCR (already pushed by the build job).
|
||||
# Trivy scans the runner's native platform; OS packages are identical
|
||||
# across architectures since they all share the same alpine base.
|
||||
image-ref: ${{ env.IMAGE }}:${{ env.RELEASE_TAG }}${{ matrix.source_suffix }}
|
||||
scanners: vuln
|
||||
vuln-type: os,library
|
||||
severity: HIGH,CRITICAL
|
||||
# Keep unfixed findings in the report so digest-pinning consumers see
|
||||
# the complete picture, not just what a patch could resolve.
|
||||
ignore-unfixed: false
|
||||
limit-severities-for-sarif: true
|
||||
format: sarif
|
||||
output: trivy-results.sarif
|
||||
exit-code: '0'
|
||||
|
||||
- name: Upload Trivy scan results to GitHub Security
|
||||
if: always()
|
||||
uses: github/codeql-action/upload-sarif@v4.39.0
|
||||
with:
|
||||
sarif_file: trivy-results.sarif
|
||||
category: trivy-${{ matrix.variant }}
|
||||
|
||||
# Point `latest` (and `latest_large_disk`) at the just-released versioned
|
||||
# image. crane tag adds an extra tag to an existing manifest — no rebuild,
|
||||
# no QEMU, no separate workflow. Replaces the old container_latest.yml
|
||||
# rebuild that often failed or lagged behind the release. Independent of
|
||||
# trivy-scan: vuln findings are reported but do not block `latest`. The
|
||||
# cosign signature is attached to the digest, so `latest` carries it too.
|
||||
tag-latest:
|
||||
runs-on: ubuntu-latest
|
||||
needs: [merge]
|
||||
if: github.event_name == 'push'
|
||||
strategy:
|
||||
matrix:
|
||||
include:
|
||||
- source_suffix: ""
|
||||
latest_tag: latest
|
||||
- source_suffix: _large_disk
|
||||
latest_tag: latest_large_disk
|
||||
steps:
|
||||
- name: Login to Docker Hub
|
||||
uses: docker/login-action@v4.6.0
|
||||
with:
|
||||
username: ${{ secrets.DOCKER_USERNAME }}
|
||||
password: ${{ secrets.DOCKER_PASSWORD }}
|
||||
|
||||
- name: Login to GHCR
|
||||
uses: docker/login-action@v4.6.0
|
||||
with:
|
||||
registry: ghcr.io
|
||||
username: ${{ secrets.GHCR_USERNAME }}
|
||||
password: ${{ secrets.GHCR_TOKEN }}
|
||||
|
||||
- name: Install crane
|
||||
run: |
|
||||
cd $(mktemp -d)
|
||||
curl -sLO https://github.com/google/go-containerregistry/releases/download/v0.22.0/go-containerregistry_Linux_x86_64.tar.gz
|
||||
echo "edb74d53fad9a596860f59d1c5d04a43dfb5f441dc71f57060dd0bf39483c833 go-containerregistry_Linux_x86_64.tar.gz" | sha256sum -c -
|
||||
tar xzf go-containerregistry_Linux_x86_64.tar.gz crane
|
||||
sudo mv crane /usr/local/bin/
|
||||
crane version
|
||||
|
||||
- name: Re-tag ${{ env.RELEASE_TAG }}${{ matrix.source_suffix }} as ${{ matrix.latest_tag }}
|
||||
run: |
|
||||
retry_with_backoff() {
|
||||
local max_attempts=5
|
||||
local timeout=1
|
||||
local attempt=1
|
||||
local exit_code=0
|
||||
while [ $attempt -le $max_attempts ]; do
|
||||
if "$@"; then
|
||||
return 0
|
||||
else
|
||||
exit_code=$?
|
||||
fi
|
||||
if [ $attempt -lt $max_attempts ]; then
|
||||
echo "Attempt $attempt failed. Retrying in ${timeout}s..." >&2
|
||||
sleep $timeout
|
||||
timeout=$((timeout * 2))
|
||||
fi
|
||||
attempt=$((attempt + 1))
|
||||
done
|
||||
echo "Command failed after $max_attempts attempts" >&2
|
||||
return $exit_code
|
||||
}
|
||||
|
||||
SRC_TAG="${{ env.RELEASE_TAG }}${{ matrix.source_suffix }}"
|
||||
DST_TAG="${{ matrix.latest_tag }}"
|
||||
|
||||
echo "Tagging ${{ env.IMAGE }}:${SRC_TAG} as ${DST_TAG}"
|
||||
retry_with_backoff crane tag "${{ env.IMAGE }}:${SRC_TAG}" "${DST_TAG}"
|
||||
|
||||
echo "Tagging chrislusf/seaweedfs:${SRC_TAG} as ${DST_TAG}"
|
||||
retry_with_backoff crane tag "chrislusf/seaweedfs:${SRC_TAG}" "${DST_TAG}"
|
||||
|
||||
helm-release:
|
||||
runs-on: ubuntu-latest
|
||||
needs: [copy-to-dockerhub]
|
||||
needs: [build]
|
||||
if: github.event_name == 'push' || github.event_name == 'workflow_dispatch'
|
||||
permissions:
|
||||
contents: write
|
||||
pages: write
|
||||
steps:
|
||||
- uses: actions/checkout@v6
|
||||
- uses: actions/checkout@v7
|
||||
- name: Publish Helm charts
|
||||
uses: stefanprodan/helm-gh-pages@v1.7.0
|
||||
with:
|
||||
with:
|
||||
token: ${{ secrets.GITHUB_TOKEN }}
|
||||
charts_dir: k8s/charts
|
||||
target_dir: helm
|
||||
|
||||
@@ -4,9 +4,9 @@ on:
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
rocksdb_version:
|
||||
description: 'RocksDB git tag or branch to build (e.g. v10.10.1)'
|
||||
description: 'RocksDB git tag or branch to build (e.g. v11.1.2)'
|
||||
required: true
|
||||
default: 'v10.10.1'
|
||||
default: 'v11.1.2'
|
||||
seaweedfs_ref:
|
||||
description: 'SeaweedFS git tag, branch, or commit to build'
|
||||
required: true
|
||||
@@ -16,16 +16,24 @@ on:
|
||||
required: false
|
||||
default: ''
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.event.inputs.image_tag || format('rocksdb_{0}_seaweedfs_{1}', github.event.inputs.rocksdb_version, github.event.inputs.seaweedfs_ref) }}
|
||||
cancel-in-progress: false
|
||||
queue: max
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
jobs:
|
||||
build-rocksdb-image:
|
||||
runs-on: ubuntu-latest
|
||||
permissions:
|
||||
contents: read
|
||||
id-token: write
|
||||
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v2
|
||||
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v2
|
||||
|
||||
- name: Prepare Docker tag
|
||||
id: tag
|
||||
@@ -82,19 +90,20 @@ jobs:
|
||||
echo "seaweedfs_ref=$seaweed" >> "$GITHUB_OUTPUT"
|
||||
|
||||
- name: Set up QEMU
|
||||
uses: docker/setup-qemu-action@c7c53464625b32c7a7e944ae62b3e17d2b600130 # v1
|
||||
uses: docker/setup-qemu-action@99012661954931238ded8c8b007157a8430204e1 # v1
|
||||
|
||||
- name: Set up Docker Buildx
|
||||
uses: docker/setup-buildx-action@e468171a9de216ec08956ac3ada2f0791b6bd435 # v1
|
||||
uses: docker/setup-buildx-action@4d04d5d9486b7bd6fa91e7baf45bbb4f8b9deedd # v1
|
||||
|
||||
- name: Login to Docker Hub
|
||||
uses: docker/login-action@5e57cd118135c172c3672efd75eb46360885c0ef # v1
|
||||
uses: docker/login-action@dbcb813823bdd20940b903addbd779551569679f # v1
|
||||
with:
|
||||
username: ${{ secrets.DOCKER_USERNAME }}
|
||||
password: ${{ secrets.DOCKER_PASSWORD }}
|
||||
|
||||
- name: Build and push image
|
||||
uses: docker/build-push-action@263435318d21b8e681c14492fe198d362a7d2c83 # v2
|
||||
id: build
|
||||
uses: docker/build-push-action@bcafcacb16a39f128d818304e6c9c0c18556b85f # v2
|
||||
with:
|
||||
context: ./docker
|
||||
push: true
|
||||
@@ -108,3 +117,8 @@ jobs:
|
||||
org.opencontainers.image.title=seaweedfs
|
||||
org.opencontainers.image.description=SeaweedFS is a distributed storage system for blobs, objects, files, and data lake, to store and serve billions of files fast!
|
||||
org.opencontainers.image.vendor=Chris Lu
|
||||
|
||||
- name: Sign
|
||||
uses: ./.github/actions/sign-image
|
||||
with:
|
||||
images: chrislusf/seaweedfs@${{ steps.build.outputs.digest }}
|
||||
@@ -21,22 +21,63 @@ jobs:
|
||||
deploy:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v6
|
||||
- uses: actions/checkout@v7
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v6
|
||||
uses: actions/setup-go@v7
|
||||
with:
|
||||
go-version: '1.24'
|
||||
go-version-file: 'telemetry/server/go.mod'
|
||||
cache: ${{ runner.environment == 'github-hosted' }}
|
||||
|
||||
- name: Build Telemetry Server
|
||||
if: github.event_name == 'workflow_dispatch' && inputs.deploy
|
||||
run: |
|
||||
# telemetry/server is its own Go module; build from within it
|
||||
cd telemetry/server
|
||||
go mod tidy
|
||||
echo "Building telemetry server..."
|
||||
GOOS=linux GOARCH=amd64 go build -o telemetry-server ./telemetry/server/main.go
|
||||
CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -o ../../telemetry-server .
|
||||
cd ../..
|
||||
ls -la telemetry-server
|
||||
echo "Build completed successfully"
|
||||
|
||||
- name: Generate Service Configuration
|
||||
if: github.event_name == 'workflow_dispatch' && (inputs.setup || inputs.deploy)
|
||||
env:
|
||||
REMOTE_USER: ${{ secrets.TELEMETRY_USER }}
|
||||
run: |
|
||||
# Create systemd service file
|
||||
echo "
|
||||
[Unit]
|
||||
Description=SeaweedFS Telemetry Server
|
||||
After=network.target
|
||||
|
||||
[Service]
|
||||
Type=simple
|
||||
User=$REMOTE_USER
|
||||
WorkingDirectory=/home/$REMOTE_USER/seaweedfs-telemetry
|
||||
ExecStart=/bin/sh -c 'exec /home/$REMOTE_USER/seaweedfs-telemetry/bin/telemetry-server -port=8353 >>/home/$REMOTE_USER/seaweedfs-telemetry/logs/telemetry.log 2>>/home/$REMOTE_USER/seaweedfs-telemetry/logs/telemetry.error.log'
|
||||
Restart=always
|
||||
RestartSec=5
|
||||
|
||||
[Install]
|
||||
WantedBy=multi-user.target" > telemetry.service
|
||||
|
||||
# Setup logrotate configuration
|
||||
echo "# SeaweedFS Telemetry service log rotation
|
||||
/home/$REMOTE_USER/seaweedfs-telemetry/logs/*.log {
|
||||
daily
|
||||
rotate 30
|
||||
compress
|
||||
delaycompress
|
||||
missingok
|
||||
notifempty
|
||||
create 644 $REMOTE_USER $REMOTE_USER
|
||||
postrotate
|
||||
systemctl restart telemetry.service
|
||||
endscript
|
||||
}" > telemetry_logrotate
|
||||
|
||||
- name: First-time Server Setup
|
||||
if: github.event_name == 'workflow_dispatch' && inputs.setup
|
||||
env:
|
||||
@@ -58,40 +99,6 @@ jobs:
|
||||
touch ~/seaweedfs-telemetry/logs/telemetry.log ~/seaweedfs-telemetry/logs/telemetry.error.log && \
|
||||
chmod 644 ~/seaweedfs-telemetry/logs/*.log"
|
||||
|
||||
# Create systemd service file
|
||||
echo "
|
||||
[Unit]
|
||||
Description=SeaweedFS Telemetry Server
|
||||
After=network.target
|
||||
|
||||
[Service]
|
||||
Type=simple
|
||||
User=$REMOTE_USER
|
||||
WorkingDirectory=/home/$REMOTE_USER/seaweedfs-telemetry
|
||||
ExecStart=/home/$REMOTE_USER/seaweedfs-telemetry/bin/telemetry-server -port=8353
|
||||
Restart=always
|
||||
RestartSec=5
|
||||
StandardOutput=append:/home/$REMOTE_USER/seaweedfs-telemetry/logs/telemetry.log
|
||||
StandardError=append:/home/$REMOTE_USER/seaweedfs-telemetry/logs/telemetry.error.log
|
||||
|
||||
[Install]
|
||||
WantedBy=multi-user.target" > telemetry.service
|
||||
|
||||
# Setup logrotate configuration
|
||||
echo "# SeaweedFS Telemetry service log rotation
|
||||
/home/$REMOTE_USER/seaweedfs-telemetry/logs/*.log {
|
||||
daily
|
||||
rotate 30
|
||||
compress
|
||||
delaycompress
|
||||
missingok
|
||||
notifempty
|
||||
create 644 $REMOTE_USER $REMOTE_USER
|
||||
postrotate
|
||||
systemctl restart telemetry.service
|
||||
endscript
|
||||
}" > telemetry_logrotate
|
||||
|
||||
# Copy configuration files
|
||||
scp -i ~/.ssh/deploy_key telemetry/grafana-dashboard.json $REMOTE_USER@$REMOTE_HOST:~/seaweedfs-telemetry/
|
||||
scp -i ~/.ssh/deploy_key telemetry/prometheus.yml $REMOTE_USER@$REMOTE_HOST:~/seaweedfs-telemetry/
|
||||
@@ -134,11 +141,18 @@ jobs:
|
||||
scp -i ~/.ssh/deploy_key telemetry/grafana-dashboard.json $REMOTE_USER@$REMOTE_HOST:~/seaweedfs-telemetry/
|
||||
scp -i ~/.ssh/deploy_key telemetry/prometheus.yml $REMOTE_USER@$REMOTE_HOST:~/seaweedfs-telemetry/
|
||||
|
||||
# Copy updated service and logrotate files
|
||||
scp -i ~/.ssh/deploy_key telemetry.service telemetry_logrotate $REMOTE_USER@$REMOTE_HOST:~/seaweedfs-telemetry/
|
||||
|
||||
# Check if service exists and deploy accordingly
|
||||
ssh -i ~/.ssh/deploy_key $REMOTE_USER@$REMOTE_HOST "
|
||||
if systemctl list-unit-files telemetry.service >/dev/null 2>&1; then
|
||||
echo 'Service exists, performing update...'
|
||||
set -e
|
||||
sudo systemctl stop telemetry.service
|
||||
sudo mv ~/seaweedfs-telemetry/telemetry.service /etc/systemd/system/
|
||||
sudo mv ~/seaweedfs-telemetry/telemetry_logrotate /etc/logrotate.d/seaweedfs-telemetry
|
||||
sudo systemctl daemon-reload
|
||||
mkdir -p ~/seaweedfs-telemetry/bin
|
||||
mv ~/seaweedfs-telemetry/tmp/telemetry-server ~/seaweedfs-telemetry/bin/
|
||||
chmod +x ~/seaweedfs-telemetry/bin/telemetry-server
|
||||
|
||||
@@ -1,6 +1,10 @@
|
||||
name: 'Dependency Review'
|
||||
on: [pull_request]
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.event.pull_request.number || (github.event_name == 'push' && github.ref) || github.run_id }}
|
||||
cancel-in-progress: ${{ github.event_name == 'pull_request' || (github.event_name == 'push' && !startsWith(github.ref, 'refs/tags/')) }}
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
@@ -9,6 +13,6 @@ jobs:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: 'Checkout Repository'
|
||||
uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8
|
||||
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1
|
||||
- name: 'Dependency Review'
|
||||
uses: actions/dependency-review-action@3c4e3dcb1aa7874d2c16be7d79418e9b7efd6261
|
||||
uses: actions/dependency-review-action@a1d282b36b6f3519aa1f3fc636f609c47dddb294
|
||||
+44
-21
@@ -3,12 +3,24 @@ name: "End to End"
|
||||
on:
|
||||
push:
|
||||
branches: [ master ]
|
||||
paths:
|
||||
- 'weed/**'
|
||||
- 'docker/**'
|
||||
- 'go.mod'
|
||||
- 'go.sum'
|
||||
- '.github/workflows/e2e.yml'
|
||||
pull_request:
|
||||
branches: [ master ]
|
||||
paths:
|
||||
- 'weed/**'
|
||||
- 'docker/**'
|
||||
- 'go.mod'
|
||||
- 'go.sum'
|
||||
- '.github/workflows/e2e.yml'
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.head_ref }}/e2e
|
||||
cancel-in-progress: true
|
||||
group: ${{ github.workflow }}-${{ github.event.pull_request.number || (github.event_name == 'push' && github.ref) || github.run_id }}
|
||||
cancel-in-progress: ${{ github.event_name == 'pull_request' || (github.event_name == 'push' && !startsWith(github.ref, 'refs/tags/')) }}
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
@@ -23,20 +35,26 @@ jobs:
|
||||
runs-on: ubuntu-22.04
|
||||
timeout-minutes: 30
|
||||
steps:
|
||||
- name: Set up Go 1.x
|
||||
uses: actions/setup-go@a5f9b05d2d216f63e13859e0d847461041025775 # v2
|
||||
with:
|
||||
go-version: ^1.13
|
||||
id: go
|
||||
|
||||
- name: Check out code into the Go module directory
|
||||
uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v2
|
||||
uses: actions/checkout@v7
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v7
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
cache: ${{ runner.environment == 'github-hosted' }}
|
||||
|
||||
- name: Configure Docker Hub mirror
|
||||
run: |
|
||||
echo '{"registry-mirrors": ["https://mirror.gcr.io"]}' | sudo tee /etc/docker/daemon.json
|
||||
sudo systemctl restart docker
|
||||
|
||||
- name: Set up Docker Buildx
|
||||
uses: docker/setup-buildx-action@v3
|
||||
uses: docker/setup-buildx-action@v4
|
||||
|
||||
- name: Cache Docker layers
|
||||
uses: actions/cache@v5
|
||||
if: runner.environment == 'github-hosted'
|
||||
uses: actions/cache@v6
|
||||
with:
|
||||
path: /tmp/.buildx-cache
|
||||
key: ${{ runner.os }}-buildx-e2e-${{ github.sha }}
|
||||
@@ -45,25 +63,22 @@ jobs:
|
||||
|
||||
- name: Install dependencies
|
||||
run: |
|
||||
# Use faster mirrors and install with timeout
|
||||
sudo rm -f /etc/apt/sources.list.d/azure-cli.list /etc/apt/sources.list.d/microsoft-prod.list
|
||||
echo "deb http://azure.archive.ubuntu.com/ubuntu/ $(lsb_release -cs) main restricted universe multiverse" | sudo tee /etc/apt/sources.list
|
||||
echo "deb http://azure.archive.ubuntu.com/ubuntu/ $(lsb_release -cs)-updates main restricted universe multiverse" | sudo tee -a /etc/apt/sources.list
|
||||
|
||||
sudo apt-get update --fix-missing
|
||||
sudo DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends fuse
|
||||
# Same helper the e2e image installs through: the runner's own list is
|
||||
# azure-only too, and an outage there fails this step outright.
|
||||
sudo ./apt-install fuse
|
||||
|
||||
# Verify FUSE installation
|
||||
echo "FUSE version: $(fusermount --version 2>&1 || echo 'fusermount not found')"
|
||||
echo "FUSE device: $(ls -la /dev/fuse 2>&1 || echo '/dev/fuse not found')"
|
||||
|
||||
- name: Start SeaweedFS
|
||||
timeout-minutes: 10
|
||||
timeout-minutes: 15
|
||||
run: |
|
||||
# Enable Docker buildkit for better caching
|
||||
export DOCKER_BUILDKIT=1
|
||||
export COMPOSE_DOCKER_CLI_BUILD=1
|
||||
|
||||
|
||||
# Build with retry logic
|
||||
for i in {1..3}; do
|
||||
echo "Build attempt $i/3"
|
||||
@@ -78,10 +93,18 @@ jobs:
|
||||
sleep 30
|
||||
fi
|
||||
done
|
||||
|
||||
|
||||
# Start services with wait
|
||||
docker compose -f ./compose/e2e-mount.yml up --wait
|
||||
|
||||
- name: Rotate buildx cache
|
||||
if: always()
|
||||
run: |
|
||||
# Without this, --cache-to writes to .buildx-cache-new but actions/cache only
|
||||
# uploads .buildx-cache, so layers (notably the slow apt RUN) never persist.
|
||||
rm -rf /tmp/.buildx-cache
|
||||
if [ -d /tmp/.buildx-cache-new ]; then mv /tmp/.buildx-cache-new /tmp/.buildx-cache; fi
|
||||
|
||||
- name: Run FIO 4k
|
||||
timeout-minutes: 15
|
||||
run: |
|
||||
@@ -135,7 +158,7 @@ jobs:
|
||||
|
||||
- name: Archive logs
|
||||
if: always()
|
||||
uses: actions/upload-artifact@v6
|
||||
uses: actions/upload-artifact@v7
|
||||
with:
|
||||
name: output-logs
|
||||
path: docker/output.log
|
||||
|
||||
@@ -3,8 +3,24 @@ name: "EC Integration Tests"
|
||||
on:
|
||||
push:
|
||||
branches: [ master ]
|
||||
paths:
|
||||
- 'weed/**'
|
||||
- 'test/erasure_coding/**'
|
||||
- 'go.mod'
|
||||
- 'go.sum'
|
||||
- '.github/workflows/ec-integration-tests.yml'
|
||||
pull_request:
|
||||
branches: [ master ]
|
||||
paths:
|
||||
- 'weed/**'
|
||||
- 'test/erasure_coding/**'
|
||||
- 'go.mod'
|
||||
- 'go.sum'
|
||||
- '.github/workflows/ec-integration-tests.yml'
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.event.pull_request.number || (github.event_name == 'push' && github.ref) || github.run_id }}
|
||||
cancel-in-progress: ${{ github.event_name == 'pull_request' || (github.event_name == 'push' && !startsWith(github.ref, 'refs/tags/')) }}
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
@@ -16,13 +32,14 @@ jobs:
|
||||
timeout-minutes: 30
|
||||
steps:
|
||||
- name: Set up Go 1.x
|
||||
uses: actions/setup-go@v6
|
||||
uses: actions/setup-go@v7
|
||||
with:
|
||||
go-version: ^1.24
|
||||
cache: ${{ runner.environment == 'github-hosted' }}
|
||||
go-version: ^1.26
|
||||
id: go
|
||||
|
||||
- name: Check out code into the Go module directory
|
||||
uses: actions/checkout@v6
|
||||
uses: actions/checkout@v7
|
||||
|
||||
- name: Build weed binary
|
||||
run: |
|
||||
@@ -31,7 +48,10 @@ jobs:
|
||||
- name: Run EC Integration Tests
|
||||
working-directory: test/erasure_coding
|
||||
run: |
|
||||
go test -v
|
||||
# The suite now includes the interruption matrix and runs close to Go's
|
||||
# default 10m binary timeout on slower runners; bound it by the job's
|
||||
# 30m budget instead.
|
||||
go test -v -timeout 25m
|
||||
|
||||
- name: Collect server logs on failure
|
||||
if: failure()
|
||||
@@ -52,7 +72,7 @@ jobs:
|
||||
|
||||
- name: Archive logs
|
||||
if: failure()
|
||||
uses: actions/upload-artifact@v6
|
||||
uses: actions/upload-artifact@v7
|
||||
with:
|
||||
name: ec-integration-test-logs
|
||||
path: |
|
||||
|
||||
@@ -1,49 +0,0 @@
|
||||
name: EC Integration Tests
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [ master ]
|
||||
paths:
|
||||
- 'weed/admin/**'
|
||||
- 'weed/worker/**'
|
||||
- 'test/erasure_coding/admin_dockertest/**'
|
||||
- '.github/workflows/ec-integration.yml'
|
||||
pull_request:
|
||||
branches: [ master ]
|
||||
paths:
|
||||
- 'weed/admin/**'
|
||||
- 'weed/worker/**'
|
||||
- 'test/erasure_coding/admin_dockertest/**'
|
||||
- '.github/workflows/ec-integration.yml'
|
||||
|
||||
jobs:
|
||||
ec-integration-test:
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 15
|
||||
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v6
|
||||
with:
|
||||
go-version: '1.24'
|
||||
|
||||
- name: Build weed binary
|
||||
run: |
|
||||
cd weed
|
||||
go build -o ../weed_bin
|
||||
|
||||
- name: Run EC integration tests
|
||||
run: |
|
||||
cd test/erasure_coding/admin_dockertest
|
||||
go test -v -timeout 15m ec_integration_test.go
|
||||
|
||||
- name: Upload test logs on failure
|
||||
if: failure()
|
||||
uses: actions/upload-artifact@v6
|
||||
with:
|
||||
name: ec-test-logs
|
||||
path: test/erasure_coding/admin_dockertest/tmp/logs/
|
||||
retention-days: 7
|
||||
@@ -0,0 +1,70 @@
|
||||
name: "FUSE DLM Integration Tests"
|
||||
|
||||
on:
|
||||
pull_request:
|
||||
paths:
|
||||
- 'weed/command/mount*.go'
|
||||
- 'weed/mount/**'
|
||||
- 'weed/cluster/**'
|
||||
- 'test/fuse_dlm/**'
|
||||
- '.github/workflows/fuse-dlm-integration.yml'
|
||||
- '.github/actions/fix-fusermount-setuid/**'
|
||||
push:
|
||||
branches: [master]
|
||||
paths:
|
||||
- 'weed/command/mount*.go'
|
||||
- 'weed/mount/**'
|
||||
- 'weed/cluster/**'
|
||||
- 'test/fuse_dlm/**'
|
||||
- '.github/actions/fix-fusermount-setuid/**'
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.event.pull_request.number || (github.event_name == 'push' && github.ref) || github.run_id }}
|
||||
cancel-in-progress: ${{ github.event_name == 'pull_request' || (github.event_name == 'push' && !startsWith(github.ref, 'refs/tags/')) }}
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
jobs:
|
||||
fuse-dlm-integration:
|
||||
name: FUSE DLM Integration Tests
|
||||
runs-on: ubuntu-22.04
|
||||
timeout-minutes: 30
|
||||
|
||||
steps:
|
||||
- name: Check out code
|
||||
uses: actions/checkout@v7
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v7
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
cache: ${{ runner.environment == 'github-hosted' }}
|
||||
|
||||
- name: Configure FUSE
|
||||
run: |
|
||||
# Nothing to install: fuse3 ships fusermount3 and is pre-installed,
|
||||
# and go-fuse is pure Go, so the libfuse headers were never linked
|
||||
# against.
|
||||
echo 'user_allow_other' | sudo tee -a /etc/fuse.conf
|
||||
sudo chmod 644 /etc/fuse.conf
|
||||
|
||||
- name: Repair the fusermount3 setuid bit
|
||||
uses: ./.github/actions/fix-fusermount-setuid
|
||||
|
||||
- name: Build SeaweedFS
|
||||
run: go build -o weed/weed -buildvcs=false ./weed
|
||||
|
||||
- name: Run DLM integration tests
|
||||
timeout-minutes: 25
|
||||
env:
|
||||
WEED_BINARY: ${{ github.workspace }}/weed/weed
|
||||
run: go test -v -count=1 -timeout=20m ./test/fuse_dlm/...
|
||||
|
||||
- name: Upload logs on failure
|
||||
if: failure()
|
||||
uses: actions/upload-artifact@v7
|
||||
with:
|
||||
name: fuse-dlm-test-logs
|
||||
path: /tmp/seaweedfs-fuse-dlm-logs/
|
||||
retention-days: 3
|
||||
@@ -0,0 +1,80 @@
|
||||
name: "FUSE Volume Server Failover Tests"
|
||||
|
||||
on:
|
||||
pull_request:
|
||||
paths:
|
||||
- 'weed/command/mount*.go'
|
||||
- 'weed/mount/**'
|
||||
- 'weed/filer/**'
|
||||
- 'weed/wdclient/**'
|
||||
- 'weed/operation/upload_content.go'
|
||||
- 'test/fuse_failover/**'
|
||||
- '.github/workflows/fuse-failover.yml'
|
||||
- '.github/actions/fix-fusermount-setuid/**'
|
||||
push:
|
||||
branches: [master]
|
||||
paths:
|
||||
- 'weed/command/mount*.go'
|
||||
- 'weed/mount/**'
|
||||
- 'weed/filer/**'
|
||||
- 'weed/wdclient/**'
|
||||
- 'weed/operation/upload_content.go'
|
||||
- 'test/fuse_failover/**'
|
||||
- '.github/workflows/fuse-failover.yml'
|
||||
- '.github/actions/fix-fusermount-setuid/**'
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.event.pull_request.number || (github.event_name == 'push' && github.ref) || github.run_id }}
|
||||
cancel-in-progress: ${{ github.event_name == 'pull_request' || (github.event_name == 'push' && !startsWith(github.ref, 'refs/tags/')) }}
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
jobs:
|
||||
fuse-failover:
|
||||
name: FUSE Volume Server Failover
|
||||
runs-on: ubuntu-22.04
|
||||
timeout-minutes: 40
|
||||
|
||||
steps:
|
||||
- name: Check out code
|
||||
uses: actions/checkout@v7
|
||||
with:
|
||||
persist-credentials: false
|
||||
|
||||
- name: Clean up stale FUSE mounts
|
||||
uses: ./.github/actions/cleanup-stale-fuse-mounts
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v7
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
cache: ${{ runner.environment == 'github-hosted' }}
|
||||
|
||||
- name: Configure FUSE
|
||||
run: |
|
||||
# Nothing to install: fuse3 ships fusermount3 and is pre-installed,
|
||||
# and go-fuse is pure Go, so the libfuse headers were never linked
|
||||
# against.
|
||||
echo 'user_allow_other' | sudo tee -a /etc/fuse.conf
|
||||
sudo chmod 644 /etc/fuse.conf
|
||||
|
||||
- name: Repair the fusermount3 setuid bit
|
||||
uses: ./.github/actions/fix-fusermount-setuid
|
||||
|
||||
- name: Build SeaweedFS
|
||||
run: go build -o weed/weed -buildvcs=false ./weed
|
||||
|
||||
- name: Run failover integration tests
|
||||
timeout-minutes: 35
|
||||
env:
|
||||
WEED_BINARY: ${{ github.workspace }}/weed/weed
|
||||
run: go test -v -count=1 -timeout=30m ./test/fuse_failover/...
|
||||
|
||||
- name: Upload logs on failure
|
||||
if: failure()
|
||||
uses: actions/upload-artifact@v7
|
||||
with:
|
||||
name: fuse-failover-test-logs
|
||||
path: /tmp/seaweedfs-fuse-failover-logs/
|
||||
retention-days: 3
|
||||
@@ -7,228 +7,79 @@ on:
|
||||
- 'weed/**'
|
||||
- 'test/fuse_integration/**'
|
||||
- '.github/workflows/fuse-integration.yml'
|
||||
- '.github/actions/fix-fusermount-setuid/**'
|
||||
pull_request:
|
||||
branches: [ master, main ]
|
||||
paths:
|
||||
- 'weed/**'
|
||||
- 'test/fuse_integration/**'
|
||||
- '.github/workflows/fuse-integration.yml'
|
||||
- '.github/actions/fix-fusermount-setuid/**'
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.head_ref }}/fuse-integration
|
||||
cancel-in-progress: true
|
||||
group: ${{ github.workflow }}-${{ github.event.pull_request.number || (github.event_name == 'push' && github.ref) || github.run_id }}
|
||||
cancel-in-progress: ${{ github.event_name == 'pull_request' || (github.event_name == 'push' && !startsWith(github.ref, 'refs/tags/')) }}
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
env:
|
||||
GO_VERSION: '1.24'
|
||||
TEST_TIMEOUT: '45m'
|
||||
|
||||
jobs:
|
||||
fuse-integration:
|
||||
name: FUSE Integration Testing
|
||||
runs-on: ubuntu-22.04
|
||||
timeout-minutes: 50
|
||||
|
||||
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: Set up Go ${{ env.GO_VERSION }}
|
||||
uses: actions/setup-go@v6
|
||||
uses: actions/checkout@v7
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v7
|
||||
with:
|
||||
go-version: ${{ env.GO_VERSION }}
|
||||
|
||||
- name: Install FUSE and dependencies
|
||||
go-version-file: 'go.mod'
|
||||
cache: ${{ runner.environment == 'github-hosted' }}
|
||||
|
||||
- name: Configure FUSE
|
||||
run: |
|
||||
sudo apt-get update
|
||||
sudo apt-get install -y fuse libfuse-dev
|
||||
# Nothing to install: fuse3 ships fusermount3 and is pre-installed, and
|
||||
# go-fuse is pure Go, so the libfuse headers were never linked against.
|
||||
# Allow non-root FUSE mounts with allow_other
|
||||
echo 'user_allow_other' | sudo tee -a /etc/fuse.conf
|
||||
sudo chmod 644 /etc/fuse.conf
|
||||
# Verify FUSE installation
|
||||
fusermount --version || true
|
||||
ls -la /dev/fuse || true
|
||||
|
||||
fusermount3 --version || fusermount --version || true
|
||||
ls -la /dev/fuse
|
||||
|
||||
- name: Repair the fusermount3 setuid bit
|
||||
uses: ./.github/actions/fix-fusermount-setuid
|
||||
|
||||
- name: Build SeaweedFS
|
||||
run: |
|
||||
cd weed
|
||||
go build -tags "elastic gocdk sqlite ydb tarantool tikv rclone" -v .
|
||||
go build -tags "elastic gocdk sqlite ydb tarantool tikv rclone" -o weed .
|
||||
chmod +x weed
|
||||
# Verify binary
|
||||
./weed version
|
||||
|
||||
- name: Prepare FUSE Integration Tests
|
||||
# Make weed binary available in PATH for the test framework
|
||||
sudo cp weed /usr/local/bin/weed
|
||||
|
||||
- name: Install test dependencies
|
||||
run: |
|
||||
# Create isolated test directory to avoid Go module conflicts
|
||||
mkdir -p /tmp/seaweedfs-fuse-tests
|
||||
|
||||
# Copy only the working test files to avoid Go module conflicts
|
||||
# These are the files we've verified work without package name issues
|
||||
cp test/fuse_integration/simple_test.go /tmp/seaweedfs-fuse-tests/ 2>/dev/null || echo "⚠️ simple_test.go not found"
|
||||
cp test/fuse_integration/working_demo_test.go /tmp/seaweedfs-fuse-tests/ 2>/dev/null || echo "⚠️ working_demo_test.go not found"
|
||||
|
||||
# Note: Other test files (framework.go, basic_operations_test.go, etc.)
|
||||
# have Go module conflicts and are skipped until resolved
|
||||
|
||||
echo "📁 Working test files copied:"
|
||||
ls -la /tmp/seaweedfs-fuse-tests/*.go 2>/dev/null || echo "ℹ️ No test files found"
|
||||
|
||||
# Initialize Go module in isolated directory
|
||||
cd /tmp/seaweedfs-fuse-tests
|
||||
go mod init seaweedfs-fuse-tests
|
||||
go mod tidy
|
||||
|
||||
# Verify setup
|
||||
echo "✅ FUSE integration test environment prepared"
|
||||
ls -la /tmp/seaweedfs-fuse-tests/
|
||||
|
||||
echo ""
|
||||
echo "ℹ️ Current Status: Running working subset of FUSE tests"
|
||||
echo " • simple_test.go: Package structure verification"
|
||||
echo " • working_demo_test.go: Framework capability demonstration"
|
||||
echo " • Full framework: Available in test/fuse_integration/ (module conflicts pending resolution)"
|
||||
|
||||
cd test/fuse_integration
|
||||
go mod download
|
||||
|
||||
- name: Run FUSE Integration Tests
|
||||
run: |
|
||||
cd /tmp/seaweedfs-fuse-tests
|
||||
|
||||
echo "🧪 Running FUSE integration tests..."
|
||||
echo "============================================"
|
||||
|
||||
# Run available working test files
|
||||
TESTS_RUN=0
|
||||
|
||||
if [ -f "simple_test.go" ]; then
|
||||
echo "📋 Running simple_test.go..."
|
||||
go test -v -timeout=${{ env.TEST_TIMEOUT }} simple_test.go
|
||||
TESTS_RUN=$((TESTS_RUN + 1))
|
||||
fi
|
||||
|
||||
if [ -f "working_demo_test.go" ]; then
|
||||
echo "📋 Running working_demo_test.go..."
|
||||
go test -v -timeout=${{ env.TEST_TIMEOUT }} working_demo_test.go
|
||||
TESTS_RUN=$((TESTS_RUN + 1))
|
||||
fi
|
||||
|
||||
# Run combined test if multiple files exist
|
||||
if [ -f "simple_test.go" ] && [ -f "working_demo_test.go" ]; then
|
||||
echo "📋 Running combined tests..."
|
||||
go test -v -timeout=${{ env.TEST_TIMEOUT }} simple_test.go working_demo_test.go
|
||||
fi
|
||||
|
||||
if [ $TESTS_RUN -eq 0 ]; then
|
||||
echo "⚠️ No working test files found, running module verification only"
|
||||
go version
|
||||
go mod verify
|
||||
else
|
||||
echo "✅ Successfully ran $TESTS_RUN test file(s)"
|
||||
fi
|
||||
|
||||
echo "============================================"
|
||||
echo "✅ FUSE integration tests completed"
|
||||
|
||||
- name: Run Extended Framework Validation
|
||||
run: |
|
||||
cd /tmp/seaweedfs-fuse-tests
|
||||
|
||||
echo "🔍 Running extended framework validation..."
|
||||
echo "============================================"
|
||||
|
||||
# Test individual components (only run tests that exist)
|
||||
if [ -f "simple_test.go" ]; then
|
||||
echo "Testing simple verification..."
|
||||
go test -v simple_test.go
|
||||
fi
|
||||
|
||||
if [ -f "working_demo_test.go" ]; then
|
||||
echo "Testing framework demo..."
|
||||
go test -v working_demo_test.go
|
||||
fi
|
||||
|
||||
# Test combined execution if both files exist
|
||||
if [ -f "simple_test.go" ] && [ -f "working_demo_test.go" ]; then
|
||||
echo "Testing combined execution..."
|
||||
go test -v simple_test.go working_demo_test.go
|
||||
elif [ -f "simple_test.go" ] || [ -f "working_demo_test.go" ]; then
|
||||
echo "✅ Individual tests already validated above"
|
||||
else
|
||||
echo "⚠️ No working test files found for combined testing"
|
||||
fi
|
||||
|
||||
echo "============================================"
|
||||
echo "✅ Extended validation completed"
|
||||
|
||||
- name: Generate Test Coverage Report
|
||||
run: |
|
||||
cd /tmp/seaweedfs-fuse-tests
|
||||
|
||||
echo "📊 Generating test coverage report..."
|
||||
go test -v -coverprofile=coverage.out .
|
||||
go tool cover -html=coverage.out -o coverage.html
|
||||
|
||||
echo "Coverage report generated: coverage.html"
|
||||
|
||||
- name: Verify SeaweedFS Binary Integration
|
||||
run: |
|
||||
# Test that SeaweedFS binary is accessible from test environment
|
||||
WEED_BINARY=$(pwd)/weed/weed
|
||||
|
||||
if [ -f "$WEED_BINARY" ]; then
|
||||
echo "✅ SeaweedFS binary found at: $WEED_BINARY"
|
||||
$WEED_BINARY version
|
||||
echo "Binary is ready for full integration testing"
|
||||
else
|
||||
echo "❌ SeaweedFS binary not found"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
- name: Upload Test Artifacts
|
||||
set -o pipefail
|
||||
cd test/fuse_integration
|
||||
echo "Running full FUSE integration test suite..."
|
||||
go test -v -count=1 -timeout=45m ./... 2>&1 | tee /tmp/fuse-test-output.log
|
||||
|
||||
- name: Upload Test Logs
|
||||
if: always()
|
||||
uses: actions/upload-artifact@v6
|
||||
uses: actions/upload-artifact@v7
|
||||
with:
|
||||
name: fuse-integration-test-results
|
||||
path: |
|
||||
/tmp/seaweedfs-fuse-tests/coverage.out
|
||||
/tmp/seaweedfs-fuse-tests/coverage.html
|
||||
/tmp/seaweedfs-fuse-tests/*.log
|
||||
/tmp/fuse-test-output.log
|
||||
/tmp/seaweedfs-fuse-logs/
|
||||
retention-days: 7
|
||||
|
||||
- name: Test Summary
|
||||
if: always()
|
||||
run: |
|
||||
echo "## 🚀 FUSE Integration Test Summary" >> $GITHUB_STEP_SUMMARY
|
||||
echo "" >> $GITHUB_STEP_SUMMARY
|
||||
echo "### Framework Status" >> $GITHUB_STEP_SUMMARY
|
||||
echo "- ✅ **Framework Design**: Complete and validated" >> $GITHUB_STEP_SUMMARY
|
||||
echo "- ✅ **Working Tests**: Core framework demonstration functional" >> $GITHUB_STEP_SUMMARY
|
||||
echo "- ⚠️ **Full Framework**: Available but requires Go module resolution" >> $GITHUB_STEP_SUMMARY
|
||||
echo "- ✅ **CI/CD Integration**: Automated testing pipeline established" >> $GITHUB_STEP_SUMMARY
|
||||
echo "" >> $GITHUB_STEP_SUMMARY
|
||||
echo "### Test Capabilities" >> $GITHUB_STEP_SUMMARY
|
||||
echo "- 📁 **File Operations**: Create, read, write, delete, permissions" >> $GITHUB_STEP_SUMMARY
|
||||
echo "- 📂 **Directory Operations**: Create, list, delete, nested structures" >> $GITHUB_STEP_SUMMARY
|
||||
echo "- 📊 **Large Files**: Multi-megabyte file handling" >> $GITHUB_STEP_SUMMARY
|
||||
echo "- 🔄 **Concurrent Operations**: Multi-threaded stress testing" >> $GITHUB_STEP_SUMMARY
|
||||
echo "- ⚠️ **Error Scenarios**: Comprehensive error handling validation" >> $GITHUB_STEP_SUMMARY
|
||||
echo "" >> $GITHUB_STEP_SUMMARY
|
||||
echo "### Comparison with Current Tests" >> $GITHUB_STEP_SUMMARY
|
||||
echo "| Aspect | Current (FIO) | This Framework |" >> $GITHUB_STEP_SUMMARY
|
||||
echo "|--------|---------------|----------------|" >> $GITHUB_STEP_SUMMARY
|
||||
echo "| **Scope** | Performance only | Functional + Performance |" >> $GITHUB_STEP_SUMMARY
|
||||
echo "| **Operations** | Read/Write only | All FUSE operations |" >> $GITHUB_STEP_SUMMARY
|
||||
echo "| **Concurrency** | Single-threaded | Multi-threaded stress tests |" >> $GITHUB_STEP_SUMMARY
|
||||
echo "| **Automation** | Manual setup | Fully automated |" >> $GITHUB_STEP_SUMMARY
|
||||
echo "| **Validation** | Speed metrics | Correctness + Performance |" >> $GITHUB_STEP_SUMMARY
|
||||
echo "" >> $GITHUB_STEP_SUMMARY
|
||||
echo "### Current Working Tests" >> $GITHUB_STEP_SUMMARY
|
||||
echo "- ✅ **Framework Structure**: Package and module verification" >> $GITHUB_STEP_SUMMARY
|
||||
echo "- ✅ **Configuration Management**: Test config validation" >> $GITHUB_STEP_SUMMARY
|
||||
echo "- ✅ **File Operations Demo**: Basic file create/read/write simulation" >> $GITHUB_STEP_SUMMARY
|
||||
echo "- ✅ **Large File Handling**: 1MB+ file processing demonstration" >> $GITHUB_STEP_SUMMARY
|
||||
echo "- ✅ **Concurrency Simulation**: Multi-file operation testing" >> $GITHUB_STEP_SUMMARY
|
||||
echo "" >> $GITHUB_STEP_SUMMARY
|
||||
echo "### Next Steps" >> $GITHUB_STEP_SUMMARY
|
||||
echo "1. **Module Resolution**: Fix Go package conflicts for full framework" >> $GITHUB_STEP_SUMMARY
|
||||
echo "2. **SeaweedFS Integration**: Connect with real cluster for end-to-end testing" >> $GITHUB_STEP_SUMMARY
|
||||
echo "3. **Performance Benchmarks**: Add performance regression testing" >> $GITHUB_STEP_SUMMARY
|
||||
echo "" >> $GITHUB_STEP_SUMMARY
|
||||
echo "📈 **Total Framework Size**: ~1,500 lines of comprehensive testing infrastructure" >> $GITHUB_STEP_SUMMARY
|
||||
@@ -0,0 +1,79 @@
|
||||
name: "FUSE P2P Peer Chunk Sharing Integration Tests"
|
||||
|
||||
on:
|
||||
pull_request:
|
||||
paths:
|
||||
- 'weed/command/mount*.go'
|
||||
- 'weed/mount/**'
|
||||
- 'weed/filer/mount_peer_registry*.go'
|
||||
- 'weed/server/filer_grpc_server_mount_peer.go'
|
||||
- 'weed/pb/mount_peer.proto'
|
||||
- 'weed/pb/filer.proto'
|
||||
- 'test/fuse_p2p/**'
|
||||
- '.github/workflows/fuse-p2p-integration.yml'
|
||||
- '.github/actions/fix-fusermount-setuid/**'
|
||||
push:
|
||||
branches: [master]
|
||||
paths:
|
||||
- 'weed/command/mount*.go'
|
||||
- 'weed/mount/**'
|
||||
- 'weed/filer/mount_peer_registry*.go'
|
||||
- 'weed/server/filer_grpc_server_mount_peer.go'
|
||||
- 'weed/pb/mount_peer.proto'
|
||||
- 'weed/pb/filer.proto'
|
||||
- 'test/fuse_p2p/**'
|
||||
- '.github/actions/fix-fusermount-setuid/**'
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.event.pull_request.number || (github.event_name == 'push' && github.ref) || github.run_id }}
|
||||
cancel-in-progress: ${{ github.event_name == 'pull_request' || (github.event_name == 'push' && !startsWith(github.ref, 'refs/tags/')) }}
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
jobs:
|
||||
fuse-p2p-integration:
|
||||
name: FUSE P2P Peer Chunk Sharing
|
||||
runs-on: ubuntu-22.04
|
||||
timeout-minutes: 20
|
||||
|
||||
steps:
|
||||
- name: Check out code
|
||||
uses: actions/checkout@v7
|
||||
|
||||
- name: Clean up stale FUSE mounts
|
||||
uses: ./.github/actions/cleanup-stale-fuse-mounts
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v7
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
cache: ${{ runner.environment == 'github-hosted' }}
|
||||
|
||||
- name: Configure FUSE
|
||||
run: |
|
||||
# Nothing to install: fuse3 ships fusermount3 and is pre-installed,
|
||||
# and go-fuse is pure Go, so the libfuse headers were never linked
|
||||
# against.
|
||||
echo 'user_allow_other' | sudo tee -a /etc/fuse.conf
|
||||
sudo chmod 644 /etc/fuse.conf
|
||||
|
||||
- name: Repair the fusermount3 setuid bit
|
||||
uses: ./.github/actions/fix-fusermount-setuid
|
||||
|
||||
- name: Build SeaweedFS
|
||||
run: go build -o weed/weed -buildvcs=false ./weed
|
||||
|
||||
- name: Run P2P integration tests
|
||||
timeout-minutes: 15
|
||||
env:
|
||||
WEED_BINARY: ${{ github.workspace }}/weed/weed
|
||||
run: go test -v -count=1 -timeout=12m ./test/fuse_p2p/...
|
||||
|
||||
- name: Upload logs on failure
|
||||
if: failure()
|
||||
uses: actions/upload-artifact@v7
|
||||
with:
|
||||
name: fuse-p2p-test-logs
|
||||
path: /tmp/seaweedfs-fuse-p2p-logs/
|
||||
retention-days: 3
|
||||
+128
-20
@@ -3,12 +3,22 @@ name: "go: build binary"
|
||||
on:
|
||||
push:
|
||||
branches: [ master ]
|
||||
paths:
|
||||
- '**/*.go'
|
||||
- 'go.mod'
|
||||
- 'go.sum'
|
||||
- '.github/workflows/go.yml'
|
||||
pull_request:
|
||||
branches: [ master ]
|
||||
paths:
|
||||
- '**/*.go'
|
||||
- 'go.mod'
|
||||
- 'go.sum'
|
||||
- '.github/workflows/go.yml'
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.head_ref }}/go
|
||||
cancel-in-progress: true
|
||||
group: ${{ github.workflow }}-${{ github.event.pull_request.number || (github.event_name == 'push' && github.ref) || github.run_id }}
|
||||
cancel-in-progress: ${{ github.event_name == 'pull_request' || (github.event_name == 'push' && !startsWith(github.ref, 'refs/tags/')) }}
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
@@ -19,13 +29,13 @@ jobs:
|
||||
name: Go Vet
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Set up Go 1.x
|
||||
uses: actions/setup-go@a5f9b05d2d216f63e13859e0d847461041025775 # v2
|
||||
with:
|
||||
go-version: ^1.13
|
||||
id: go
|
||||
- name: Check out code into the Go module directory
|
||||
uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v2
|
||||
uses: actions/checkout@v7
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v7
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
cache: ${{ runner.environment == 'github-hosted' }}
|
||||
- name: Get dependencies
|
||||
run: |
|
||||
cd weed; go get -v -t -d ./...
|
||||
@@ -38,30 +48,128 @@ jobs:
|
||||
# Fail only if there are actual vet errors (not counting the filtered lock warnings)
|
||||
if grep -q "vet:" vet-output.txt; then exit 1; fi
|
||||
|
||||
vet-32bit:
|
||||
name: Go Vet 32-bit
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Check out code into the Go module directory
|
||||
uses: actions/checkout@v7
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v7
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
cache: ${{ runner.environment == 'github-hosted' }}
|
||||
- name: Go Vet linux/386 (type-checks code and tests for 32-bit int overflows)
|
||||
run: |
|
||||
GOOS=linux GOARCH=386 go vet ./... 2>&1 | grep -v "MessageState contains sync.Mutex" | grep -v "IdentityAccessManagement contains sync.RWMutex" | tee vet-32bit-output.txt
|
||||
if grep -q "vet:" vet-32bit-output.txt; then exit 1; fi
|
||||
|
||||
build:
|
||||
name: Build
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Set up Go 1.x
|
||||
uses: actions/setup-go@a5f9b05d2d216f63e13859e0d847461041025775 # v2
|
||||
with:
|
||||
go-version: ^1.13
|
||||
id: go
|
||||
- name: Check out code into the Go module directory
|
||||
uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v2
|
||||
uses: actions/checkout@v7
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v7
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
cache: ${{ runner.environment == 'github-hosted' }}
|
||||
- name: Build
|
||||
run: cd weed; go build -tags "elastic gocdk sqlite ydb tarantool tikv rclone" -v .
|
||||
|
||||
build-cross:
|
||||
name: Build cross-platform (${{ matrix.goos }}/${{ matrix.goarch }})
|
||||
runs-on: ubuntu-latest
|
||||
strategy:
|
||||
# One target's breakage should not hide the other three.
|
||||
fail-fast: false
|
||||
matrix:
|
||||
include:
|
||||
- { goos: windows, goarch: amd64 }
|
||||
- { goos: windows, goarch: arm64 }
|
||||
- { goos: freebsd, goarch: amd64 }
|
||||
- { goos: darwin, goarch: arm64 }
|
||||
steps:
|
||||
- name: Check out code into the Go module directory
|
||||
uses: actions/checkout@v7
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v7
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
cache: ${{ runner.environment == 'github-hosted' }}
|
||||
# Nothing else on a PR compiles these, so per-OS syscall constants creep
|
||||
# back into shared files unnoticed and only a release build catches it.
|
||||
- name: Cross-compile
|
||||
env:
|
||||
GOOS: ${{ matrix.goos }}
|
||||
GOARCH: ${{ matrix.goarch }}
|
||||
run: |
|
||||
go build ./weed/...
|
||||
# Tests too: they reach for per-OS syscall constants the build does
|
||||
# not, and only compiling them catches an untagged one.
|
||||
go vet ./weed/mount/... ./weed/command/... 2>&1 |
|
||||
grep -v "MessageState contains sync.Mutex" | tee /tmp/vet.txt
|
||||
if grep -q "vet:" /tmp/vet.txt; then exit 1; fi
|
||||
|
||||
test:
|
||||
name: Test
|
||||
runs-on: ubuntu-latest
|
||||
services:
|
||||
redis:
|
||||
image: redis:8
|
||||
ports:
|
||||
- 6379:6379
|
||||
options: >-
|
||||
--health-cmd "redis-cli ping"
|
||||
--health-interval 10s
|
||||
--health-timeout 5s
|
||||
--health-retries 5
|
||||
steps:
|
||||
- name: Set up Go 1.x
|
||||
uses: actions/setup-go@a5f9b05d2d216f63e13859e0d847461041025775 # v2
|
||||
with:
|
||||
go-version: ^1.13
|
||||
id: go
|
||||
- name: Check out code into the Go module directory
|
||||
uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v2
|
||||
uses: actions/checkout@v7
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v7
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
cache: ${{ runner.environment == 'github-hosted' }}
|
||||
- name: Test
|
||||
env:
|
||||
RUN_REDIS_TESTS: "1"
|
||||
run: cd weed; go test -tags "elastic gocdk sqlite ydb tarantool tikv rclone" -v ./...
|
||||
|
||||
test-32bit:
|
||||
name: Test 32-bit
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Check out code into the Go module directory
|
||||
uses: actions/checkout@v7
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v7
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
cache: ${{ runner.environment == 'github-hosted' }}
|
||||
# 386 test binaries run natively on the amd64 runner. This catches what vet
|
||||
# can't: unaligned 64-bit atomics and arithmetic that wraps at runtime.
|
||||
# -short skips the e2e suites already covered on amd64.
|
||||
- name: Test linux/386
|
||||
run: cd weed; GOOS=linux GOARCH=386 go test -short ./...
|
||||
|
||||
build-azurekms:
|
||||
name: Build and test with azurekms tag
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Check out code into the Go module directory
|
||||
uses: actions/checkout@v7
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v7
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
cache: ${{ runner.environment == 'github-hosted' }}
|
||||
# weed/kms/azure is excluded from every default build, so without this
|
||||
# nothing compiles it and tag-gated code silently rots.
|
||||
- name: Build and test with azurekms
|
||||
run: |
|
||||
cd weed
|
||||
go build -tags azurekms ./...
|
||||
go test -tags azurekms ./kms/...
|
||||
+2020
-26
File diff suppressed because it is too large.
Load diff
@@ -1,21 +1,25 @@
|
||||
name: "helm: manual release"
|
||||
name: "helm: release"
|
||||
|
||||
on:
|
||||
push:
|
||||
tags:
|
||||
- '*'
|
||||
workflow_dispatch:
|
||||
|
||||
permissions:
|
||||
contents: write
|
||||
pages: write
|
||||
packages: write
|
||||
|
||||
jobs:
|
||||
helm-release:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v6
|
||||
- uses: actions/checkout@v7
|
||||
with:
|
||||
fetch-depth: 0
|
||||
|
||||
- name: Publish Helm charts
|
||||
- name: Publish Helm charts to github pages
|
||||
uses: stefanprodan/helm-gh-pages@v1.7.0
|
||||
with:
|
||||
token: ${{ secrets.GITHUB_TOKEN }}
|
||||
@@ -23,3 +27,15 @@ jobs:
|
||||
target_dir: helm
|
||||
branch: gh-pages
|
||||
helm_version: "3.18.4"
|
||||
|
||||
- name: Publish Helm charts to github container registry (ghcr.io)
|
||||
uses: bitdeps/helm-oci-charts-releaser@v0.1.5
|
||||
with:
|
||||
charts_dir: k8s/charts
|
||||
github_token: ${{ secrets.GITHUB_TOKEN }}
|
||||
oci_registry: ghcr.io/${{ github.repository_owner }}
|
||||
oci_username: github-actions
|
||||
oci_password: ${{ secrets.GITHUB_TOKEN }}
|
||||
skip_dependencies: true
|
||||
skip_helm_install: true
|
||||
skip_gh_release: true
|
||||
@@ -14,6 +14,10 @@ on:
|
||||
- 'weed/**'
|
||||
- '.github/workflows/java_integration_tests.yml'
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.event.pull_request.number || (github.event_name == 'push' && github.ref) || github.run_id }}
|
||||
cancel-in-progress: ${{ github.event_name == 'pull_request' || (github.event_name == 'push' && !startsWith(github.ref, 'refs/tags/')) }}
|
||||
|
||||
jobs:
|
||||
test:
|
||||
name: Java Integration Tests
|
||||
@@ -25,20 +29,21 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v6
|
||||
uses: actions/checkout@v7
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v6
|
||||
uses: actions/setup-go@v7
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
cache: ${{ runner.environment == 'github-hosted' }}
|
||||
id: go
|
||||
|
||||
- name: Set up Java
|
||||
uses: actions/setup-java@v5
|
||||
uses: actions/setup-java@v6
|
||||
with:
|
||||
java-version: ${{ matrix.java }}
|
||||
distribution: 'temurin'
|
||||
cache: 'maven'
|
||||
cache: ${{ runner.environment == 'github-hosted' && 'maven' || '' }}
|
||||
|
||||
- name: Build SeaweedFS
|
||||
run: |
|
||||
@@ -55,6 +60,7 @@ jobs:
|
||||
# Start SeaweedFS with optimized settings for CI
|
||||
# Include S3 API for s3copier integration tests
|
||||
weed server -dir="$WEED_DATA_DIR" \
|
||||
-ip=127.0.0.1 -ip.bind=127.0.0.1 \
|
||||
-master.raftHashicorp \
|
||||
-master.electionTimeout=1s \
|
||||
-master.volumeSizeLimitMB=100 \
|
||||
@@ -111,7 +117,7 @@ jobs:
|
||||
|
||||
# Wait for S3 API
|
||||
for i in {1..30}; do
|
||||
if curl -s http://localhost:8333/ > /dev/null 2>&1; then
|
||||
if curl -s http://localhost:8333/healthz > /dev/null 2>&1; then
|
||||
echo "✓ S3 API is ready"
|
||||
break
|
||||
fi
|
||||
|
||||
@@ -0,0 +1,183 @@
|
||||
name: "release: java clients"
|
||||
|
||||
# Publishes the SeaweedFS Java clients (seaweedfs-client and
|
||||
# seaweedfs-hadoop3-client) to Maven Central via the Sonatype Central Portal.
|
||||
#
|
||||
# Required repository secrets:
|
||||
# MAVEN_CENTRAL_USERNAME - Central Portal user token username (central.sonatype.com -> Account -> Generate User Token)
|
||||
# MAVEN_CENTRAL_PASSWORD - Central Portal user token password
|
||||
# MAVEN_GPG_PRIVATE_KEY - ASCII-armored signing key:
|
||||
# gpg --homedir <keyring> --armor --export-secret-keys <KEYID> > key.asc
|
||||
# MAVEN_GPG_PASSPHRASE - passphrase for that key
|
||||
#
|
||||
# A plain run publishes to Central. Check dry_run to exercise secrets, GPG
|
||||
# signing, and the build without committing or publishing.
|
||||
|
||||
on:
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
version:
|
||||
description: "Version to publish, e.g. 4.39. Leave blank to use the current SeaweedFS version."
|
||||
type: string
|
||||
required: false
|
||||
dry_run:
|
||||
description: "Build and GPG-sign only; skip the version commit and the Central upload"
|
||||
type: boolean
|
||||
default: false
|
||||
|
||||
permissions:
|
||||
contents: write
|
||||
|
||||
concurrency:
|
||||
group: java-release
|
||||
cancel-in-progress: false
|
||||
|
||||
env:
|
||||
# Java 17 needs these opens for the Maven publishing plugins' reflective access.
|
||||
MAVEN_OPTS: >-
|
||||
--add-opens=java.base/java.util=ALL-UNNAMED
|
||||
--add-opens=java.base/java.lang.reflect=ALL-UNNAMED
|
||||
--add-opens=java.base/java.text=ALL-UNNAMED
|
||||
--add-opens=java.desktop/java.awt.font=ALL-UNNAMED
|
||||
|
||||
jobs:
|
||||
publish:
|
||||
name: Publish to Maven Central
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v7
|
||||
with:
|
||||
# Full history so the version-bump commit can rebase onto a moved master.
|
||||
fetch-depth: 0
|
||||
|
||||
# An explicit input wins; otherwise fall back to the current SeaweedFS
|
||||
# version (MAJOR.MINOR) from constants.go, matching its %d.%02d formatting.
|
||||
- name: Resolve version
|
||||
id: resolve
|
||||
env:
|
||||
INPUT_VERSION: ${{ inputs.version }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
VERSION="$INPUT_VERSION"
|
||||
if [ -z "$VERSION" ]; then
|
||||
CONST=weed/util/version/constants.go
|
||||
MAJOR=$(grep -oP 'MAJOR_VERSION\s*=\s*int32\(\K[0-9]+' "$CONST")
|
||||
MINOR=$(grep -oP 'MINOR_VERSION\s*=\s*int32\(\K[0-9]+' "$CONST")
|
||||
VERSION=$(printf '%d.%02d' "$MAJOR" "$MINOR")
|
||||
echo "No version given; using current SeaweedFS version ${VERSION}."
|
||||
fi
|
||||
if ! [[ "$VERSION" =~ ^[0-9]+\.[0-9]+$ ]]; then
|
||||
echo "::error::version must be MAJOR.MINOR, e.g. 4.39 (got '$VERSION')"
|
||||
exit 1
|
||||
fi
|
||||
echo "version=${VERSION}" >> "$GITHUB_OUTPUT"
|
||||
|
||||
- name: Set up JDK 17
|
||||
uses: actions/setup-java@v6
|
||||
with:
|
||||
java-version: '17'
|
||||
distribution: 'temurin'
|
||||
cache: ${{ runner.environment == 'github-hosted' && 'maven' || '' }}
|
||||
server-id: central
|
||||
server-username: MAVEN_CENTRAL_USERNAME
|
||||
server-password: MAVEN_CENTRAL_PASSWORD
|
||||
gpg-private-key: ${{ secrets.MAVEN_GPG_PRIVATE_KEY }}
|
||||
gpg-passphrase: MAVEN_GPG_PASSPHRASE
|
||||
|
||||
# client carries the version literally; hdfs3 derives its own version and
|
||||
# its seaweedfs-client dependency from the seaweedfs.client.version property.
|
||||
- name: Set versions in poms
|
||||
env:
|
||||
VERSION: ${{ steps.resolve.outputs.version }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
V=org.codehaus.mojo:versions-maven-plugin:2.16.2
|
||||
mvn -B -ntp -f other/java/client/pom.xml "${V}:set" \
|
||||
-DnewVersion="$VERSION" -DgenerateBackupPoms=false
|
||||
mvn -B -ntp -f other/java/hdfs3/pom.xml "${V}:set-property" \
|
||||
-Dproperty=seaweedfs.client.version -DnewVersion="$VERSION" -DgenerateBackupPoms=false
|
||||
|
||||
- name: Commit version bump
|
||||
if: ${{ !inputs.dry_run }}
|
||||
env:
|
||||
VERSION: ${{ steps.resolve.outputs.version }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
if git diff --quiet; then
|
||||
echo "Poms already at ${VERSION}; nothing to commit."
|
||||
exit 0
|
||||
fi
|
||||
git config user.name "github-actions[bot]"
|
||||
git config user.email "41898282+github-actions[bot]@users.noreply.github.com"
|
||||
git add other/java/client/pom.xml other/java/hdfs3/pom.xml
|
||||
git commit -m "java ${VERSION}"
|
||||
# Rebase and retry so a concurrent push to master doesn't lose the bump.
|
||||
for attempt in 1 2 3 4 5; do
|
||||
if git push origin HEAD:master; then
|
||||
exit 0
|
||||
fi
|
||||
echo "push rejected (attempt ${attempt}); rebasing onto latest master"
|
||||
git pull --rebase origin master
|
||||
done
|
||||
echo "::error::could not push version bump after retries"
|
||||
exit 1
|
||||
|
||||
# Tests are skipped here: the integration tests need a live filer, and the
|
||||
# offline unit tests already run on every push in java_unit_tests.yml.
|
||||
# dry_run stops at `install` (build + GPG sign, no upload). A real run deploys.
|
||||
# client goes first either way so its install populates the local repo for hdfs3.
|
||||
- name: Publish seaweedfs-client
|
||||
working-directory: other/java/client
|
||||
env:
|
||||
MAVEN_CENTRAL_USERNAME: ${{ secrets.MAVEN_CENTRAL_USERNAME }}
|
||||
MAVEN_CENTRAL_PASSWORD: ${{ secrets.MAVEN_CENTRAL_PASSWORD }}
|
||||
MAVEN_GPG_PASSPHRASE: ${{ secrets.MAVEN_GPG_PASSPHRASE }}
|
||||
DRY_RUN: ${{ inputs.dry_run }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
[ "$DRY_RUN" = "true" ] && GOAL="clean install" || GOAL="clean deploy"
|
||||
mvn -B -ntp $GOAL -DskipTests
|
||||
|
||||
- name: Publish seaweedfs-hadoop3-client
|
||||
working-directory: other/java/hdfs3
|
||||
env:
|
||||
MAVEN_CENTRAL_USERNAME: ${{ secrets.MAVEN_CENTRAL_USERNAME }}
|
||||
MAVEN_CENTRAL_PASSWORD: ${{ secrets.MAVEN_CENTRAL_PASSWORD }}
|
||||
MAVEN_GPG_PASSPHRASE: ${{ secrets.MAVEN_GPG_PASSPHRASE }}
|
||||
DRY_RUN: ${{ inputs.dry_run }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
[ "$DRY_RUN" = "true" ] && GOAL="clean install" || GOAL="clean deploy"
|
||||
mvn -B -ntp $GOAL -DskipTests
|
||||
|
||||
# Bump every version the wiki quotes for the Java clients. Runs only after
|
||||
# a successful real publish. Uses GITHUB_TOKEN; set a WIKI_TOKEN secret if
|
||||
# the token can't push to the wiki.
|
||||
- name: Update wiki client versions
|
||||
if: ${{ !inputs.dry_run }}
|
||||
env:
|
||||
VERSION: ${{ steps.resolve.outputs.version }}
|
||||
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
WIKI_TOKEN: ${{ secrets.WIKI_TOKEN }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
TOKEN="${WIKI_TOKEN:-$GH_TOKEN}"
|
||||
git clone -q "https://x-access-token:${TOKEN}@github.com/${{ github.repository }}.wiki.git" wiki
|
||||
cd wiki
|
||||
for f in $(grep -rlE "seaweedfs-(client|hadoop3-client)" --include="*.md" .); do
|
||||
NEWV="$VERSION" perl -0777 -i -pe '
|
||||
my $v = $ENV{NEWV};
|
||||
s{(<artifactId>seaweedfs-(?:client|hadoop3-client)</artifactId>\s*<version>)[0-9]+\.[0-9]+(</version>)}{$1$v$2}g;
|
||||
s{(seaweedfs-(?:client|hadoop3-client):)[0-9]+\.[0-9]+}{$1$v}g;
|
||||
s{(seaweedfs-(?:client|hadoop3-client)-)[0-9]+\.[0-9]+(\.jar)}{$1$v$2}g;
|
||||
s{(seaweedfs-(?:client|hadoop3-client)/)[0-9]+\.[0-9]+(/)}{$1$v$2}g;
|
||||
' "$f"
|
||||
done
|
||||
if git diff --quiet; then
|
||||
echo "Wiki already references ${VERSION}."
|
||||
exit 0
|
||||
fi
|
||||
git config user.name "github-actions[bot]"
|
||||
git config user.email "41898282+github-actions[bot]@users.noreply.github.com"
|
||||
git commit -am "java clients ${VERSION}"
|
||||
git push
|
||||
@@ -12,6 +12,10 @@ on:
|
||||
- 'other/java/**'
|
||||
- '.github/workflows/java_unit_tests.yml'
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.event.pull_request.number || (github.event_name == 'push' && github.ref) || github.run_id }}
|
||||
cancel-in-progress: ${{ github.event_name == 'pull_request' || (github.event_name == 'push' && !startsWith(github.ref, 'refs/tags/')) }}
|
||||
|
||||
jobs:
|
||||
test:
|
||||
name: Java Unit Tests
|
||||
@@ -23,14 +27,14 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v6
|
||||
uses: actions/checkout@v7
|
||||
|
||||
- name: Set up Java
|
||||
uses: actions/setup-java@v5
|
||||
uses: actions/setup-java@v6
|
||||
with:
|
||||
java-version: ${{ matrix.java }}
|
||||
distribution: 'temurin'
|
||||
cache: 'maven'
|
||||
cache: ${{ runner.environment == 'github-hosted' && 'maven' || '' }}
|
||||
|
||||
- name: Build and Install SeaweedFS Client
|
||||
working-directory: other/java/client
|
||||
@@ -49,7 +53,7 @@ jobs:
|
||||
|
||||
- name: Upload Test Reports
|
||||
if: always()
|
||||
uses: actions/upload-artifact@v6
|
||||
uses: actions/upload-artifact@v7
|
||||
with:
|
||||
name: test-reports-java-${{ matrix.java }}
|
||||
path: |
|
||||
|
||||
@@ -3,13 +3,29 @@ name: "Kafka Quick Test (Load Test with Schema Registry)"
|
||||
on:
|
||||
push:
|
||||
branches: [ master ]
|
||||
paths:
|
||||
- 'weed/mq/**'
|
||||
- 'weed/pb/mq_pb/**'
|
||||
- 'weed/pb/schema_pb/**'
|
||||
- 'test/kafka/**'
|
||||
- 'go.mod'
|
||||
- 'go.sum'
|
||||
- '.github/workflows/kafka-quicktest.yml'
|
||||
pull_request:
|
||||
branches: [ master ]
|
||||
paths:
|
||||
- 'weed/mq/**'
|
||||
- 'weed/pb/mq_pb/**'
|
||||
- 'weed/pb/schema_pb/**'
|
||||
- 'test/kafka/**'
|
||||
- 'go.mod'
|
||||
- 'go.sum'
|
||||
- '.github/workflows/kafka-quicktest.yml'
|
||||
workflow_dispatch: # Allow manual trigger
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.head_ref }}/kafka-quicktest
|
||||
cancel-in-progress: true
|
||||
group: ${{ github.workflow }}-${{ github.event.pull_request.number || (github.event_name == 'push' && github.ref) || github.run_id }}
|
||||
cancel-in-progress: ${{ github.event_name == 'pull_request' || (github.event_name == 'push' && !startsWith(github.ref, 'refs/tags/')) }}
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
@@ -21,19 +37,24 @@ jobs:
|
||||
timeout-minutes: 15
|
||||
steps:
|
||||
- name: Check out code
|
||||
uses: actions/checkout@v6
|
||||
uses: actions/checkout@v7
|
||||
|
||||
- name: Set up Go 1.x
|
||||
uses: actions/setup-go@v6
|
||||
uses: actions/setup-go@v7
|
||||
with:
|
||||
go-version: ^1.24
|
||||
cache: true
|
||||
go-version: ^1.26
|
||||
cache: ${{ runner.environment == 'github-hosted' }}
|
||||
cache-dependency-path: |
|
||||
**/go.sum
|
||||
id: go
|
||||
|
||||
- name: Configure Docker Hub mirror
|
||||
run: |
|
||||
echo '{"registry-mirrors": ["https://mirror.gcr.io"]}' | sudo tee /etc/docker/daemon.json
|
||||
sudo systemctl restart docker
|
||||
|
||||
- name: Set up Docker Buildx
|
||||
uses: docker/setup-buildx-action@v3
|
||||
uses: docker/setup-buildx-action@v4
|
||||
|
||||
- name: Install dependencies
|
||||
run: |
|
||||
|
||||
@@ -3,12 +3,28 @@ name: "Kafka Gateway Tests"
|
||||
on:
|
||||
push:
|
||||
branches: [ master ]
|
||||
paths:
|
||||
- 'weed/mq/**'
|
||||
- 'weed/pb/mq_pb/**'
|
||||
- 'weed/pb/schema_pb/**'
|
||||
- 'test/kafka/**'
|
||||
- 'go.mod'
|
||||
- 'go.sum'
|
||||
- '.github/workflows/kafka-tests.yml'
|
||||
pull_request:
|
||||
branches: [ master ]
|
||||
paths:
|
||||
- 'weed/mq/**'
|
||||
- 'weed/pb/mq_pb/**'
|
||||
- 'weed/pb/schema_pb/**'
|
||||
- 'test/kafka/**'
|
||||
- 'go.mod'
|
||||
- 'go.sum'
|
||||
- '.github/workflows/kafka-tests.yml'
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.head_ref }}/kafka-tests
|
||||
cancel-in-progress: true
|
||||
group: ${{ github.workflow }}-${{ github.event.pull_request.number || (github.event_name == 'push' && github.ref) || github.run_id }}
|
||||
cancel-in-progress: ${{ github.event_name == 'pull_request' || (github.event_name == 'push' && !startsWith(github.ref, 'refs/tags/')) }}
|
||||
|
||||
# Force different runners for better isolation
|
||||
env:
|
||||
@@ -27,7 +43,7 @@ jobs:
|
||||
matrix:
|
||||
container-id: [unit-tests-1]
|
||||
container:
|
||||
image: golang:1.24-alpine
|
||||
image: golang:1.26-alpine
|
||||
options: --cpus 1.0 --memory 1g --hostname kafka-unit-${{ matrix.container-id }}
|
||||
env:
|
||||
GOMAXPROCS: 1
|
||||
@@ -35,13 +51,14 @@ jobs:
|
||||
CONTAINER_ID: ${{ matrix.container-id }}
|
||||
steps:
|
||||
- name: Set up Go 1.x
|
||||
uses: actions/setup-go@v6
|
||||
uses: actions/setup-go@v7
|
||||
with:
|
||||
go-version: ^1.24
|
||||
cache: ${{ runner.environment == 'github-hosted' }}
|
||||
go-version: ^1.26
|
||||
id: go
|
||||
|
||||
- name: Check out code
|
||||
uses: actions/checkout@v6
|
||||
uses: actions/checkout@v7
|
||||
|
||||
- name: Setup Container Environment
|
||||
run: |
|
||||
@@ -71,7 +88,7 @@ jobs:
|
||||
matrix:
|
||||
container-id: [integration-1]
|
||||
container:
|
||||
image: golang:1.24-alpine
|
||||
image: golang:1.26-alpine
|
||||
options: --cpus 2.0 --memory 2g --ulimit nofile=1024:1024 --hostname kafka-integration-${{ matrix.container-id }}
|
||||
env:
|
||||
GOMAXPROCS: 2
|
||||
@@ -80,13 +97,14 @@ jobs:
|
||||
CONTAINER_ID: ${{ matrix.container-id }}
|
||||
steps:
|
||||
- name: Set up Go 1.x
|
||||
uses: actions/setup-go@v6
|
||||
uses: actions/setup-go@v7
|
||||
with:
|
||||
go-version: ^1.24
|
||||
cache: ${{ runner.environment == 'github-hosted' }}
|
||||
go-version: ^1.26
|
||||
id: go
|
||||
|
||||
- name: Check out code
|
||||
uses: actions/checkout@v6
|
||||
uses: actions/checkout@v7
|
||||
|
||||
- name: Setup Integration Container Environment
|
||||
run: |
|
||||
@@ -118,7 +136,7 @@ jobs:
|
||||
matrix:
|
||||
container-id: [e2e-1]
|
||||
container:
|
||||
image: golang:1.24-alpine
|
||||
image: golang:1.26-alpine
|
||||
options: --cpus 2.0 --memory 2g --hostname kafka-e2e-${{ matrix.container-id }}
|
||||
env:
|
||||
GOMAXPROCS: 2
|
||||
@@ -127,13 +145,13 @@ jobs:
|
||||
CONTAINER_ID: ${{ matrix.container-id }}
|
||||
steps:
|
||||
- name: Check out code
|
||||
uses: actions/checkout@v6
|
||||
uses: actions/checkout@v7
|
||||
|
||||
- name: Set up Go 1.x
|
||||
uses: actions/setup-go@v6
|
||||
uses: actions/setup-go@v7
|
||||
with:
|
||||
go-version: ^1.24
|
||||
cache: true
|
||||
go-version: ^1.26
|
||||
cache: ${{ runner.environment == 'github-hosted' }}
|
||||
cache-dependency-path: |
|
||||
**/go.sum
|
||||
id: go
|
||||
@@ -297,7 +315,7 @@ jobs:
|
||||
matrix:
|
||||
container-id: [consumer-group-1]
|
||||
container:
|
||||
image: golang:1.24-alpine
|
||||
image: golang:1.26-alpine
|
||||
options: --cpus 1.0 --memory 2g --ulimit nofile=512:512 --hostname kafka-consumer-${{ matrix.container-id }}
|
||||
env:
|
||||
GOMAXPROCS: 1
|
||||
@@ -306,13 +324,13 @@ jobs:
|
||||
CONTAINER_ID: ${{ matrix.container-id }}
|
||||
steps:
|
||||
- name: Check out code
|
||||
uses: actions/checkout@v6
|
||||
uses: actions/checkout@v7
|
||||
|
||||
- name: Set up Go 1.x
|
||||
uses: actions/setup-go@v6
|
||||
uses: actions/setup-go@v7
|
||||
with:
|
||||
go-version: ^1.24
|
||||
cache: true
|
||||
go-version: ^1.26
|
||||
cache: ${{ runner.environment == 'github-hosted' }}
|
||||
cache-dependency-path: |
|
||||
**/go.sum
|
||||
id: go
|
||||
@@ -445,7 +463,7 @@ jobs:
|
||||
# Test consumer group functionality with explicit timeout
|
||||
ulimit -n 512 || echo "Warning: Could not set file descriptor limit"
|
||||
ulimit -u 100 || echo "Warning: Could not set process limit"
|
||||
timeout 240s go test -v -run "^TestConsumerGroups" -timeout 180s ./integration/... || echo "Test execution timed out or failed"
|
||||
timeout 240s go test -v -run "^TestConsumerGroups" -timeout 180s ./integration/...
|
||||
env:
|
||||
GOMAXPROCS: 1
|
||||
SEAWEEDFS_MASTERS: 127.0.0.1:9333
|
||||
@@ -459,7 +477,7 @@ jobs:
|
||||
matrix:
|
||||
container-id: [client-compat-1]
|
||||
container:
|
||||
image: golang:1.24-alpine
|
||||
image: golang:1.26-alpine
|
||||
options: --cpus 1.0 --memory 1.5g --shm-size 256m --hostname kafka-client-${{ matrix.container-id }}
|
||||
env:
|
||||
GOMAXPROCS: 1
|
||||
@@ -468,13 +486,13 @@ jobs:
|
||||
CONTAINER_ID: ${{ matrix.container-id }}
|
||||
steps:
|
||||
- name: Check out code
|
||||
uses: actions/checkout@v6
|
||||
uses: actions/checkout@v7
|
||||
|
||||
- name: Set up Go 1.x
|
||||
uses: actions/setup-go@v6
|
||||
uses: actions/setup-go@v7
|
||||
with:
|
||||
go-version: ^1.24
|
||||
cache: true
|
||||
go-version: ^1.26
|
||||
cache: ${{ runner.environment == 'github-hosted' }}
|
||||
cache-dependency-path: |
|
||||
**/go.sum
|
||||
id: go
|
||||
@@ -617,7 +635,7 @@ jobs:
|
||||
matrix:
|
||||
container-id: [smq-integration-1]
|
||||
container:
|
||||
image: golang:1.24-alpine
|
||||
image: golang:1.26-alpine
|
||||
options: --cpus 1.0 --memory 2g --hostname kafka-smq-${{ matrix.container-id }}
|
||||
env:
|
||||
GOMAXPROCS: 1
|
||||
@@ -626,13 +644,13 @@ jobs:
|
||||
CONTAINER_ID: ${{ matrix.container-id }}
|
||||
steps:
|
||||
- name: Check out code
|
||||
uses: actions/checkout@v6
|
||||
uses: actions/checkout@v7
|
||||
|
||||
- name: Set up Go 1.x
|
||||
uses: actions/setup-go@v6
|
||||
uses: actions/setup-go@v7
|
||||
with:
|
||||
go-version: ^1.24
|
||||
cache: true
|
||||
go-version: ^1.26
|
||||
cache: ${{ runner.environment == 'github-hosted' }}
|
||||
cache-dependency-path: |
|
||||
**/go.sum
|
||||
id: go
|
||||
@@ -778,7 +796,7 @@ jobs:
|
||||
matrix:
|
||||
container-id: [protocol-1]
|
||||
container:
|
||||
image: golang:1.24-alpine
|
||||
image: golang:1.26-alpine
|
||||
options: --cpus 1.0 --memory 1g --tmpfs /tmp:exec --hostname kafka-protocol-${{ matrix.container-id }}
|
||||
env:
|
||||
GOMAXPROCS: 1
|
||||
@@ -787,13 +805,14 @@ jobs:
|
||||
CONTAINER_ID: ${{ matrix.container-id }}
|
||||
steps:
|
||||
- name: Set up Go 1.x
|
||||
uses: actions/setup-go@v6
|
||||
uses: actions/setup-go@v7
|
||||
with:
|
||||
go-version: ^1.24
|
||||
cache: ${{ runner.environment == 'github-hosted' }}
|
||||
go-version: ^1.26
|
||||
id: go
|
||||
|
||||
- name: Check out code
|
||||
uses: actions/checkout@v6
|
||||
uses: actions/checkout@v7
|
||||
|
||||
- name: Setup Protocol Container Environment
|
||||
run: |
|
||||
|
||||
@@ -0,0 +1,142 @@
|
||||
name: "KMS Tests"
|
||||
|
||||
on:
|
||||
pull_request:
|
||||
paths:
|
||||
- 'weed/kms/**'
|
||||
- 'weed/s3api/s3_sse_*.go'
|
||||
- 'weed/s3api/s3api_object_handlers.go'
|
||||
- 'weed/s3api/s3api_object_handlers_put.go'
|
||||
- 'test/kms/**'
|
||||
- '.github/workflows/kms-tests.yml'
|
||||
push:
|
||||
branches: [ master, main ]
|
||||
paths:
|
||||
- 'weed/kms/**'
|
||||
- 'weed/s3api/s3_sse_*.go'
|
||||
- 'weed/s3api/s3api_object_handlers.go'
|
||||
- 'weed/s3api/s3api_object_handlers_put.go'
|
||||
- 'test/kms/**'
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.event.pull_request.number || (github.event_name == 'push' && github.ref) || github.run_id }}
|
||||
cancel-in-progress: ${{ github.event_name == 'pull_request' || (github.event_name == 'push' && !startsWith(github.ref, 'refs/tags/')) }}
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
defaults:
|
||||
run:
|
||||
working-directory: weed
|
||||
|
||||
jobs:
|
||||
kms-provider-tests:
|
||||
name: KMS Provider Integration Tests
|
||||
runs-on: ubuntu-22.04
|
||||
timeout-minutes: 20
|
||||
|
||||
steps:
|
||||
- name: Check out code
|
||||
uses: actions/checkout@v7
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v7
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
cache: ${{ runner.environment == 'github-hosted' }}
|
||||
id: go
|
||||
|
||||
- name: Install SeaweedFS
|
||||
run: |
|
||||
go install -buildvcs=false
|
||||
|
||||
- name: Run KMS provider integration tests
|
||||
timeout-minutes: 15
|
||||
working-directory: test/kms
|
||||
run: |
|
||||
set -x
|
||||
echo "=== System Information ==="
|
||||
uname -a
|
||||
free -h
|
||||
docker --version
|
||||
|
||||
make test-provider-ci
|
||||
|
||||
- name: Show OpenBao logs on failure
|
||||
if: failure()
|
||||
run: |
|
||||
echo "=== OpenBao Container Logs ==="
|
||||
docker logs openbao-ci 2>&1 | tail -50 || echo "No OpenBao container found"
|
||||
echo "=== Setup Logs ==="
|
||||
cat /tmp/openbao-ci-setup.log 2>/dev/null || echo "No setup log found"
|
||||
|
||||
- name: Cleanup
|
||||
if: always()
|
||||
working-directory: test/kms
|
||||
run: |
|
||||
make stop-openbao-ci || true
|
||||
|
||||
s3-kms-e2e-tests:
|
||||
name: S3 KMS End-to-End Tests
|
||||
runs-on: ubuntu-22.04
|
||||
timeout-minutes: 25
|
||||
|
||||
steps:
|
||||
- name: Check out code
|
||||
uses: actions/checkout@v7
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v7
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
cache: ${{ runner.environment == 'github-hosted' }}
|
||||
id: go
|
||||
|
||||
- name: Install SeaweedFS
|
||||
run: |
|
||||
go install -buildvcs=false
|
||||
|
||||
- name: Run S3 KMS end-to-end tests
|
||||
timeout-minutes: 20
|
||||
working-directory: test/kms
|
||||
run: |
|
||||
set -x
|
||||
echo "=== System Information ==="
|
||||
uname -a
|
||||
free -h
|
||||
docker --version
|
||||
aws --version
|
||||
|
||||
make test-s3-kms-ci
|
||||
|
||||
- name: Show logs on failure
|
||||
if: failure()
|
||||
working-directory: test/kms
|
||||
run: |
|
||||
echo "=== OpenBao Container Logs ==="
|
||||
cat /tmp/openbao-ci-container.log 2>/dev/null || docker logs openbao-ci 2>&1 | tail -50 || echo "No OpenBao logs found"
|
||||
echo "=== SeaweedFS Server Logs ==="
|
||||
tail -100 /tmp/seaweedfs-kms-mini.log 2>/dev/null || echo "No server log found"
|
||||
echo "=== Setup Logs ==="
|
||||
cat /tmp/weed-kms-ci-setup.log 2>/dev/null || echo "No weed setup log"
|
||||
echo "=== Process Information ==="
|
||||
ps aux | grep -E "(weed|test)" || true
|
||||
|
||||
- name: Upload test logs on failure
|
||||
if: failure()
|
||||
uses: actions/upload-artifact@v7
|
||||
with:
|
||||
name: s3-kms-e2e-logs
|
||||
path: |
|
||||
/tmp/seaweedfs-kms-mini.log
|
||||
/tmp/openbao-ci-container.log
|
||||
/tmp/weed-kms-ci-setup.log
|
||||
retention-days: 3
|
||||
|
||||
- name: Cleanup
|
||||
if: always()
|
||||
working-directory: test/kms
|
||||
run: |
|
||||
make stop-seaweedfs-ci || true
|
||||
make stop-openbao-ci || true
|
||||
make clean-ci || true
|
||||
@@ -0,0 +1,80 @@
|
||||
name: "Master Cold Start Tests"
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [ master ]
|
||||
paths:
|
||||
- 'weed/server/master_*.go'
|
||||
- 'weed/topology/**'
|
||||
- 'weed/operation/**'
|
||||
- 'test/master_cold_start/**'
|
||||
- 'test/testutil/**'
|
||||
- '.github/workflows/master-cold-start-tests.yml'
|
||||
pull_request:
|
||||
branches: [ master ]
|
||||
paths:
|
||||
- 'weed/server/master_*.go'
|
||||
- 'weed/topology/**'
|
||||
- 'weed/operation/**'
|
||||
- 'test/master_cold_start/**'
|
||||
- 'test/testutil/**'
|
||||
- '.github/workflows/master-cold-start-tests.yml'
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.event.pull_request.number || (github.event_name == 'push' && github.ref) || github.run_id }}
|
||||
cancel-in-progress: ${{ github.event_name == 'pull_request' || (github.event_name == 'push' && !startsWith(github.ref, 'refs/tags/')) }}
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
jobs:
|
||||
master-cold-start-tests:
|
||||
name: Master Cold Start Tests
|
||||
runs-on: ubuntu-22.04
|
||||
timeout-minutes: 10
|
||||
steps:
|
||||
- name: Check out code
|
||||
uses: actions/checkout@v7
|
||||
with:
|
||||
persist-credentials: false
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v7
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
cache: ${{ runner.environment == 'github-hosted' }}
|
||||
|
||||
- name: Build weed binary
|
||||
run: |
|
||||
cd weed && go install -buildvcs=false
|
||||
|
||||
- name: Run master cold start tests
|
||||
# test/master_cold_start boots a fresh master plus empty volume
|
||||
# servers and requires the very first assign (HTTP and gRPC, no
|
||||
# client retries) to complete a write: the assign that triggers
|
||||
# volume growth must wait for it instead of failing with
|
||||
# "volume growth in progress".
|
||||
run: |
|
||||
export WEED_BINARY=$(go env GOPATH)/bin/weed
|
||||
go test -v -timeout=8m ./test/master_cold_start/...
|
||||
|
||||
- name: Collect server logs on failure
|
||||
if: failure()
|
||||
run: |
|
||||
# test/master_cold_start/cluster.go keeps failing-test dirs created
|
||||
# via os.MkdirTemp("", "seaweedfs_master_cold_start_it_") with each
|
||||
# process log under <baseDir>/logs/.
|
||||
mkdir -p /tmp/master-cold-start-logs
|
||||
find /tmp -maxdepth 1 -type d -name "seaweedfs_master_cold_start_it_*" 2>/dev/null | while read dir; do
|
||||
echo "Found test directory: $dir"
|
||||
cp -r "$dir" /tmp/master-cold-start-logs/ 2>/dev/null || true
|
||||
done
|
||||
find /tmp/master-cold-start-logs -type f -name "*.log" -print -exec tail -n 100 {} \; 2>/dev/null || echo "No logs found"
|
||||
|
||||
- name: Archive logs
|
||||
if: failure()
|
||||
uses: actions/upload-artifact@v7
|
||||
with:
|
||||
name: master-cold-start-test-logs
|
||||
path: /tmp/master-cold-start-logs/
|
||||
retention-days: 7
|
||||
@@ -8,6 +8,7 @@ on:
|
||||
- 'weed/pb/filer_pb/**'
|
||||
- 'weed/util/log_buffer/**'
|
||||
- 'weed/server/filer_grpc_server_sub_meta.go'
|
||||
- 'weed/server/master_grpc_server.go'
|
||||
- 'weed/command/filer_backup.go'
|
||||
- 'test/metadata_subscribe/**'
|
||||
- '.github/workflows/metadata-subscribe-tests.yml'
|
||||
@@ -18,19 +19,19 @@ on:
|
||||
- 'weed/pb/filer_pb/**'
|
||||
- 'weed/util/log_buffer/**'
|
||||
- 'weed/server/filer_grpc_server_sub_meta.go'
|
||||
- 'weed/server/master_grpc_server.go'
|
||||
- 'weed/command/filer_backup.go'
|
||||
- 'test/metadata_subscribe/**'
|
||||
- '.github/workflows/metadata-subscribe-tests.yml'
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.head_ref }}/metadata-subscribe-tests
|
||||
cancel-in-progress: true
|
||||
group: ${{ github.workflow }}-${{ github.event.pull_request.number || (github.event_name == 'push' && github.ref) || github.run_id }}
|
||||
cancel-in-progress: ${{ github.event_name == 'pull_request' || (github.event_name == 'push' && !startsWith(github.ref, 'refs/tags/')) }}
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
env:
|
||||
GO_VERSION: '1.24'
|
||||
TEST_TIMEOUT: '10m'
|
||||
|
||||
jobs:
|
||||
@@ -41,12 +42,13 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v6
|
||||
uses: actions/checkout@v7
|
||||
|
||||
- name: Set up Go ${{ env.GO_VERSION }}
|
||||
uses: actions/setup-go@v6
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v7
|
||||
with:
|
||||
go-version: ${{ env.GO_VERSION }}
|
||||
go-version-file: 'go.mod'
|
||||
cache: ${{ runner.environment == 'github-hosted' }}
|
||||
|
||||
- name: Build SeaweedFS
|
||||
run: |
|
||||
@@ -70,7 +72,7 @@ jobs:
|
||||
|
||||
- name: Archive logs on failure
|
||||
if: failure()
|
||||
uses: actions/upload-artifact@v6
|
||||
uses: actions/upload-artifact@v7
|
||||
with:
|
||||
name: metadata-subscribe-test-logs
|
||||
path: |
|
||||
|
||||
@@ -0,0 +1,205 @@
|
||||
name: "mount: benchmark"
|
||||
|
||||
# Manual benchmark: native WinFsp mount vs rclone+WebDAV on the same Windows
|
||||
# runner, with a Linux FUSE mount of the same build as a reference. Numbers
|
||||
# from shared runners are noisy; this is for finding factor-of-N gaps, not
|
||||
# regressions of a few percent.
|
||||
on:
|
||||
workflow_dispatch:
|
||||
push:
|
||||
branches: [ 'winfsp-bench**' ]
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.event.pull_request.number || (github.event_name == 'push' && github.ref) || github.run_id }}
|
||||
cancel-in-progress: ${{ github.event_name == 'pull_request' || (github.event_name == 'push' && !startsWith(github.ref, 'refs/tags/')) }}
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
jobs:
|
||||
bench-windows:
|
||||
name: Windows native vs rclone
|
||||
runs-on: windows-latest
|
||||
timeout-minutes: 60
|
||||
env:
|
||||
CGO_ENABLED: 0
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@v7
|
||||
with:
|
||||
persist-credentials: false
|
||||
|
||||
- uses: actions/setup-go@v7
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
cache: ${{ runner.environment == 'github-hosted' }}
|
||||
|
||||
- name: Install WinFsp and rclone
|
||||
run: choco install winfsp rclone -y --no-progress
|
||||
|
||||
- name: Build weed.exe
|
||||
run: go build -o weed.exe ./weed
|
||||
|
||||
- name: Benchmark both mounts
|
||||
shell: pwsh
|
||||
run: |
|
||||
$ErrorActionPreference = 'Stop'
|
||||
|
||||
function Test-Port($port) {
|
||||
$client = New-Object System.Net.Sockets.TcpClient
|
||||
try { $client.Connect('127.0.0.1', $port); return $client.Connected }
|
||||
catch { return $false }
|
||||
finally { $client.Dispose() }
|
||||
}
|
||||
|
||||
function Wait-Drive($drive, $what) {
|
||||
$deadline = (Get-Date).AddMinutes(2)
|
||||
while ((Get-Date) -lt $deadline) {
|
||||
if (Test-Path "${drive}\") { Write-Host "$what is mounted on $drive"; return }
|
||||
Start-Sleep -Seconds 2
|
||||
}
|
||||
throw "$what never appeared on $drive"
|
||||
}
|
||||
|
||||
function Invoke-Bench($dir, $label, $out) {
|
||||
Write-Host "::group::bench $label"
|
||||
& go run ./test/mount_bench -dir $dir -label $label -filer 127.0.0.1:8888 -out $out
|
||||
$code = $LASTEXITCODE
|
||||
Write-Host "::endgroup::"
|
||||
if ($code -ne 0) { throw "bench $label failed with exit $code" }
|
||||
}
|
||||
|
||||
New-Item -ItemType Directory -Force -Path C:\seaweed-data | Out-Null
|
||||
Start-Process -FilePath .\weed.exe `
|
||||
-ArgumentList '-logtostderr','mini','-dir=C:\seaweed-data','-ip=127.0.0.1' `
|
||||
-RedirectStandardOutput C:\seaweed-mini.log -RedirectStandardError C:\seaweed-mini.err.log
|
||||
|
||||
$deadline = (Get-Date).AddMinutes(3)
|
||||
while ((Get-Date) -lt $deadline) {
|
||||
if ((Test-Port 8888) -and (Test-Port 18888) -and (Test-Port 7333)) { break }
|
||||
Start-Sleep -Seconds 3
|
||||
}
|
||||
if (-not ((Test-Port 8888) -and (Test-Port 18888) -and (Test-Port 7333))) {
|
||||
Get-Content C:\seaweed-mini.log, C:\seaweed-mini.err.log -ErrorAction SilentlyContinue
|
||||
throw "mini cluster never came up"
|
||||
}
|
||||
Write-Host "filer on 8888/18888, webdav on 7333"
|
||||
|
||||
# --- native WinFsp mount ---
|
||||
Start-Process -FilePath .\weed.exe `
|
||||
-ArgumentList '-logtostderr','mount','-filer=127.0.0.1:8888','-dir=S:' `
|
||||
-RedirectStandardOutput C:\seaweed-mount.log -RedirectStandardError C:\seaweed-mount.err.log
|
||||
Wait-Drive 'S:' 'weed mount'
|
||||
|
||||
Invoke-Bench 'S:\bench-native' 'winfsp-native' 'C:\results-native.json'
|
||||
|
||||
Get-CimInstance Win32_Process -Filter "Name = 'weed.exe'" |
|
||||
Where-Object { $_.CommandLine -like '*mount*' } |
|
||||
ForEach-Object { Stop-Process -Id $_.ProcessId -Force }
|
||||
$deadline = (Get-Date).AddMinutes(1)
|
||||
while ((Get-Date) -lt $deadline -and (Test-Path S:\)) { Start-Sleep -Seconds 2 }
|
||||
|
||||
# --- rclone + WebDAV on the same WinFsp ---
|
||||
# rclone serves listings from a directory cache it fills lazily, so the
|
||||
# big-listing files have to exist before it mounts or it never sees them.
|
||||
& go run ./test/mount_bench -filer 127.0.0.1:8888 -seed bench-rclone/biglist
|
||||
if ($LASTEXITCODE -ne 0) { throw "seeding failed" }
|
||||
|
||||
$env:RCLONE_CONFIG_SEAWEED_TYPE = 'webdav'
|
||||
$env:RCLONE_CONFIG_SEAWEED_URL = 'http://127.0.0.1:7333'
|
||||
$env:RCLONE_CONFIG_SEAWEED_VENDOR = 'other'
|
||||
Start-Process -FilePath rclone `
|
||||
-ArgumentList 'mount','seaweed:','T:','--vfs-cache-mode=writes','-v','--log-file=C:\rclone.log'
|
||||
Wait-Drive 'T:' 'rclone mount'
|
||||
|
||||
Invoke-Bench 'T:\bench-rclone' 'rclone-webdav' 'C:\results-rclone.json'
|
||||
|
||||
Stop-Process -Name rclone -Force -ErrorAction SilentlyContinue
|
||||
|
||||
# --- comparison ---
|
||||
$table = & go run ./test/mount_bench -compare C:\results-native.json,C:\results-rclone.json
|
||||
$table | Write-Host
|
||||
"## Windows: native WinFsp vs rclone+WebDAV" | Out-File -Append $env:GITHUB_STEP_SUMMARY
|
||||
$table | Out-File -Append $env:GITHUB_STEP_SUMMARY
|
||||
|
||||
- name: Logs
|
||||
if: always()
|
||||
shell: pwsh
|
||||
run: |
|
||||
foreach ($f in 'C:\seaweed-mount.log','C:\seaweed-mount.err.log','C:\rclone.log','C:\seaweed-mini.log','C:\seaweed-mini.err.log') {
|
||||
if (Test-Path $f) { Write-Host "===== $f"; Get-Content $f -Tail 100 }
|
||||
}
|
||||
|
||||
- name: Results
|
||||
if: always()
|
||||
uses: actions/upload-artifact@v7
|
||||
with:
|
||||
name: results-windows
|
||||
path: C:\results-*.json
|
||||
if-no-files-found: ignore
|
||||
|
||||
bench-linux:
|
||||
name: Linux FUSE reference
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 60
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@v7
|
||||
with:
|
||||
persist-credentials: false
|
||||
|
||||
- uses: actions/setup-go@v7
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
cache: ${{ runner.environment == 'github-hosted' }}
|
||||
|
||||
- name: Repair the fusermount3 setuid bit
|
||||
uses: ./.github/actions/fix-fusermount-setuid
|
||||
|
||||
- name: Allow non-root FUSE mounts with allow_other
|
||||
run: |
|
||||
echo 'user_allow_other' | sudo tee -a /etc/fuse.conf
|
||||
sudo chmod 644 /etc/fuse.conf
|
||||
|
||||
- name: Build weed
|
||||
run: go build -o /tmp/weed ./weed
|
||||
|
||||
- name: Benchmark FUSE mount
|
||||
run: |
|
||||
set -e
|
||||
mkdir -p /tmp/seaweed-data
|
||||
/tmp/weed -logtostderr mini -dir=/tmp/seaweed-data -ip=127.0.0.1 > /tmp/mini.log 2>&1 &
|
||||
|
||||
for i in $(seq 1 60); do
|
||||
if nc -z 127.0.0.1 8888 && nc -z 127.0.0.1 18888; then break; fi
|
||||
sleep 3
|
||||
done
|
||||
nc -z 127.0.0.1 8888 || { cat /tmp/mini.log; echo "filer never came up"; exit 1; }
|
||||
|
||||
mkdir -p "$HOME/mnt"
|
||||
/tmp/weed -logtostderr mount -filer=127.0.0.1:8888 -dir="$HOME/mnt" > /tmp/mount.log 2>&1 &
|
||||
for i in $(seq 1 60); do
|
||||
if mountpoint -q "$HOME/mnt"; then break; fi
|
||||
sleep 2
|
||||
done
|
||||
mountpoint -q "$HOME/mnt" || { cat /tmp/mount.log; echo "mount never appeared"; exit 1; }
|
||||
|
||||
go run ./test/mount_bench -dir "$HOME/mnt/bench-linux" -label linux-fuse -filer 127.0.0.1:8888 -out /tmp/results-linux.json
|
||||
|
||||
{
|
||||
echo "## Linux FUSE reference"
|
||||
go run ./test/mount_bench -compare /tmp/results-linux.json
|
||||
} >> "$GITHUB_STEP_SUMMARY"
|
||||
|
||||
- name: Logs
|
||||
if: always()
|
||||
run: |
|
||||
tail -n 100 /tmp/mount.log /tmp/mini.log 2>/dev/null || true
|
||||
|
||||
- name: Results
|
||||
if: always()
|
||||
uses: actions/upload-artifact@v7
|
||||
with:
|
||||
name: results-linux
|
||||
path: /tmp/results-linux.json
|
||||
if-no-files-found: ignore
|
||||
@@ -0,0 +1,125 @@
|
||||
name: "mount: windows conformance"
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [ master ]
|
||||
paths:
|
||||
- 'weed/mount/**'
|
||||
- 'weed/command/mount*.go'
|
||||
- 'weed/storage/volume_vacuum*.go'
|
||||
- 'weed/storage/volume_loading.go'
|
||||
- 'weed/storage/disk_location.go'
|
||||
- 'test/winfsp-conformance/**'
|
||||
- '.github/workflows/mount-windows-conformance.yml'
|
||||
# No base branch filter: this is the only thing that runs the Windows mount,
|
||||
# so it should cover a pull request stacked on another one too.
|
||||
pull_request:
|
||||
paths:
|
||||
- 'weed/mount/**'
|
||||
- 'weed/command/mount*.go'
|
||||
- 'weed/storage/volume_vacuum*.go'
|
||||
- 'weed/storage/volume_loading.go'
|
||||
- 'weed/storage/disk_location.go'
|
||||
- 'test/winfsp-conformance/**'
|
||||
- '.github/workflows/mount-windows-conformance.yml'
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.event.pull_request.number || (github.event_name == 'push' && github.ref) || github.run_id }}
|
||||
cancel-in-progress: ${{ github.event_name == 'pull_request' || (github.event_name == 'push' && !startsWith(github.ref, 'refs/tags/')) }}
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
jobs:
|
||||
conformance:
|
||||
name: WinFsp conformance
|
||||
runs-on: windows-latest
|
||||
timeout-minutes: 60
|
||||
env:
|
||||
# The runner ships MinGW, so cgo is on by default and cgofuse picks its
|
||||
# cgo variant, which wants WinFsp's headers. The nocgo variant loads
|
||||
# winfsp-x64.dll at run time instead, which is how weed.exe is released.
|
||||
CGO_ENABLED: 0
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@v7
|
||||
with:
|
||||
persist-credentials: false
|
||||
|
||||
- uses: actions/setup-go@v7
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
cache: ${{ runner.environment == 'github-hosted' }}
|
||||
|
||||
# cgofuse loads winfsp-x64.dll at run time, so WinFsp is needed here but
|
||||
# not to build.
|
||||
- name: Install WinFsp
|
||||
run: choco install winfsp -y --no-progress
|
||||
|
||||
- name: Build weed.exe
|
||||
run: go build -o weed.exe ./weed
|
||||
|
||||
# The runner tears down a step's process tree when its shell exits, so a
|
||||
# cluster started in one step is gone by the next. Everything that needs
|
||||
# the cluster and the mount alive has to share a step.
|
||||
- name: Mount and run winfsp-tests
|
||||
shell: pwsh
|
||||
run: |
|
||||
$ErrorActionPreference = 'Stop'
|
||||
|
||||
function Test-Port($port) {
|
||||
# A plain connect, because Test-NetConnection has reported success
|
||||
# here for a port nothing was listening on.
|
||||
$client = New-Object System.Net.Sockets.TcpClient
|
||||
try { $client.Connect('127.0.0.1', $port); return $client.Connected }
|
||||
catch { return $false }
|
||||
finally { $client.Dispose() }
|
||||
}
|
||||
|
||||
function Start-Mount($log) {
|
||||
Start-Process -FilePath .\weed.exe `
|
||||
-ArgumentList '-logtostderr','mount','-filer=127.0.0.1:8888','-dir=S:' `
|
||||
-RedirectStandardOutput "C:\$log.log" -RedirectStandardError "C:\$log.err.log"
|
||||
$deadline = (Get-Date).AddMinutes(2)
|
||||
while ((Get-Date) -lt $deadline) {
|
||||
if (Test-Path S:\) { Write-Host "S: is mounted"; return }
|
||||
Start-Sleep -Seconds 2
|
||||
}
|
||||
Get-Content "C:\$log.log", "C:\$log.err.log" -ErrorAction SilentlyContinue
|
||||
throw "S: never appeared"
|
||||
}
|
||||
|
||||
New-Item -ItemType Directory -Force -Path C:\seaweed-data | Out-Null
|
||||
# -ip pins the cluster to loopback; it otherwise advertises and binds
|
||||
# the runner's LAN address, which 127.0.0.1 cannot reach.
|
||||
Start-Process -FilePath .\weed.exe `
|
||||
-ArgumentList '-logtostderr','mini','-dir=C:\seaweed-data','-ip=127.0.0.1' `
|
||||
-RedirectStandardOutput C:\seaweed-mini.log -RedirectStandardError C:\seaweed-mini.err.log
|
||||
|
||||
$deadline = (Get-Date).AddMinutes(3)
|
||||
while ((Get-Date) -lt $deadline) {
|
||||
# The mount dials grpc, not http, so both ports have to answer.
|
||||
if ((Test-Port 8888) -and (Test-Port 18888)) { break }
|
||||
Start-Sleep -Seconds 3
|
||||
}
|
||||
if (-not ((Test-Port 8888) -and (Test-Port 18888))) {
|
||||
Get-Content C:\seaweed-mini.log, C:\seaweed-mini.err.log -ErrorAction SilentlyContinue
|
||||
throw "filer never came up"
|
||||
}
|
||||
Write-Host "filer is up on http 8888 and grpc 18888"
|
||||
|
||||
Start-Mount 'seaweed-mount'
|
||||
|
||||
Write-Host "::group::winfsp-tests"
|
||||
& pwsh -File test/winfsp-conformance/run.ps1 -MountPoint S:\
|
||||
$code = $LASTEXITCODE
|
||||
Write-Host "::endgroup::"
|
||||
if ($code -ne 0) { throw "winfsp-tests failed with exit $code" }
|
||||
|
||||
- name: Logs
|
||||
if: always()
|
||||
shell: pwsh
|
||||
run: |
|
||||
foreach ($f in 'C:\seaweed-mount.log','C:\seaweed-mount.err.log','C:\seaweed-mini.log','C:\seaweed-mini.err.log') {
|
||||
if (Test-Path $f) { Write-Host "===== $f"; Get-Content $f -Tail 200 }
|
||||
}
|
||||
@@ -0,0 +1,206 @@
|
||||
name: "mount: windows"
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [ master ]
|
||||
paths:
|
||||
- 'weed/mount/**'
|
||||
- 'weed/command/mount*.go'
|
||||
- 'weed/storage/volume_vacuum*.go'
|
||||
- 'weed/storage/volume_loading.go'
|
||||
- 'weed/storage/disk_location.go'
|
||||
- 'test/winfsp/**'
|
||||
- '.github/workflows/mount-windows.yml'
|
||||
# No base branch filter: this is the only thing that runs the Windows mount,
|
||||
# so it should cover a pull request stacked on another one too.
|
||||
pull_request:
|
||||
paths:
|
||||
- 'weed/mount/**'
|
||||
- 'weed/command/mount*.go'
|
||||
- 'weed/storage/volume_vacuum*.go'
|
||||
- 'weed/storage/volume_loading.go'
|
||||
- 'weed/storage/disk_location.go'
|
||||
- 'test/winfsp/**'
|
||||
- '.github/workflows/mount-windows.yml'
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.event.pull_request.number || (github.event_name == 'push' && github.ref) || github.run_id }}
|
||||
cancel-in-progress: ${{ github.event_name == 'pull_request' || (github.event_name == 'push' && !startsWith(github.ref, 'refs/tags/')) }}
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
jobs:
|
||||
mount-windows:
|
||||
name: Mount on Windows
|
||||
runs-on: windows-latest
|
||||
timeout-minutes: 40
|
||||
env:
|
||||
# The runner ships MinGW, so cgo is on by default and cgofuse picks its
|
||||
# cgo variant, which wants WinFsp's headers. The nocgo variant loads
|
||||
# winfsp-x64.dll at run time instead, which is how weed.exe is released.
|
||||
CGO_ENABLED: 0
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@v7
|
||||
with:
|
||||
persist-credentials: false
|
||||
|
||||
- uses: actions/setup-go@v7
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
cache: ${{ runner.environment == 'github-hosted' }}
|
||||
|
||||
# cgofuse loads winfsp-x64.dll at run time, so WinFsp is needed here but
|
||||
# not to build.
|
||||
- name: Install WinFsp
|
||||
run: choco install winfsp -y --no-progress
|
||||
|
||||
- name: Build weed.exe
|
||||
run: go build -o weed.exe ./weed
|
||||
|
||||
# The runner tears down a step's process tree when its shell exits, so a
|
||||
# cluster started in one step is gone by the next. Everything that needs
|
||||
# the cluster and the mount alive has to share a step.
|
||||
- name: Mount and exercise
|
||||
shell: pwsh
|
||||
run: |
|
||||
$ErrorActionPreference = 'Stop'
|
||||
|
||||
function Test-Port($port) {
|
||||
# A plain connect, because Test-NetConnection has reported success
|
||||
# here for a port nothing was listening on.
|
||||
$client = New-Object System.Net.Sockets.TcpClient
|
||||
try { $client.Connect('127.0.0.1', $port); return $client.Connected }
|
||||
catch { return $false }
|
||||
finally { $client.Dispose() }
|
||||
}
|
||||
|
||||
function Start-Mount($log) {
|
||||
Start-Process -FilePath .\weed.exe `
|
||||
-ArgumentList '-logtostderr','mount','-filer=127.0.0.1:8888','-dir=S:' `
|
||||
-RedirectStandardOutput "C:\$log.log" -RedirectStandardError "C:\$log.err.log"
|
||||
$deadline = (Get-Date).AddMinutes(2)
|
||||
while ((Get-Date) -lt $deadline) {
|
||||
if (Test-Path S:\) { Write-Host "S: is mounted"; return }
|
||||
Start-Sleep -Seconds 2
|
||||
}
|
||||
Get-Content "C:\$log.log", "C:\$log.err.log" -ErrorAction SilentlyContinue
|
||||
throw "S: never appeared"
|
||||
}
|
||||
|
||||
function Stop-Mount {
|
||||
Get-CimInstance Win32_Process -Filter "Name = 'weed.exe'" |
|
||||
Where-Object { $_.CommandLine -like '*mount*' } |
|
||||
ForEach-Object { Stop-Process -Id $_.ProcessId -Force }
|
||||
$deadline = (Get-Date).AddMinutes(1)
|
||||
while ((Get-Date) -lt $deadline -and (Test-Path S:\)) { Start-Sleep -Seconds 2 }
|
||||
if (Test-Path S:\) { throw "S: still present after stopping the mount" }
|
||||
Write-Host "unmounted"
|
||||
}
|
||||
|
||||
function Invoke-Tests($label, [string[]]$goArgs) {
|
||||
Write-Host "::group::$label"
|
||||
& go @goArgs
|
||||
$code = $LASTEXITCODE
|
||||
Write-Host "::endgroup::"
|
||||
if ($code -ne 0) { throw "$label failed with exit $code" }
|
||||
}
|
||||
|
||||
New-Item -ItemType Directory -Force -Path C:\seaweed-data | Out-Null
|
||||
# -ip pins the cluster to loopback; it otherwise advertises and binds
|
||||
# the runner's LAN address, which 127.0.0.1 cannot reach.
|
||||
Start-Process -FilePath .\weed.exe `
|
||||
-ArgumentList '-logtostderr','mini','-dir=C:\seaweed-data','-ip=127.0.0.1' `
|
||||
-RedirectStandardOutput C:\seaweed-mini.log -RedirectStandardError C:\seaweed-mini.err.log
|
||||
|
||||
$deadline = (Get-Date).AddMinutes(3)
|
||||
while ((Get-Date) -lt $deadline) {
|
||||
# The mount dials grpc, not http, so both ports have to answer.
|
||||
if ((Test-Port 8888) -and (Test-Port 18888)) { break }
|
||||
Start-Sleep -Seconds 3
|
||||
}
|
||||
if (-not ((Test-Port 8888) -and (Test-Port 18888))) {
|
||||
Get-Content C:\seaweed-mini.log, C:\seaweed-mini.err.log -ErrorAction SilentlyContinue
|
||||
throw "filer never came up"
|
||||
}
|
||||
Write-Host "filer is up on http 8888 and grpc 18888"
|
||||
|
||||
Start-Mount 'seaweed-mount'
|
||||
|
||||
Invoke-Tests 'exercise' @('test','-v','-timeout','20m','./test/winfsp','-mountpoint=S:\')
|
||||
Invoke-Tests 'persist-write' @('test','-v','-timeout','15m','./test/winfsp','-run','TestPersistence','-mountpoint=S:\','-phase=write','-filer=127.0.0.1:8888')
|
||||
|
||||
Stop-Mount
|
||||
Start-Mount 'seaweed-remount'
|
||||
|
||||
Invoke-Tests 'persist-verify' @('test','-v','-timeout','15m','./test/winfsp','-run','TestPersistence','-mountpoint=S:\','-phase=verify')
|
||||
|
||||
# WinFsp creates the mount directory itself, so the path must not
|
||||
# exist; only its parent has to.
|
||||
Write-Host "::group::mount over a directory"
|
||||
Stop-Mount
|
||||
Remove-Item C:\seaweed-mnt -Recurse -Force -ErrorAction SilentlyContinue
|
||||
Start-Process -FilePath .\weed.exe `
|
||||
-ArgumentList '-logtostderr','mount','-filer=127.0.0.1:8888','-dir=C:\seaweed-mnt' `
|
||||
-RedirectStandardOutput C:\seaweed-dirmount.log -RedirectStandardError C:\seaweed-dirmount.err.log
|
||||
$deadline = (Get-Date).AddMinutes(2)
|
||||
$ok = $false
|
||||
while ((Get-Date) -lt $deadline) {
|
||||
# Listing succeeds on the plain empty directory too, so wait for the
|
||||
# reparse point WinFsp turns it into. Otherwise this step passes
|
||||
# without a mount and writes to local disk.
|
||||
$item = Get-Item C:\seaweed-mnt -Force -ErrorAction SilentlyContinue
|
||||
if ($null -ne $item -and $item.Attributes.ToString() -like '*ReparsePoint*') { $ok = $true; break }
|
||||
Start-Sleep -Seconds 2
|
||||
}
|
||||
if (-not $ok) {
|
||||
Get-Content C:\seaweed-dirmount.log, C:\seaweed-dirmount.err.log -ErrorAction SilentlyContinue
|
||||
throw "mounting over a directory failed"
|
||||
}
|
||||
Set-Content -Path C:\seaweed-mnt\dirmount.txt -Value 'via directory mount'
|
||||
if ((Get-Content C:\seaweed-mnt\dirmount.txt) -ne 'via directory mount') { throw "readback through the directory mount differs" }
|
||||
Remove-Item C:\seaweed-mnt\dirmount.txt -Force
|
||||
Get-CimInstance Win32_Process -Filter "Name = 'weed.exe'" |
|
||||
Where-Object { $_.CommandLine -like '*mount*' } |
|
||||
ForEach-Object { Stop-Process -Id $_.ProcessId -Force }
|
||||
Start-Sleep -Seconds 5
|
||||
Write-Host "::endgroup::"
|
||||
|
||||
Start-Mount 'seaweed-remount2'
|
||||
|
||||
Write-Host "::group::explorer-style walk"
|
||||
New-Item -ItemType Directory -Force -Path S:\walk | Out-Null
|
||||
1..200 | ForEach-Object { Set-Content -Path "S:\walk\f$_.txt" -Value "line $_" }
|
||||
$names = @(Get-ChildItem S:\walk | ForEach-Object { $_.Name })
|
||||
if ($names.Count -ne 200) {
|
||||
# Name the strays: a dot entry surfacing here is a different problem
|
||||
# from a missing or duplicated file.
|
||||
$unexpected = $names | Where-Object { $_ -notmatch '^f\d+\.txt$' }
|
||||
throw "listed $($names.Count) entries, expected 200; unexpected: $($unexpected -join ', ')"
|
||||
}
|
||||
$body = Get-Content S:\walk\f42.txt
|
||||
if ($body -ne 'line 42') { throw "unexpected content: $body" }
|
||||
Copy-Item S:\walk\f42.txt S:\walk\copy.txt
|
||||
Remove-Item S:\walk -Recurse -Force
|
||||
if (Test-Path S:\walk) { throw "directory survived recursive delete" }
|
||||
Write-Host "::endgroup::"
|
||||
|
||||
# Tear down everything we started so the runner's later steps (Logs,
|
||||
# post-checkout) launch into a clean environment. A WinFsp mount left
|
||||
# active and a weed.exe holding winfsp-x64.dll have been seen to make
|
||||
# the next step's pwsh.exe fail with STATUS_DLL_INIT_FAILED
|
||||
# (0xC0000142), which fails a job whose actual test step passed.
|
||||
try { Stop-Mount } catch { Write-Host "no mount to stop at teardown: $_" }
|
||||
Get-CimInstance Win32_Process -Filter "Name = 'weed.exe'" |
|
||||
ForEach-Object { Stop-Process -Id $_.ProcessId -Force -ErrorAction SilentlyContinue }
|
||||
Start-Sleep -Seconds 3
|
||||
|
||||
- name: Logs
|
||||
if: always()
|
||||
continue-on-error: true
|
||||
shell: pwsh
|
||||
run: |
|
||||
foreach ($f in 'C:\seaweed-mount.log','C:\seaweed-mount.err.log','C:\seaweed-remount.log','C:\seaweed-remount.err.log','C:\seaweed-remount2.log','C:\seaweed-remount2.err.log','C:\seaweed-dirmount.log','C:\seaweed-dirmount.err.log','C:\seaweed-mini.log','C:\seaweed-mini.err.log') {
|
||||
if (Test-Path $f) { Write-Host "===== $f"; Get-Content $f -Tail 200 }
|
||||
}
|
||||
@@ -0,0 +1,78 @@
|
||||
name: "Multi-Master Tests"
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [ master ]
|
||||
paths:
|
||||
- 'weed/server/master_*.go'
|
||||
- 'weed/server/raft_*.go'
|
||||
- 'weed/topology/**'
|
||||
- 'test/multi_master/**'
|
||||
- 'test/testutil/**'
|
||||
- '.github/workflows/multi-master-tests.yml'
|
||||
pull_request:
|
||||
branches: [ master ]
|
||||
paths:
|
||||
- 'weed/server/master_*.go'
|
||||
- 'weed/server/raft_*.go'
|
||||
- 'weed/topology/**'
|
||||
- 'test/multi_master/**'
|
||||
- 'test/testutil/**'
|
||||
- '.github/workflows/multi-master-tests.yml'
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.event.pull_request.number || (github.event_name == 'push' && github.ref) || github.run_id }}
|
||||
cancel-in-progress: ${{ github.event_name == 'pull_request' || (github.event_name == 'push' && !startsWith(github.ref, 'refs/tags/')) }}
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
jobs:
|
||||
multi-master-failover-tests:
|
||||
name: Multi-Master Failover Tests
|
||||
runs-on: ubuntu-22.04
|
||||
timeout-minutes: 10
|
||||
steps:
|
||||
- name: Check out code
|
||||
uses: actions/checkout@v7
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v7
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
cache: ${{ runner.environment == 'github-hosted' }}
|
||||
|
||||
- name: Install SeaweedFS
|
||||
run: |
|
||||
cd weed && go install -buildvcs=false
|
||||
|
||||
- name: Run multi-master failover tests
|
||||
# The tests in test/multi_master spin up their own 3-node master raft
|
||||
# cluster (using the freshly-installed `weed` binary) and exercise
|
||||
# leader-election, failover and recovery scenarios. The shared
|
||||
# test/testutil port-allocator regression test runs alongside since it
|
||||
# is a prerequisite for the cluster fixtures.
|
||||
run: |
|
||||
go test -v -timeout=8m ./test/multi_master/... ./test/testutil/...
|
||||
|
||||
- name: Collect server logs on failure
|
||||
if: failure()
|
||||
run: |
|
||||
# test/multi_master/cluster.go creates per-test dirs via
|
||||
# os.MkdirTemp("", "seaweedfs_multi_master_it_") and writes each
|
||||
# node's log into <baseDir>/logs/master*.log.
|
||||
echo "Collecting per-node master logs from temp directories..."
|
||||
mkdir -p /tmp/multi-master-logs
|
||||
find /tmp -maxdepth 1 -type d -name "seaweedfs_multi_master_it_*" 2>/dev/null | while read dir; do
|
||||
echo "Found test directory: $dir"
|
||||
cp -r "$dir" /tmp/multi-master-logs/ 2>/dev/null || true
|
||||
done
|
||||
find /tmp/multi-master-logs -type f -name "*.log" -print -exec tail -n 100 {} \; 2>/dev/null || echo "No logs found"
|
||||
|
||||
- name: Archive logs
|
||||
if: failure()
|
||||
uses: actions/upload-artifact@v7
|
||||
with:
|
||||
name: multi-master-test-logs
|
||||
path: /tmp/multi-master-logs/
|
||||
retention-days: 7
|
||||
@@ -0,0 +1,473 @@
|
||||
name: "Performance"
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [ master ]
|
||||
paths:
|
||||
- '**/*.go'
|
||||
- 'go.mod'
|
||||
- 'go.sum'
|
||||
- 'seaweed-volume/**'
|
||||
- 'seaweed-common/**'
|
||||
- 'test/perf/**'
|
||||
- '.github/workflows/performance.yml'
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
profile_duration:
|
||||
description: "CPU profiling duration in seconds"
|
||||
required: false
|
||||
default: "30"
|
||||
type: string
|
||||
benchmark_files:
|
||||
description: "Number of files for the throughput benchmark"
|
||||
required: false
|
||||
default: "100000"
|
||||
type: string
|
||||
benchmark_concurrency:
|
||||
description: "Concurrent read/write workers"
|
||||
required: false
|
||||
default: "16"
|
||||
type: string
|
||||
benchmark_size:
|
||||
description: "Simulated file size in bytes"
|
||||
required: false
|
||||
default: "1024"
|
||||
type: string
|
||||
s3_objects:
|
||||
description: "Number of objects for the S3 benchmark"
|
||||
required: false
|
||||
default: "20000"
|
||||
type: string
|
||||
s3_size:
|
||||
description: "S3 object size in bytes"
|
||||
required: false
|
||||
default: "4096"
|
||||
type: string
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.event.pull_request.number || (github.event_name == 'push' && github.ref) || github.run_id }}
|
||||
cancel-in-progress: ${{ github.event_name == 'pull_request' || (github.event_name == 'push' && !startsWith(github.ref, 'refs/tags/')) }}
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
env:
|
||||
VOL_SIZE_LIMIT: "1024"
|
||||
|
||||
jobs:
|
||||
performance-profile:
|
||||
name: CPU and Heap Profile
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 30
|
||||
steps:
|
||||
- name: Check out code
|
||||
uses: actions/checkout@v7
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v7
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
cache: ${{ runner.environment == 'github-hosted' }}
|
||||
|
||||
- name: Build weed
|
||||
run: go build -o weed_bin ./weed
|
||||
|
||||
- name: Start server with profiling enabled
|
||||
run: |
|
||||
mkdir -p ./perfdata
|
||||
./weed_bin -v=1 server -debug -debug.port=6060 -dir=./perfdata \
|
||||
-s3 -filer -volume.max=0 -master.volumeSizeLimitMB=100 \
|
||||
-s3.port=8000 -s3.config=./docker/compose/s3.json \
|
||||
> weed.log 2>&1 &
|
||||
echo "WEED_PID=$!" >> "$GITHUB_ENV"
|
||||
for i in $(seq 1 60); do
|
||||
if curl -sf http://localhost:9333/dir/status >/dev/null 2>&1; then
|
||||
echo "master is ready"
|
||||
break
|
||||
fi
|
||||
sleep 1
|
||||
done
|
||||
# give the volume server a moment to register with the master
|
||||
sleep 3
|
||||
|
||||
- name: Start memory sampler
|
||||
run: |
|
||||
bash test/perf/mem_sample.sh mem-profile.csv "server=${WEED_PID}" &
|
||||
echo "SAMPLER_PID=$!" >> "$GITHUB_ENV"
|
||||
|
||||
- name: Capture profiles under load
|
||||
run: |
|
||||
DURATION="${{ github.event.inputs.profile_duration || '30' }}"
|
||||
# drive write load so the sampled profile reflects real work
|
||||
./weed_bin benchmark -master=localhost:9333 -writeOnly \
|
||||
-c=16 -n=5000000 -size=1024 > benchmark-load.log 2>&1 &
|
||||
echo "Sampling CPU profile for ${DURATION}s..."
|
||||
curl -s "http://localhost:6060/debug/pprof/profile?seconds=${DURATION}" -o cpu.pprof
|
||||
curl -s "http://localhost:6060/debug/pprof/heap" -o heap.pprof
|
||||
curl -s "http://localhost:6060/debug/pprof/goroutine?debug=1" -o goroutine.txt
|
||||
go tool pprof -top -nodecount=50 cpu.pprof > cpu-top.txt 2>/dev/null || true
|
||||
go tool pprof -top -nodecount=50 -sample_index=inuse_space heap.pprof > heap-top.txt 2>/dev/null || true
|
||||
|
||||
- name: Record memory usage
|
||||
if: always()
|
||||
run: |
|
||||
kill -TERM "${SAMPLER_PID}" 2>/dev/null || true
|
||||
sleep 2
|
||||
{
|
||||
echo "## Memory usage (peak RSS)"
|
||||
echo '```'
|
||||
if [ -f mem-profile.csv.peak ]; then
|
||||
awk -F'\t' '{printf "%-10s %8d KB (%.1f MB)\n", $1, $2, $2/1024}' mem-profile.csv.peak
|
||||
else
|
||||
echo "no memory samples captured"
|
||||
fi
|
||||
echo '```'
|
||||
} >> "$GITHUB_STEP_SUMMARY"
|
||||
|
||||
- name: Profile summary
|
||||
if: always()
|
||||
run: |
|
||||
{
|
||||
echo "## CPU profile (top functions)"
|
||||
echo '```'
|
||||
head -45 cpu-top.txt 2>/dev/null || echo "no cpu profile captured"
|
||||
echo '```'
|
||||
} >> "$GITHUB_STEP_SUMMARY"
|
||||
|
||||
- name: Stop server
|
||||
if: always()
|
||||
run: kill "${WEED_PID}" 2>/dev/null || true
|
||||
|
||||
- name: Show server log on failure
|
||||
if: failure()
|
||||
run: tail -200 weed.log || true
|
||||
|
||||
- name: Upload profiles
|
||||
if: always()
|
||||
uses: actions/upload-artifact@v7
|
||||
with:
|
||||
name: performance-profile-${{ github.run_number }}
|
||||
path: |
|
||||
cpu.pprof
|
||||
heap.pprof
|
||||
cpu-top.txt
|
||||
heap-top.txt
|
||||
goroutine.txt
|
||||
mem-profile.csv
|
||||
mem-profile.csv.peak
|
||||
weed.log
|
||||
retention-days: 30
|
||||
|
||||
benchmark:
|
||||
name: Throughput Benchmark (${{ matrix.impl }} volume)
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 60
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
impl: [go, rust]
|
||||
env:
|
||||
IMPL: ${{ matrix.impl }}
|
||||
steps:
|
||||
- name: Check out code
|
||||
uses: actions/checkout@v7
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v7
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
cache: ${{ runner.environment == 'github-hosted' }}
|
||||
|
||||
- name: Install Rust toolchain
|
||||
if: matrix.impl == 'rust'
|
||||
uses: dtolnay/rust-toolchain@stable
|
||||
|
||||
- name: Cache cargo registry and target
|
||||
if: matrix.impl == 'rust' && runner.environment == 'github-hosted'
|
||||
uses: actions/cache@v6
|
||||
with:
|
||||
path: |
|
||||
~/.cargo/registry
|
||||
~/.cargo/git
|
||||
seaweed-volume/target
|
||||
key: rust-${{ hashFiles('seaweed-volume/Cargo.lock') }}
|
||||
restore-keys: |
|
||||
rust-
|
||||
|
||||
- name: Build weed
|
||||
run: go build -o weed_bin ./weed
|
||||
|
||||
- name: Build Rust volume server
|
||||
if: matrix.impl == 'rust'
|
||||
run: cd seaweed-volume && cargo build --release
|
||||
|
||||
- name: Start master and volume server
|
||||
run: |
|
||||
mkdir -p ./perfdata/master ./perfdata/vol
|
||||
./weed_bin -v=1 master -ip=127.0.0.1 -port=9333 \
|
||||
-mdir=./perfdata/master -peers=none \
|
||||
-volumeSizeLimitMB="${VOL_SIZE_LIMIT}" -defaultReplication=000 \
|
||||
> master.log 2>&1 &
|
||||
echo "MASTER_PID=$!" >> "$GITHUB_ENV"
|
||||
for i in $(seq 1 60); do
|
||||
if curl -sf http://localhost:9333/dir/status >/dev/null 2>&1; then
|
||||
echo "master is ready"
|
||||
break
|
||||
fi
|
||||
sleep 1
|
||||
done
|
||||
if [ "${IMPL}" = "rust" ]; then
|
||||
./seaweed-volume/target/release/weed-volume \
|
||||
--master 127.0.0.1:9333 --ip 127.0.0.1 --ip.bind 127.0.0.1 \
|
||||
--port 8080 --dir ./perfdata/vol --max 100 --preStopSeconds 0 \
|
||||
> volume.log 2>&1 &
|
||||
else
|
||||
./weed_bin -v=1 volume -master=127.0.0.1:9333 -ip=127.0.0.1 \
|
||||
-port=8080 -dir=./perfdata/vol -max=100 \
|
||||
> volume.log 2>&1 &
|
||||
fi
|
||||
echo "VOLUME_PID=$!" >> "$GITHUB_ENV"
|
||||
for i in $(seq 1 60); do
|
||||
if curl -sf http://localhost:8080/status >/dev/null 2>&1; then
|
||||
echo "volume server is ready"
|
||||
break
|
||||
fi
|
||||
sleep 1
|
||||
done
|
||||
# let the volume server register with the master via heartbeat
|
||||
sleep 3
|
||||
|
||||
- name: Start memory sampler
|
||||
run: |
|
||||
bash test/perf/mem_sample.sh mem-benchmark.csv \
|
||||
"master=${MASTER_PID}" "volume=${VOLUME_PID}" &
|
||||
echo "SAMPLER_PID=$!" >> "$GITHUB_ENV"
|
||||
|
||||
- name: Run throughput benchmark
|
||||
run: |
|
||||
N="${{ github.event.inputs.benchmark_files || '100000' }}"
|
||||
C="${{ github.event.inputs.benchmark_concurrency || '16' }}"
|
||||
SIZE="${{ github.event.inputs.benchmark_size || '1024' }}"
|
||||
./weed_bin benchmark -master=localhost:9333 \
|
||||
-c="${C}" -n="${N}" -size="${SIZE}" 2>&1 | tee benchmark-results.txt
|
||||
|
||||
- name: Run Go micro-benchmarks
|
||||
if: matrix.impl == 'go'
|
||||
continue-on-error: true
|
||||
run: |
|
||||
go test -run='^$' -bench=. -benchmem -benchtime=10x \
|
||||
./weed/topology/... ./weed/util/log_buffer/... ./weed/util/buffered_queue/... \
|
||||
2>&1 | tee go-benchmarks.txt
|
||||
|
||||
- name: Record memory usage
|
||||
if: always()
|
||||
run: |
|
||||
kill -TERM "${SAMPLER_PID}" 2>/dev/null || true
|
||||
sleep 2
|
||||
{
|
||||
echo "## Memory usage (peak RSS, ${IMPL} volume)"
|
||||
echo '```'
|
||||
if [ -f mem-benchmark.csv.peak ]; then
|
||||
awk -F'\t' '{printf "%-10s %8d KB (%.1f MB)\n", $1, $2, $2/1024}' mem-benchmark.csv.peak
|
||||
else
|
||||
echo "no memory samples captured"
|
||||
fi
|
||||
echo '```'
|
||||
} >> "$GITHUB_STEP_SUMMARY"
|
||||
|
||||
- name: Benchmark summary
|
||||
if: always()
|
||||
run: |
|
||||
{
|
||||
echo "## Throughput benchmark (${IMPL} volume)"
|
||||
echo '```'
|
||||
grep -E "Concurrency Level|Time taken|Completed requests|Failed requests|Requests per second|Transfer rate" \
|
||||
benchmark-results.txt 2>/dev/null || echo "no benchmark results captured"
|
||||
echo '```'
|
||||
} >> "$GITHUB_STEP_SUMMARY"
|
||||
|
||||
- name: Stop processes
|
||||
if: always()
|
||||
run: |
|
||||
kill "${VOLUME_PID}" "${MASTER_PID}" 2>/dev/null || true
|
||||
|
||||
- name: Show logs on failure
|
||||
if: failure()
|
||||
run: |
|
||||
echo "=== master.log ==="; tail -100 master.log 2>/dev/null || true
|
||||
echo "=== volume.log ==="; tail -200 volume.log 2>/dev/null || true
|
||||
|
||||
- name: Upload benchmark results
|
||||
if: always()
|
||||
uses: actions/upload-artifact@v7
|
||||
with:
|
||||
name: benchmark-results-${{ matrix.impl }}-${{ github.run_number }}
|
||||
path: |
|
||||
benchmark-results.txt
|
||||
go-benchmarks.txt
|
||||
mem-benchmark.csv
|
||||
mem-benchmark.csv.peak
|
||||
master.log
|
||||
volume.log
|
||||
retention-days: 7
|
||||
|
||||
s3-benchmark:
|
||||
name: S3 Read/Write Benchmark (${{ matrix.impl }} volume)
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 60
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
impl: [go, rust]
|
||||
env:
|
||||
IMPL: ${{ matrix.impl }}
|
||||
steps:
|
||||
- name: Check out code
|
||||
uses: actions/checkout@v7
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v7
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
cache: ${{ runner.environment == 'github-hosted' }}
|
||||
|
||||
- name: Install Rust toolchain
|
||||
if: matrix.impl == 'rust'
|
||||
uses: dtolnay/rust-toolchain@stable
|
||||
|
||||
- name: Cache cargo registry and target
|
||||
if: matrix.impl == 'rust' && runner.environment == 'github-hosted'
|
||||
uses: actions/cache@v6
|
||||
with:
|
||||
path: |
|
||||
~/.cargo/registry
|
||||
~/.cargo/git
|
||||
seaweed-volume/target
|
||||
key: rust-${{ hashFiles('seaweed-volume/Cargo.lock') }}
|
||||
restore-keys: |
|
||||
rust-
|
||||
|
||||
- name: Build weed and S3 load tool
|
||||
run: |
|
||||
go build -o weed_bin ./weed
|
||||
go build -o s3bench ./test/s3/benchmark
|
||||
|
||||
- name: Build Rust volume server
|
||||
if: matrix.impl == 'rust'
|
||||
run: cd seaweed-volume && cargo build --release
|
||||
|
||||
- name: Start cluster with S3 gateway
|
||||
run: |
|
||||
mkdir -p ./perfdata/master ./perfdata/vol ./perfdata/filer
|
||||
./weed_bin -v=1 master -ip=127.0.0.1 -port=9333 \
|
||||
-mdir=./perfdata/master -peers=none \
|
||||
-volumeSizeLimitMB="${VOL_SIZE_LIMIT}" -defaultReplication=000 \
|
||||
> master.log 2>&1 &
|
||||
echo "MASTER_PID=$!" >> "$GITHUB_ENV"
|
||||
for i in $(seq 1 60); do
|
||||
if curl -sf http://localhost:9333/dir/status >/dev/null 2>&1; then
|
||||
echo "master is ready"
|
||||
break
|
||||
fi
|
||||
sleep 1
|
||||
done
|
||||
if [ "${IMPL}" = "rust" ]; then
|
||||
./seaweed-volume/target/release/weed-volume \
|
||||
--master 127.0.0.1:9333 --ip 127.0.0.1 --ip.bind 127.0.0.1 \
|
||||
--port 8080 --dir ./perfdata/vol --max 100 --preStopSeconds 0 \
|
||||
> volume.log 2>&1 &
|
||||
else
|
||||
./weed_bin -v=1 volume -master=127.0.0.1:9333 -ip=127.0.0.1 \
|
||||
-port=8080 -dir=./perfdata/vol -max=100 \
|
||||
> volume.log 2>&1 &
|
||||
fi
|
||||
echo "VOLUME_PID=$!" >> "$GITHUB_ENV"
|
||||
for i in $(seq 1 60); do
|
||||
if curl -sf http://localhost:8080/status >/dev/null 2>&1; then
|
||||
echo "volume server is ready"
|
||||
break
|
||||
fi
|
||||
sleep 1
|
||||
done
|
||||
sleep 3
|
||||
./weed_bin -v=1 filer -master=127.0.0.1:9333 -ip=127.0.0.1 -port=8888 \
|
||||
-s3 -s3.port=8000 -s3.config=./docker/compose/s3.json \
|
||||
> filer.log 2>&1 &
|
||||
echo "FILER_PID=$!" >> "$GITHUB_ENV"
|
||||
for i in $(seq 1 30); do
|
||||
if nc -z localhost 8000 2>/dev/null; then
|
||||
echo "s3 gateway is ready"
|
||||
break
|
||||
fi
|
||||
sleep 1
|
||||
done
|
||||
sleep 2
|
||||
|
||||
- name: Start memory sampler
|
||||
run: |
|
||||
bash test/perf/mem_sample.sh mem-s3.csv \
|
||||
"master=${MASTER_PID}" "volume=${VOLUME_PID}" "filer=${FILER_PID}" &
|
||||
echo "SAMPLER_PID=$!" >> "$GITHUB_ENV"
|
||||
|
||||
- name: Run S3 read/write benchmark
|
||||
run: |
|
||||
OBJECTS="${{ github.event.inputs.s3_objects || '20000' }}"
|
||||
C="${{ github.event.inputs.benchmark_concurrency || '16' }}"
|
||||
SIZE="${{ github.event.inputs.s3_size || '4096' }}"
|
||||
./s3bench -endpoint=http://localhost:8000 \
|
||||
-access-key=some_access_key1 -secret-key=some_secret_key1 \
|
||||
-objects="${OBJECTS}" -size="${SIZE}" -concurrency="${C}" -mode=both \
|
||||
2>&1 | tee s3-benchmark-results.txt
|
||||
|
||||
- name: Record memory usage
|
||||
if: always()
|
||||
run: |
|
||||
kill -TERM "${SAMPLER_PID}" 2>/dev/null || true
|
||||
sleep 2
|
||||
{
|
||||
echo "## Memory usage (peak RSS, ${IMPL} volume)"
|
||||
echo '```'
|
||||
if [ -f mem-s3.csv.peak ]; then
|
||||
awk -F'\t' '{printf "%-10s %8d KB (%.1f MB)\n", $1, $2, $2/1024}' mem-s3.csv.peak
|
||||
else
|
||||
echo "no memory samples captured"
|
||||
fi
|
||||
echo '```'
|
||||
} >> "$GITHUB_STEP_SUMMARY"
|
||||
|
||||
- name: S3 benchmark summary
|
||||
if: always()
|
||||
run: |
|
||||
{
|
||||
echo "## S3 read/write benchmark (${IMPL} volume)"
|
||||
echo '```'
|
||||
grep -E "results:|Concurrency Level|Time taken|Completed requests|Failed requests|Requests per second|Transfer rate|Latency" \
|
||||
s3-benchmark-results.txt 2>/dev/null || echo "no S3 benchmark results captured"
|
||||
echo '```'
|
||||
} >> "$GITHUB_STEP_SUMMARY"
|
||||
|
||||
- name: Stop processes
|
||||
if: always()
|
||||
run: |
|
||||
kill "${FILER_PID}" "${VOLUME_PID}" "${MASTER_PID}" 2>/dev/null || true
|
||||
|
||||
- name: Show logs on failure
|
||||
if: failure()
|
||||
run: |
|
||||
echo "=== master.log ==="; tail -100 master.log 2>/dev/null || true
|
||||
echo "=== volume.log ==="; tail -200 volume.log 2>/dev/null || true
|
||||
echo "=== filer.log ==="; tail -200 filer.log 2>/dev/null || true
|
||||
|
||||
- name: Upload S3 benchmark results
|
||||
if: always()
|
||||
uses: actions/upload-artifact@v7
|
||||
with:
|
||||
name: s3-benchmark-results-${{ matrix.impl }}-${{ github.run_number }}
|
||||
path: |
|
||||
s3-benchmark-results.txt
|
||||
mem-s3.csv
|
||||
mem-s3.csv.peak
|
||||
master.log
|
||||
volume.log
|
||||
filer.log
|
||||
retention-days: 7
|
||||
@@ -0,0 +1,126 @@
|
||||
name: "pjdfstest POSIX Compliance"
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [ master, main ]
|
||||
paths:
|
||||
- 'weed/mount/**'
|
||||
- 'weed/filer/**'
|
||||
- 'test/pjdfstest/**'
|
||||
- '.github/workflows/pjdfstest.yml'
|
||||
pull_request:
|
||||
branches: [ master, main ]
|
||||
paths:
|
||||
- 'weed/mount/**'
|
||||
- 'weed/filer/**'
|
||||
- 'test/pjdfstest/**'
|
||||
- '.github/workflows/pjdfstest.yml'
|
||||
workflow_dispatch:
|
||||
|
||||
concurrency:
|
||||
group: pjdfstest/${{ github.ref }}
|
||||
cancel-in-progress: true
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
jobs:
|
||||
pjdfstest:
|
||||
name: pjdfstest
|
||||
runs-on: ubuntu-22.04
|
||||
timeout-minutes: 60
|
||||
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v7
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v7
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
cache: ${{ runner.environment == 'github-hosted' }}
|
||||
|
||||
- name: Start local Docker registry
|
||||
run: |
|
||||
if ! docker start registry 2>/dev/null; then
|
||||
docker rm -f registry >/dev/null 2>&1 || true
|
||||
docker run -d --restart=always -p 5000:5000 --name registry registry:2
|
||||
fi
|
||||
|
||||
- name: Set up Docker Buildx
|
||||
uses: docker/setup-buildx-action@v4
|
||||
with:
|
||||
driver-opts: network=host
|
||||
|
||||
- name: Set up Docker Compose
|
||||
uses: docker/setup-compose-action@v2
|
||||
|
||||
- name: Build weed race binary
|
||||
run: |
|
||||
cd docker
|
||||
make binary_race
|
||||
|
||||
- name: Build SeaweedFS e2e image
|
||||
uses: docker/build-push-action@v7
|
||||
with:
|
||||
context: docker
|
||||
file: docker/Dockerfile.e2e
|
||||
tags: localhost:5000/chrislusf/seaweedfs:e2e
|
||||
push: true
|
||||
cache-from: type=gha,scope=pjdfstest-e2e
|
||||
cache-to: type=gha,mode=max,scope=pjdfstest-e2e
|
||||
|
||||
- name: Tag e2e image for docker compose
|
||||
run: |
|
||||
docker pull localhost:5000/chrislusf/seaweedfs:e2e
|
||||
docker tag localhost:5000/chrislusf/seaweedfs:e2e chrislusf/seaweedfs:e2e
|
||||
|
||||
- name: Build pjdfstest image
|
||||
uses: docker/build-push-action@v7
|
||||
with:
|
||||
context: test/pjdfstest
|
||||
build-contexts: |
|
||||
chrislusf/seaweedfs:e2e=docker-image://localhost:5000/chrislusf/seaweedfs:e2e
|
||||
tags: localhost:5000/chrislusf/seaweedfs:pjdfstest
|
||||
push: true
|
||||
cache-from: type=gha,scope=pjdfstest-harness
|
||||
cache-to: type=gha,mode=max,scope=pjdfstest-harness
|
||||
|
||||
- name: Tag pjdfstest image for docker compose
|
||||
run: |
|
||||
docker pull localhost:5000/chrislusf/seaweedfs:pjdfstest
|
||||
docker tag localhost:5000/chrislusf/seaweedfs:pjdfstest chrislusf/seaweedfs:pjdfstest
|
||||
|
||||
- name: Start SeaweedFS cluster
|
||||
run: |
|
||||
docker compose -f test/pjdfstest/docker-compose.yml up --wait
|
||||
|
||||
- name: Run pjdfstest
|
||||
run: |
|
||||
set -o pipefail
|
||||
docker compose -f test/pjdfstest/docker-compose.yml exec -T mount \
|
||||
/run.sh 2>&1 | tee /tmp/pjdfstest-output.log
|
||||
|
||||
- name: Collect logs
|
||||
if: always()
|
||||
run: |
|
||||
mkdir -p /tmp/pjdfstest-docker-logs
|
||||
for svc in master volume filer mount; do
|
||||
docker compose -f test/pjdfstest/docker-compose.yml logs "$svc" \
|
||||
> "/tmp/pjdfstest-docker-logs/${svc}.log" 2>&1 || true
|
||||
done
|
||||
|
||||
- name: Tear down
|
||||
if: always()
|
||||
run: |
|
||||
docker compose -f test/pjdfstest/docker-compose.yml down -v
|
||||
|
||||
- name: Upload logs
|
||||
if: always()
|
||||
uses: actions/upload-artifact@v7
|
||||
with:
|
||||
name: pjdfstest-results
|
||||
path: |
|
||||
/tmp/pjdfstest-output.log
|
||||
/tmp/pjdfstest-docker-logs/
|
||||
retention-days: 7
|
||||
@@ -0,0 +1,56 @@
|
||||
name: "Plugin Worker Integration Tests"
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [ master ]
|
||||
paths:
|
||||
- 'weed/**'
|
||||
- 'test/plugin_workers/**'
|
||||
- 'go.mod'
|
||||
- 'go.sum'
|
||||
- '.github/workflows/plugin-workers.yml'
|
||||
pull_request:
|
||||
branches: [ master ]
|
||||
paths:
|
||||
- 'weed/**'
|
||||
- 'test/plugin_workers/**'
|
||||
- 'go.mod'
|
||||
- 'go.sum'
|
||||
- '.github/workflows/plugin-workers.yml'
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.event.pull_request.number || (github.event_name == 'push' && github.ref) || github.run_id }}
|
||||
cancel-in-progress: ${{ github.event_name == 'pull_request' || (github.event_name == 'push' && !startsWith(github.ref, 'refs/tags/')) }}
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
jobs:
|
||||
plugin-worker:
|
||||
name: "Plugin Worker: ${{ matrix.worker }}"
|
||||
runs-on: ubuntu-22.04
|
||||
timeout-minutes: 10
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
include:
|
||||
- worker: erasure_coding
|
||||
path: test/plugin_workers/erasure_coding
|
||||
- worker: vacuum
|
||||
path: test/plugin_workers/vacuum
|
||||
- worker: volume_balance
|
||||
path: test/plugin_workers/volume_balance
|
||||
|
||||
steps:
|
||||
- name: Set up Go 1.x
|
||||
uses: actions/setup-go@v7
|
||||
with:
|
||||
cache: ${{ runner.environment == 'github-hosted' }}
|
||||
go-version: ^1.26
|
||||
id: go
|
||||
|
||||
- name: Check out code into the Go module directory
|
||||
uses: actions/checkout@v7
|
||||
|
||||
- name: Run plugin worker tests
|
||||
run: go test -v ./${{ matrix.path }}
|
||||
@@ -0,0 +1,791 @@
|
||||
name: POSIX Compliance Tests
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [ main, master, develop ]
|
||||
paths:
|
||||
- 'weed/mount/**'
|
||||
- 'weed/command/mount*'
|
||||
- 'weed/command/fuse*'
|
||||
- 'test/fuse_integration/**'
|
||||
pull_request:
|
||||
branches: [ main, master, develop ]
|
||||
paths:
|
||||
- 'weed/mount/**'
|
||||
- 'weed/command/mount*'
|
||||
- 'weed/command/fuse*'
|
||||
- 'test/fuse_integration/**'
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
test_type:
|
||||
description: 'Type of POSIX tests to run (default: full for comprehensive testing)'
|
||||
required: true
|
||||
default: 'full'
|
||||
type: choice
|
||||
options:
|
||||
- critical
|
||||
- basic
|
||||
- extended
|
||||
- full
|
||||
enable_external_tests:
|
||||
description: 'Run external test suites (slower)'
|
||||
required: false
|
||||
default: false
|
||||
type: boolean
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
pull-requests: write
|
||||
issues: write
|
||||
|
||||
env:
|
||||
GO_VERSION: '1.26'
|
||||
TIMEOUT: '45m'
|
||||
|
||||
jobs:
|
||||
posix-compliance-ubuntu:
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 60
|
||||
|
||||
strategy:
|
||||
matrix:
|
||||
fuse-version: ['2.9', '3.0']
|
||||
fail-fast: false
|
||||
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v7
|
||||
with:
|
||||
fetch-depth: 1
|
||||
|
||||
- name: Clean up stale FUSE mounts
|
||||
uses: ./.github/actions/cleanup-stale-fuse-mounts
|
||||
|
||||
- name: Set up Go ${{ env.GO_VERSION }}
|
||||
uses: actions/setup-go@v7
|
||||
with:
|
||||
go-version: ${{ env.GO_VERSION }}
|
||||
cache: ${{ runner.environment == 'github-hosted' }}
|
||||
cache-dependency-path: |
|
||||
go.sum
|
||||
test/fuse_integration/go.sum
|
||||
|
||||
- name: Install system dependencies
|
||||
run: |
|
||||
sudo apt-get update
|
||||
sudo apt-get install -y \
|
||||
attr \
|
||||
acl \
|
||||
build-essential \
|
||||
git \
|
||||
python3-pip
|
||||
|
||||
# Install FUSE version specific packages
|
||||
if [ "${{ matrix.fuse-version }}" = "2.9" ]; then
|
||||
echo "Installing FUSE 2.9..."
|
||||
sudo apt-get install -y fuse libfuse-dev
|
||||
elif [ "${{ matrix.fuse-version }}" = "3.0" ]; then
|
||||
echo "Installing FUSE 3.0..."
|
||||
sudo apt-get install -y fuse3 libfuse3-dev
|
||||
# Also install fuse2 for compatibility if needed
|
||||
sudo apt-get install -y fuse libfuse-dev
|
||||
fi
|
||||
|
||||
- name: Set up user permissions for FUSE
|
||||
run: |
|
||||
# Create fuse group if it doesn't exist
|
||||
sudo groupadd -f fuse
|
||||
|
||||
# Add user to fuse group
|
||||
sudo usermod -a -G fuse $USER
|
||||
|
||||
# Set permissions on FUSE device
|
||||
sudo chmod 666 /dev/fuse
|
||||
|
||||
# Allow non-root mounts to pass -o allow_other (weed -allowOthers)
|
||||
echo 'user_allow_other' | sudo tee -a /etc/fuse.conf
|
||||
sudo chmod 644 /etc/fuse.conf
|
||||
|
||||
# Ensure fuse module is loaded
|
||||
sudo modprobe fuse || true
|
||||
|
||||
# Verify setup
|
||||
echo "FUSE setup verification:"
|
||||
ls -la /dev/fuse
|
||||
groups $USER | grep fuse || echo "User not in fuse group yet (will be effective after login)"
|
||||
lsmod | grep fuse || echo "FUSE module not loaded"
|
||||
|
||||
- name: Install external test tools
|
||||
if: ${{ github.event.inputs.enable_external_tests == 'true' }}
|
||||
run: |
|
||||
# Install nfstest for POSIX API verification
|
||||
pip3 install --user nfstest
|
||||
|
||||
# Install FIO for performance testing
|
||||
sudo apt-get install -y fio
|
||||
|
||||
# Install additional test utilities
|
||||
sudo apt-get install -y stress-ng
|
||||
|
||||
- name: Build SeaweedFS
|
||||
run: |
|
||||
make
|
||||
# Verify binary exists and is executable (it's installed to GOPATH/bin)
|
||||
weed version
|
||||
|
||||
# Make weed binary available in PATH (it's already there from go install)
|
||||
sudo cp $(which weed) /usr/local/bin/weed
|
||||
which weed
|
||||
weed version
|
||||
|
||||
- name: Set up SeaweedFS cluster
|
||||
run: |
|
||||
start_and_wait() {
|
||||
local name=$1
|
||||
local url=$2
|
||||
local pidfile=$3
|
||||
shift 3
|
||||
local cmd="$@"
|
||||
|
||||
echo "Starting $name..."
|
||||
$cmd > "/tmp/seaweedfs/$name.log" 2>&1 &
|
||||
local pid=$!
|
||||
echo $pid > "$pidfile"
|
||||
|
||||
echo "Waiting for $name to start..."
|
||||
for i in {1..30}; do
|
||||
if curl -sf "$url" > /dev/null 2>&1; then
|
||||
echo "$name is ready"
|
||||
return 0
|
||||
fi
|
||||
if [ $i -eq 30 ]; then
|
||||
echo "$name failed to start"
|
||||
cat "/tmp/seaweedfs/$name.log"
|
||||
exit 1
|
||||
fi
|
||||
sleep 2
|
||||
done
|
||||
}
|
||||
|
||||
mkdir -p /tmp/seaweedfs/{master,volume,filer,mount}
|
||||
|
||||
start_and_wait "master" "http://127.0.0.1:9333/cluster/status" "/tmp/seaweedfs/master.pid" \
|
||||
"weed master -ip=127.0.0.1 -port=9333 -mdir=/tmp/seaweedfs/master -raftBootstrap=true"
|
||||
|
||||
start_and_wait "volume" "http://127.0.0.1:8080/status" "/tmp/seaweedfs/volume.pid" \
|
||||
"weed volume -mserver=127.0.0.1:9333 -ip=127.0.0.1 -port=8080 -dir=/tmp/seaweedfs/volume -max=100"
|
||||
|
||||
start_and_wait "filer" "http://127.0.0.1:8888/" "/tmp/seaweedfs/filer.pid" \
|
||||
"weed filer -master=127.0.0.1:9333 -ip=127.0.0.1 -port=8888"
|
||||
|
||||
echo "SeaweedFS cluster status:"
|
||||
curl -s http://127.0.0.1:9333/cluster/status || true
|
||||
|
||||
- name: Set up FUSE mount
|
||||
run: |
|
||||
# Create mount point
|
||||
MOUNT_POINT="/tmp/seaweedfs/mount"
|
||||
mkdir -p $MOUNT_POINT
|
||||
|
||||
echo "Mounting SeaweedFS FUSE filesystem..."
|
||||
# Mount SeaweedFS FUSE filesystem in background
|
||||
weed mount \
|
||||
-filer=127.0.0.1:8888 \
|
||||
-dir=$MOUNT_POINT \
|
||||
-filer.path=/ \
|
||||
-dirAutoCreate=true \
|
||||
-allowOthers=true \
|
||||
-nonempty=true \
|
||||
> /tmp/seaweedfs/mount.log 2>&1 &
|
||||
MOUNT_PID=$!
|
||||
echo $MOUNT_PID > /tmp/seaweedfs/mount.pid
|
||||
|
||||
# Wait for mount to be ready. The mount point directory already
|
||||
# exists, so only a real mount counts — a directory listing would
|
||||
# pass on the local disk even if weed mount died.
|
||||
echo "Waiting for FUSE mount to be ready..."
|
||||
for i in {1..30}; do
|
||||
if mountpoint -q $MOUNT_POINT 2>/dev/null; then
|
||||
echo "FUSE mount is ready"
|
||||
break
|
||||
fi
|
||||
if ! kill -0 $MOUNT_PID 2>/dev/null; then
|
||||
echo "FUSE mount process exited"
|
||||
cat /tmp/seaweedfs/mount.log
|
||||
exit 1
|
||||
fi
|
||||
if [ $i -eq 30 ]; then
|
||||
echo "FUSE mount failed to be ready"
|
||||
cat /tmp/seaweedfs/mount.log
|
||||
echo "Mount status:"
|
||||
mount | grep fuse || echo "No FUSE mounts found"
|
||||
ps aux | grep weed || echo "No weed processes found"
|
||||
exit 1
|
||||
fi
|
||||
sleep 2
|
||||
done
|
||||
|
||||
# Verify mount is working
|
||||
echo "Testing FUSE mount functionality..."
|
||||
echo "test" > $MOUNT_POINT/test_file.txt
|
||||
if [ "$(cat $MOUNT_POINT/test_file.txt)" = "test" ]; then
|
||||
echo "FUSE mount is working correctly"
|
||||
rm $MOUNT_POINT/test_file.txt
|
||||
else
|
||||
echo "FUSE mount is not working properly"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Export mount point for tests
|
||||
echo "SEAWEEDFS_MOUNT_POINT=$MOUNT_POINT" >> $GITHUB_ENV
|
||||
|
||||
- name: Set up test environment
|
||||
run: |
|
||||
cd test/fuse_integration
|
||||
|
||||
# Initialize Go module for tests
|
||||
if [ ! -f go.mod ]; then
|
||||
go mod init seaweedfs-posix-tests
|
||||
go mod tidy
|
||||
fi
|
||||
|
||||
# Create reports directory
|
||||
mkdir -p reports
|
||||
|
||||
# Set up external tools if requested
|
||||
# Note: External tools setup is now optional and won't fail the build
|
||||
if [ "${{ github.event.inputs.enable_external_tests }}" = "true" ]; then
|
||||
echo "Setting up external POSIX testing tools..."
|
||||
make -f posix_Makefile setup-external-tools || true
|
||||
# setup-external-tools exits 0 even when a sub-target failed, so
|
||||
# require the tools only when they are actually installed
|
||||
if [ -d external_tools/pjdfstest ] && command -v nfstest_posix >/dev/null 2>&1; then
|
||||
export POSIX_REQUIRE_EXTERNAL_TOOLS="true"
|
||||
echo "POSIX_REQUIRE_EXTERNAL_TOOLS=true" >> $GITHUB_ENV
|
||||
echo "✅ External tools setup completed successfully"
|
||||
else
|
||||
echo "⚠️ External tools setup incomplete - continuing with basic tests only"
|
||||
echo "External tools will be skipped but tests will continue"
|
||||
echo "This is expected behavior when build tools are not available"
|
||||
fi
|
||||
fi
|
||||
|
||||
# Verify SeaweedFS cluster is accessible
|
||||
echo "Verifying SeaweedFS cluster accessibility..."
|
||||
curl -sf http://127.0.0.1:9333/cluster/status
|
||||
curl -sf http://127.0.0.1:8080/status
|
||||
curl -sf http://127.0.0.1:8888/
|
||||
|
||||
# Verify mount point
|
||||
echo "Mount point: $SEAWEEDFS_MOUNT_POINT"
|
||||
ls -la $SEAWEEDFS_MOUNT_POINT
|
||||
|
||||
# Verify mount is actually working by testing file operations
|
||||
echo "Verifying SeaweedFS mount functionality..."
|
||||
TEST_FILE="$SEAWEEDFS_MOUNT_POINT/.mount_verification_test"
|
||||
echo "SeaweedFS mount test" > "$TEST_FILE"
|
||||
if [ "$(cat "$TEST_FILE")" = "SeaweedFS mount test" ]; then
|
||||
echo "✅ Mount verification successful"
|
||||
rm -f "$TEST_FILE"
|
||||
else
|
||||
echo "❌ Mount verification failed - mount may not be working properly"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Check if this is actually a FUSE mount — the plain directory created
|
||||
# above passes the file test, so without this the suite can run
|
||||
# against the local disk and report a pass it never earned
|
||||
if mount | grep -q "$SEAWEEDFS_MOUNT_POINT"; then
|
||||
echo "✅ FUSE mount detected:"
|
||||
mount | grep "$SEAWEEDFS_MOUNT_POINT"
|
||||
else
|
||||
echo "❌ Mount point not found in mount table - refusing to test a local directory"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
- name: Run POSIX compliance tests
|
||||
id: posix-tests
|
||||
env:
|
||||
SEAWEEDFS_MOUNT_POINT: ${{ env.SEAWEEDFS_MOUNT_POINT }}
|
||||
POSIX_REQUIRE_EXTERNAL_TOOLS: ${{ env.POSIX_REQUIRE_EXTERNAL_TOOLS }}
|
||||
run: |
|
||||
cd test/fuse_integration
|
||||
|
||||
# Determine which tests to run
|
||||
TEST_TYPE="${{ github.event.inputs.test_type }}"
|
||||
if [ -z "$TEST_TYPE" ]; then
|
||||
TEST_TYPE="full"
|
||||
fi
|
||||
|
||||
echo "Running POSIX tests: $TEST_TYPE"
|
||||
if [ "$TEST_TYPE" = "full" ]; then
|
||||
echo "🚀 Running COMPREHENSIVE POSIX test suite (including external test suites)"
|
||||
else
|
||||
echo "Using mount point: $SEAWEEDFS_MOUNT_POINT"
|
||||
fi
|
||||
|
||||
# Set test configuration to use our mounted filesystem
|
||||
export TEST_MOUNT_POINT="$SEAWEEDFS_MOUNT_POINT"
|
||||
export TEST_SKIP_CLUSTER_SETUP="true"
|
||||
|
||||
# Debug: Show environment variables
|
||||
echo "🔍 Test Environment:"
|
||||
echo " TEST_MOUNT_POINT=$TEST_MOUNT_POINT"
|
||||
echo " TEST_SKIP_CLUSTER_SETUP=$TEST_SKIP_CLUSTER_SETUP"
|
||||
echo " SEAWEEDFS_MOUNT_POINT=$SEAWEEDFS_MOUNT_POINT"
|
||||
|
||||
case "$TEST_TYPE" in
|
||||
"critical")
|
||||
echo "Running critical POSIX tests (basic compliance only)..."
|
||||
make -f posix_Makefile test-posix-critical
|
||||
;;
|
||||
"basic")
|
||||
echo "Running basic POSIX tests (core file operations)..."
|
||||
make -f posix_Makefile test-posix-basic
|
||||
;;
|
||||
"extended")
|
||||
echo "Running extended POSIX tests (including advanced features)..."
|
||||
make -f posix_Makefile test-posix-extended
|
||||
;;
|
||||
"full")
|
||||
echo "🚀 Running COMPREHENSIVE POSIX test suite (all tests including external)..."
|
||||
echo " This includes: basic, extended, and external test suites"
|
||||
echo " External suites: pjdfstest, nfstest_posix, and custom stress tests"
|
||||
make -f posix_Makefile test-posix-full
|
||||
;;
|
||||
*)
|
||||
echo "Unknown test type: $TEST_TYPE, defaulting to full suite"
|
||||
echo "🚀 Running full POSIX test suite (all tests including external)..."
|
||||
make -f posix_Makefile test-posix-full
|
||||
;;
|
||||
esac
|
||||
|
||||
- name: Run external test suites
|
||||
if: ${{ github.event.inputs.enable_external_tests == 'true' }}
|
||||
continue-on-error: true
|
||||
run: |
|
||||
cd test/fuse_integration
|
||||
|
||||
# Point the external suites at the same mount as the Go tests;
|
||||
# without these the Makefile falls back to a local-disk path.
|
||||
export TEST_MOUNT_POINT="$SEAWEEDFS_MOUNT_POINT"
|
||||
export TEST_SKIP_CLUSTER_SETUP="true"
|
||||
|
||||
# Run external tests (may fail on some systems)
|
||||
make -f posix_Makefile test-nfstest-posix || echo "nfstest failed or not available"
|
||||
make -f posix_Makefile test-fio-posix || echo "FIO tests failed or not available"
|
||||
|
||||
- name: Generate compliance report
|
||||
if: always()
|
||||
run: |
|
||||
cd test/fuse_integration
|
||||
make -f posix_Makefile generate-report
|
||||
|
||||
- name: Collect system information for debugging
|
||||
if: failure()
|
||||
run: |
|
||||
cd test/fuse_integration
|
||||
mkdir -p reports/debug
|
||||
|
||||
echo "System Information" > reports/debug/system_info.txt
|
||||
uname -a >> reports/debug/system_info.txt
|
||||
echo "" >> reports/debug/system_info.txt
|
||||
|
||||
echo "Mount Information" >> reports/debug/system_info.txt
|
||||
mount >> reports/debug/system_info.txt
|
||||
echo "" >> reports/debug/system_info.txt
|
||||
|
||||
echo "Disk Space" >> reports/debug/system_info.txt
|
||||
timeout 15 df -h >> reports/debug/system_info.txt || true
|
||||
echo "" >> reports/debug/system_info.txt
|
||||
|
||||
echo "Memory Information" >> reports/debug/system_info.txt
|
||||
free -h >> reports/debug/system_info.txt
|
||||
echo "" >> reports/debug/system_info.txt
|
||||
|
||||
echo "FUSE Information" >> reports/debug/system_info.txt
|
||||
ls -la /dev/fuse >> reports/debug/system_info.txt
|
||||
lsmod | grep fuse >> reports/debug/system_info.txt
|
||||
echo "" >> reports/debug/system_info.txt
|
||||
|
||||
echo "SeaweedFS Version" >> reports/debug/system_info.txt
|
||||
weed version >> reports/debug/system_info.txt
|
||||
echo "" >> reports/debug/system_info.txt
|
||||
|
||||
echo "Go Version" >> reports/debug/system_info.txt
|
||||
go version >> reports/debug/system_info.txt
|
||||
echo "" >> reports/debug/system_info.txt
|
||||
|
||||
echo "Running Processes" >> reports/debug/system_info.txt
|
||||
ps aux | grep -E "(weed|fuse)" >> reports/debug/system_info.txt
|
||||
echo "" >> reports/debug/system_info.txt
|
||||
|
||||
# Collect SeaweedFS service logs
|
||||
echo "=== SeaweedFS Service Logs ===" >> reports/debug/system_info.txt
|
||||
if [ -f /tmp/seaweedfs/master.log ]; then
|
||||
echo "Master Log:" >> reports/debug/system_info.txt
|
||||
tail -50 /tmp/seaweedfs/master.log >> reports/debug/system_info.txt
|
||||
echo "" >> reports/debug/system_info.txt
|
||||
fi
|
||||
|
||||
if [ -f /tmp/seaweedfs/volume.log ]; then
|
||||
echo "Volume Log:" >> reports/debug/system_info.txt
|
||||
tail -50 /tmp/seaweedfs/volume.log >> reports/debug/system_info.txt
|
||||
echo "" >> reports/debug/system_info.txt
|
||||
fi
|
||||
|
||||
if [ -f /tmp/seaweedfs/filer.log ]; then
|
||||
echo "Filer Log:" >> reports/debug/system_info.txt
|
||||
tail -50 /tmp/seaweedfs/filer.log >> reports/debug/system_info.txt
|
||||
echo "" >> reports/debug/system_info.txt
|
||||
fi
|
||||
|
||||
if [ -f /tmp/seaweedfs/mount.log ]; then
|
||||
echo "Mount Log:" >> reports/debug/system_info.txt
|
||||
tail -50 /tmp/seaweedfs/mount.log >> reports/debug/system_info.txt
|
||||
echo "" >> reports/debug/system_info.txt
|
||||
fi
|
||||
|
||||
# Copy full logs as separate files
|
||||
if [ -f /tmp/seaweedfs/master.log ]; then
|
||||
cp /tmp/seaweedfs/master.log reports/debug/master.log
|
||||
fi
|
||||
if [ -f /tmp/seaweedfs/volume.log ]; then
|
||||
cp /tmp/seaweedfs/volume.log reports/debug/volume.log
|
||||
fi
|
||||
if [ -f /tmp/seaweedfs/filer.log ]; then
|
||||
cp /tmp/seaweedfs/filer.log reports/debug/filer.log
|
||||
fi
|
||||
if [ -f /tmp/seaweedfs/mount.log ]; then
|
||||
cp /tmp/seaweedfs/mount.log reports/debug/mount.log
|
||||
fi
|
||||
|
||||
- name: Cleanup SeaweedFS cluster and FUSE mount
|
||||
if: always()
|
||||
run: |
|
||||
stop_process() {
|
||||
local name=$1
|
||||
local pidfile=$2
|
||||
if [ -f "$pidfile" ]; then
|
||||
local pid=$(cat "$pidfile")
|
||||
if kill -0 $pid 2>/dev/null; then
|
||||
echo "Stopping $name process (PID: $pid)..."
|
||||
kill -TERM $pid || true
|
||||
sleep 2
|
||||
kill -KILL $pid 2>/dev/null || true
|
||||
fi
|
||||
fi
|
||||
}
|
||||
|
||||
echo "Cleaning up SeaweedFS cluster and FUSE mount..."
|
||||
|
||||
MOUNT_POINT="/tmp/seaweedfs/mount"
|
||||
if mountpoint -q $MOUNT_POINT 2>/dev/null; then
|
||||
echo "Unmounting FUSE filesystem..."
|
||||
fusermount -u $MOUNT_POINT || umount $MOUNT_POINT || true
|
||||
fi
|
||||
|
||||
stop_process "mount" "/tmp/seaweedfs/mount.pid"
|
||||
stop_process "filer" "/tmp/seaweedfs/filer.pid"
|
||||
stop_process "volume" "/tmp/seaweedfs/volume.pid"
|
||||
stop_process "master" "/tmp/seaweedfs/master.pid"
|
||||
|
||||
pkill -f "weed " || true
|
||||
|
||||
fusermount -u $MOUNT_POINT 2>/dev/null || true
|
||||
umount $MOUNT_POINT 2>/dev/null || true
|
||||
|
||||
rm -rf /tmp/seaweedfs || true
|
||||
|
||||
echo "Cleanup completed"
|
||||
|
||||
|
||||
- name: Upload test results
|
||||
uses: actions/upload-artifact@v7
|
||||
if: always()
|
||||
with:
|
||||
name: posix-test-results-ubuntu-fuse${{ matrix.fuse-version }}
|
||||
path: |
|
||||
test/fuse_integration/reports/
|
||||
test/fuse_integration/*.log
|
||||
retention-days: 30
|
||||
|
||||
- name: Upload test coverage
|
||||
uses: actions/upload-artifact@v7
|
||||
if: always() && (github.event.inputs.test_type == 'full' || github.event.inputs.test_type == 'extended')
|
||||
with:
|
||||
name: posix-coverage-ubuntu
|
||||
path: test/fuse_integration/reports/posix_coverage.html
|
||||
retention-days: 14
|
||||
|
||||
- name: Comment PR with results
|
||||
if: github.event_name == 'pull_request' && always()
|
||||
uses: actions/github-script@v7
|
||||
with:
|
||||
script: |
|
||||
const fs = require('fs');
|
||||
const path = require('path');
|
||||
|
||||
const reportPath = 'test/fuse_integration/reports/posix_compliance_summary.txt';
|
||||
|
||||
if (fs.existsSync(reportPath)) {
|
||||
const report = fs.readFileSync(reportPath, 'utf8');
|
||||
const comment = `## POSIX Compliance Test Results
|
||||
|
||||
**Go Version:** ${{ env.GO_VERSION }}
|
||||
**FUSE Version:** ${{ matrix.fuse-version }}
|
||||
**Test Type:** ${{ github.event.inputs.test_type || 'full' }}
|
||||
|
||||
<details>
|
||||
<summary>Test Summary</summary>
|
||||
|
||||
\`\`\`
|
||||
${report}
|
||||
\`\`\`
|
||||
</details>
|
||||
|
||||
📊 Full results available in [test artifacts](https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }})
|
||||
`;
|
||||
|
||||
try {
|
||||
await github.rest.issues.createComment({
|
||||
issue_number: context.issue.number,
|
||||
owner: context.repo.owner,
|
||||
repo: context.repo.repo,
|
||||
body: comment
|
||||
});
|
||||
} catch (err) {
|
||||
console.log('Could not post PR comment: ' + err.message);
|
||||
}
|
||||
}
|
||||
|
||||
posix-compliance-macos:
|
||||
runs-on: macos-latest
|
||||
timeout-minutes: 60
|
||||
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v7
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v7
|
||||
with:
|
||||
go-version: ${{ env.GO_VERSION }}
|
||||
cache: ${{ runner.environment == 'github-hosted' }}
|
||||
|
||||
- name: Install macFUSE
|
||||
run: |
|
||||
# Install macFUSE
|
||||
brew install macfuse
|
||||
|
||||
# Note: macFUSE may require system extension approval on macOS
|
||||
# This step may need manual intervention in some cases
|
||||
|
||||
- name: Build SeaweedFS
|
||||
run: |
|
||||
make
|
||||
# Verify binary exists and is executable (it's installed to GOPATH/bin)
|
||||
weed version
|
||||
|
||||
- name: Run critical POSIX tests (macOS)
|
||||
continue-on-error: true # macOS FUSE can be more restrictive
|
||||
run: |
|
||||
cd test/fuse_integration
|
||||
|
||||
if [ ! -f go.mod ]; then
|
||||
go mod init seaweedfs-posix-tests
|
||||
go mod tidy
|
||||
fi
|
||||
|
||||
mkdir -p reports
|
||||
|
||||
# Run basic tests only on macOS due to FUSE limitations
|
||||
make -f posix_Makefile test-posix-critical
|
||||
|
||||
- name: Upload macOS test results
|
||||
uses: actions/upload-artifact@v7
|
||||
if: always()
|
||||
with:
|
||||
name: posix-test-results-macos
|
||||
path: |
|
||||
test/fuse_integration/reports/
|
||||
test/fuse_integration/*.log
|
||||
retention-days: 30
|
||||
|
||||
security-analysis:
|
||||
runs-on: ubuntu-latest
|
||||
if: github.event_name == 'pull_request'
|
||||
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v7
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v7
|
||||
with:
|
||||
go-version: ${{ env.GO_VERSION }}
|
||||
cache: ${{ runner.environment == 'github-hosted' }}
|
||||
|
||||
- name: Install security analysis tools
|
||||
run: |
|
||||
echo "Installing staticcheck for security analysis..."
|
||||
go install honnef.co/go/tools/cmd/staticcheck@latest
|
||||
|
||||
- name: Run security analysis on FUSE code
|
||||
run: |
|
||||
# Analyze mount and FUSE-related code for security and quality issues
|
||||
echo "Running staticcheck security analysis..."
|
||||
|
||||
# Run staticcheck on FUSE-related code
|
||||
echo "Analyzing FUSE mount code..."
|
||||
staticcheck ./weed/mount/... > staticcheck-report.txt 2>&1 || true
|
||||
staticcheck ./weed/command/mount* ./weed/command/fuse* >> staticcheck-report.txt 2>&1 || true
|
||||
|
||||
# Convert to JSON format for consistency
|
||||
echo "{" > gosec-report.json
|
||||
echo " \"tool\": \"staticcheck\"," >> gosec-report.json
|
||||
echo " \"timestamp\": \"$(date -Iseconds)\"," >> gosec-report.json
|
||||
echo " \"issues\": [" >> gosec-report.json
|
||||
|
||||
if [ -s staticcheck-report.txt ]; then
|
||||
echo " \"$(cat staticcheck-report.txt | head -20 | sed 's/"/\\"/g' | tr '\n' ' ')\"" >> gosec-report.json
|
||||
fi
|
||||
|
||||
echo " ]," >> gosec-report.json
|
||||
echo " \"stats\": {" >> gosec-report.json
|
||||
echo " \"files_analyzed\": $(find ./weed/mount ./weed/command -name '*.go' | wc -l)," >> gosec-report.json
|
||||
echo " \"issues_found\": $(wc -l < staticcheck-report.txt 2>/dev/null || echo 0)" >> gosec-report.json
|
||||
echo " }" >> gosec-report.json
|
||||
echo "}" >> gosec-report.json
|
||||
|
||||
echo "Security analysis completed"
|
||||
echo "Issues found: $(wc -l < staticcheck-report.txt 2>/dev/null || echo 0)"
|
||||
|
||||
- name: Upload security analysis results
|
||||
uses: actions/upload-artifact@v7
|
||||
if: always()
|
||||
with:
|
||||
name: security-analysis-results
|
||||
path: gosec-report.json
|
||||
retention-days: 30
|
||||
|
||||
performance-baseline:
|
||||
runs-on: ubuntu-latest
|
||||
if: github.event.inputs.test_type == 'full' || github.event_name == 'schedule'
|
||||
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v7
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v7
|
||||
with:
|
||||
go-version: ${{ env.GO_VERSION }}
|
||||
cache: ${{ runner.environment == 'github-hosted' }}
|
||||
|
||||
- name: Install dependencies
|
||||
run: |
|
||||
sudo apt-get update
|
||||
sudo apt-get install -y fuse libfuse-dev fio
|
||||
sudo usermod -a -G fuse $USER
|
||||
sudo chmod 666 /dev/fuse
|
||||
# The posix suites mount with -allowOthers; allow_other needs this
|
||||
echo 'user_allow_other' | sudo tee -a /etc/fuse.conf
|
||||
|
||||
- name: Build SeaweedFS
|
||||
run: make
|
||||
|
||||
- name: Run performance baseline tests
|
||||
run: |
|
||||
cd test/fuse_integration
|
||||
|
||||
if [ ! -f go.mod ]; then
|
||||
go mod init seaweedfs-posix-tests
|
||||
go mod tidy
|
||||
fi
|
||||
|
||||
mkdir -p reports
|
||||
|
||||
# Run performance benchmarks
|
||||
make -f posix_Makefile benchmark-posix
|
||||
# This job has no FUSE mount, so the fio suite can only skip; keep the
|
||||
# run green but honest about producing no mounted-FS results
|
||||
make -f posix_Makefile test-fio-posix || echo "FIO skipped - no mounted filesystem in this job"
|
||||
|
||||
- name: Store performance baseline
|
||||
uses: actions/upload-artifact@v7
|
||||
with:
|
||||
name: performance-baseline-results
|
||||
path: |
|
||||
test/fuse_integration/reports/posix_benchmark_results.log
|
||||
test/fuse_integration/reports/fio_*_results.log
|
||||
retention-days: 90
|
||||
|
||||
# Schedule regular compliance checks
|
||||
scheduled-compliance-check:
|
||||
runs-on: ubuntu-latest
|
||||
if: github.event_name == 'schedule'
|
||||
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v7
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v7
|
||||
with:
|
||||
go-version: ${{ env.GO_VERSION }}
|
||||
cache: ${{ runner.environment == 'github-hosted' }}
|
||||
|
||||
- name: Install dependencies
|
||||
run: |
|
||||
sudo apt-get update
|
||||
sudo apt-get install -y fuse libfuse-dev
|
||||
pip3 install --user nfstest
|
||||
# The posix suites mount with -allowOthers; allow_other needs this
|
||||
echo 'user_allow_other' | sudo tee -a /etc/fuse.conf
|
||||
|
||||
- name: Build SeaweedFS
|
||||
run: make
|
||||
|
||||
- name: Run comprehensive compliance check
|
||||
run: |
|
||||
cd test/fuse_integration
|
||||
make -f posix_Makefile test-posix-full
|
||||
make -f posix_Makefile generate-report
|
||||
|
||||
- name: Create compliance issue if tests fail
|
||||
if: failure()
|
||||
uses: actions/github-script@v7
|
||||
with:
|
||||
script: |
|
||||
const issue = await github.rest.issues.create({
|
||||
owner: context.repo.owner,
|
||||
repo: context.repo.repo,
|
||||
title: 'POSIX Compliance Check Failed - ' + new Date().toISOString().split('T')[0],
|
||||
body: `## POSIX Compliance Check Failure
|
||||
|
||||
The scheduled POSIX compliance check has failed. This may indicate:
|
||||
|
||||
- Regression in FUSE mount functionality
|
||||
- Changes affecting POSIX compatibility
|
||||
- Infrastructure issues with the test environment
|
||||
|
||||
**Action Required:** Review the test results and investigate any failures.
|
||||
|
||||
**Test Run:** [Workflow Run](https://github.com/${context.repo.owner}/${context.repo.repo}/actions/runs/${context.runId})
|
||||
|
||||
**Date:** ${new Date().toISOString()}
|
||||
|
||||
---
|
||||
This issue was automatically created by the POSIX compliance workflow.
|
||||
`,
|
||||
labels: ['bug', 'posix-compliance', 'automated-issue']
|
||||
});
|
||||
|
||||
console.log('Created issue:', issue.data.number);
|
||||
|
||||
# Schedule this workflow to run weekly
|
||||
# Uncomment the following lines to enable scheduled runs:
|
||||
# schedule:
|
||||
# - cron: '0 2 * * 1' # Every Monday at 2 AM UTC
|
||||
|
||||
@@ -3,12 +3,28 @@ name: "PostgreSQL Gateway Tests"
|
||||
on:
|
||||
push:
|
||||
branches: [ master ]
|
||||
paths:
|
||||
- 'weed/server/postgres/**'
|
||||
- 'weed/query/**'
|
||||
- 'weed/mq/**'
|
||||
- 'test/postgres/**'
|
||||
- 'go.mod'
|
||||
- 'go.sum'
|
||||
- '.github/workflows/postgres-tests.yml'
|
||||
pull_request:
|
||||
branches: [ master ]
|
||||
paths:
|
||||
- 'weed/server/postgres/**'
|
||||
- 'weed/query/**'
|
||||
- 'weed/mq/**'
|
||||
- 'test/postgres/**'
|
||||
- 'go.mod'
|
||||
- 'go.sum'
|
||||
- '.github/workflows/postgres-tests.yml'
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.head_ref }}/postgres-tests
|
||||
cancel-in-progress: true
|
||||
group: ${{ github.workflow }}-${{ github.event.pull_request.number || (github.event_name == 'push' && github.ref) || github.run_id }}
|
||||
cancel-in-progress: ${{ github.event_name == 'pull_request' || (github.event_name == 'push' && !startsWith(github.ref, 'refs/tags/')) }}
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
@@ -23,19 +39,26 @@ jobs:
|
||||
working-directory: test/postgres
|
||||
steps:
|
||||
- name: Set up Go 1.x
|
||||
uses: actions/setup-go@v6
|
||||
uses: actions/setup-go@v7
|
||||
with:
|
||||
go-version: ^1.24
|
||||
cache: ${{ runner.environment == 'github-hosted' }}
|
||||
go-version: ^1.26
|
||||
id: go
|
||||
|
||||
- name: Check out code
|
||||
uses: actions/checkout@v6
|
||||
uses: actions/checkout@v7
|
||||
|
||||
- name: Configure Docker Hub mirror
|
||||
run: |
|
||||
echo '{"registry-mirrors": ["https://mirror.gcr.io"]}' | sudo tee /etc/docker/daemon.json
|
||||
sudo systemctl restart docker
|
||||
|
||||
- name: Set up Docker Buildx
|
||||
uses: docker/setup-buildx-action@v3
|
||||
uses: docker/setup-buildx-action@v4
|
||||
|
||||
- name: Cache Docker layers
|
||||
uses: actions/cache@v5
|
||||
if: runner.environment == 'github-hosted'
|
||||
uses: actions/cache@v6
|
||||
with:
|
||||
path: /tmp/.buildx-cache
|
||||
key: ${{ runner.os }}-buildx-postgres-${{ github.sha }}
|
||||
@@ -62,7 +85,7 @@ jobs:
|
||||
|
||||
- name: Archive logs
|
||||
if: always()
|
||||
uses: actions/upload-artifact@v6
|
||||
uses: actions/upload-artifact@v7
|
||||
with:
|
||||
name: postgres-logs
|
||||
path: test/postgres/postgres-output.log
|
||||
|
||||
@@ -0,0 +1,278 @@
|
||||
name: "release: bump version and cut the release"
|
||||
|
||||
# One entry point for a SeaweedFS release:
|
||||
# 1. bump MAJOR/MINOR in constants.go and the Helm Chart.yaml, commit to master
|
||||
# 2. push the <appVersion> tag, which fans out to the workflows that trigger on
|
||||
# `push: tags` (binaries_release*, container_release_unified, helm_manual_release)
|
||||
# 3. create the GitHub release, with GitHub's generated notes
|
||||
# 4. dispatch "Prepare release" in seaweedfs-csi-driver and seaweedfs-operator,
|
||||
# which pick up the new master through `go get -u`, and wait for both
|
||||
#
|
||||
# Events raised by the default GITHUB_TOKEN do not start other workflows, and it
|
||||
# cannot reach the other two repositories at all. Add a repo secret RELEASE_PAT
|
||||
# with `contents: write` here and `actions: write` on the csi-driver and operator
|
||||
# repos. Without it the tag is still pushed, but nothing downstream of it runs.
|
||||
|
||||
on:
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
bump:
|
||||
description: "Which part to increment (ignored when 'version' is set)"
|
||||
type: choice
|
||||
options:
|
||||
- minor
|
||||
- major
|
||||
default: minor
|
||||
version:
|
||||
description: "Explicit MAJOR.MINOR to set, e.g. 4.36 (overrides 'bump')"
|
||||
type: string
|
||||
required: false
|
||||
downstream:
|
||||
description: "Also release the CSI driver and the operator"
|
||||
type: boolean
|
||||
default: true
|
||||
dry_run:
|
||||
description: "Show the version bump, but change nothing"
|
||||
type: boolean
|
||||
default: false
|
||||
|
||||
jobs:
|
||||
release:
|
||||
runs-on: ubuntu-latest
|
||||
permissions:
|
||||
contents: write
|
||||
outputs:
|
||||
app_version: ${{ steps.compute.outputs.app_version }}
|
||||
sha: ${{ steps.tag.outputs.sha }}
|
||||
steps:
|
||||
- uses: actions/checkout@v7
|
||||
with:
|
||||
ref: master
|
||||
fetch-depth: 0
|
||||
token: ${{ secrets.RELEASE_PAT || secrets.GITHUB_TOKEN }}
|
||||
|
||||
- name: Check the release token
|
||||
env:
|
||||
HAS_PAT: ${{ secrets.RELEASE_PAT != '' }}
|
||||
run: |
|
||||
if [ "$HAS_PAT" != "true" ]; then
|
||||
echo "::warning::RELEASE_PAT is not set. The tag will be pushed with GITHUB_TOKEN, so the binary, container and helm workflows will not start on their own."
|
||||
fi
|
||||
|
||||
- name: Compute new version
|
||||
id: compute
|
||||
env:
|
||||
BUMP: ${{ inputs.bump }}
|
||||
INPUT_VERSION: ${{ inputs.version }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
CONST=weed/util/version/constants.go
|
||||
|
||||
MAJOR=$(grep -oP 'MAJOR_VERSION\s*=\s*int32\(\K[0-9]+' "$CONST")
|
||||
MINOR=$(grep -oP 'MINOR_VERSION\s*=\s*int32\(\K[0-9]+' "$CONST")
|
||||
echo "current: ${MAJOR}.${MINOR}"
|
||||
|
||||
if [ -n "$INPUT_VERSION" ]; then
|
||||
if ! [[ "$INPUT_VERSION" =~ ^[0-9]+\.[0-9]+$ ]]; then
|
||||
echo "::error::version must be MAJOR.MINOR, e.g. 5.01 (got '$INPUT_VERSION')"
|
||||
exit 1
|
||||
fi
|
||||
# 10# forces base 10 so 08/09 are not parsed as octal.
|
||||
MAJOR=$((10#${INPUT_VERSION%%.*}))
|
||||
MINOR=$((10#${INPUT_VERSION##*.}))
|
||||
if [ "$MINOR" -gt 99 ]; then
|
||||
echo "::error::minor must be 0-99 (got $MINOR); it rolls into the next major at 99"
|
||||
exit 1
|
||||
fi
|
||||
else
|
||||
case "$BUMP" in
|
||||
# Minor is a 2-digit field: 4.99 -> 5.00 -> 5.01.
|
||||
major) MAJOR=$((MAJOR + 1)); MINOR=0 ;;
|
||||
minor)
|
||||
if [ "$MINOR" -ge 99 ]; then
|
||||
MAJOR=$((MAJOR + 1)); MINOR=0
|
||||
else
|
||||
MINOR=$((MINOR + 1))
|
||||
fi
|
||||
;;
|
||||
*) echo "::error::unknown bump '$BUMP'"; exit 1 ;;
|
||||
esac
|
||||
fi
|
||||
|
||||
# appVersion mirrors the Go VERSION_NUMBER (zero-padded minor);
|
||||
# chart version is plain SemVer (no leading zeros).
|
||||
APP_VERSION=$(printf '%d.%02d' "$MAJOR" "$MINOR")
|
||||
CHART_VERSION="${MAJOR}.${MINOR}.0"
|
||||
echo "new: app=${APP_VERSION} chart=${CHART_VERSION}"
|
||||
|
||||
{
|
||||
echo "major=${MAJOR}"
|
||||
echo "minor=${MINOR}"
|
||||
echo "app_version=${APP_VERSION}"
|
||||
echo "chart_version=${CHART_VERSION}"
|
||||
} >> "$GITHUB_OUTPUT"
|
||||
|
||||
- name: Apply version to constants.go
|
||||
env:
|
||||
MAJOR: ${{ steps.compute.outputs.major }}
|
||||
MINOR: ${{ steps.compute.outputs.minor }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
CONST=weed/util/version/constants.go
|
||||
sed -i -E "s/(MAJOR_VERSION[[:space:]]*=[[:space:]]*int32\()[0-9]+(\))/\1${MAJOR}\2/" "$CONST"
|
||||
sed -i -E "s/(MINOR_VERSION[[:space:]]*=[[:space:]]*int32\()[0-9]+(\))/\1${MINOR}\2/" "$CONST"
|
||||
grep -E 'MAJOR_VERSION|MINOR_VERSION' "$CONST"
|
||||
|
||||
- name: Apply version to Chart.yaml
|
||||
env:
|
||||
APP_VERSION: ${{ steps.compute.outputs.app_version }}
|
||||
CHART_VERSION: ${{ steps.compute.outputs.chart_version }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
CHART=k8s/charts/seaweedfs/Chart.yaml
|
||||
sed -i -E "s/^appVersion:.*/appVersion: \"${APP_VERSION}\"/" "$CHART"
|
||||
sed -i -E "s/^version:.*/version: ${CHART_VERSION}/" "$CHART"
|
||||
cat "$CHART"
|
||||
|
||||
- name: Commit, and push the tag
|
||||
id: tag
|
||||
env:
|
||||
TAG: ${{ steps.compute.outputs.app_version }}
|
||||
DRY_RUN: ${{ inputs.dry_run }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
if git ls-remote --exit-code --tags origin "refs/tags/${TAG}" >/dev/null 2>&1; then
|
||||
echo "::error::Tag ${TAG} already exists."
|
||||
exit 1
|
||||
fi
|
||||
if [ "$DRY_RUN" = "true" ]; then
|
||||
git --no-pager diff --stat
|
||||
echo "sha=$(git rev-parse HEAD)" >> "$GITHUB_OUTPUT"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
git config user.name "github-actions[bot]"
|
||||
git config user.email "41898282+github-actions[bot]@users.noreply.github.com"
|
||||
if git diff --quiet; then
|
||||
echo "::warning::Version files are already at ${TAG}; tagging the current HEAD."
|
||||
else
|
||||
git add weed/util/version/constants.go k8s/charts/seaweedfs/Chart.yaml
|
||||
git commit -m "${TAG}"
|
||||
git push
|
||||
fi
|
||||
|
||||
# Push the tag with git so the `push: tags` triggers fire. Creating the
|
||||
# tag through the release API alone would only emit a `create` event.
|
||||
git tag "$TAG"
|
||||
git push origin "$TAG"
|
||||
echo "sha=$(git rev-parse HEAD)" >> "$GITHUB_OUTPUT"
|
||||
|
||||
- name: Create the release
|
||||
if: ${{ !inputs.dry_run }}
|
||||
env:
|
||||
GH_TOKEN: ${{ secrets.RELEASE_PAT || secrets.GITHUB_TOKEN }}
|
||||
TAG: ${{ steps.compute.outputs.app_version }}
|
||||
run: gh release create "$TAG" --title "$TAG" --generate-notes --verify-tag
|
||||
|
||||
downstream:
|
||||
needs: release
|
||||
if: ${{ inputs.downstream && !inputs.dry_run }}
|
||||
runs-on: ubuntu-latest
|
||||
# The wait below puts no bound of its own on runner-queue time; this does.
|
||||
timeout-minutes: 120
|
||||
permissions: {}
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
include:
|
||||
- repo: seaweedfs/seaweedfs-csi-driver
|
||||
workflow: prepare_release.yaml
|
||||
- repo: seaweedfs/seaweedfs-operator
|
||||
workflow: prepare_release.yml
|
||||
steps:
|
||||
- name: Release ${{ matrix.repo }}
|
||||
env:
|
||||
GH_TOKEN: ${{ secrets.RELEASE_PAT }}
|
||||
REPO: ${{ matrix.repo }}
|
||||
WORKFLOW: ${{ matrix.workflow }}
|
||||
SHA: ${{ needs.release.outputs.sha }}
|
||||
MODULE: github.com/seaweedfs/seaweedfs
|
||||
run: |
|
||||
set -euo pipefail
|
||||
if [ -z "${GH_TOKEN}" ]; then
|
||||
echo "::error::RELEASE_PAT with actions:write on ${REPO} is required to release it"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# The dispatched workflow pins seaweedfs with `go get -u ...@latest`, so
|
||||
# wait until the proxy serves the release commit as the tip. Asking for
|
||||
# the commit by name is what makes the proxy fetch it. The proxy can
|
||||
# take longer than five minutes to refresh @latest after a new tag.
|
||||
for _ in $(seq 120); do
|
||||
curl -sf "https://proxy.golang.org/${MODULE}/@v/${SHA}.info" >/dev/null || true
|
||||
TIP=$(curl -sf "https://proxy.golang.org/${MODULE}/@latest" | jq -r '.Origin.Hash // ""' || true)
|
||||
[ "$TIP" = "$SHA" ] && break
|
||||
sleep 10
|
||||
done
|
||||
if [ "$TIP" != "$SHA" ]; then
|
||||
echo "::error::the module proxy still serves ${TIP} as the tip, so ${REPO} would pin a pre-release commit. Run ${WORKFLOW} there once it catches up."
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# The dispatched workflow publishes the release as its last step, so
|
||||
# its conclusion decides success. Wait on the run, not on a wall
|
||||
# clock: time it spends queued for a runner must not count against
|
||||
# the budget. A dispatch does not return its run id, so take the
|
||||
# newest workflow_dispatch run created since ours; a concurrent
|
||||
# dispatch would be performing this same release, and waiting on it
|
||||
# is just as good.
|
||||
released() { gh api "repos/${REPO}/releases?per_page=30" --jq '[.[].tag_name]'; }
|
||||
|
||||
BEFORE=$(released)
|
||||
# A minute early, so runner clock skew cannot hide the run.
|
||||
DISPATCHED_AT=$(date -u -d '1 minute ago' '+%Y-%m-%dT%H:%M:%SZ')
|
||||
gh workflow run -R "$REPO" "$WORKFLOW" --ref master -f bump=patch -f update_seaweedfs=true
|
||||
|
||||
RUN_ID=""
|
||||
for _ in $(seq 12); do
|
||||
sleep 10
|
||||
RUN_ID=$(gh api -X GET "repos/${REPO}/actions/workflows/${WORKFLOW}/runs" \
|
||||
-f event=workflow_dispatch -f "created=>=${DISPATCHED_AT}" \
|
||||
--jq '(.workflow_runs | sort_by(.created_at) | last | .id) // empty' || true)
|
||||
[ -n "$RUN_ID" ] && break
|
||||
done
|
||||
if [ -z "$RUN_ID" ]; then
|
||||
echo "::error::the dispatch created no ${WORKFLOW} run in ${REPO}; see https://github.com/${REPO}/actions/workflows/${WORKFLOW}"
|
||||
exit 1
|
||||
fi
|
||||
RUN_URL="https://github.com/${REPO}/actions/runs/${RUN_ID}"
|
||||
echo "waiting on ${RUN_URL}"
|
||||
|
||||
# 20 minutes of execution; polls that find the run still queued do
|
||||
# not consume it.
|
||||
RUNNING=0
|
||||
STATE=""
|
||||
while :; do
|
||||
sleep 15
|
||||
STATE=$(gh api "repos/${REPO}/actions/runs/${RUN_ID}" \
|
||||
--jq '.status + "/" + (.conclusion // "")' || true)
|
||||
case "$STATE" in
|
||||
completed/*) break ;;
|
||||
in_progress/*) RUNNING=$((RUNNING + 1)) ;;
|
||||
esac
|
||||
if [ "$RUNNING" -gt 80 ]; then
|
||||
echo "::error::${RUN_URL} has been executing for over 20 minutes; giving up on it"
|
||||
exit 1
|
||||
fi
|
||||
done
|
||||
if [ "$STATE" != "completed/success" ]; then
|
||||
echo "::error::${RUN_URL} concluded '${STATE#completed/}'"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
NEW=$(released | jq -c --argjson before "$BEFORE" '. - $before')
|
||||
if [ "$(jq length <<<"$NEW")" -eq 0 ]; then
|
||||
echo "::error::${RUN_URL} succeeded but ${REPO} shows no new release"
|
||||
exit 1
|
||||
fi
|
||||
echo "${REPO} released $(jq -r 'join(", ")' <<<"$NEW")"
|
||||
@@ -0,0 +1,351 @@
|
||||
name: "Rust Volume Server Tests"
|
||||
|
||||
on:
|
||||
pull_request:
|
||||
branches: [ master ]
|
||||
paths:
|
||||
- 'seaweed-volume/**'
|
||||
- 'seaweed-common/**'
|
||||
- 'test/volume_server/**'
|
||||
- 'weed/pb/volume_server.proto'
|
||||
- 'weed/pb/volume_server_pb/**'
|
||||
- '.github/workflows/rust-volume-server-tests.yml'
|
||||
push:
|
||||
branches: [ master, main ]
|
||||
paths:
|
||||
- 'seaweed-volume/**'
|
||||
- 'seaweed-common/**'
|
||||
- 'test/volume_server/**'
|
||||
- 'weed/pb/volume_server.proto'
|
||||
- 'weed/pb/volume_server_pb/**'
|
||||
- '.github/workflows/rust-volume-server-tests.yml'
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.event.pull_request.number || (github.event_name == 'push' && github.ref) || github.run_id }}
|
||||
cancel-in-progress: ${{ github.event_name == 'pull_request' || (github.event_name == 'push' && !startsWith(github.ref, 'refs/tags/')) }}
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
|
||||
jobs:
|
||||
changes:
|
||||
name: Detect changed paths
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 5
|
||||
permissions:
|
||||
contents: read
|
||||
outputs:
|
||||
rust: ${{ steps.filter.outputs.rust }}
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v7
|
||||
with:
|
||||
fetch-depth: 0
|
||||
|
||||
- name: Filter changed paths
|
||||
id: filter
|
||||
uses: dorny/paths-filter@v4
|
||||
with:
|
||||
filters: |
|
||||
rust:
|
||||
- 'seaweed-volume/**'
|
||||
- '.github/workflows/rust-volume-server-tests.yml'
|
||||
|
||||
build-test-binaries:
|
||||
name: Build Test Binaries
|
||||
runs-on: ubuntu-22.04
|
||||
timeout-minutes: 30
|
||||
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v7
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v7
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
cache: ${{ runner.environment == 'github-hosted' }}
|
||||
|
||||
- name: Install Rust toolchain
|
||||
uses: dtolnay/rust-toolchain@stable
|
||||
|
||||
# cargo tracks its own inputs but not the runner's C toolchain, so a cached
|
||||
# target/ can carry C objects built against a different glibc than we link against.
|
||||
- name: Fingerprint build toolchain
|
||||
id: toolchain
|
||||
run: echo "fingerprint=$(getconf GNU_LIBC_VERSION | tr ' ' '-')-rustc-$(rustc -V | awk '{print $2}')" >> "$GITHUB_OUTPUT"
|
||||
|
||||
- name: Cache cargo registry and target
|
||||
if: runner.environment == 'github-hosted'
|
||||
uses: actions/cache@v6
|
||||
with:
|
||||
path: |
|
||||
~/.cargo/registry
|
||||
~/.cargo/git
|
||||
seaweed-volume/target
|
||||
key: rust-${{ steps.toolchain.outputs.fingerprint }}-${{ hashFiles('seaweed-volume/Cargo.lock', 'seaweed-common/Cargo.lock') }}
|
||||
restore-keys: |
|
||||
rust-${{ steps.toolchain.outputs.fingerprint }}-
|
||||
|
||||
- name: Build Go weed binary
|
||||
run: |
|
||||
cd weed
|
||||
go build -tags 5BytesOffset -o weed .
|
||||
chmod +x weed
|
||||
./weed version
|
||||
|
||||
- name: Build Rust volume binary
|
||||
run: cd seaweed-volume && cargo build --release
|
||||
|
||||
- name: Package binaries
|
||||
run: tar cf rust-test-binaries.tar weed/weed seaweed-volume/target/release/weed-volume
|
||||
|
||||
- name: Upload test binaries
|
||||
uses: actions/upload-artifact@v7
|
||||
with:
|
||||
# archive layout mirrors the workspace: weed/weed and
|
||||
# seaweed-volume/target/release/weed-volume
|
||||
name: rust-test-binaries
|
||||
if-no-files-found: error
|
||||
retention-days: 1
|
||||
path: rust-test-binaries.tar
|
||||
|
||||
rust-unit-tests:
|
||||
name: Rust Unit Tests
|
||||
runs-on: ubuntu-22.04
|
||||
timeout-minutes: 15
|
||||
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v7
|
||||
|
||||
- name: Install Rust toolchain
|
||||
uses: dtolnay/rust-toolchain@stable
|
||||
|
||||
# cargo tracks its own inputs but not the runner's C toolchain, so a cached
|
||||
# target/ can carry C objects built against a different glibc than we link against.
|
||||
- name: Fingerprint build toolchain
|
||||
id: toolchain
|
||||
run: echo "fingerprint=$(getconf GNU_LIBC_VERSION | tr ' ' '-')-rustc-$(rustc -V | awk '{print $2}')" >> "$GITHUB_OUTPUT"
|
||||
|
||||
- name: Cache cargo registry and target
|
||||
if: runner.environment == 'github-hosted'
|
||||
uses: actions/cache@v6
|
||||
with:
|
||||
path: |
|
||||
~/.cargo/registry
|
||||
~/.cargo/git
|
||||
seaweed-volume/target
|
||||
key: rust-${{ steps.toolchain.outputs.fingerprint }}-${{ hashFiles('seaweed-volume/Cargo.lock', 'seaweed-common/Cargo.lock') }}
|
||||
restore-keys: |
|
||||
rust-${{ steps.toolchain.outputs.fingerprint }}-
|
||||
|
||||
- name: Build Rust volume server
|
||||
run: cd seaweed-volume && cargo build --release
|
||||
|
||||
# The crate is warning-free under clippy as of the sweep that added
|
||||
# this step. Uncomment to make that a gate; `[lints.clippy]` in
|
||||
# seaweed-volume/Cargo.toml is where crate-wide exceptions live.
|
||||
# - name: Clippy
|
||||
# run: cd seaweed-volume && cargo clippy --all-targets -- -D warnings
|
||||
|
||||
# The crate is rustfmt-clean as of the PR that added this step.
|
||||
# Uncomment to keep it that way.
|
||||
# - name: Check formatting
|
||||
# run: cd seaweed-volume && cargo fmt --check
|
||||
|
||||
# seaweed-common is a path dependency of this crate, not a member of its
|
||||
# workspace, so the run below does not reach its own tests. It builds into
|
||||
# this job's cached target directory, and the cache key above covers the
|
||||
# shared crate's lock, so the aws-lc-sys that rustls pulls in is restored
|
||||
# with the cache instead of compiled from scratch on every run.
|
||||
- name: Run shared-crate unit tests
|
||||
env:
|
||||
CARGO_TARGET_DIR: ${{ github.workspace }}/seaweed-volume/target
|
||||
run: cd seaweed-common && cargo test
|
||||
|
||||
- name: Run Rust unit tests
|
||||
run: cd seaweed-volume && cargo test
|
||||
|
||||
- name: Run Rust unit tests (redb experimental cursor)
|
||||
run: cd seaweed-volume && cargo test --features redb-experimental-cursor --lib storage::needle_map
|
||||
|
||||
rust-unit-tests-windows:
|
||||
name: Rust Unit Tests (Windows)
|
||||
runs-on: windows-latest
|
||||
timeout-minutes: 30
|
||||
needs: [changes]
|
||||
if: needs.changes.outputs.rust == 'true'
|
||||
defaults:
|
||||
run:
|
||||
shell: bash
|
||||
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v7
|
||||
|
||||
- name: Install Rust toolchain
|
||||
uses: dtolnay/rust-toolchain@stable
|
||||
|
||||
# No glibc on Windows: key the cache on the toolchain and OS only.
|
||||
- name: Fingerprint build toolchain
|
||||
id: toolchain
|
||||
run: echo "fingerprint=windows-rustc-$(rustc -V | awk '{print $2}')" >> "$GITHUB_OUTPUT"
|
||||
|
||||
- name: Cache cargo registry and target
|
||||
if: runner.environment == 'github-hosted'
|
||||
uses: actions/cache@v6
|
||||
with:
|
||||
path: |
|
||||
~/.cargo/registry
|
||||
~/.cargo/git
|
||||
seaweed-volume/target
|
||||
key: rust-windows-${{ steps.toolchain.outputs.fingerprint }}-${{ hashFiles('seaweed-volume/Cargo.lock') }}
|
||||
restore-keys: |
|
||||
rust-windows-${{ steps.toolchain.outputs.fingerprint }}-
|
||||
|
||||
- name: Run Rust unit tests
|
||||
run: cd seaweed-volume && cargo test
|
||||
|
||||
- name: Run Rust unit tests (redb experimental cursor)
|
||||
run: cd seaweed-volume && cargo test --features redb-experimental-cursor --lib storage::needle_map
|
||||
|
||||
rust-integration-tests:
|
||||
name: Rust Integration Tests
|
||||
runs-on: ubuntu-22.04
|
||||
timeout-minutes: 30
|
||||
needs: [build-test-binaries]
|
||||
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v7
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v7
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
cache: ${{ runner.environment == 'github-hosted' }}
|
||||
|
||||
- name: Download test binaries
|
||||
uses: actions/download-artifact@v7
|
||||
with:
|
||||
name: rust-test-binaries
|
||||
|
||||
- name: Restore binaries
|
||||
run: |
|
||||
tar xf rust-test-binaries.tar
|
||||
chmod +x weed/weed seaweed-volume/target/release/weed-volume
|
||||
./weed/weed version
|
||||
|
||||
- name: Run integration tests
|
||||
env:
|
||||
WEED_BINARY: ${{ github.workspace }}/weed/weed
|
||||
RUST_VOLUME_BINARY: ${{ github.workspace }}/seaweed-volume/target/release/weed-volume
|
||||
run: |
|
||||
echo "Running Rust volume server integration tests..."
|
||||
go test -v -count=1 -timeout=15m ./test/volume_server/rust/...
|
||||
|
||||
- name: Collect logs on failure
|
||||
if: failure()
|
||||
run: |
|
||||
mkdir -p /tmp/rust-volume-server-it-logs
|
||||
find /tmp -maxdepth 1 -type d -name "seaweedfs_volume_server_it_*" -print -exec cp -r {} /tmp/rust-volume-server-it-logs/ \; || true
|
||||
|
||||
- name: Archive logs on failure
|
||||
if: failure()
|
||||
uses: actions/upload-artifact@v7
|
||||
with:
|
||||
name: rust-volume-server-integration-test-logs
|
||||
path: /tmp/rust-volume-server-it-logs/
|
||||
if-no-files-found: warn
|
||||
retention-days: 7
|
||||
|
||||
- name: Test summary
|
||||
if: always()
|
||||
run: |
|
||||
echo "## Rust Volume Server Integration Test Summary" >> "$GITHUB_STEP_SUMMARY"
|
||||
echo "- Suite: test/volume_server/rust" >> "$GITHUB_STEP_SUMMARY"
|
||||
echo "- Command: go test -v -count=1 -timeout=15m ./test/volume_server/rust/..." >> "$GITHUB_STEP_SUMMARY"
|
||||
|
||||
rust-volume-go-tests:
|
||||
name: Go Tests with Rust Volume (${{ matrix.test-type }} - Shard ${{ matrix.shard }})
|
||||
runs-on: ubuntu-22.04
|
||||
timeout-minutes: 45
|
||||
needs: [build-test-binaries]
|
||||
env:
|
||||
# Keep in step with the length of matrix.shard below.
|
||||
SHARD_COUNT: 3
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
test-type: [grpc, http]
|
||||
shard: [1, 2, 3]
|
||||
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v7
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v7
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
cache: ${{ runner.environment == 'github-hosted' }}
|
||||
|
||||
- name: Download test binaries
|
||||
uses: actions/download-artifact@v7
|
||||
with:
|
||||
name: rust-test-binaries
|
||||
|
||||
- name: Restore binaries
|
||||
run: |
|
||||
tar xf rust-test-binaries.tar
|
||||
chmod +x weed/weed seaweed-volume/target/release/weed-volume
|
||||
./weed/weed version
|
||||
./seaweed-volume/target/release/weed-volume --help >/dev/null || true
|
||||
|
||||
# Dealing the listed tests out one by one keeps the shards even. Bucketing
|
||||
# them by first letter did not: names cluster, so ^Test[I-S] drew 50 of
|
||||
# the 114 grpc tests and ran nearly twice as long as the other two shards.
|
||||
- name: Select this shard's tests
|
||||
env:
|
||||
TEST_TYPE: ${{ matrix.test-type }}
|
||||
SHARD: ${{ matrix.shard }}
|
||||
run: |
|
||||
tests=$(go test -tags 5BytesOffset ./test/volume_server/"$TEST_TYPE"/... -list '.*' | grep '^Test' | sort -u)
|
||||
# An empty list would make -run match nothing and the shard pass vacuously.
|
||||
[ -n "$tests" ] || { echo "listed no tests in test/volume_server/$TEST_TYPE"; exit 1; }
|
||||
selected=$(echo "$tests" | awk -v n="$SHARD_COUNT" -v i="$SHARD" 'NR % n == i - 1')
|
||||
echo "shard $SHARD of $SHARD_COUNT runs $(echo "$selected" | wc -l) of $(echo "$tests" | wc -l) tests"
|
||||
echo "TEST_PATTERN=^($(echo "$selected" | paste -sd'|' -))\$" >> "$GITHUB_ENV"
|
||||
|
||||
- name: Run volume server integration tests with Rust volume
|
||||
env:
|
||||
WEED_BINARY: ${{ github.workspace }}/weed/weed
|
||||
RUST_VOLUME_BINARY: ${{ github.workspace }}/seaweed-volume/target/release/weed-volume
|
||||
VOLUME_SERVER_IMPL: rust
|
||||
run: |
|
||||
echo "Running Go volume server tests with Rust volume for ${{ matrix.test-type }} (Shard ${{ matrix.shard }} of ${SHARD_COUNT})..."
|
||||
go test -v -count=1 -tags 5BytesOffset -timeout=30m ./test/volume_server/${{ matrix.test-type }}/... -run "${TEST_PATTERN}"
|
||||
|
||||
- name: Collect logs on failure
|
||||
if: failure()
|
||||
run: |
|
||||
mkdir -p /tmp/rust-volume-go-test-logs
|
||||
find /tmp -maxdepth 1 -type d -name "seaweedfs_volume_server_it_*" -print -exec cp -r {} /tmp/rust-volume-go-test-logs/ \; || true
|
||||
|
||||
- name: Archive logs on failure
|
||||
if: failure()
|
||||
uses: actions/upload-artifact@v7
|
||||
with:
|
||||
name: rust-volume-go-test-logs-${{ matrix.test-type }}-shard${{ matrix.shard }}
|
||||
path: /tmp/rust-volume-go-test-logs/
|
||||
if-no-files-found: warn
|
||||
retention-days: 7
|
||||
|
||||
- name: Test summary
|
||||
if: always()
|
||||
run: |
|
||||
echo "## Rust Volume - Go Test Summary (${{ matrix.test-type }} - Shard ${{ matrix.shard }})" >> "$GITHUB_STEP_SUMMARY"
|
||||
echo "- Suite: test/volume_server/${{ matrix.test-type }} (shard ${{ matrix.shard }} of ${SHARD_COUNT}, see 'Select this shard's tests' for the split)" >> "$GITHUB_STEP_SUMMARY"
|
||||
echo "- Volume server: Rust (VOLUME_SERVER_IMPL=rust)" >> "$GITHUB_STEP_SUMMARY"
|
||||
@@ -0,0 +1,107 @@
|
||||
name: "Rust Plugin Worker Tests"
|
||||
|
||||
on:
|
||||
pull_request:
|
||||
branches: [ master ]
|
||||
paths:
|
||||
- 'seaweed-worker/**'
|
||||
- 'seaweed-common/**'
|
||||
- 'weed/pb/plugin.proto'
|
||||
- '.github/workflows/rust-worker-tests.yml'
|
||||
push:
|
||||
branches: [ master, main ]
|
||||
paths:
|
||||
- 'seaweed-worker/**'
|
||||
- 'seaweed-common/**'
|
||||
- 'weed/pb/plugin.proto'
|
||||
- '.github/workflows/rust-worker-tests.yml'
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.event.pull_request.number || (github.event_name == 'push' && github.ref) || github.run_id }}
|
||||
cancel-in-progress: ${{ github.event_name == 'pull_request' || (github.event_name == 'push' && !startsWith(github.ref, 'refs/tags/')) }}
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
jobs:
|
||||
rust-worker-build:
|
||||
name: Rust Plugin Worker Build and Unit Tests
|
||||
runs-on: ubuntu-22.04
|
||||
timeout-minutes: 45
|
||||
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v7
|
||||
with:
|
||||
persist-credentials: false
|
||||
|
||||
- name: Install Rust toolchain
|
||||
uses: dtolnay/rust-toolchain@stable
|
||||
|
||||
# cargo tracks its own inputs but not the runner's C toolchain, so a cached
|
||||
# target/ can carry C objects built against a different glibc than we link against.
|
||||
- name: Fingerprint build toolchain
|
||||
id: toolchain
|
||||
run: echo "fingerprint=$(getconf GNU_LIBC_VERSION | tr ' ' '-')-rustc-$(rustc -V | awk '{print $2}')" >> "$GITHUB_OUTPUT"
|
||||
|
||||
- name: Cache cargo registry and target
|
||||
if: runner.environment == 'github-hosted'
|
||||
uses: actions/cache@v6
|
||||
with:
|
||||
path: |
|
||||
~/.cargo/registry
|
||||
~/.cargo/git
|
||||
seaweed-worker/target/release
|
||||
key: rust-worker-${{ steps.toolchain.outputs.fingerprint }}-${{ hashFiles('seaweed-worker/Cargo.lock', 'seaweed-common/Cargo.lock') }}
|
||||
restore-keys: |
|
||||
rust-worker-${{ steps.toolchain.outputs.fingerprint }}-
|
||||
|
||||
# lance's build scripts compile their own protos and look for a protoc.
|
||||
# Point them at the one protoc-bin-vendored ships, which seaweed-worker's
|
||||
# own build already uses, so no job depends on a system package and every
|
||||
# build sees the same version.
|
||||
- name: Use the vendored protoc
|
||||
run: |
|
||||
cd seaweed-worker
|
||||
cargo fetch
|
||||
# The version from the lock, not whatever else a restored cache holds.
|
||||
version=$(awk '/^name = "protoc-bin-vendored-linux-x86_64"$/{found=1; next} found && /^version = /{gsub(/"/,"",$3); print $3; exit}' Cargo.lock)
|
||||
test -n "$version" || { echo "protoc-bin-vendored-linux-x86_64 is not in Cargo.lock" >&2; exit 1; }
|
||||
protoc=$(find ~/.cargo/registry/src -path "*protoc-bin-vendored-linux-x86_64-$version/bin/protoc" | head -1)
|
||||
test -x "$protoc" || { echo "no vendored protoc $version in the registry" >&2; exit 1; }
|
||||
echo "PROTOC=$protoc" >> "$GITHUB_ENV"
|
||||
|
||||
# The release profile is what ships, and it is where the release and the
|
||||
# container builds would otherwise discover a break for the first time.
|
||||
- name: Build the plugin workers
|
||||
run: cd seaweed-worker && cargo build --release
|
||||
|
||||
# The workspace is warning-free under clippy as of the sweep that added
|
||||
# this step. Uncomment to make that a gate; `[workspace.lints.clippy]`
|
||||
# in seaweed-worker/Cargo.toml is where crate-wide exceptions live.
|
||||
# - name: Clippy
|
||||
# run: cd seaweed-worker && cargo clippy --workspace --all-targets -- -D warnings
|
||||
|
||||
# The workspace is rustfmt-clean as of the PR that added this step.
|
||||
# Uncomment to keep it that way.
|
||||
# - name: Check formatting
|
||||
# run: cd seaweed-worker && cargo fmt --all --check
|
||||
|
||||
# seaweed-common is a path dependency of core and lance, not a member of
|
||||
# this workspace, so `--workspace` below does not reach its own tests.
|
||||
# Release and this job's cached target directory, and the cache key above
|
||||
# covers the shared crate's lock. That lock pins the same rustls and
|
||||
# aws-lc-sys this workspace resolves, so the release build above has
|
||||
# already paid for them.
|
||||
- name: Run shared-crate unit tests
|
||||
env:
|
||||
CARGO_TARGET_DIR: ${{ github.workspace }}/seaweed-worker/target
|
||||
run: cd seaweed-common && cargo test --release
|
||||
|
||||
# The tests that need a live gateway skip themselves without one, the way
|
||||
# the Go integration tests skip without Docker; the lifecycle suite in
|
||||
# test/s3tables/lifecycle is what runs them against a real cluster.
|
||||
# Release, so this reuses the build above rather than compiling lance,
|
||||
# arrow and datafusion a second time in another profile.
|
||||
- name: Run unit tests
|
||||
run: cd seaweed-worker && cargo test --release --workspace
|
||||
@@ -0,0 +1,163 @@
|
||||
name: "rust: build dev volume server binaries"
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [ master ]
|
||||
paths:
|
||||
- 'seaweed-volume/**'
|
||||
- 'seaweed-common/**'
|
||||
- '.github/workflows/rust_binaries_dev.yml'
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
jobs:
|
||||
|
||||
cleanup:
|
||||
permissions:
|
||||
contents: write
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Delete old Rust volume dev assets
|
||||
uses: mknejp/delete-release-assets@v1
|
||||
continue-on-error: true
|
||||
with:
|
||||
token: ${{ github.token }}
|
||||
tag: dev
|
||||
fail-if-no-assets: false
|
||||
assets: |
|
||||
weed-volume-*
|
||||
|
||||
build-rust-volume-dev-linux:
|
||||
permissions:
|
||||
contents: write
|
||||
needs: cleanup
|
||||
runs-on: ubuntu-22.04
|
||||
strategy:
|
||||
matrix:
|
||||
include:
|
||||
- target: x86_64-unknown-linux-gnu
|
||||
asset_suffix: linux-amd64
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@v7
|
||||
|
||||
- name: Install Rust toolchain
|
||||
uses: dtolnay/rust-toolchain@stable
|
||||
|
||||
- name: Cache cargo registry and target
|
||||
if: runner.environment == 'github-hosted'
|
||||
uses: actions/cache@v6
|
||||
with:
|
||||
path: |
|
||||
~/.cargo/registry
|
||||
~/.cargo/git
|
||||
seaweed-volume/target
|
||||
key: rust-dev-${{ matrix.target }}-${{ hashFiles('seaweed-volume/Cargo.lock') }}
|
||||
restore-keys: |
|
||||
rust-dev-${{ matrix.target }}-
|
||||
|
||||
- name: Set BUILD_TIME
|
||||
run: echo BUILD_TIME=$(date -u +%Y%m%d-%H%M) >> "$GITHUB_ENV"
|
||||
|
||||
- name: Build Rust volume server (large disk)
|
||||
env:
|
||||
SEAWEEDFS_COMMIT: ${{ github.sha }}
|
||||
run: cd seaweed-volume && cargo build --release
|
||||
|
||||
- name: Package large disk binary
|
||||
run: |
|
||||
cp seaweed-volume/target/release/weed-volume weed-volume-large-disk
|
||||
tar czf "weed-volume-large-disk-${{ env.BUILD_TIME }}-${{ matrix.asset_suffix }}.tar.gz" weed-volume-large-disk
|
||||
rm weed-volume-large-disk
|
||||
|
||||
- name: Build Rust volume server (normal)
|
||||
env:
|
||||
SEAWEEDFS_COMMIT: ${{ github.sha }}
|
||||
run: cd seaweed-volume && cargo build --release --no-default-features
|
||||
|
||||
- name: Package normal binary
|
||||
run: |
|
||||
cp seaweed-volume/target/release/weed-volume weed-volume-normal
|
||||
tar czf "weed-volume-${{ env.BUILD_TIME }}-${{ matrix.asset_suffix }}.tar.gz" weed-volume-normal
|
||||
rm weed-volume-normal
|
||||
|
||||
- name: Upload dev release assets
|
||||
uses: softprops/action-gh-release@v3
|
||||
with:
|
||||
tag_name: dev
|
||||
prerelease: true
|
||||
files: |
|
||||
weed-volume-large-disk-${{ env.BUILD_TIME }}-${{ matrix.asset_suffix }}.tar.gz
|
||||
weed-volume-${{ env.BUILD_TIME }}-${{ matrix.asset_suffix }}.tar.gz
|
||||
env:
|
||||
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
|
||||
build-rust-volume-dev-darwin:
|
||||
permissions:
|
||||
contents: write
|
||||
needs: build-rust-volume-dev-linux
|
||||
runs-on: macos-latest
|
||||
strategy:
|
||||
matrix:
|
||||
include:
|
||||
- target: aarch64-apple-darwin
|
||||
asset_suffix: darwin-arm64
|
||||
- target: x86_64-apple-darwin
|
||||
asset_suffix: darwin-amd64
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@v7
|
||||
|
||||
- name: Install Rust toolchain
|
||||
uses: dtolnay/rust-toolchain@stable
|
||||
with:
|
||||
targets: ${{ matrix.target }}
|
||||
|
||||
- name: Cache cargo registry and target
|
||||
if: runner.environment == 'github-hosted'
|
||||
uses: actions/cache@v6
|
||||
with:
|
||||
path: |
|
||||
~/.cargo/registry
|
||||
~/.cargo/git
|
||||
seaweed-volume/target
|
||||
key: rust-dev-${{ matrix.target }}-${{ hashFiles('seaweed-volume/Cargo.lock') }}
|
||||
restore-keys: |
|
||||
rust-dev-${{ matrix.target }}-
|
||||
|
||||
- name: Set BUILD_TIME
|
||||
run: echo BUILD_TIME=$(date -u +%Y%m%d-%H%M) >> "$GITHUB_ENV"
|
||||
|
||||
- name: Build Rust volume server (large disk)
|
||||
env:
|
||||
SEAWEEDFS_COMMIT: ${{ github.sha }}
|
||||
run: cd seaweed-volume && cargo build --release --target ${{ matrix.target }}
|
||||
|
||||
- name: Package large disk binary
|
||||
run: |
|
||||
cp seaweed-volume/target/${{ matrix.target }}/release/weed-volume weed-volume-large-disk
|
||||
tar czf "weed-volume-large-disk-${{ env.BUILD_TIME }}-${{ matrix.asset_suffix }}.tar.gz" weed-volume-large-disk
|
||||
rm weed-volume-large-disk
|
||||
|
||||
- name: Build Rust volume server (normal)
|
||||
env:
|
||||
SEAWEEDFS_COMMIT: ${{ github.sha }}
|
||||
run: cd seaweed-volume && cargo build --release --target ${{ matrix.target }} --no-default-features
|
||||
|
||||
- name: Package normal binary
|
||||
run: |
|
||||
cp seaweed-volume/target/${{ matrix.target }}/release/weed-volume weed-volume-normal
|
||||
tar czf "weed-volume-${{ env.BUILD_TIME }}-${{ matrix.asset_suffix }}.tar.gz" weed-volume-normal
|
||||
rm weed-volume-normal
|
||||
|
||||
- name: Upload dev release assets
|
||||
uses: softprops/action-gh-release@v3
|
||||
with:
|
||||
tag_name: dev
|
||||
prerelease: true
|
||||
files: |
|
||||
weed-volume-large-disk-${{ env.BUILD_TIME }}-${{ matrix.asset_suffix }}.tar.gz
|
||||
weed-volume-${{ env.BUILD_TIME }}-${{ matrix.asset_suffix }}.tar.gz
|
||||
env:
|
||||
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
@@ -0,0 +1,375 @@
|
||||
name: "rust: build versioned binaries"
|
||||
|
||||
on:
|
||||
push:
|
||||
tags:
|
||||
- '*'
|
||||
|
||||
workflow_dispatch:
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
jobs:
|
||||
|
||||
build-rust-volume-linux:
|
||||
permissions:
|
||||
contents: write
|
||||
runs-on: ubuntu-22.04
|
||||
strategy:
|
||||
matrix:
|
||||
include:
|
||||
- target: x86_64-unknown-linux-gnu
|
||||
asset_suffix: linux_amd64
|
||||
- target: aarch64-unknown-linux-gnu
|
||||
asset_suffix: linux_arm64
|
||||
cross: true
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@v7
|
||||
|
||||
- name: Install Rust toolchain
|
||||
uses: dtolnay/rust-toolchain@stable
|
||||
with:
|
||||
targets: ${{ matrix.target }}
|
||||
|
||||
- name: Install cross-compilation tools
|
||||
if: matrix.cross
|
||||
run: |
|
||||
sudo dpkg --add-architecture arm64
|
||||
sudo sed -i 's/^deb /deb [arch=amd64] /' /etc/apt/sources.list
|
||||
echo "deb [arch=arm64] http://ports.ubuntu.com/ jammy main restricted universe multiverse" | sudo tee /etc/apt/sources.list.d/arm64.list
|
||||
echo "deb [arch=arm64] http://ports.ubuntu.com/ jammy-updates main restricted universe multiverse" | sudo tee -a /etc/apt/sources.list.d/arm64.list
|
||||
sudo apt-get update
|
||||
sudo apt-get install -y gcc-aarch64-linux-gnu libssl-dev:arm64
|
||||
echo "CARGO_TARGET_AARCH64_UNKNOWN_LINUX_GNU_LINKER=aarch64-linux-gnu-gcc" >> "$GITHUB_ENV"
|
||||
echo "OPENSSL_DIR=/usr" >> "$GITHUB_ENV"
|
||||
echo "OPENSSL_INCLUDE_DIR=/usr/include" >> "$GITHUB_ENV"
|
||||
echo "OPENSSL_LIB_DIR=/usr/lib/aarch64-linux-gnu" >> "$GITHUB_ENV"
|
||||
|
||||
- name: Cache cargo registry and target
|
||||
if: runner.environment == 'github-hosted'
|
||||
uses: actions/cache@v6
|
||||
with:
|
||||
path: |
|
||||
~/.cargo/registry
|
||||
~/.cargo/git
|
||||
seaweed-volume/target
|
||||
key: rust-release-${{ matrix.target }}-${{ hashFiles('seaweed-volume/Cargo.lock') }}
|
||||
restore-keys: |
|
||||
rust-release-${{ matrix.target }}-
|
||||
|
||||
- name: Build Rust volume server (large disk)
|
||||
env:
|
||||
SEAWEEDFS_COMMIT: ${{ github.sha }}
|
||||
run: |
|
||||
cd seaweed-volume
|
||||
cargo build --release --target ${{ matrix.target }} --target-dir target/large-disk
|
||||
|
||||
- name: Build Rust volume server (normal)
|
||||
env:
|
||||
SEAWEEDFS_COMMIT: ${{ github.sha }}
|
||||
run: |
|
||||
cd seaweed-volume
|
||||
cargo build --release --target ${{ matrix.target }} --no-default-features --target-dir target/normal
|
||||
|
||||
- name: Package binaries
|
||||
run: |
|
||||
# Large disk (default, 5bytes feature)
|
||||
cp seaweed-volume/target/large-disk/${{ matrix.target }}/release/weed-volume weed-volume-large-disk
|
||||
tar czf weed-volume_large_disk_${{ matrix.asset_suffix }}.tar.gz weed-volume-large-disk
|
||||
rm weed-volume-large-disk
|
||||
|
||||
# Normal volume size
|
||||
cp seaweed-volume/target/normal/${{ matrix.target }}/release/weed-volume weed-volume-normal
|
||||
tar czf weed-volume_${{ matrix.asset_suffix }}.tar.gz weed-volume-normal
|
||||
rm weed-volume-normal
|
||||
|
||||
- name: Generate md5 checksums
|
||||
run: |
|
||||
for f in weed-volume_large_disk_${{ matrix.asset_suffix }}.tar.gz weed-volume_${{ matrix.asset_suffix }}.tar.gz; do
|
||||
md5sum "$f" > "$f.md5"
|
||||
done
|
||||
|
||||
- name: Upload release assets
|
||||
if: startsWith(github.ref, 'refs/tags/')
|
||||
uses: softprops/action-gh-release@v3
|
||||
with:
|
||||
files: |
|
||||
weed-volume_large_disk_${{ matrix.asset_suffix }}.tar.gz
|
||||
weed-volume_large_disk_${{ matrix.asset_suffix }}.tar.gz.md5
|
||||
weed-volume_${{ matrix.asset_suffix }}.tar.gz
|
||||
weed-volume_${{ matrix.asset_suffix }}.tar.gz.md5
|
||||
env:
|
||||
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
|
||||
- name: Upload artifacts
|
||||
if: ${{ !startsWith(github.ref, 'refs/tags/') }}
|
||||
uses: actions/upload-artifact@v7
|
||||
with:
|
||||
name: rust-volume-${{ matrix.asset_suffix }}
|
||||
path: |
|
||||
weed-volume_large_disk_${{ matrix.asset_suffix }}.tar.gz
|
||||
weed-volume_large_disk_${{ matrix.asset_suffix }}.tar.gz.md5
|
||||
weed-volume_${{ matrix.asset_suffix }}.tar.gz
|
||||
weed-volume_${{ matrix.asset_suffix }}.tar.gz.md5
|
||||
|
||||
# The Rust maintenance worker: Linux only, because it runs beside the cluster
|
||||
# it maintains rather than on a laptop, and its dependency tree (lance, arrow,
|
||||
# datafusion) makes every extra target an expensive build.
|
||||
build-rust-worker-linux:
|
||||
permissions:
|
||||
contents: write
|
||||
runs-on: ubuntu-22.04
|
||||
strategy:
|
||||
matrix:
|
||||
include:
|
||||
- target: x86_64-unknown-linux-gnu
|
||||
asset_suffix: linux_amd64
|
||||
- target: aarch64-unknown-linux-gnu
|
||||
asset_suffix: linux_arm64
|
||||
cross: true
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@v7
|
||||
with:
|
||||
# The upload step is handed a token explicitly; a cargo build script
|
||||
# should not find another one sitting in the checkout's git config.
|
||||
persist-credentials: false
|
||||
|
||||
- name: Install Rust toolchain
|
||||
uses: dtolnay/rust-toolchain@stable
|
||||
with:
|
||||
targets: ${{ matrix.target }}
|
||||
|
||||
- name: Install cross-compilation tools
|
||||
if: matrix.cross
|
||||
run: |
|
||||
sudo dpkg --add-architecture arm64
|
||||
sudo sed -i 's/^deb /deb [arch=amd64] /' /etc/apt/sources.list
|
||||
echo "deb [arch=arm64] http://ports.ubuntu.com/ jammy main restricted universe multiverse" | sudo tee /etc/apt/sources.list.d/arm64.list
|
||||
echo "deb [arch=arm64] http://ports.ubuntu.com/ jammy-updates main restricted universe multiverse" | sudo tee -a /etc/apt/sources.list.d/arm64.list
|
||||
sudo apt-get update
|
||||
sudo apt-get install -y gcc-aarch64-linux-gnu
|
||||
echo "CARGO_TARGET_AARCH64_UNKNOWN_LINUX_GNU_LINKER=aarch64-linux-gnu-gcc" >> "$GITHUB_ENV"
|
||||
|
||||
- name: Cache cargo registry and target
|
||||
if: runner.environment == 'github-hosted'
|
||||
uses: actions/cache@v6
|
||||
with:
|
||||
path: |
|
||||
~/.cargo/registry
|
||||
~/.cargo/git
|
||||
seaweed-worker/target/${{ matrix.target }}/release
|
||||
key: rust-worker-release-${{ matrix.target }}-${{ hashFiles('seaweed-worker/Cargo.lock') }}
|
||||
restore-keys: |
|
||||
rust-worker-release-${{ matrix.target }}-
|
||||
|
||||
# lance's build scripts compile their own protos and look for a protoc.
|
||||
# Point them at the one protoc-bin-vendored ships, which seaweed-worker's
|
||||
# own build already uses, so no job depends on a system package and every
|
||||
# build sees the same version.
|
||||
- name: Use the vendored protoc
|
||||
run: |
|
||||
cd seaweed-worker
|
||||
cargo fetch
|
||||
# The version from the lock, not whatever else a restored cache holds.
|
||||
version=$(awk '/^name = "protoc-bin-vendored-linux-x86_64"$/{found=1; next} found && /^version = /{gsub(/"/,"",$3); print $3; exit}' Cargo.lock)
|
||||
test -n "$version" || { echo "protoc-bin-vendored-linux-x86_64 is not in Cargo.lock" >&2; exit 1; }
|
||||
protoc=$(find ~/.cargo/registry/src -path "*protoc-bin-vendored-linux-x86_64-$version/bin/protoc" | head -1)
|
||||
test -x "$protoc" || { echo "no vendored protoc $version in the registry" >&2; exit 1; }
|
||||
echo "PROTOC=$protoc" >> "$GITHUB_ENV"
|
||||
|
||||
- name: Build the Rust maintenance worker
|
||||
run: |
|
||||
cd seaweed-worker
|
||||
cargo build --release -p weed-lance-worker --target ${{ matrix.target }}
|
||||
|
||||
- name: Package binary
|
||||
run: |
|
||||
cp seaweed-worker/target/${{ matrix.target }}/release/weed-worker weed-worker
|
||||
tar czf weed-worker_${{ matrix.asset_suffix }}.tar.gz weed-worker
|
||||
rm weed-worker
|
||||
md5sum weed-worker_${{ matrix.asset_suffix }}.tar.gz > weed-worker_${{ matrix.asset_suffix }}.tar.gz.md5
|
||||
|
||||
- name: Upload release assets
|
||||
if: startsWith(github.ref, 'refs/tags/')
|
||||
uses: softprops/action-gh-release@v3
|
||||
with:
|
||||
files: |
|
||||
weed-worker_${{ matrix.asset_suffix }}.tar.gz
|
||||
weed-worker_${{ matrix.asset_suffix }}.tar.gz.md5
|
||||
env:
|
||||
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
|
||||
- name: Upload artifacts
|
||||
if: ${{ !startsWith(github.ref, 'refs/tags/') }}
|
||||
uses: actions/upload-artifact@v7
|
||||
with:
|
||||
name: rust-worker-${{ matrix.asset_suffix }}
|
||||
path: |
|
||||
weed-worker_${{ matrix.asset_suffix }}.tar.gz
|
||||
weed-worker_${{ matrix.asset_suffix }}.tar.gz.md5
|
||||
|
||||
build-rust-volume-darwin:
|
||||
permissions:
|
||||
contents: write
|
||||
runs-on: macos-latest
|
||||
strategy:
|
||||
matrix:
|
||||
include:
|
||||
- target: x86_64-apple-darwin
|
||||
asset_suffix: darwin_amd64
|
||||
- target: aarch64-apple-darwin
|
||||
asset_suffix: darwin_arm64
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@v7
|
||||
|
||||
- name: Install Rust toolchain
|
||||
uses: dtolnay/rust-toolchain@stable
|
||||
with:
|
||||
targets: ${{ matrix.target }}
|
||||
|
||||
- name: Cache cargo registry and target
|
||||
if: runner.environment == 'github-hosted'
|
||||
uses: actions/cache@v6
|
||||
with:
|
||||
path: |
|
||||
~/.cargo/registry
|
||||
~/.cargo/git
|
||||
seaweed-volume/target
|
||||
key: rust-release-${{ matrix.target }}-${{ hashFiles('seaweed-volume/Cargo.lock') }}
|
||||
restore-keys: |
|
||||
rust-release-${{ matrix.target }}-
|
||||
|
||||
- name: Build Rust volume server (large disk)
|
||||
env:
|
||||
SEAWEEDFS_COMMIT: ${{ github.sha }}
|
||||
run: |
|
||||
cd seaweed-volume
|
||||
cargo build --release --target ${{ matrix.target }} --target-dir target/large-disk
|
||||
|
||||
- name: Build Rust volume server (normal)
|
||||
env:
|
||||
SEAWEEDFS_COMMIT: ${{ github.sha }}
|
||||
run: |
|
||||
cd seaweed-volume
|
||||
cargo build --release --target ${{ matrix.target }} --no-default-features --target-dir target/normal
|
||||
|
||||
- name: Package binaries
|
||||
run: |
|
||||
cp seaweed-volume/target/large-disk/${{ matrix.target }}/release/weed-volume weed-volume-large-disk
|
||||
tar czf weed-volume_large_disk_${{ matrix.asset_suffix }}.tar.gz weed-volume-large-disk
|
||||
rm weed-volume-large-disk
|
||||
|
||||
cp seaweed-volume/target/normal/${{ matrix.target }}/release/weed-volume weed-volume-normal
|
||||
tar czf weed-volume_${{ matrix.asset_suffix }}.tar.gz weed-volume-normal
|
||||
rm weed-volume-normal
|
||||
|
||||
- name: Generate md5 checksums
|
||||
run: |
|
||||
for f in weed-volume_large_disk_${{ matrix.asset_suffix }}.tar.gz weed-volume_${{ matrix.asset_suffix }}.tar.gz; do
|
||||
md5 -r "$f" > "$f.md5"
|
||||
done
|
||||
|
||||
- name: Upload release assets
|
||||
if: startsWith(github.ref, 'refs/tags/')
|
||||
uses: softprops/action-gh-release@v3
|
||||
with:
|
||||
files: |
|
||||
weed-volume_large_disk_${{ matrix.asset_suffix }}.tar.gz
|
||||
weed-volume_large_disk_${{ matrix.asset_suffix }}.tar.gz.md5
|
||||
weed-volume_${{ matrix.asset_suffix }}.tar.gz
|
||||
weed-volume_${{ matrix.asset_suffix }}.tar.gz.md5
|
||||
env:
|
||||
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
|
||||
- name: Upload artifacts
|
||||
if: ${{ !startsWith(github.ref, 'refs/tags/') }}
|
||||
uses: actions/upload-artifact@v7
|
||||
with:
|
||||
name: rust-volume-${{ matrix.asset_suffix }}
|
||||
path: |
|
||||
weed-volume_large_disk_${{ matrix.asset_suffix }}.tar.gz
|
||||
weed-volume_large_disk_${{ matrix.asset_suffix }}.tar.gz.md5
|
||||
weed-volume_${{ matrix.asset_suffix }}.tar.gz
|
||||
weed-volume_${{ matrix.asset_suffix }}.tar.gz.md5
|
||||
|
||||
build-rust-volume-windows:
|
||||
permissions:
|
||||
contents: write
|
||||
runs-on: windows-latest
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@v7
|
||||
|
||||
- name: Install Rust toolchain
|
||||
uses: dtolnay/rust-toolchain@stable
|
||||
|
||||
- name: Cache cargo registry and target
|
||||
if: runner.environment == 'github-hosted'
|
||||
uses: actions/cache@v6
|
||||
with:
|
||||
path: |
|
||||
~/.cargo/registry
|
||||
~/.cargo/git
|
||||
seaweed-volume/target
|
||||
key: rust-release-windows-${{ hashFiles('seaweed-volume/Cargo.lock') }}
|
||||
restore-keys: |
|
||||
rust-release-windows-
|
||||
|
||||
- name: Build Rust volume server (large disk)
|
||||
env:
|
||||
SEAWEEDFS_COMMIT: ${{ github.sha }}
|
||||
run: |
|
||||
cd seaweed-volume
|
||||
cargo build --release --target-dir target/large-disk
|
||||
|
||||
- name: Build Rust volume server (normal)
|
||||
env:
|
||||
SEAWEEDFS_COMMIT: ${{ github.sha }}
|
||||
run: |
|
||||
cd seaweed-volume
|
||||
cargo build --release --no-default-features --target-dir target/normal
|
||||
|
||||
- name: Package binaries
|
||||
shell: bash
|
||||
run: |
|
||||
cp seaweed-volume/target/large-disk/release/weed-volume.exe weed-volume-large-disk.exe
|
||||
7z a weed-volume_large_disk_windows_amd64.zip weed-volume-large-disk.exe
|
||||
rm weed-volume-large-disk.exe
|
||||
|
||||
cp seaweed-volume/target/normal/release/weed-volume.exe weed-volume-normal.exe
|
||||
7z a weed-volume_windows_amd64.zip weed-volume-normal.exe
|
||||
rm weed-volume-normal.exe
|
||||
|
||||
- name: Generate md5 checksums
|
||||
shell: bash
|
||||
run: |
|
||||
for f in weed-volume_large_disk_windows_amd64.zip weed-volume_windows_amd64.zip; do
|
||||
md5sum "$f" > "$f.md5"
|
||||
done
|
||||
|
||||
- name: Upload release assets
|
||||
if: startsWith(github.ref, 'refs/tags/')
|
||||
uses: softprops/action-gh-release@v3
|
||||
with:
|
||||
files: |
|
||||
weed-volume_large_disk_windows_amd64.zip
|
||||
weed-volume_large_disk_windows_amd64.zip.md5
|
||||
weed-volume_windows_amd64.zip
|
||||
weed-volume_windows_amd64.zip.md5
|
||||
env:
|
||||
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
|
||||
- name: Upload artifacts
|
||||
if: ${{ !startsWith(github.ref, 'refs/tags/') }}
|
||||
uses: actions/upload-artifact@v7
|
||||
with:
|
||||
name: rust-volume-windows_amd64
|
||||
path: |
|
||||
weed-volume_large_disk_windows_amd64.zip
|
||||
weed-volume_large_disk_windows_amd64.zip.md5
|
||||
weed-volume_windows_amd64.zip
|
||||
weed-volume_windows_amd64.zip.md5
|
||||
@@ -0,0 +1,130 @@
|
||||
name: "S3 ETag and ACL Tests"
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [ master ]
|
||||
paths:
|
||||
- 'weed/s3api/**'
|
||||
- 'weed/filer/etag*.go'
|
||||
- 'weed/server/filer_server_handlers_*.go'
|
||||
- 'test/s3/etag/**'
|
||||
- 'test/s3/acl/**'
|
||||
- '.github/workflows/s3-etag-acl-tests.yml'
|
||||
pull_request:
|
||||
branches: [ master ]
|
||||
paths:
|
||||
- 'weed/s3api/**'
|
||||
- 'weed/filer/etag*.go'
|
||||
- 'weed/server/filer_server_handlers_*.go'
|
||||
- 'test/s3/etag/**'
|
||||
- 'test/s3/acl/**'
|
||||
- '.github/workflows/s3-etag-acl-tests.yml'
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.event.pull_request.number || (github.event_name == 'push' && github.ref) || github.run_id }}
|
||||
cancel-in-progress: ${{ github.event_name == 'pull_request' || (github.event_name == 'push' && !startsWith(github.ref, 'refs/tags/')) }}
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
jobs:
|
||||
s3-etag-acl-tests:
|
||||
name: S3 ETag + ACL Integration Tests
|
||||
runs-on: ubuntu-22.04
|
||||
timeout-minutes: 15
|
||||
steps:
|
||||
- name: Check out code
|
||||
uses: actions/checkout@v7
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v7
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
cache: ${{ runner.environment == 'github-hosted' }}
|
||||
|
||||
- name: Install SeaweedFS
|
||||
run: |
|
||||
cd weed && go install -buildvcs=false
|
||||
|
||||
- name: Start weed mini (S3 on :8333)
|
||||
run: |
|
||||
mkdir -p /tmp/seaweedfs-etag-acl
|
||||
# Minimal identity config so SSE-aware tests under acl/ can authenticate.
|
||||
cat > /tmp/seaweedfs-etag-acl-s3.json <<'JSON'
|
||||
{
|
||||
"identities": [
|
||||
{
|
||||
"name": "admin",
|
||||
"credentials": [
|
||||
{"accessKey": "some_access_key1", "secretKey": "some_secret_key1"}
|
||||
],
|
||||
"actions": ["Admin", "Read", "Write"]
|
||||
}
|
||||
]
|
||||
}
|
||||
JSON
|
||||
AWS_ACCESS_KEY_ID=some_access_key1 \
|
||||
AWS_SECRET_ACCESS_KEY=some_secret_key1 \
|
||||
weed mini \
|
||||
-dir=/tmp/seaweedfs-etag-acl \
|
||||
-s3.port=8333 \
|
||||
-s3.config=/tmp/seaweedfs-etag-acl-s3.json \
|
||||
-ip=127.0.0.1 \
|
||||
> /tmp/weed-mini.log 2>&1 &
|
||||
echo $! > /tmp/weed-mini.pid
|
||||
|
||||
# Wait for the S3 endpoint to come up (returns 403 unauth before any
|
||||
# request is signed; that's fine — it means the server is listening).
|
||||
for i in $(seq 1 30); do
|
||||
if curl -s -o /dev/null -w "%{http_code}" http://127.0.0.1:8333/ | grep -qE "^(200|403)$"; then
|
||||
echo "weed mini is ready"
|
||||
exit 0
|
||||
fi
|
||||
sleep 1
|
||||
done
|
||||
echo "weed mini failed to start within 30s"
|
||||
tail -50 /tmp/weed-mini.log
|
||||
exit 1
|
||||
|
||||
- name: Run ETag tests
|
||||
# Pins the regression for #7768: PutObject of an auto-chunked file (>8MB)
|
||||
# must return a pure MD5 hex ETag, not a `<md5>-N` composite — the AWS
|
||||
# SDK for Java v2 rejects the latter on the PutObject path.
|
||||
env:
|
||||
S3_ENDPOINT: http://127.0.0.1:8333
|
||||
AWS_ACCESS_KEY_ID: some_access_key1
|
||||
AWS_SECRET_ACCESS_KEY: some_secret_key1
|
||||
AWS_REGION: us-east-1
|
||||
run: go test -v -timeout=5m ./test/s3/etag/...
|
||||
|
||||
- name: Run ACL versioning tests
|
||||
# Pins object-ACL behavior on a versioned bucket: GetObjectAcl /
|
||||
# PutObjectAcl with and without versionId, modifying ACLs on different
|
||||
# versions independently.
|
||||
env:
|
||||
S3_ENDPOINT: http://127.0.0.1:8333
|
||||
AWS_ACCESS_KEY_ID: some_access_key1
|
||||
AWS_SECRET_ACCESS_KEY: some_secret_key1
|
||||
AWS_REGION: us-east-1
|
||||
run: go test -v -timeout=5m ./test/s3/acl/...
|
||||
|
||||
- name: Stop weed mini
|
||||
if: always()
|
||||
run: |
|
||||
if [ -f /tmp/weed-mini.pid ]; then
|
||||
kill "$(cat /tmp/weed-mini.pid)" 2>/dev/null || true
|
||||
fi
|
||||
|
||||
- name: Show server log on failure
|
||||
if: failure()
|
||||
run: |
|
||||
echo "=== weed mini log (last 200 lines) ==="
|
||||
tail -n 200 /tmp/weed-mini.log 2>/dev/null || echo "no log available"
|
||||
|
||||
- name: Archive log
|
||||
if: failure()
|
||||
uses: actions/upload-artifact@v7
|
||||
with:
|
||||
name: s3-etag-acl-server-log
|
||||
path: /tmp/weed-mini.log
|
||||
retention-days: 3
|
||||
@@ -2,10 +2,19 @@ name: "S3 Authenticated Integration Tests"
|
||||
|
||||
on:
|
||||
pull_request:
|
||||
|
||||
paths:
|
||||
- 'weed/s3api/**'
|
||||
- 'weed/iam/**'
|
||||
- 'weed/filer/**'
|
||||
- 'weed/server/**'
|
||||
- 'test/s3/normal/**'
|
||||
- 'go.mod'
|
||||
- 'go.sum'
|
||||
- '.github/workflows/s3-example-integration-tests.yml'
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.head_ref }}/s3-integration-tests
|
||||
cancel-in-progress: true
|
||||
group: ${{ github.workflow }}-${{ github.event.pull_request.number || (github.event_name == 'push' && github.ref) || github.run_id }}
|
||||
cancel-in-progress: ${{ github.event_name == 'pull_request' || (github.event_name == 'push' && !startsWith(github.ref, 'refs/tags/')) }}
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
@@ -18,12 +27,13 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Check out code
|
||||
uses: actions/checkout@v6
|
||||
uses: actions/checkout@v7
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v6
|
||||
uses: actions/setup-go@v7
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
cache: ${{ runner.environment == 'github-hosted' }}
|
||||
id: go
|
||||
|
||||
- name: Build SeaweedFS
|
||||
@@ -39,6 +49,30 @@ jobs:
|
||||
echo "=== Running S3 Integration Tests ==="
|
||||
go test -v -timeout=60s -run TestS3Integration ./...
|
||||
|
||||
- name: Run S3 DeleteBucketNotEmpty Tests
|
||||
timeout-minutes: 15
|
||||
working-directory: test/s3/normal
|
||||
run: |
|
||||
set -x
|
||||
echo "=== Running S3 DeleteBucketNotEmpty Tests ==="
|
||||
go test -v -timeout=60s -run TestS3DeleteBucketNotEmpty ./...
|
||||
|
||||
- name: Run S3 Empty Directory Marker Tests
|
||||
timeout-minutes: 15
|
||||
working-directory: test/s3/normal
|
||||
run: |
|
||||
set -x
|
||||
echo "=== Running S3 Empty Directory Marker Tests ==="
|
||||
go test -v -timeout=180s -run TestS3ListObjectsEmptyDirectoryMarkers ./...
|
||||
|
||||
- name: Run S3 Prefix Object Tests
|
||||
timeout-minutes: 15
|
||||
working-directory: test/s3/normal
|
||||
run: |
|
||||
set -x
|
||||
echo "=== Running S3 Prefix Object Tests ==="
|
||||
go test -v -timeout=180s -run TestS3PrefixObjectKeys ./...
|
||||
|
||||
- name: Run IAM Integration Tests
|
||||
timeout-minutes: 15
|
||||
working-directory: test/s3/normal
|
||||
@@ -49,7 +83,7 @@ jobs:
|
||||
|
||||
- name: Upload test logs on failure
|
||||
if: failure()
|
||||
uses: actions/upload-artifact@v6
|
||||
uses: actions/upload-artifact@v7
|
||||
with:
|
||||
name: integration-test-logs
|
||||
path: test/s3/normal/*.log
|
||||
|
||||
@@ -2,10 +2,18 @@ name: "S3 Filer Group Tests"
|
||||
|
||||
on:
|
||||
pull_request:
|
||||
|
||||
paths:
|
||||
- 'weed/s3api/**'
|
||||
- 'weed/filer/**'
|
||||
- 'weed/server/**'
|
||||
- 'test/s3/filer_group/**'
|
||||
- 'go.mod'
|
||||
- 'go.sum'
|
||||
- '.github/workflows/s3-filer-group-tests.yml'
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.head_ref }}/s3-filer-group-tests
|
||||
cancel-in-progress: true
|
||||
group: ${{ github.workflow }}-${{ github.event.pull_request.number || (github.event_name == 'push' && github.ref) || github.run_id }}
|
||||
cancel-in-progress: ${{ github.event_name == 'pull_request' || (github.event_name == 'push' && !startsWith(github.ref, 'refs/tags/')) }}
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
@@ -22,12 +30,16 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Check out code
|
||||
uses: actions/checkout@v6
|
||||
uses: actions/checkout@v7
|
||||
|
||||
- name: Clean up stale FUSE mounts
|
||||
uses: ./.github/actions/cleanup-stale-fuse-mounts
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v6
|
||||
uses: actions/setup-go@v7
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
cache: ${{ runner.environment == 'github-hosted' }}
|
||||
id: go
|
||||
|
||||
- name: Install SeaweedFS
|
||||
@@ -42,7 +54,7 @@ jobs:
|
||||
echo "=== System Information ==="
|
||||
uname -a
|
||||
free -h
|
||||
df -h
|
||||
timeout 15 df -h || true
|
||||
echo "=== Starting Filer Group Tests ==="
|
||||
|
||||
# Run tests with automatic server management
|
||||
@@ -77,7 +89,7 @@ jobs:
|
||||
|
||||
- name: Upload test logs on failure
|
||||
if: failure()
|
||||
uses: actions/upload-artifact@v6
|
||||
uses: actions/upload-artifact@v7
|
||||
with:
|
||||
name: s3-filer-group-test-logs
|
||||
path: test/s3/filer_group/weed-test*.log
|
||||
|
||||
@@ -2,10 +2,19 @@ name: "S3 Go Tests"
|
||||
|
||||
on:
|
||||
pull_request:
|
||||
|
||||
paths:
|
||||
- 'weed/s3api/**'
|
||||
- 'weed/remote_storage/**'
|
||||
- 'weed/filer/**'
|
||||
- 'weed/server/**'
|
||||
- 'test/s3/**'
|
||||
- 'go.mod'
|
||||
- 'go.sum'
|
||||
- '.github/workflows/s3-go-tests.yml'
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.head_ref }}/s3-go-tests
|
||||
cancel-in-progress: true
|
||||
group: ${{ github.workflow }}-${{ github.event.pull_request.number || (github.event_name == 'push' && github.ref) || github.run_id }}
|
||||
cancel-in-progress: ${{ github.event_name == 'pull_request' || (github.event_name == 'push' && !startsWith(github.ref, 'refs/tags/')) }}
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
@@ -25,12 +34,16 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Check out code
|
||||
uses: actions/checkout@v6
|
||||
uses: actions/checkout@v7
|
||||
|
||||
- name: Clean up stale FUSE mounts
|
||||
uses: ./.github/actions/cleanup-stale-fuse-mounts
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v6
|
||||
uses: actions/setup-go@v7
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
cache: ${{ runner.environment == 'github-hosted' }}
|
||||
id: go
|
||||
|
||||
- name: Install SeaweedFS
|
||||
@@ -45,7 +58,7 @@ jobs:
|
||||
echo "=== System Information ==="
|
||||
uname -a
|
||||
free -h
|
||||
df -h
|
||||
timeout 15 df -h || true
|
||||
echo "=== Starting Tests ==="
|
||||
|
||||
# Run tests with automatic server management
|
||||
@@ -76,7 +89,7 @@ jobs:
|
||||
|
||||
- name: Upload test logs on failure
|
||||
if: failure()
|
||||
uses: actions/upload-artifact@v6
|
||||
uses: actions/upload-artifact@v7
|
||||
with:
|
||||
name: s3-versioning-test-logs-${{ matrix.test-type }}
|
||||
path: test/s3/versioning/weed-test*.log
|
||||
@@ -89,12 +102,13 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Check out code
|
||||
uses: actions/checkout@v6
|
||||
uses: actions/checkout@v7
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v6
|
||||
uses: actions/setup-go@v7
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
cache: ${{ runner.environment == 'github-hosted' }}
|
||||
id: go
|
||||
|
||||
- name: Install SeaweedFS
|
||||
@@ -124,7 +138,7 @@ jobs:
|
||||
|
||||
- name: Upload server logs on failure
|
||||
if: failure()
|
||||
uses: actions/upload-artifact@v6
|
||||
uses: actions/upload-artifact@v7
|
||||
with:
|
||||
name: s3-versioning-compatibility-logs
|
||||
path: test/s3/versioning/weed-test*.log
|
||||
@@ -137,12 +151,13 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Check out code
|
||||
uses: actions/checkout@v6
|
||||
uses: actions/checkout@v7
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v6
|
||||
uses: actions/setup-go@v7
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
cache: ${{ runner.environment == 'github-hosted' }}
|
||||
id: go
|
||||
|
||||
- name: Install SeaweedFS
|
||||
@@ -172,7 +187,7 @@ jobs:
|
||||
|
||||
- name: Upload server logs on failure
|
||||
if: failure()
|
||||
uses: actions/upload-artifact@v6
|
||||
uses: actions/upload-artifact@v7
|
||||
with:
|
||||
name: s3-cors-compatibility-logs
|
||||
path: test/s3/cors/weed-test*.log
|
||||
@@ -188,12 +203,16 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Check out code
|
||||
uses: actions/checkout@v6
|
||||
uses: actions/checkout@v7
|
||||
|
||||
- name: Clean up stale FUSE mounts
|
||||
uses: ./.github/actions/cleanup-stale-fuse-mounts
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v6
|
||||
uses: actions/setup-go@v7
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
cache: ${{ runner.environment == 'github-hosted' }}
|
||||
id: go
|
||||
|
||||
- name: Install SeaweedFS
|
||||
@@ -208,7 +227,7 @@ jobs:
|
||||
echo "=== System Information ==="
|
||||
uname -a
|
||||
free -h
|
||||
df -h
|
||||
timeout 15 df -h || true
|
||||
echo "=== Starting Tests ==="
|
||||
|
||||
# Run tests with automatic server management
|
||||
@@ -239,12 +258,144 @@ jobs:
|
||||
|
||||
- name: Upload test logs on failure
|
||||
if: failure()
|
||||
uses: actions/upload-artifact@v6
|
||||
uses: actions/upload-artifact@v7
|
||||
with:
|
||||
name: s3-retention-test-logs-${{ matrix.test-type }}
|
||||
path: test/s3/retention/weed-test*.log
|
||||
retention-days: 3
|
||||
|
||||
s3-lifecycle-tests:
|
||||
name: S3 Lifecycle Tests
|
||||
runs-on: ubuntu-22.04
|
||||
timeout-minutes: 10
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
# One job per test so each gets a fresh `weed mini` server, avoiding
|
||||
# the cross-test volume-pool exhaustion that surfaced when several
|
||||
# TTL-pinned bucket collections piled up in a single run.
|
||||
test:
|
||||
- TestLifecycleAbortIncompleteMultipartUpload
|
||||
- TestLifecycleAdminDispatchSucceedsWithCustomFilerGrpcPort
|
||||
- TestLifecycleBootstrapWalkOnExistingObjects
|
||||
- TestLifecycleConfigUpdateBetweenSweeps
|
||||
- TestLifecycleDeleteBucketLifecycleStopsDispatching
|
||||
- TestLifecycleDisabledRuleSkipsObject
|
||||
- TestLifecycleEmptyBucketSweepIsNoOp
|
||||
- TestLifecycleExpirationDateInThePast
|
||||
- TestLifecycleExpirationFiresOnBackdatedObject
|
||||
- TestLifecycleExpiredDeleteMarkerCleanup
|
||||
- TestLifecycleMultipleBucketsInOneSweep
|
||||
- TestLifecycleMultipleRulesInOneBucket
|
||||
- TestLifecycleNewerNoncurrentVersions
|
||||
- TestLifecycleNoncurrentVersionExpiration
|
||||
- TestLifecycleSizeFilterGreaterThan
|
||||
- TestLifecycleSkipsObjectLockedObjects
|
||||
- TestLifecycleSuspendedVersioningExpiration
|
||||
- TestLifecycleTagFilter
|
||||
- TestLifecycleVersionedBucketCreatesDeleteMarker
|
||||
- TestLifecycleVersionedBucketExpiresOnSecondRun
|
||||
|
||||
steps:
|
||||
- name: Check out code
|
||||
uses: actions/checkout@v7
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v7
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
cache: ${{ runner.environment == 'github-hosted' }}
|
||||
id: go
|
||||
|
||||
- name: Install SeaweedFS
|
||||
run: |
|
||||
go install -buildvcs=false
|
||||
|
||||
- name: Run ${{ matrix.test }}
|
||||
timeout-minutes: 8
|
||||
working-directory: test/s3/lifecycle
|
||||
run: |
|
||||
set -x
|
||||
make test-with-server TEST_PATTERN='^${{ matrix.test }}$$'
|
||||
|
||||
- name: Show server logs on failure
|
||||
if: failure()
|
||||
working-directory: test/s3/lifecycle
|
||||
run: |
|
||||
if [ -f weed-test.log ]; then
|
||||
echo "=== Last 200 lines of server logs ==="
|
||||
tail -200 weed-test.log
|
||||
fi
|
||||
ps aux | grep -E "(weed|test)" || true
|
||||
netstat -tlnp 2>/dev/null | grep -E "(8333|9333|8080|8888)" || true
|
||||
|
||||
- name: Upload test logs on failure
|
||||
if: failure()
|
||||
uses: actions/upload-artifact@v7
|
||||
with:
|
||||
name: s3-lifecycle-test-logs-${{ matrix.test }}
|
||||
path: test/s3/lifecycle/weed-test*.log
|
||||
retention-days: 3
|
||||
|
||||
s3-checksum-tests:
|
||||
name: S3 Checksum Tests
|
||||
runs-on: ubuntu-22.04
|
||||
timeout-minutes: 20
|
||||
|
||||
steps:
|
||||
- name: Check out code
|
||||
uses: actions/checkout@v7
|
||||
|
||||
- name: Clean up stale FUSE mounts
|
||||
uses: ./.github/actions/cleanup-stale-fuse-mounts
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v7
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
cache: ${{ runner.environment == 'github-hosted' }}
|
||||
id: go
|
||||
|
||||
- name: Install SeaweedFS
|
||||
run: |
|
||||
go install -buildvcs=false
|
||||
|
||||
- name: Run S3 Checksum Tests
|
||||
timeout-minutes: 16
|
||||
working-directory: test/s3/checksum
|
||||
run: |
|
||||
set -x
|
||||
echo "=== System Information ==="
|
||||
uname -a
|
||||
free -h
|
||||
timeout 15 df -h || true
|
||||
echo "=== Starting Tests ==="
|
||||
make test-with-server
|
||||
|
||||
- name: Show server logs on failure
|
||||
if: failure()
|
||||
working-directory: test/s3/checksum
|
||||
run: |
|
||||
echo "=== Server Logs ==="
|
||||
if [ -f weed-test.log ]; then
|
||||
echo "Last 100 lines of server logs:"
|
||||
tail -100 weed-test.log
|
||||
else
|
||||
echo "No server log file found"
|
||||
fi
|
||||
|
||||
echo "=== Test Environment ==="
|
||||
ps aux | grep -E "(weed|test)" || true
|
||||
netstat -tlnp | grep -E "(8333|9333|8080)" || true
|
||||
|
||||
- name: Upload test logs on failure
|
||||
if: failure()
|
||||
uses: actions/upload-artifact@v7
|
||||
with:
|
||||
name: s3-checksum-test-logs
|
||||
path: test/s3/checksum/weed-test*.log
|
||||
retention-days: 3
|
||||
|
||||
s3-cors-tests:
|
||||
name: S3 CORS Tests
|
||||
runs-on: ubuntu-22.04
|
||||
@@ -255,12 +406,16 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Check out code
|
||||
uses: actions/checkout@v6
|
||||
uses: actions/checkout@v7
|
||||
|
||||
- name: Clean up stale FUSE mounts
|
||||
uses: ./.github/actions/cleanup-stale-fuse-mounts
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v6
|
||||
uses: actions/setup-go@v7
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
cache: ${{ runner.environment == 'github-hosted' }}
|
||||
id: go
|
||||
|
||||
- name: Install SeaweedFS
|
||||
@@ -275,7 +430,7 @@ jobs:
|
||||
echo "=== System Information ==="
|
||||
uname -a
|
||||
free -h
|
||||
df -h
|
||||
timeout 15 df -h || true
|
||||
echo "=== Starting Tests ==="
|
||||
|
||||
# Run tests with automatic server management
|
||||
@@ -306,7 +461,7 @@ jobs:
|
||||
|
||||
- name: Upload test logs on failure
|
||||
if: failure()
|
||||
uses: actions/upload-artifact@v6
|
||||
uses: actions/upload-artifact@v7
|
||||
with:
|
||||
name: s3-cors-test-logs-${{ matrix.test-type }}
|
||||
path: test/s3/cors/weed-test*.log
|
||||
@@ -319,12 +474,13 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Check out code
|
||||
uses: actions/checkout@v6
|
||||
uses: actions/checkout@v7
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v6
|
||||
uses: actions/setup-go@v7
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
cache: ${{ runner.environment == 'github-hosted' }}
|
||||
id: go
|
||||
|
||||
- name: Install SeaweedFS
|
||||
@@ -355,7 +511,7 @@ jobs:
|
||||
|
||||
- name: Upload server logs on failure
|
||||
if: failure()
|
||||
uses: actions/upload-artifact@v6
|
||||
uses: actions/upload-artifact@v7
|
||||
with:
|
||||
name: s3-retention-worm-logs
|
||||
path: test/s3/retention/weed-test*.log
|
||||
@@ -368,12 +524,13 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Check out code
|
||||
uses: actions/checkout@v6
|
||||
uses: actions/checkout@v7
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v6
|
||||
uses: actions/setup-go@v7
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
cache: ${{ runner.environment == 'github-hosted' }}
|
||||
id: go
|
||||
|
||||
- name: Install SeaweedFS
|
||||
@@ -422,7 +579,7 @@ jobs:
|
||||
|
||||
- name: Upload stress test logs
|
||||
if: always()
|
||||
uses: actions/upload-artifact@v6
|
||||
uses: actions/upload-artifact@v7
|
||||
with:
|
||||
name: s3-versioning-stress-logs
|
||||
path: test/s3/versioning/weed-test*.log
|
||||
@@ -436,12 +593,13 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Check out code
|
||||
uses: actions/checkout@v6
|
||||
uses: actions/checkout@v7
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v6
|
||||
uses: actions/setup-go@v7
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
cache: ${{ runner.environment == 'github-hosted' }}
|
||||
id: go
|
||||
|
||||
- name: Install SeaweedFS
|
||||
@@ -461,6 +619,8 @@ jobs:
|
||||
export S3_ENDPOINT="http://localhost:8006"
|
||||
export S3_ACCESS_KEY="0555b35654ad1656d804"
|
||||
export S3_SECRET_KEY="h7GhxuBLTrlhVUyxSPUKUV8r/2EI4ngqJxD7iBdBYLhwluN30JaT3Q=="
|
||||
export AWS_ACCESS_KEY_ID="$S3_ACCESS_KEY"
|
||||
export AWS_SECRET_ACCESS_KEY="$S3_SECRET_KEY"
|
||||
|
||||
# Run the specific test that is equivalent to AWS S3 tagging behavior
|
||||
make test-with-server || {
|
||||
@@ -476,7 +636,7 @@ jobs:
|
||||
|
||||
- name: Upload test logs on failure
|
||||
if: failure()
|
||||
uses: actions/upload-artifact@v6
|
||||
uses: actions/upload-artifact@v7
|
||||
with:
|
||||
name: s3-tagging-test-logs
|
||||
path: test/s3/tagging/weed-test*.log
|
||||
@@ -489,12 +649,13 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Check out code
|
||||
uses: actions/checkout@v6
|
||||
uses: actions/checkout@v7
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v6
|
||||
uses: actions/setup-go@v7
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
cache: ${{ runner.environment == 'github-hosted' }}
|
||||
id: go
|
||||
|
||||
- name: Install SeaweedFS
|
||||
@@ -529,7 +690,7 @@ jobs:
|
||||
|
||||
- name: Upload test logs on failure
|
||||
if: failure()
|
||||
uses: actions/upload-artifact@v6
|
||||
uses: actions/upload-artifact@v7
|
||||
with:
|
||||
name: s3-remote-cache-test-logs
|
||||
path: |
|
||||
|
||||
@@ -5,6 +5,8 @@ on:
|
||||
paths:
|
||||
- 'weed/iam/**'
|
||||
- 'weed/s3api/**'
|
||||
- 'weed/credential/**'
|
||||
- 'weed/pb/**'
|
||||
- 'test/s3/iam/**'
|
||||
- '.github/workflows/s3-iam-tests.yml'
|
||||
push:
|
||||
@@ -12,12 +14,14 @@ on:
|
||||
paths:
|
||||
- 'weed/iam/**'
|
||||
- 'weed/s3api/**'
|
||||
- 'weed/credential/**'
|
||||
- 'weed/pb/**'
|
||||
- 'test/s3/iam/**'
|
||||
- '.github/workflows/s3-iam-tests.yml'
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.head_ref }}/s3-iam-tests
|
||||
cancel-in-progress: true
|
||||
group: ${{ github.workflow }}-${{ github.event.pull_request.number || (github.event_name == 'push' && github.ref) || github.run_id }}
|
||||
cancel-in-progress: ${{ github.event_name == 'pull_request' || (github.event_name == 'push' && !startsWith(github.ref, 'refs/tags/')) }}
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
@@ -35,12 +39,13 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Check out code
|
||||
uses: actions/checkout@v6
|
||||
uses: actions/checkout@v7
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v6
|
||||
uses: actions/setup-go@v7
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
cache: ${{ runner.environment == 'github-hosted' }}
|
||||
id: go
|
||||
|
||||
- name: Get dependencies
|
||||
@@ -65,7 +70,7 @@ jobs:
|
||||
|
||||
- name: Upload test results on failure
|
||||
if: failure()
|
||||
uses: actions/upload-artifact@v6
|
||||
uses: actions/upload-artifact@v7
|
||||
with:
|
||||
name: iam-unit-test-results
|
||||
path: |
|
||||
@@ -80,16 +85,20 @@ jobs:
|
||||
timeout-minutes: 25
|
||||
strategy:
|
||||
matrix:
|
||||
test-type: ["basic", "advanced", "policy-enforcement"]
|
||||
test-type: ["basic", "advanced", "policy-enforcement", "group", "sts"]
|
||||
|
||||
steps:
|
||||
- name: Check out code
|
||||
uses: actions/checkout@v6
|
||||
uses: actions/checkout@v7
|
||||
|
||||
- name: Clean up stale FUSE mounts
|
||||
uses: ./.github/actions/cleanup-stale-fuse-mounts
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v6
|
||||
uses: actions/setup-go@v7
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
cache: ${{ runner.environment == 'github-hosted' }}
|
||||
id: go
|
||||
|
||||
- name: Install SeaweedFS
|
||||
@@ -105,7 +114,7 @@ jobs:
|
||||
echo "=== System Information ==="
|
||||
uname -a
|
||||
free -h
|
||||
df -h
|
||||
timeout 15 df -h || true
|
||||
echo "=== Starting S3 IAM Integration Tests (${{ matrix.test-type }}) ==="
|
||||
|
||||
# Set WEED_BINARY to use the installed version
|
||||
@@ -117,7 +126,7 @@ jobs:
|
||||
"basic")
|
||||
echo "Running basic IAM functionality tests..."
|
||||
make clean setup start-services wait-for-services
|
||||
go test -v -timeout 15m -run "TestS3IAMAuthentication|TestS3IAMBasicWorkflow|TestS3IAMTokenValidation|TestIAM" ./...
|
||||
go test -v -timeout 15m -run "TestS3IAMAuthentication|TestS3IAMBasicWorkflow|TestS3IAMTokenValidation|TestIAMUserManagement|TestIAMAccessKeyManagement|TestIAMPolicyManagement" ./...
|
||||
;;
|
||||
"advanced")
|
||||
echo "Running advanced IAM feature tests..."
|
||||
@@ -129,6 +138,28 @@ jobs:
|
||||
make clean setup start-services wait-for-services
|
||||
go test -v -timeout 15m -run "TestS3IAMPolicyEnforcement|TestS3IAMBucketPolicy|TestS3IAMContextual" ./...
|
||||
;;
|
||||
"group")
|
||||
echo "Running IAM group management tests..."
|
||||
make clean setup start-services wait-for-services
|
||||
go test -v -timeout 15m -run "TestIAMGroup" ./...
|
||||
;;
|
||||
"sts")
|
||||
echo "Running STS and service account tests..."
|
||||
make clean setup start-services wait-for-services
|
||||
# SigV4-signed STS calls need admin credentials matching test_config.json.
|
||||
# Tests default to "admin"/"admin" when env vars are unset, which don't exist.
|
||||
export STS_TEST_ACCESS_KEY=test-access-key
|
||||
export STS_TEST_SECRET_KEY=test-secret-key
|
||||
# The use_service_account_credentials subtest is excluded because
|
||||
# newly-created service-account access keys are not currently
|
||||
# persisted to the filer after CreateServiceAccount — a
|
||||
# pre-existing sync issue tracked separately from the
|
||||
# GetFederationToken routing fix this PR addresses.
|
||||
go test -v -timeout 15m \
|
||||
-run "TestSTS|TestAssumeRoleWithWebIdentity|TestServiceAccount" \
|
||||
-skip "TestServiceAccountLifecycle/use_service_account_credentials" \
|
||||
./...
|
||||
;;
|
||||
*)
|
||||
echo "Unknown test type: ${{ matrix.test-type }}"
|
||||
exit 1
|
||||
@@ -162,7 +193,7 @@ jobs:
|
||||
|
||||
- name: Upload test logs on failure
|
||||
if: failure()
|
||||
uses: actions/upload-artifact@v6
|
||||
uses: actions/upload-artifact@v7
|
||||
with:
|
||||
name: s3-iam-integration-logs-${{ matrix.test-type }}
|
||||
path: test/s3/iam/weed-*.log
|
||||
@@ -176,12 +207,13 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Check out code
|
||||
uses: actions/checkout@v6
|
||||
uses: actions/checkout@v7
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v6
|
||||
uses: actions/setup-go@v7
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
cache: ${{ runner.environment == 'github-hosted' }}
|
||||
id: go
|
||||
|
||||
- name: Install SeaweedFS
|
||||
@@ -222,7 +254,7 @@ jobs:
|
||||
|
||||
- name: Upload distributed test logs
|
||||
if: always()
|
||||
uses: actions/upload-artifact@v6
|
||||
uses: actions/upload-artifact@v7
|
||||
with:
|
||||
name: s3-iam-distributed-logs
|
||||
path: test/s3/iam/weed-*.log
|
||||
@@ -236,12 +268,13 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Check out code
|
||||
uses: actions/checkout@v6
|
||||
uses: actions/checkout@v7
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v6
|
||||
uses: actions/setup-go@v7
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
cache: ${{ runner.environment == 'github-hosted' }}
|
||||
id: go
|
||||
|
||||
- name: Install SeaweedFS
|
||||
@@ -274,7 +307,7 @@ jobs:
|
||||
|
||||
- name: Upload performance test results
|
||||
if: always()
|
||||
uses: actions/upload-artifact@v6
|
||||
uses: actions/upload-artifact@v7
|
||||
with:
|
||||
name: s3-iam-performance-results
|
||||
path: |
|
||||
|
||||
@@ -16,8 +16,8 @@ on:
|
||||
- '.github/workflows/s3-keycloak-tests.yml'
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.head_ref }}/s3-keycloak-tests
|
||||
cancel-in-progress: true
|
||||
group: ${{ github.workflow }}-${{ github.event.pull_request.number || (github.event_name == 'push' && github.ref) || github.run_id }}
|
||||
cancel-in-progress: ${{ github.event_name == 'pull_request' || (github.event_name == 'push' && !startsWith(github.ref, 'refs/tags/')) }}
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
@@ -35,12 +35,16 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Check out code
|
||||
uses: actions/checkout@v6
|
||||
uses: actions/checkout@v7
|
||||
|
||||
- name: Clean up stale FUSE mounts
|
||||
uses: ./.github/actions/cleanup-stale-fuse-mounts
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v6
|
||||
uses: actions/setup-go@v7
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
cache: ${{ runner.environment == 'github-hosted' }}
|
||||
id: go
|
||||
|
||||
- name: Install SeaweedFS
|
||||
@@ -56,7 +60,7 @@ jobs:
|
||||
echo "=== System Information ==="
|
||||
uname -a
|
||||
free -h
|
||||
df -h
|
||||
timeout 15 df -h || true
|
||||
echo "=== Starting S3 Keycloak Integration Tests ==="
|
||||
|
||||
# Set WEED_BINARY to use the installed version
|
||||
@@ -97,7 +101,7 @@ jobs:
|
||||
# Verify service accessibility
|
||||
echo "=== Verifying Service Accessibility ==="
|
||||
curl -f http://localhost:8080/realms/master
|
||||
curl -s http://localhost:8333
|
||||
curl -s http://localhost:8333/healthz
|
||||
echo "✅ SeaweedFS S3 API is responding (IAM-protected endpoint)"
|
||||
|
||||
# Run Keycloak-specific tests
|
||||
@@ -152,7 +156,7 @@ jobs:
|
||||
|
||||
- name: Upload test logs on failure
|
||||
if: failure()
|
||||
uses: actions/upload-artifact@v6
|
||||
uses: actions/upload-artifact@v7
|
||||
with:
|
||||
name: s3-keycloak-test-logs
|
||||
path: |
|
||||
|
||||
@@ -0,0 +1,151 @@
|
||||
name: "S3 Mutation Regression Tests"
|
||||
|
||||
on:
|
||||
pull_request:
|
||||
paths:
|
||||
- 'weed/s3api/**'
|
||||
- 'test/s3/delete/**'
|
||||
- 'test/s3/distributed_lock/**'
|
||||
- 'test/s3/versioning/**'
|
||||
- 'test/volume_server/framework/**'
|
||||
- 'docker/compose/s3.json'
|
||||
- '.github/workflows/s3-mutation-regression-tests.yml'
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.event.pull_request.number || (github.event_name == 'push' && github.ref) || github.run_id }}
|
||||
cancel-in-progress: ${{ github.event_name == 'pull_request' || (github.event_name == 'push' && !startsWith(github.ref, 'refs/tags/')) }}
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
jobs:
|
||||
s3-versioning-regressions:
|
||||
name: S3 Versioning Regression Tests
|
||||
runs-on: ubuntu-22.04
|
||||
timeout-minutes: 25
|
||||
|
||||
steps:
|
||||
- name: Check out code
|
||||
uses: actions/checkout@v7
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v7
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
cache: ${{ runner.environment == 'github-hosted' }}
|
||||
|
||||
- name: Run S3 versioning regression tests
|
||||
timeout-minutes: 20
|
||||
working-directory: test/s3/versioning
|
||||
run: |
|
||||
set -x
|
||||
# Run every versioning test, so a regression test lands covered instead
|
||||
# of waiting for someone to remember this file. Name a test in EXCLUDE,
|
||||
# with the reason, to keep it out.
|
||||
#
|
||||
# TestVersioningPagination*: opt-in stress tests that build 1500+
|
||||
# versions. They self-skip without ENABLE_STRESS_TESTS and have their
|
||||
# own make target, so this gate should not carry them.
|
||||
EXCLUDE='TestVersioningPagination.*'
|
||||
tests=$(go test . -list '.*' | grep '^Test' | sort -u)
|
||||
# An empty list would make -run match nothing and pass this job vacuously.
|
||||
[ -n "$tests" ] || { echo "listed no versioning tests"; exit 1; }
|
||||
selected=$(echo "$tests" | grep -vE "^($EXCLUDE)$")
|
||||
[ -n "$selected" ] || { echo "EXCLUDE matched every test"; exit 1; }
|
||||
echo "running $(echo "$selected" | wc -l) of $(echo "$tests" | wc -l) versioning tests"
|
||||
# make swallows a lone trailing $, taking the anchor with it, so escape it.
|
||||
make test-with-server TEST_PATTERN="^($(echo "$selected" | paste -sd'|' -))"'$$'
|
||||
|
||||
- name: Show server logs on failure
|
||||
if: failure()
|
||||
working-directory: test/s3/versioning
|
||||
run: |
|
||||
echo "=== Server Logs ==="
|
||||
if [ -f weed-test.log ]; then
|
||||
tail -100 weed-test.log
|
||||
fi
|
||||
|
||||
- name: Upload versioning logs on failure
|
||||
if: failure()
|
||||
uses: actions/upload-artifact@v7
|
||||
with:
|
||||
name: s3-versioning-regression-logs
|
||||
path: test/s3/versioning/weed-test*.log
|
||||
retention-days: 3
|
||||
|
||||
s3-delete-regressions:
|
||||
name: S3 Delete Regression Tests
|
||||
runs-on: ubuntu-22.04
|
||||
timeout-minutes: 20
|
||||
|
||||
steps:
|
||||
- name: Check out code
|
||||
uses: actions/checkout@v7
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v7
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
cache: ${{ runner.environment == 'github-hosted' }}
|
||||
|
||||
- name: Run S3 delete regression tests
|
||||
timeout-minutes: 15
|
||||
working-directory: test/s3/delete
|
||||
run: |
|
||||
set -x
|
||||
make test-with-server
|
||||
|
||||
- name: Show server logs on failure
|
||||
if: failure()
|
||||
working-directory: test/s3/delete
|
||||
run: |
|
||||
echo "=== Server Logs ==="
|
||||
if [ -f weed-test.log ]; then
|
||||
tail -100 weed-test.log
|
||||
fi
|
||||
|
||||
- name: Upload delete logs on failure
|
||||
if: failure()
|
||||
uses: actions/upload-artifact@v7
|
||||
with:
|
||||
name: s3-delete-regression-logs
|
||||
path: test/s3/delete/weed-test*.log
|
||||
retention-days: 3
|
||||
|
||||
s3-distributed-lock-regressions:
|
||||
name: S3 Distributed Lock Regression Tests
|
||||
runs-on: ubuntu-22.04
|
||||
timeout-minutes: 30
|
||||
|
||||
steps:
|
||||
- name: Check out code
|
||||
uses: actions/checkout@v7
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v7
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
cache: ${{ runner.environment == 'github-hosted' }}
|
||||
|
||||
- name: Build SeaweedFS
|
||||
run: |
|
||||
go build -o weed/weed -buildvcs=false ./weed
|
||||
|
||||
- name: Run distributed lock regressions
|
||||
timeout-minutes: 25
|
||||
env:
|
||||
TMPDIR: ${{ github.workspace }}/test/s3/distributed_lock/tmp
|
||||
S3_DISTRIBUTED_LOCK_KEEP_LOGS: "1"
|
||||
WEED_BINARY: ${{ github.workspace }}/weed/weed
|
||||
run: |
|
||||
set -x
|
||||
mkdir -p "$TMPDIR"
|
||||
go test -v -count=1 -timeout=20m ./test/s3/distributed_lock
|
||||
|
||||
- name: Upload distributed lock logs on failure
|
||||
if: failure()
|
||||
uses: actions/upload-artifact@v7
|
||||
with:
|
||||
name: s3-distributed-lock-regression-logs
|
||||
path: test/s3/distributed_lock/tmp/seaweedfs_s3_distributed_lock_*
|
||||
retention-days: 3
|
||||
@@ -23,6 +23,10 @@ env:
|
||||
S3_ENDPOINT_URL: http://localhost:8333
|
||||
BUCKET_NAME: test-parquet-bucket
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.event.pull_request.number || (github.event_name == 'push' && github.ref) || github.run_id }}
|
||||
cancel-in-progress: ${{ github.event_name == 'pull_request' || (github.event_name == 'push' && !startsWith(github.ref, 'refs/tags/')) }}
|
||||
|
||||
jobs:
|
||||
parquet-integration-tests:
|
||||
name: PyArrow Parquet Tests (Python ${{ matrix.python-version }})
|
||||
@@ -36,19 +40,19 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v6
|
||||
uses: actions/checkout@v7
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v6
|
||||
uses: actions/setup-go@v7
|
||||
with:
|
||||
go-version: ^1.24
|
||||
cache: true
|
||||
go-version: ^1.26
|
||||
cache: ${{ runner.environment == 'github-hosted' }}
|
||||
|
||||
- name: Set up Python ${{ matrix.python-version }}
|
||||
uses: actions/setup-python@v6
|
||||
uses: actions/setup-python@v7
|
||||
with:
|
||||
python-version: ${{ matrix.python-version }}
|
||||
cache: 'pip'
|
||||
cache: ${{ runner.environment == 'github-hosted' && 'pip' || '' }}
|
||||
cache-dependency-path: 'test/s3/parquet/requirements.txt'
|
||||
|
||||
- name: Install system dependencies
|
||||
@@ -121,7 +125,7 @@ jobs:
|
||||
|
||||
- name: Upload test logs on failure
|
||||
if: failure()
|
||||
uses: actions/upload-artifact@v6
|
||||
uses: actions/upload-artifact@v7
|
||||
with:
|
||||
name: test-logs-python-${{ matrix.python-version }}
|
||||
path: |
|
||||
@@ -139,17 +143,18 @@ jobs:
|
||||
unit-tests:
|
||||
name: Go Unit Tests (Implicit Directory)
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 10
|
||||
# Leave time for setup and the focused run before the nine-minute suite.
|
||||
timeout-minutes: 20
|
||||
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v6
|
||||
uses: actions/checkout@v7
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v6
|
||||
uses: actions/setup-go@v7
|
||||
with:
|
||||
go-version: ^1.24
|
||||
cache: true
|
||||
go-version: ^1.26
|
||||
cache: ${{ runner.environment == 'github-hosted' }}
|
||||
|
||||
- name: Run Go unit tests
|
||||
run: |
|
||||
@@ -159,5 +164,5 @@ jobs:
|
||||
- name: Run all S3 API tests
|
||||
run: |
|
||||
cd weed/s3api
|
||||
go test -v -timeout 5m
|
||||
go test -v -timeout 9m
|
||||
|
||||
@@ -24,8 +24,8 @@ on:
|
||||
- '.github/workflows/s3-policy-tests.yml'
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.head_ref }}/s3-policy-tests
|
||||
cancel-in-progress: true
|
||||
group: ${{ github.workflow }}-${{ github.event.pull_request.number || (github.event_name == 'push' && github.ref) || github.run_id }}
|
||||
cancel-in-progress: ${{ github.event_name == 'pull_request' || (github.event_name == 'push' && !startsWith(github.ref, 'refs/tags/')) }}
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
@@ -43,12 +43,13 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Check out code
|
||||
uses: actions/checkout@v6
|
||||
uses: actions/checkout@v7
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v6
|
||||
uses: actions/setup-go@v7
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
cache: ${{ runner.environment == 'github-hosted' }}
|
||||
id: go
|
||||
|
||||
- name: Get dependencies
|
||||
@@ -70,7 +71,7 @@ jobs:
|
||||
|
||||
- name: Upload test results on failure
|
||||
if: failure()
|
||||
uses: actions/upload-artifact@v6
|
||||
uses: actions/upload-artifact@v7
|
||||
with:
|
||||
name: policy-unit-test-results
|
||||
path: |
|
||||
@@ -86,12 +87,16 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Check out code
|
||||
uses: actions/checkout@v6
|
||||
uses: actions/checkout@v7
|
||||
|
||||
- name: Clean up stale FUSE mounts
|
||||
uses: ./.github/actions/cleanup-stale-fuse-mounts
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v6
|
||||
uses: actions/setup-go@v7
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
cache: ${{ runner.environment == 'github-hosted' }}
|
||||
id: go
|
||||
|
||||
- name: Install SeaweedFS
|
||||
@@ -106,7 +111,7 @@ jobs:
|
||||
echo "=== System Information ==="
|
||||
uname -a
|
||||
free -h
|
||||
df -h
|
||||
timeout 15 df -h || true
|
||||
|
||||
echo "=== Starting S3 Policy Variables Integration Tests ==="
|
||||
|
||||
@@ -178,7 +183,7 @@ jobs:
|
||||
|
||||
- name: Upload test logs on failure
|
||||
if: failure()
|
||||
uses: actions/upload-artifact@v6
|
||||
uses: actions/upload-artifact@v7
|
||||
with:
|
||||
name: s3-policy-variables-test-logs
|
||||
path: /tmp/weed_policy_test_server.log
|
||||
@@ -195,12 +200,16 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Check out code
|
||||
uses: actions/checkout@v6
|
||||
uses: actions/checkout@v7
|
||||
|
||||
- name: Clean up stale FUSE mounts
|
||||
uses: ./.github/actions/cleanup-stale-fuse-mounts
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v6
|
||||
uses: actions/setup-go@v7
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
cache: ${{ runner.environment == 'github-hosted' }}
|
||||
id: go
|
||||
|
||||
- name: Install SeaweedFS
|
||||
@@ -215,7 +224,7 @@ jobs:
|
||||
echo "=== System Information ==="
|
||||
uname -a
|
||||
free -h
|
||||
df -h
|
||||
timeout 15 df -h || true
|
||||
|
||||
echo "=== Starting S3 Policy Enforcement Tests (${{ matrix.test-case }}) ==="
|
||||
|
||||
@@ -299,7 +308,7 @@ jobs:
|
||||
|
||||
- name: Upload test logs on failure
|
||||
if: failure()
|
||||
uses: actions/upload-artifact@v6
|
||||
uses: actions/upload-artifact@v7
|
||||
with:
|
||||
name: s3-policy-enforcement-logs-${{ matrix.test-case }}
|
||||
path: /tmp/weed_policy_enforcement_${{ matrix.test-case }}.log
|
||||
@@ -313,12 +322,13 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Check out code
|
||||
uses: actions/checkout@v6
|
||||
uses: actions/checkout@v7
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v6
|
||||
uses: actions/setup-go@v7
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
cache: ${{ runner.environment == 'github-hosted' }}
|
||||
id: go
|
||||
|
||||
- name: Install SeaweedFS
|
||||
@@ -386,7 +396,7 @@ jobs:
|
||||
|
||||
- name: Upload test logs on failure
|
||||
if: failure()
|
||||
uses: actions/upload-artifact@v6
|
||||
uses: actions/upload-artifact@v7
|
||||
with:
|
||||
name: trusted-proxy-test-logs
|
||||
path: /tmp/weed_proxy_test.log
|
||||
@@ -400,12 +410,13 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Check out code
|
||||
uses: actions/checkout@v6
|
||||
uses: actions/checkout@v7
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v6
|
||||
uses: actions/setup-go@v7
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
cache: ${{ runner.environment == 'github-hosted' }}
|
||||
id: go
|
||||
|
||||
- name: Install SeaweedFS
|
||||
|
||||
@@ -0,0 +1,144 @@
|
||||
name: "S3 Proxy Signature Tests"
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [ master ]
|
||||
paths:
|
||||
- 'weed/s3api/**'
|
||||
- 'weed/server/**'
|
||||
- 'test/s3/proxy_signature/**'
|
||||
- 'go.mod'
|
||||
- 'go.sum'
|
||||
- '.github/workflows/s3-proxy-signature-tests.yml'
|
||||
pull_request:
|
||||
branches: [ master ]
|
||||
paths:
|
||||
- 'weed/s3api/**'
|
||||
- 'weed/server/**'
|
||||
- 'test/s3/proxy_signature/**'
|
||||
- 'go.mod'
|
||||
- 'go.sum'
|
||||
- '.github/workflows/s3-proxy-signature-tests.yml'
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.event.pull_request.number || (github.event_name == 'push' && github.ref) || github.run_id }}
|
||||
cancel-in-progress: ${{ github.event_name == 'pull_request' || (github.event_name == 'push' && !startsWith(github.ref, 'refs/tags/')) }}
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
jobs:
|
||||
proxy-signature-tests:
|
||||
name: S3 Proxy Signature Verification Tests
|
||||
runs-on: ubuntu-22.04
|
||||
timeout-minutes: 15
|
||||
steps:
|
||||
- name: Check out code into the Go module directory
|
||||
uses: actions/checkout@v7
|
||||
|
||||
- name: Set up Go 1.x
|
||||
uses: actions/setup-go@v7
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
cache: ${{ runner.environment == 'github-hosted' }}
|
||||
id: go
|
||||
|
||||
- name: Configure Docker Hub mirror
|
||||
run: |
|
||||
echo '{"registry-mirrors": ["https://mirror.gcr.io"]}' | sudo tee /etc/docker/daemon.json
|
||||
sudo systemctl restart docker
|
||||
|
||||
- name: Set up Docker Buildx
|
||||
uses: docker/setup-buildx-action@v4
|
||||
|
||||
- name: Build SeaweedFS binary for Linux
|
||||
run: |
|
||||
set -x
|
||||
CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -buildvcs=false -v -o test/s3/proxy_signature/weed ./weed
|
||||
|
||||
- name: Run S3 Proxy Signature Tests
|
||||
timeout-minutes: 10
|
||||
working-directory: test/s3/proxy_signature
|
||||
run: |
|
||||
set -x
|
||||
echo "Starting Docker Compose services..."
|
||||
docker compose up -d --build
|
||||
|
||||
# Check if containers are running
|
||||
echo "Checking container status..."
|
||||
docker compose ps
|
||||
|
||||
# Wait for services to be ready
|
||||
echo "Waiting for nginx proxy to be ready..."
|
||||
PROXY_READY=0
|
||||
for i in $(seq 1 30); do
|
||||
if curl -s http://localhost:9000/ > /dev/null 2>&1; then
|
||||
echo "Proxy is ready"
|
||||
PROXY_READY=1
|
||||
break
|
||||
fi
|
||||
echo "Waiting for proxy... ($i/30)"
|
||||
sleep 1
|
||||
done
|
||||
if [ $PROXY_READY -eq 0 ]; then
|
||||
echo "ERROR: Proxy failed to become ready after 30 seconds"
|
||||
echo "Docker compose logs:"
|
||||
docker compose logs --no-color || true
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Wait for SeaweedFS to be ready
|
||||
echo "Waiting for SeaweedFS S3 gateway to be ready via proxy..."
|
||||
S3_READY=0
|
||||
for i in $(seq 1 30); do
|
||||
# Check logs first for the readiness line. weed mini's progress
|
||||
# board prints " S3 ready (Xs)"; older builds and the
|
||||
# standalone S3 binary log "S3 (gateway|service) ... ready".
|
||||
if docker compose logs seaweedfs 2>&1 | grep -qE "S3 (gateway|service).*(started|ready)|S3[[:space:]]+ready"; then
|
||||
echo "SeaweedFS S3 gateway is ready"
|
||||
S3_READY=1
|
||||
break
|
||||
fi
|
||||
# Fallback: check headers via proxy (which is already ready)
|
||||
if curl -s -I http://localhost:9000/ | grep -qi "SeaweedFS"; then
|
||||
echo "SeaweedFS S3 gateway is responding via proxy"
|
||||
S3_READY=1
|
||||
break
|
||||
fi
|
||||
echo "Waiting for S3 gateway... ($i/30)"
|
||||
sleep 1
|
||||
done
|
||||
if [ $S3_READY -eq 0 ]; then
|
||||
echo "ERROR: SeaweedFS S3 gateway failed to become ready after 30 seconds"
|
||||
echo "Latest seaweedfs logs:"
|
||||
docker compose logs --no-color --tail 20 seaweedfs || true
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Run the test script inside AWS CLI container
|
||||
echo "Running test script..."
|
||||
docker run --rm --network host \
|
||||
--entrypoint bash \
|
||||
amazon/aws-cli:latest \
|
||||
-c "$(cat test.sh)"
|
||||
|
||||
TEST_RESULT=$?
|
||||
|
||||
# Cleanup
|
||||
docker compose down
|
||||
|
||||
exit $TEST_RESULT
|
||||
|
||||
- name: Cleanup on failure
|
||||
if: failure()
|
||||
working-directory: test/s3/proxy_signature
|
||||
run: |
|
||||
echo "Cleaning up Docker containers..."
|
||||
ls -al weed || true
|
||||
ldd weed || true
|
||||
echo "Docker compose logs:"
|
||||
docker compose logs --no-color || true
|
||||
echo "Container status before cleanup:"
|
||||
docker ps -a
|
||||
echo "Stopping services..."
|
||||
docker compose down || true
|
||||
@@ -0,0 +1,111 @@
|
||||
name: "S3 SDK V2 Route Disambiguation Tests"
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [ master ]
|
||||
paths:
|
||||
- 'weed/s3api/**'
|
||||
- 'test/s3/sdk_v2_routing/**'
|
||||
- '.github/workflows/s3-sdk-v2-routing-tests.yml'
|
||||
pull_request:
|
||||
branches: [ master ]
|
||||
paths:
|
||||
- 'weed/s3api/**'
|
||||
- 'test/s3/sdk_v2_routing/**'
|
||||
- '.github/workflows/s3-sdk-v2-routing-tests.yml'
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.event.pull_request.number || (github.event_name == 'push' && github.ref) || github.run_id }}
|
||||
cancel-in-progress: ${{ github.event_name == 'pull_request' || (github.event_name == 'push' && !startsWith(github.ref, 'refs/tags/')) }}
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
jobs:
|
||||
s3-sdk-v2-routing-tests:
|
||||
name: S3 SDK V2 Routing Tests
|
||||
runs-on: ubuntu-22.04
|
||||
timeout-minutes: 10
|
||||
steps:
|
||||
- name: Check out code
|
||||
uses: actions/checkout@v7
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v7
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
cache: ${{ runner.environment == 'github-hosted' }}
|
||||
|
||||
- name: Install SeaweedFS
|
||||
run: |
|
||||
cd weed && go install -buildvcs=false
|
||||
|
||||
- name: Start weed mini (S3 on :8333)
|
||||
# Pins the regression for issue #9559: AWS SDK V2 / Hadoop s3a
|
||||
# listing a bucket literally named "buckets" must get an XML
|
||||
# ListObjectsV2 response, not the JSON ListTableBuckets body
|
||||
# served by the S3 Tables REST endpoint on the same path.
|
||||
run: |
|
||||
mkdir -p /tmp/seaweedfs-sdk-v2-routing
|
||||
cat > /tmp/seaweedfs-sdk-v2-routing-s3.json <<'JSON'
|
||||
{
|
||||
"identities": [
|
||||
{
|
||||
"name": "admin",
|
||||
"credentials": [
|
||||
{"accessKey": "some_access_key1", "secretKey": "some_secret_key1"}
|
||||
],
|
||||
"actions": ["Admin", "Read", "Write"]
|
||||
}
|
||||
]
|
||||
}
|
||||
JSON
|
||||
AWS_ACCESS_KEY_ID=some_access_key1 \
|
||||
AWS_SECRET_ACCESS_KEY=some_secret_key1 \
|
||||
weed mini \
|
||||
-dir=/tmp/seaweedfs-sdk-v2-routing \
|
||||
-s3.port=8333 \
|
||||
-s3.config=/tmp/seaweedfs-sdk-v2-routing-s3.json \
|
||||
-ip=127.0.0.1 \
|
||||
> /tmp/weed-mini.log 2>&1 &
|
||||
echo $! > /tmp/weed-mini.pid
|
||||
|
||||
for i in $(seq 1 30); do
|
||||
if curl -s -o /dev/null -w "%{http_code}" http://127.0.0.1:8333/ | grep -qE "^(200|403)$"; then
|
||||
echo "weed mini is ready"
|
||||
exit 0
|
||||
fi
|
||||
sleep 1
|
||||
done
|
||||
echo "weed mini failed to start within 30s"
|
||||
tail -50 /tmp/weed-mini.log
|
||||
exit 1
|
||||
|
||||
- name: Run SDK V2 routing tests
|
||||
env:
|
||||
S3_ENDPOINT: http://127.0.0.1:8333
|
||||
AWS_ACCESS_KEY_ID: some_access_key1
|
||||
AWS_SECRET_ACCESS_KEY: some_secret_key1
|
||||
AWS_REGION: us-east-1
|
||||
run: go test -v -timeout=5m ./test/s3/sdk_v2_routing/...
|
||||
|
||||
- name: Stop weed mini
|
||||
if: always()
|
||||
run: |
|
||||
if [ -f /tmp/weed-mini.pid ]; then
|
||||
kill "$(cat /tmp/weed-mini.pid)" 2>/dev/null || true
|
||||
fi
|
||||
|
||||
- name: Show server log on failure
|
||||
if: failure()
|
||||
run: |
|
||||
echo "=== weed mini log (last 200 lines) ==="
|
||||
tail -n 200 /tmp/weed-mini.log 2>/dev/null || echo "no log available"
|
||||
|
||||
- name: Archive log
|
||||
if: failure()
|
||||
uses: actions/upload-artifact@v7
|
||||
with:
|
||||
name: s3-sdk-v2-routing-server-log
|
||||
path: /tmp/weed-mini.log
|
||||
retention-days: 3
|
||||
@@ -0,0 +1,107 @@
|
||||
name: "Snowflake S3Compat API tests"
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [ master ]
|
||||
paths:
|
||||
- 'weed/s3api/**'
|
||||
- 'weed/filer/**'
|
||||
- 'weed/server/**'
|
||||
- 'weed/iam/**'
|
||||
- 'weed/command/**'
|
||||
- 'weed/storage/**'
|
||||
- 'weed/operation/**'
|
||||
- 'weed/wdclient/**'
|
||||
- 'weed/cluster/**'
|
||||
- 'weed/pb/**'
|
||||
- 'test/s3/snowflake/**'
|
||||
- 'go.mod'
|
||||
- 'go.sum'
|
||||
- '.github/workflows/s3-snowflake-tests.yml'
|
||||
pull_request:
|
||||
branches: [ master ]
|
||||
paths:
|
||||
- 'weed/s3api/**'
|
||||
- 'weed/filer/**'
|
||||
- 'weed/server/**'
|
||||
- 'weed/iam/**'
|
||||
- 'weed/command/**'
|
||||
- 'weed/storage/**'
|
||||
- 'weed/operation/**'
|
||||
- 'weed/wdclient/**'
|
||||
- 'weed/cluster/**'
|
||||
- 'weed/pb/**'
|
||||
- 'test/s3/snowflake/**'
|
||||
- 'go.mod'
|
||||
- 'go.sum'
|
||||
- '.github/workflows/s3-snowflake-tests.yml'
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.event.pull_request.number || github.ref }}/s3-snowflake-tests
|
||||
cancel-in-progress: true
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
jobs:
|
||||
snowflake-s3compat-tests:
|
||||
name: Snowflake S3Compat API tests
|
||||
runs-on: ubuntu-22.04
|
||||
timeout-minutes: 30
|
||||
env:
|
||||
WORK_DIR: /tmp/seaweedfs-snowflake-tests
|
||||
steps:
|
||||
- name: Check out code
|
||||
uses: actions/checkout@v7
|
||||
with:
|
||||
persist-credentials: false
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v7
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
cache: ${{ runner.environment == 'github-hosted' }}
|
||||
id: go
|
||||
|
||||
- name: Set up Java
|
||||
uses: actions/setup-java@v6
|
||||
with:
|
||||
java-version: '17'
|
||||
distribution: 'temurin'
|
||||
cache: ${{ runner.environment == 'github-hosted' && 'maven' || '' }}
|
||||
|
||||
- name: Install SeaweedFS
|
||||
run: |
|
||||
cd weed
|
||||
go install -buildvcs=false
|
||||
weed version
|
||||
|
||||
- name: Run Snowflake S3Compat API tests
|
||||
timeout-minutes: 20
|
||||
run: |
|
||||
# Starts weed server, creates the buckets/objects the suite needs,
|
||||
# clones the upstream suite, and runs mvn -Dtest=S3CompatApiTest.
|
||||
bash test/s3/snowflake/run.sh
|
||||
|
||||
- name: Show logs on failure
|
||||
if: failure()
|
||||
run: |
|
||||
echo "=== SeaweedFS Server Log ==="
|
||||
tail -200 "$WORK_DIR/weed.log" || echo "No server log"
|
||||
echo ""
|
||||
echo "=== Surefire results ==="
|
||||
cat "$WORK_DIR"/snowflake-s3compat-api-test-suite/s3compatapi/target/surefire-reports/*.txt 2>/dev/null || echo "No surefire reports"
|
||||
|
||||
- name: Upload test results
|
||||
if: always()
|
||||
uses: actions/upload-artifact@v7
|
||||
with:
|
||||
name: snowflake-s3compat-surefire-reports
|
||||
path: /tmp/seaweedfs-snowflake-tests/snowflake-s3compat-api-test-suite/s3compatapi/target/surefire-reports/
|
||||
retention-days: 14
|
||||
|
||||
- name: Cleanup
|
||||
if: always()
|
||||
run: |
|
||||
pkill -9 -f "weed server" || true
|
||||
rm -rf "$WORK_DIR" || true
|
||||
@@ -4,14 +4,15 @@ on:
|
||||
pull_request:
|
||||
paths:
|
||||
- 'weed/s3api/**'
|
||||
- 'weed/filer/**'
|
||||
- 'test/s3/spark/**'
|
||||
- 'test/s3tables/testutil/**'
|
||||
- '.github/workflows/s3-spark-tests.yml'
|
||||
workflow_dispatch:
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.head_ref }}/s3-spark-tests
|
||||
cancel-in-progress: true
|
||||
group: ${{ github.workflow }}-${{ github.event.pull_request.number || (github.event_name == 'push' && github.ref) || github.run_id }}
|
||||
cancel-in-progress: ${{ github.event_name == 'pull_request' || (github.event_name == 'push' && !startsWith(github.ref, 'refs/tags/')) }}
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
@@ -24,23 +25,31 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Check out code
|
||||
uses: actions/checkout@v6
|
||||
uses: actions/checkout@v7
|
||||
|
||||
- name: Clean up stale FUSE mounts
|
||||
uses: ./.github/actions/cleanup-stale-fuse-mounts
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v6
|
||||
uses: actions/setup-go@v7
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
cache: ${{ runner.environment == 'github-hosted' }}
|
||||
id: go
|
||||
|
||||
- name: Set up Docker
|
||||
uses: docker/setup-buildx-action@v3
|
||||
- name: Configure Docker Hub mirror
|
||||
run: |
|
||||
echo '{"registry-mirrors": ["https://mirror.gcr.io"]}' | sudo tee /etc/docker/daemon.json
|
||||
sudo systemctl restart docker
|
||||
|
||||
- name: Install SeaweedFS
|
||||
run: |
|
||||
go install -buildvcs=false ./weed
|
||||
|
||||
- name: Pre-pull Spark image
|
||||
run: docker pull apache/spark:3.5.8
|
||||
run: |
|
||||
pull() { for i in 1 2 3; do docker pull "$1" && return 0; sleep 15; done; return 1; }
|
||||
pull apache/spark:3.5.1
|
||||
|
||||
- name: Run S3 Spark integration tests
|
||||
working-directory: test/s3/spark
|
||||
@@ -51,7 +60,7 @@ jobs:
|
||||
echo "=== System Information ==="
|
||||
uname -a
|
||||
free -h
|
||||
df -h
|
||||
timeout 15 df -h || true
|
||||
echo "=== Starting S3 Spark Integration Tests ==="
|
||||
|
||||
go test -v -timeout 30m . 2>&1 | tee test-output.log || {
|
||||
@@ -73,7 +82,7 @@ jobs:
|
||||
|
||||
- name: Upload test logs on failure
|
||||
if: failure()
|
||||
uses: actions/upload-artifact@v6
|
||||
uses: actions/upload-artifact@v7
|
||||
with:
|
||||
name: s3-spark-test-logs
|
||||
path: test/s3/spark/test-output.log
|
||||
|
||||
@@ -23,8 +23,8 @@ on:
|
||||
- 'test/s3/sse/**'
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.head_ref }}/s3-sse-tests
|
||||
cancel-in-progress: true
|
||||
group: ${{ github.workflow }}-${{ github.event.pull_request.number || (github.event_name == 'push' && github.ref) || github.run_id }}
|
||||
cancel-in-progress: ${{ github.event_name == 'pull_request' || (github.event_name == 'push' && !startsWith(github.ref, 'refs/tags/')) }}
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
@@ -44,12 +44,16 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Check out code
|
||||
uses: actions/checkout@v6
|
||||
uses: actions/checkout@v7
|
||||
|
||||
- name: Clean up stale FUSE mounts
|
||||
uses: ./.github/actions/cleanup-stale-fuse-mounts
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v6
|
||||
uses: actions/setup-go@v7
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
cache: ${{ runner.environment == 'github-hosted' }}
|
||||
id: go
|
||||
|
||||
- name: Install SeaweedFS
|
||||
@@ -64,7 +68,7 @@ jobs:
|
||||
echo "=== System Information ==="
|
||||
uname -a
|
||||
free -h
|
||||
df -h
|
||||
timeout 15 df -h || true
|
||||
echo "=== Starting SSE Tests ==="
|
||||
|
||||
# Run tests with automatic server management
|
||||
@@ -73,8 +77,12 @@ jobs:
|
||||
# Quick tests - basic SSE-C and SSE-KMS functionality + Range requests
|
||||
make test-with-server TEST_PATTERN="TestSSECIntegrationBasic|TestSSEKMSIntegrationBasic|TestSimpleSSECIntegration|.*RangeRequestsServerBehavior"
|
||||
else
|
||||
# Comprehensive tests - SSE-C/KMS functionality, excluding copy operations (pre-existing SSE-C issues)
|
||||
make test-with-server TEST_PATTERN="TestSSECIntegrationBasic|TestSSECIntegrationVariousDataSizes|TestSSEKMSIntegrationBasic|TestSSEKMSIntegrationVariousDataSizes|.*Multipart.*Integration|TestSimpleSSECIntegration|.*RangeRequestsServerBehavior"
|
||||
# Comprehensive tests - SSE-C/KMS functionality plus cross-SSE copy.
|
||||
# The copy-operation tests (`.*ObjectCopyIntegration`, `TestCrossSSECopy`,
|
||||
# `TestSSEMultipartCopy`) were excluded for a long time as "pre-existing
|
||||
# SSE-C issues" (#9281); fixed and brought back into CI as part of the
|
||||
# same change that fixed them.
|
||||
make test-with-server TEST_PATTERN="TestSSECIntegrationBasic|TestSSECIntegrationVariousDataSizes|TestSSEKMSIntegrationBasic|TestSSEKMSIntegrationVariousDataSizes|.*Multipart.*Integration|TestSimpleSSECIntegration|.*RangeRequestsServerBehavior|.*ObjectCopyIntegration|TestCrossSSECopy|TestSSEMultipartCopy"
|
||||
fi
|
||||
|
||||
- name: Show server logs on failure
|
||||
@@ -95,7 +103,7 @@ jobs:
|
||||
|
||||
- name: Upload test logs on failure
|
||||
if: failure()
|
||||
uses: actions/upload-artifact@v6
|
||||
uses: actions/upload-artifact@v7
|
||||
with:
|
||||
name: s3-sse-test-logs-${{ matrix.test-type }}
|
||||
path: test/s3/sse/weed-test*.log
|
||||
@@ -108,12 +116,13 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Check out code
|
||||
uses: actions/checkout@v6
|
||||
uses: actions/checkout@v7
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v6
|
||||
uses: actions/setup-go@v7
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
cache: ${{ runner.environment == 'github-hosted' }}
|
||||
id: go
|
||||
|
||||
- name: Install SeaweedFS
|
||||
@@ -143,7 +152,7 @@ jobs:
|
||||
|
||||
- name: Upload server logs on failure
|
||||
if: failure()
|
||||
uses: actions/upload-artifact@v6
|
||||
uses: actions/upload-artifact@v7
|
||||
with:
|
||||
name: s3-sse-compatibility-logs
|
||||
path: test/s3/sse/weed-test*.log
|
||||
@@ -156,12 +165,13 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Check out code
|
||||
uses: actions/checkout@v6
|
||||
uses: actions/checkout@v7
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v6
|
||||
uses: actions/setup-go@v7
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
cache: ${{ runner.environment == 'github-hosted' }}
|
||||
id: go
|
||||
|
||||
- name: Install SeaweedFS
|
||||
@@ -192,7 +202,7 @@ jobs:
|
||||
|
||||
- name: Upload server logs on failure
|
||||
if: failure()
|
||||
uses: actions/upload-artifact@v6
|
||||
uses: actions/upload-artifact@v7
|
||||
with:
|
||||
name: s3-sse-metadata-persistence-logs
|
||||
path: test/s3/sse/weed-test*.log
|
||||
@@ -205,12 +215,13 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Check out code
|
||||
uses: actions/checkout@v6
|
||||
uses: actions/checkout@v7
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v6
|
||||
uses: actions/setup-go@v7
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
cache: ${{ runner.environment == 'github-hosted' }}
|
||||
id: go
|
||||
|
||||
- name: Install SeaweedFS
|
||||
@@ -241,7 +252,7 @@ jobs:
|
||||
|
||||
- name: Upload server logs on failure
|
||||
if: failure()
|
||||
uses: actions/upload-artifact@v6
|
||||
uses: actions/upload-artifact@v7
|
||||
with:
|
||||
name: s3-sse-copy-operations-logs
|
||||
path: test/s3/sse/weed-test*.log
|
||||
@@ -254,12 +265,13 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Check out code
|
||||
uses: actions/checkout@v6
|
||||
uses: actions/checkout@v7
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v6
|
||||
uses: actions/setup-go@v7
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
cache: ${{ runner.environment == 'github-hosted' }}
|
||||
id: go
|
||||
|
||||
- name: Install SeaweedFS
|
||||
@@ -290,7 +302,7 @@ jobs:
|
||||
|
||||
- name: Upload server logs on failure
|
||||
if: failure()
|
||||
uses: actions/upload-artifact@v6
|
||||
uses: actions/upload-artifact@v7
|
||||
with:
|
||||
name: s3-sse-multipart-logs
|
||||
path: test/s3/sse/weed-test*.log
|
||||
@@ -305,12 +317,13 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Check out code
|
||||
uses: actions/checkout@v6
|
||||
uses: actions/checkout@v7
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v6
|
||||
uses: actions/setup-go@v7
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
cache: ${{ runner.environment == 'github-hosted' }}
|
||||
id: go
|
||||
|
||||
- name: Install SeaweedFS
|
||||
@@ -340,7 +353,7 @@ jobs:
|
||||
|
||||
- name: Upload performance test logs
|
||||
if: always()
|
||||
uses: actions/upload-artifact@v6
|
||||
uses: actions/upload-artifact@v7
|
||||
with:
|
||||
name: s3-sse-performance-logs
|
||||
path: test/s3/sse/weed-test*.log
|
||||
@@ -353,12 +366,13 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Check out code
|
||||
uses: actions/checkout@v6
|
||||
uses: actions/checkout@v7
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v6
|
||||
uses: actions/setup-go@v7
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
cache: ${{ runner.environment == 'github-hosted' }}
|
||||
id: go
|
||||
|
||||
- name: Install SeaweedFS
|
||||
@@ -389,7 +403,7 @@ jobs:
|
||||
|
||||
- name: Upload server logs on failure
|
||||
if: failure()
|
||||
uses: actions/upload-artifact@v6
|
||||
uses: actions/upload-artifact@v7
|
||||
with:
|
||||
name: s3-volume-encryption-logs
|
||||
path: /tmp/seaweedfs-sse-*.log
|
||||
|
||||
File diff suppressed because it is too large.
Load diff
+163
-43
@@ -3,12 +3,30 @@ name: "Ceph S3 tests"
|
||||
on:
|
||||
push:
|
||||
branches: [ master ]
|
||||
paths:
|
||||
- 'weed/s3api/**'
|
||||
- 'weed/filer/**'
|
||||
- 'weed/server/**'
|
||||
- 'weed/iam/**'
|
||||
- 'test/s3/compatibility/**'
|
||||
- 'go.mod'
|
||||
- 'go.sum'
|
||||
- '.github/workflows/s3tests.yml'
|
||||
pull_request:
|
||||
branches: [ master ]
|
||||
paths:
|
||||
- 'weed/s3api/**'
|
||||
- 'weed/filer/**'
|
||||
- 'weed/server/**'
|
||||
- 'weed/iam/**'
|
||||
- 'test/s3/compatibility/**'
|
||||
- 'go.mod'
|
||||
- 'go.sum'
|
||||
- '.github/workflows/s3tests.yml'
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.head_ref }}/s3tests
|
||||
cancel-in-progress: true
|
||||
group: ${{ github.workflow }}-${{ github.event.pull_request.number || (github.event_name == 'push' && github.ref) || github.run_id }}
|
||||
cancel-in-progress: ${{ github.event_name == 'pull_request' || (github.event_name == 'push' && !startsWith(github.ref, 'refs/tags/')) }}
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
@@ -20,16 +38,17 @@ jobs:
|
||||
timeout-minutes: 15
|
||||
steps:
|
||||
- name: Check out code into the Go module directory
|
||||
uses: actions/checkout@v6
|
||||
uses: actions/checkout@v7
|
||||
|
||||
- name: Set up Go 1.x
|
||||
uses: actions/setup-go@v6
|
||||
uses: actions/setup-go@v7
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
cache: ${{ runner.environment == 'github-hosted' }}
|
||||
id: go
|
||||
|
||||
- name: Set up Python
|
||||
uses: actions/setup-python@v6
|
||||
uses: actions/setup-python@v7
|
||||
with:
|
||||
python-version: '3.9'
|
||||
|
||||
@@ -37,13 +56,15 @@ jobs:
|
||||
run: |
|
||||
git clone https://github.com/ceph/s3-tests.git
|
||||
cd s3-tests
|
||||
pip install -r requirements.txt
|
||||
pip install tox
|
||||
pip install -e .
|
||||
sudo apt-get update -qq
|
||||
sudo apt-get install -y -qq libxml2-dev libxslt1-dev zlib1g-dev
|
||||
python3 -m pip install -r requirements.txt
|
||||
python3 -m pip install tox
|
||||
python3 -m pip install -e .
|
||||
|
||||
- name: Fix S3 tests bucket creation conflicts
|
||||
run: |
|
||||
python3 test/s3/fix_s3_tests_bucket_conflicts.py
|
||||
python3 test/s3/compatibility/fix_s3_tests_bucket_conflicts.py
|
||||
env:
|
||||
S3_TESTS_PATH: s3-tests
|
||||
|
||||
@@ -108,6 +129,9 @@ jobs:
|
||||
echo "All SeaweedFS components are ready!"
|
||||
cd ../s3-tests
|
||||
sed -i "s/assert prefixes == \['foo%2B1\/', 'foo\/', 'quux%20ab\/'\]/assert prefixes == \['foo\/', 'foo%2B1\/', 'quux%20ab\/'\]/" s3tests/functional/test_s3.py
|
||||
# The suite expects RGW's 400 InvalidPart for a partNumber past the last
|
||||
# part; AWS answers 416 InvalidPartNumber, which is what we return.
|
||||
sed -i "/# request PartNumber out of range/,+4{s/assert status == 400/assert status == 416/; s/assert error_code == 'InvalidPart'/assert error_code == 'InvalidPartNumber'/}" s3tests/functional/test_s3.py
|
||||
|
||||
# Debug: Show the config file contents
|
||||
echo "=== S3 Config File Contents ==="
|
||||
@@ -130,7 +154,49 @@ jobs:
|
||||
done
|
||||
|
||||
echo "✅ S3 server is responding, starting tests..."
|
||||
|
||||
|
||||
# Spawn the lifecycle worker so test_lifecycle_expiration etc. have
|
||||
# something driving deletions. The s3tests build tag rescales one
|
||||
# day to LifeCycleInterval=10s, so a 1d rule fires within ~10s of
|
||||
# the upload's mtime; -dispatch / -checkpoint defaults are already
|
||||
# tightened under the same build tag.
|
||||
LC_LOG=/tmp/lifecycle-worker.log
|
||||
# -debug routes glog to stderr so the bootstrap walker's progress
|
||||
# shows up in $LC_LOG; without it weed shell silences glog.
|
||||
(echo "s3.lifecycle.run-shard -shards 0-15 -s3 localhost:18000 -events 0 -runtime 1800s -refresh 2s" && echo exit) \
|
||||
| weed shell -debug -master=localhost:9333 \
|
||||
> "$LC_LOG" 2>&1 &
|
||||
lc_pid=$!
|
||||
# Aliveness check: a bad shell command exits in <1s and the suite
|
||||
# would otherwise just timeout the expiration tests with no signal.
|
||||
sleep 2
|
||||
if ! kill -0 "$lc_pid" 2>/dev/null; then
|
||||
echo "lifecycle worker died on startup"
|
||||
tail -50 "$LC_LOG" 2>/dev/null || true
|
||||
exit 1
|
||||
fi
|
||||
echo "lifecycle worker pid=$lc_pid"
|
||||
|
||||
# bash -e exits the step on the first tox failure, so move teardown
|
||||
# into a trap to guarantee the worker log + data dir reach the runner.
|
||||
cleanup() {
|
||||
status=$?
|
||||
# SIGTERM first so the worker's stdout flushes; SIGKILL is the
|
||||
# bash fallback if it ignores TERM. Reading the log AFTER the
|
||||
# graceful-stop window catches the bootstrap walker's progress.
|
||||
kill -TERM "$lc_pid" 2>/dev/null || true
|
||||
kill -TERM "$pid" 2>/dev/null || true
|
||||
sleep 1
|
||||
if [ "$status" -ne 0 ]; then
|
||||
echo "=== lifecycle worker log (tail) ==="
|
||||
tail -200 "$LC_LOG" 2>/dev/null || true
|
||||
fi
|
||||
kill -9 "$lc_pid" 2>/dev/null || true
|
||||
kill -9 "$pid" 2>/dev/null || true
|
||||
rm -rf "$WEED_DATA_DIR" 2>/dev/null || true
|
||||
}
|
||||
trap cleanup EXIT
|
||||
|
||||
tox -- \
|
||||
s3tests/functional/test_s3.py::test_bucket_list_empty \
|
||||
s3tests/functional/test_s3.py::test_bucket_list_distinct \
|
||||
@@ -224,6 +290,11 @@ jobs:
|
||||
s3tests/functional/test_s3.py::test_object_write_check_etag \
|
||||
s3tests/functional/test_s3.py::test_object_write_cache_control \
|
||||
s3tests/functional/test_s3.py::test_object_write_expires \
|
||||
s3tests/functional/test_s3.py::test_put_obj_enc_conflict_c_s3 \
|
||||
s3tests/functional/test_s3.py::test_put_obj_enc_conflict_c_kms \
|
||||
s3tests/functional/test_s3.py::test_put_obj_enc_conflict_s3_kms \
|
||||
s3tests/functional/test_s3.py::test_put_obj_enc_conflict_bad_enc_kms \
|
||||
s3tests/functional/test_s3.py::test_object_content_encoding_aws_chunked \
|
||||
s3tests/functional/test_s3.py::test_object_write_read_update_read_delete \
|
||||
s3tests/functional/test_s3.py::test_object_metadata_replaced_on_put \
|
||||
s3tests/functional/test_s3.py::test_object_write_file \
|
||||
@@ -246,6 +317,7 @@ jobs:
|
||||
s3tests/functional/test_s3.py::test_get_object_ifmodifiedsince_good \
|
||||
s3tests/functional/test_s3.py::test_get_object_ifmodifiedsince_failed \
|
||||
s3tests/functional/test_s3.py::test_get_object_ifunmodifiedsince_failed \
|
||||
s3tests/functional/test_s3.py::test_get_checksum_object_attributes \
|
||||
s3tests/functional/test_s3.py::test_bucket_head \
|
||||
s3tests/functional/test_s3.py::test_bucket_head_notexist \
|
||||
s3tests/functional/test_s3.py::test_object_raw_authenticated \
|
||||
@@ -310,11 +382,8 @@ jobs:
|
||||
s3tests/functional/test_s3.py::test_lifecycle_get \
|
||||
s3tests/functional/test_s3.py::test_lifecycle_set_filter \
|
||||
s3tests/functional/test_s3.py::test_lifecycle_expiration \
|
||||
s3tests/functional/test_s3.py::test_lifecyclev2_expiration \
|
||||
s3tests/functional/test_s3.py::test_lifecycle_expiration_versioning_enabled
|
||||
kill -9 $pid || true
|
||||
# Clean up data directory
|
||||
rm -rf "$WEED_DATA_DIR" || true
|
||||
s3tests/functional/test_s3.py::test_lifecyclev2_expiration
|
||||
# cleanup() trap handles worker/server kill + data dir wipe.
|
||||
|
||||
versioning-tests:
|
||||
name: S3 Versioning & Object Lock tests
|
||||
@@ -322,16 +391,17 @@ jobs:
|
||||
timeout-minutes: 15
|
||||
steps:
|
||||
- name: Check out code into the Go module directory
|
||||
uses: actions/checkout@v6
|
||||
uses: actions/checkout@v7
|
||||
|
||||
- name: Set up Go 1.x
|
||||
uses: actions/setup-go@v6
|
||||
uses: actions/setup-go@v7
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
cache: ${{ runner.environment == 'github-hosted' }}
|
||||
id: go
|
||||
|
||||
- name: Set up Python
|
||||
uses: actions/setup-python@v6
|
||||
uses: actions/setup-python@v7
|
||||
with:
|
||||
python-version: '3.9'
|
||||
|
||||
@@ -339,13 +409,15 @@ jobs:
|
||||
run: |
|
||||
git clone https://github.com/ceph/s3-tests.git
|
||||
cd s3-tests
|
||||
pip install -r requirements.txt
|
||||
pip install tox
|
||||
pip install -e .
|
||||
sudo apt-get update -qq
|
||||
sudo apt-get install -y -qq libxml2-dev libxslt1-dev zlib1g-dev
|
||||
python3 -m pip install -r requirements.txt
|
||||
python3 -m pip install tox
|
||||
python3 -m pip install -e .
|
||||
|
||||
- name: Fix S3 tests bucket creation conflicts
|
||||
run: |
|
||||
python3 test/s3/fix_s3_tests_bucket_conflicts.py
|
||||
python3 test/s3/compatibility/fix_s3_tests_bucket_conflicts.py
|
||||
env:
|
||||
S3_TESTS_PATH: s3-tests
|
||||
|
||||
@@ -490,16 +562,17 @@ jobs:
|
||||
timeout-minutes: 10
|
||||
steps:
|
||||
- name: Check out code into the Go module directory
|
||||
uses: actions/checkout@v6
|
||||
uses: actions/checkout@v7
|
||||
|
||||
- name: Set up Go 1.x
|
||||
uses: actions/setup-go@v6
|
||||
uses: actions/setup-go@v7
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
cache: ${{ runner.environment == 'github-hosted' }}
|
||||
id: go
|
||||
|
||||
- name: Set up Python
|
||||
uses: actions/setup-python@v6
|
||||
uses: actions/setup-python@v7
|
||||
with:
|
||||
python-version: '3.9'
|
||||
|
||||
@@ -507,9 +580,11 @@ jobs:
|
||||
run: |
|
||||
git clone https://github.com/ceph/s3-tests.git
|
||||
cd s3-tests
|
||||
pip install -r requirements.txt
|
||||
pip install tox
|
||||
pip install -e .
|
||||
sudo apt-get update -qq
|
||||
sudo apt-get install -y -qq libxml2-dev libxslt1-dev zlib1g-dev
|
||||
python3 -m pip install -r requirements.txt
|
||||
python3 -m pip install tox
|
||||
python3 -m pip install -e .
|
||||
|
||||
- name: Run S3 CORS tests
|
||||
timeout-minutes: 10
|
||||
@@ -613,12 +688,13 @@ jobs:
|
||||
timeout-minutes: 10
|
||||
steps:
|
||||
- name: Check out code into the Go module directory
|
||||
uses: actions/checkout@v6
|
||||
uses: actions/checkout@v7
|
||||
|
||||
- name: Set up Go 1.x
|
||||
uses: actions/setup-go@v6
|
||||
uses: actions/setup-go@v7
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
cache: ${{ runner.environment == 'github-hosted' }}
|
||||
id: go
|
||||
|
||||
- name: Run SeaweedFS Custom S3 Copy tests
|
||||
@@ -631,10 +707,12 @@ jobs:
|
||||
export WEED_DATA_DIR="/tmp/seaweedfs-copy-test-$(date +%s)"
|
||||
mkdir -p "$WEED_DATA_DIR"
|
||||
set -x
|
||||
# Each test bucket is its own collection and grows 7 volumes; the suite runs
|
||||
# faster than the heartbeat that returns slots from the deleted collections.
|
||||
weed -v 0 server -filer -filer.maxMB=64 -s3 -ip.bind 0.0.0.0 \
|
||||
-dir="$WEED_DATA_DIR" \
|
||||
-master.raftHashicorp -master.electionTimeout 1s -master.volumeSizeLimitMB=100 \
|
||||
-volume.max=100 -volume.preStopSeconds=1 \
|
||||
-volume.max=300 -volume.preStopSeconds=1 \
|
||||
-master.port=9336 -volume.port=8083 -filer.port=8891 -s3.port=8003 -metricsPort=9327 \
|
||||
-s3.allowDeleteBucketNotEmpty=true -s3.config="$GITHUB_WORKSPACE/docker/compose/s3.json" -master.peers=none &
|
||||
pid=$!
|
||||
@@ -702,7 +780,7 @@ jobs:
|
||||
sleep 2
|
||||
done
|
||||
|
||||
go test -v
|
||||
MASTER_ENDPOINT="http://127.0.0.1:9336" go test -v
|
||||
kill -9 $pid || true
|
||||
# Clean up data directory
|
||||
rm -rf "$WEED_DATA_DIR" || true
|
||||
@@ -713,16 +791,17 @@ jobs:
|
||||
timeout-minutes: 15
|
||||
steps:
|
||||
- name: Check out code into the Go module directory
|
||||
uses: actions/checkout@v6
|
||||
uses: actions/checkout@v7
|
||||
|
||||
- name: Set up Go 1.x
|
||||
uses: actions/setup-go@v6
|
||||
uses: actions/setup-go@v7
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
cache: ${{ runner.environment == 'github-hosted' }}
|
||||
id: go
|
||||
|
||||
- name: Set up Python
|
||||
uses: actions/setup-python@v6
|
||||
uses: actions/setup-python@v7
|
||||
with:
|
||||
python-version: '3.9'
|
||||
|
||||
@@ -730,9 +809,11 @@ jobs:
|
||||
run: |
|
||||
git clone https://github.com/ceph/s3-tests.git
|
||||
cd s3-tests
|
||||
pip install -r requirements.txt
|
||||
pip install tox
|
||||
pip install -e .
|
||||
sudo apt-get update -qq
|
||||
sudo apt-get install -y -qq libxml2-dev libxslt1-dev zlib1g-dev
|
||||
python3 -m pip install -r requirements.txt
|
||||
python3 -m pip install tox
|
||||
python3 -m pip install -e .
|
||||
|
||||
- name: Run Ceph S3 tests with SQL store
|
||||
timeout-minutes: 15
|
||||
@@ -901,6 +982,9 @@ jobs:
|
||||
echo "All SeaweedFS components are ready!"
|
||||
cd ../s3-tests
|
||||
sed -i "s/assert prefixes == \['foo%2B1\/', 'foo\/', 'quux%20ab\/'\]/assert prefixes == \['foo\/', 'foo%2B1\/', 'quux%20ab\/'\]/" s3tests/functional/test_s3.py
|
||||
# The suite expects RGW's 400 InvalidPart for a partNumber past the last
|
||||
# part; AWS answers 416 InvalidPartNumber, which is what we return.
|
||||
sed -i "/# request PartNumber out of range/,+4{s/assert status == 400/assert status == 416/; s/assert error_code == 'InvalidPart'/assert error_code == 'InvalidPartNumber'/}" s3tests/functional/test_s3.py
|
||||
# Create and update s3tests.conf to use port 8004
|
||||
cp ../docker/compose/s3tests.conf ../docker/compose/s3tests-sql.conf
|
||||
sed -i 's/port = 8000/port = 8004/g' ../docker/compose/s3tests-sql.conf
|
||||
@@ -950,6 +1034,39 @@ jobs:
|
||||
|
||||
sleep 2
|
||||
done
|
||||
|
||||
# Spawn the lifecycle worker (see basic-tests block for context).
|
||||
LC_LOG=/tmp/lifecycle-worker-sql.log
|
||||
(echo "s3.lifecycle.run-shard -shards 0-15 -s3 localhost:18004 -events 0 -runtime 1800s -refresh 2s" && echo exit) \
|
||||
| weed shell -debug -master=localhost:9337 \
|
||||
> "$LC_LOG" 2>&1 &
|
||||
lc_pid=$!
|
||||
sleep 2
|
||||
if ! kill -0 "$lc_pid" 2>/dev/null; then
|
||||
echo "lifecycle worker died on startup"
|
||||
tail -50 "$LC_LOG" 2>/dev/null || true
|
||||
exit 1
|
||||
fi
|
||||
echo "lifecycle worker pid=$lc_pid"
|
||||
|
||||
cleanup() {
|
||||
status=$?
|
||||
# SIGTERM first so the worker's stdout flushes; SIGKILL is the
|
||||
# bash fallback if it ignores TERM. Reading the log AFTER the
|
||||
# graceful-stop window catches the bootstrap walker's progress.
|
||||
kill -TERM "$lc_pid" 2>/dev/null || true
|
||||
kill -TERM "$pid" 2>/dev/null || true
|
||||
sleep 1
|
||||
if [ "$status" -ne 0 ]; then
|
||||
echo "=== lifecycle worker log (tail) ==="
|
||||
tail -200 "$LC_LOG" 2>/dev/null || true
|
||||
fi
|
||||
kill -9 "$lc_pid" 2>/dev/null || true
|
||||
kill -9 "$pid" 2>/dev/null || true
|
||||
rm -rf "$WEED_DATA_DIR" 2>/dev/null || true
|
||||
}
|
||||
trap cleanup EXIT
|
||||
|
||||
tox -- \
|
||||
s3tests/functional/test_s3.py::test_bucket_list_empty \
|
||||
s3tests/functional/test_s3.py::test_bucket_list_distinct \
|
||||
@@ -1043,6 +1160,11 @@ jobs:
|
||||
s3tests/functional/test_s3.py::test_object_write_check_etag \
|
||||
s3tests/functional/test_s3.py::test_object_write_cache_control \
|
||||
s3tests/functional/test_s3.py::test_object_write_expires \
|
||||
s3tests/functional/test_s3.py::test_put_obj_enc_conflict_c_s3 \
|
||||
s3tests/functional/test_s3.py::test_put_obj_enc_conflict_c_kms \
|
||||
s3tests/functional/test_s3.py::test_put_obj_enc_conflict_s3_kms \
|
||||
s3tests/functional/test_s3.py::test_put_obj_enc_conflict_bad_enc_kms \
|
||||
s3tests/functional/test_s3.py::test_object_content_encoding_aws_chunked \
|
||||
s3tests/functional/test_s3.py::test_object_write_read_update_read_delete \
|
||||
s3tests/functional/test_s3.py::test_object_metadata_replaced_on_put \
|
||||
s3tests/functional/test_s3.py::test_object_write_file \
|
||||
@@ -1065,6 +1187,7 @@ jobs:
|
||||
s3tests/functional/test_s3.py::test_get_object_ifmodifiedsince_good \
|
||||
s3tests/functional/test_s3.py::test_get_object_ifmodifiedsince_failed \
|
||||
s3tests/functional/test_s3.py::test_get_object_ifunmodifiedsince_failed \
|
||||
s3tests/functional/test_s3.py::test_get_checksum_object_attributes \
|
||||
s3tests/functional/test_s3.py::test_bucket_head \
|
||||
s3tests/functional/test_s3.py::test_bucket_head_notexist \
|
||||
s3tests/functional/test_s3.py::test_object_raw_authenticated \
|
||||
@@ -1129,10 +1252,7 @@ jobs:
|
||||
s3tests/functional/test_s3.py::test_lifecycle_get \
|
||||
s3tests/functional/test_s3.py::test_lifecycle_set_filter \
|
||||
s3tests/functional/test_s3.py::test_lifecycle_expiration \
|
||||
s3tests/functional/test_s3.py::test_lifecyclev2_expiration \
|
||||
s3tests/functional/test_s3.py::test_lifecycle_expiration_versioning_enabled
|
||||
kill -9 $pid || true
|
||||
# Clean up data directory
|
||||
rm -rf "$WEED_DATA_DIR" || true
|
||||
s3tests/functional/test_s3.py::test_lifecyclev2_expiration
|
||||
# cleanup() trap handles worker/server kill + data dir wipe.
|
||||
|
||||
|
||||
@@ -0,0 +1,128 @@
|
||||
name: "Samba on FUSE Integration"
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [ master, main ]
|
||||
paths:
|
||||
- 'weed/mount/**'
|
||||
- 'weed/filer/**'
|
||||
- 'weed/cluster/**'
|
||||
- 'test/samba/**'
|
||||
- '.github/workflows/samba-integration.yml'
|
||||
pull_request:
|
||||
branches: [ master, main ]
|
||||
paths:
|
||||
- 'weed/mount/**'
|
||||
- 'weed/filer/**'
|
||||
- 'weed/cluster/**'
|
||||
- 'test/samba/**'
|
||||
- '.github/workflows/samba-integration.yml'
|
||||
workflow_dispatch:
|
||||
|
||||
concurrency:
|
||||
group: samba-integration/${{ github.ref }}
|
||||
cancel-in-progress: true
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
jobs:
|
||||
samba-integration:
|
||||
name: samba-integration
|
||||
runs-on: ubuntu-22.04
|
||||
timeout-minutes: 45
|
||||
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v7
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v7
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
cache: ${{ runner.environment == 'github-hosted' }}
|
||||
|
||||
- name: Start local Docker registry
|
||||
run: |
|
||||
if ! docker start registry 2>/dev/null; then
|
||||
docker rm -f registry >/dev/null 2>&1 || true
|
||||
docker run -d --restart=always -p 5000:5000 --name registry registry:2
|
||||
fi
|
||||
|
||||
- name: Set up Docker Buildx
|
||||
uses: docker/setup-buildx-action@v4
|
||||
with:
|
||||
driver-opts: network=host
|
||||
|
||||
- name: Set up Docker Compose
|
||||
uses: docker/setup-compose-action@v2
|
||||
|
||||
- name: Build weed race binary
|
||||
run: |
|
||||
cd docker
|
||||
make binary_race
|
||||
|
||||
- name: Build SeaweedFS e2e image
|
||||
uses: docker/build-push-action@v7
|
||||
with:
|
||||
context: docker
|
||||
file: docker/Dockerfile.e2e
|
||||
tags: localhost:5000/chrislusf/seaweedfs:e2e
|
||||
push: true
|
||||
cache-from: type=gha,scope=samba-e2e
|
||||
cache-to: type=gha,mode=max,scope=samba-e2e
|
||||
|
||||
- name: Tag e2e image for docker compose
|
||||
run: |
|
||||
docker pull localhost:5000/chrislusf/seaweedfs:e2e
|
||||
docker tag localhost:5000/chrislusf/seaweedfs:e2e chrislusf/seaweedfs:e2e
|
||||
|
||||
- name: Build samba image
|
||||
uses: docker/build-push-action@v7
|
||||
with:
|
||||
context: test/samba
|
||||
build-contexts: |
|
||||
chrislusf/seaweedfs:e2e=docker-image://localhost:5000/chrislusf/seaweedfs:e2e
|
||||
tags: localhost:5000/chrislusf/seaweedfs:samba
|
||||
push: true
|
||||
cache-from: type=gha,scope=samba-harness
|
||||
cache-to: type=gha,mode=max,scope=samba-harness
|
||||
|
||||
- name: Tag samba image for docker compose
|
||||
run: |
|
||||
docker pull localhost:5000/chrislusf/seaweedfs:samba
|
||||
docker tag localhost:5000/chrislusf/seaweedfs:samba chrislusf/seaweedfs:samba
|
||||
|
||||
- name: Start SeaweedFS cluster and Samba
|
||||
run: |
|
||||
docker compose -f test/samba/docker-compose.yml up --wait
|
||||
|
||||
- name: Run Samba test battery
|
||||
run: |
|
||||
set -o pipefail
|
||||
docker compose -f test/samba/docker-compose.yml exec -T samba \
|
||||
/run_inside_container.sh 2>&1 | tee /tmp/samba-output.log
|
||||
|
||||
- name: Collect logs
|
||||
if: always()
|
||||
run: |
|
||||
mkdir -p /tmp/samba-docker-logs
|
||||
for svc in master volume filer samba; do
|
||||
docker compose -f test/samba/docker-compose.yml logs "$svc" \
|
||||
> "/tmp/samba-docker-logs/${svc}.log" 2>&1 || true
|
||||
done
|
||||
|
||||
- name: Tear down
|
||||
if: always()
|
||||
run: |
|
||||
docker compose -f test/samba/docker-compose.yml down -v
|
||||
|
||||
- name: Upload logs
|
||||
if: always()
|
||||
uses: actions/upload-artifact@v7
|
||||
with:
|
||||
name: samba-integration-results
|
||||
path: |
|
||||
/tmp/samba-output.log
|
||||
/tmp/samba-docker-logs/
|
||||
retention-days: 7
|
||||
@@ -17,14 +17,13 @@ on:
|
||||
- '.github/workflows/sftp-tests.yml'
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.head_ref }}/sftp-tests
|
||||
cancel-in-progress: true
|
||||
group: ${{ github.workflow }}-${{ github.event.pull_request.number || (github.event_name == 'push' && github.ref) || github.run_id }}
|
||||
cancel-in-progress: ${{ github.event_name == 'pull_request' || (github.event_name == 'push' && !startsWith(github.ref, 'refs/tags/')) }}
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
env:
|
||||
GO_VERSION: '1.24'
|
||||
TEST_TIMEOUT: '15m'
|
||||
|
||||
jobs:
|
||||
@@ -35,12 +34,13 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v6
|
||||
uses: actions/checkout@v7
|
||||
|
||||
- name: Set up Go ${{ env.GO_VERSION }}
|
||||
uses: actions/setup-go@v6
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v7
|
||||
with:
|
||||
go-version: ${{ env.GO_VERSION }}
|
||||
go-version-file: 'go.mod'
|
||||
cache: ${{ runner.environment == 'github-hosted' }}
|
||||
|
||||
- name: Install dependencies
|
||||
run: |
|
||||
|
||||
@@ -15,6 +15,10 @@ on:
|
||||
- '.github/workflows/spark-integration-tests.yml'
|
||||
workflow_dispatch:
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.event.pull_request.number || (github.event_name == 'push' && github.ref) || github.run_id }}
|
||||
cancel-in-progress: ${{ github.event_name == 'pull_request' || (github.event_name == 'push' && !startsWith(github.ref, 'refs/tags/')) }}
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
checks: write
|
||||
@@ -31,19 +35,20 @@ jobs:
|
||||
# SETUP & BUILD
|
||||
# ========================================
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v6
|
||||
uses: actions/checkout@v7
|
||||
|
||||
- name: Set up JDK 11
|
||||
uses: actions/setup-java@v5
|
||||
uses: actions/setup-java@v6
|
||||
with:
|
||||
java-version: '11'
|
||||
distribution: 'temurin'
|
||||
cache: maven
|
||||
cache: ${{ runner.environment == 'github-hosted' && 'maven' || '' }}
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v6
|
||||
uses: actions/setup-go@v7
|
||||
with:
|
||||
go-version: '1.24'
|
||||
go-version-file: 'go.mod'
|
||||
cache: ${{ runner.environment == 'github-hosted' }}
|
||||
|
||||
- name: Build SeaweedFS binary
|
||||
run: |
|
||||
@@ -125,7 +130,7 @@ jobs:
|
||||
|
||||
- name: Upload test results
|
||||
if: always()
|
||||
uses: actions/upload-artifact@v6
|
||||
uses: actions/upload-artifact@v7
|
||||
with:
|
||||
name: spark-test-results
|
||||
path: test/java/spark/target/surefire-reports/
|
||||
@@ -133,7 +138,7 @@ jobs:
|
||||
|
||||
- name: Publish test report
|
||||
if: always()
|
||||
uses: dorny/test-reporter@v2
|
||||
uses: dorny/test-reporter@v3
|
||||
with:
|
||||
name: Spark Test Results
|
||||
path: test/java/spark/target/surefire-reports/*.xml
|
||||
@@ -150,7 +155,7 @@ jobs:
|
||||
- name: Cache Apache Spark
|
||||
if: false && (github.event_name == 'push' || github.event_name == 'workflow_dispatch')
|
||||
id: cache-spark
|
||||
uses: actions/cache@v5
|
||||
uses: actions/cache@v6
|
||||
with:
|
||||
path: spark-3.5.0-bin-hadoop3
|
||||
key: spark-3.5.0-hadoop3
|
||||
|
||||
@@ -0,0 +1,73 @@
|
||||
---
|
||||
name: Star History
|
||||
|
||||
on:
|
||||
schedule:
|
||||
- cron: "0 0 * * *" # daily, 00:00 UTC
|
||||
workflow_dispatch:
|
||||
|
||||
permissions:
|
||||
contents: write
|
||||
|
||||
concurrency:
|
||||
# Only one chart regeneration per branch at a time so overlapping runs never
|
||||
# conflict on note/star_history.svg during rebase. Scoped by ref so a manual
|
||||
# run on another branch can't cancel the daily master update.
|
||||
group: star-history-${{ github.ref }}
|
||||
cancel-in-progress: ${{ github.event_name == 'pull_request' || (github.event_name == 'push' && !startsWith(github.ref, 'refs/tags/')) }}
|
||||
|
||||
jobs:
|
||||
render:
|
||||
name: Regenerate star history chart
|
||||
runs-on: ubuntu-latest
|
||||
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v7
|
||||
with:
|
||||
# Full history so the chart commit can rebase onto a moved master.
|
||||
fetch-depth: 0
|
||||
|
||||
- name: Set up Python
|
||||
uses: actions/setup-python@v7
|
||||
with:
|
||||
python-version: "3.x"
|
||||
cache: ${{ runner.environment == 'github-hosted' && 'pip' || '' }}
|
||||
cache-dependency-path: .github/scripts/star_history.py
|
||||
|
||||
- name: Install matplotlib
|
||||
run: pip install matplotlib
|
||||
|
||||
- name: Render chart
|
||||
env:
|
||||
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
run: python .github/scripts/star_history.py
|
||||
|
||||
- name: Commit if changed
|
||||
run: |
|
||||
git config user.name "github-actions[bot]"
|
||||
git config user.email "41898282+github-actions[bot]@users.noreply.github.com"
|
||||
if git diff --quiet -- note/star_history.svg; then
|
||||
echo "No changes to the chart."
|
||||
exit 0
|
||||
fi
|
||||
git add note/star_history.svg
|
||||
git commit -m "docs: regenerate star history chart"
|
||||
# Rebase and retry so a concurrent push to master doesn't lose the chart.
|
||||
for attempt in 1 2 3 4 5; do
|
||||
if [ "$attempt" -gt 1 ]; then
|
||||
# Guard the rebase: a transient fetch error or conflict must not
|
||||
# abort the fail-fast shell before the remaining attempts run.
|
||||
if ! git pull --rebase origin "$GITHUB_REF_NAME"; then
|
||||
echo "rebase failed (attempt ${attempt}); aborting and retrying"
|
||||
git rebase --abort || true
|
||||
continue
|
||||
fi
|
||||
fi
|
||||
if git push origin HEAD:"$GITHUB_REF_NAME"; then
|
||||
exit 0
|
||||
fi
|
||||
echo "push rejected (attempt ${attempt}); will rebase and retry"
|
||||
done
|
||||
echo "::error::could not push star history chart after retries"
|
||||
exit 1
|
||||
@@ -0,0 +1,51 @@
|
||||
name: Telemetry Integration Tests
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [ master ]
|
||||
paths:
|
||||
- 'telemetry/**'
|
||||
- 'weed/telemetry/**'
|
||||
- '.github/workflows/telemetry-integration.yml'
|
||||
pull_request:
|
||||
branches: [ master ]
|
||||
paths:
|
||||
- 'telemetry/**'
|
||||
- 'weed/telemetry/**'
|
||||
- '.github/workflows/telemetry-integration.yml'
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.event.pull_request.number || (github.event_name == 'push' && github.ref) || github.run_id }}
|
||||
cancel-in-progress: ${{ github.event_name == 'pull_request' || (github.event_name == 'push' && !startsWith(github.ref, 'refs/tags/')) }}
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
jobs:
|
||||
telemetry-integration-test:
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 5
|
||||
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v7
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v7
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
cache: ${{ runner.environment == 'github-hosted' }}
|
||||
|
||||
- name: Build telemetry server
|
||||
run: cd telemetry/server && go build -o telemetry-server .
|
||||
|
||||
- name: Run telemetry integration test
|
||||
run: go run telemetry/test/integration.go
|
||||
|
||||
- name: Upload test logs on failure
|
||||
if: failure()
|
||||
uses: actions/upload-artifact@v7
|
||||
with:
|
||||
name: telemetry-test-logs
|
||||
path: telemetry-server-test.log
|
||||
retention-days: 7
|
||||
@@ -0,0 +1,85 @@
|
||||
name: "terraform: validate and test modules"
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [ master ]
|
||||
paths: ['terraform/**', '.github/workflows/terraform_ci.yml']
|
||||
pull_request:
|
||||
branches: [ master ]
|
||||
paths: ['terraform/**', '.github/workflows/terraform_ci.yml']
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.event.pull_request.number || (github.event_name == 'push' && github.ref) || github.run_id }}
|
||||
cancel-in-progress: ${{ github.event_name == 'pull_request' || (github.event_name == 'push' && !startsWith(github.ref, 'refs/tags/')) }}
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
jobs:
|
||||
validate:
|
||||
name: fmt, validate, plan-level tests
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v7
|
||||
|
||||
- name: Set up OpenTofu
|
||||
uses: opentofu/setup-opentofu@v2
|
||||
with:
|
||||
tofu_version: 1.12.1
|
||||
|
||||
- name: fmt check
|
||||
working-directory: terraform
|
||||
run: tofu fmt -recursive -check -diff
|
||||
|
||||
- name: validate core
|
||||
working-directory: terraform/modules/core
|
||||
run: |
|
||||
tofu init -backend=false -input=false
|
||||
tofu validate
|
||||
|
||||
- name: validate security
|
||||
working-directory: terraform/modules/security
|
||||
run: |
|
||||
tofu init -backend=false -input=false
|
||||
tofu validate
|
||||
|
||||
- name: plan-level tests (core)
|
||||
working-directory: terraform/modules/core
|
||||
run: tofu test
|
||||
|
||||
- name: validate examples
|
||||
run: |
|
||||
set -e
|
||||
for ex in terraform/examples/*/; do
|
||||
echo "== validate $ex =="
|
||||
tofu -chdir="$ex" init -backend=false -input=false
|
||||
tofu -chdir="$ex" validate
|
||||
done
|
||||
|
||||
smoke:
|
||||
name: local cluster smoke test (real weed)
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v7
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v7
|
||||
with:
|
||||
go-version-file: go.mod
|
||||
cache: ${{ runner.environment == 'github-hosted' }}
|
||||
|
||||
- name: Build weed
|
||||
run: go build -o "$RUNNER_TEMP/weed" ./weed
|
||||
|
||||
- name: Set up OpenTofu
|
||||
uses: opentofu/setup-opentofu@v2
|
||||
with:
|
||||
tofu_version: 1.12.1
|
||||
|
||||
- name: Run local cluster harness
|
||||
working-directory: terraform/test/local
|
||||
run: WEED="$RUNNER_TEMP/weed" ./run_local_cluster.sh
|
||||
|
||||
- name: Run local mTLS cluster harness
|
||||
working-directory: terraform/test/local-secure
|
||||
run: WEED="$RUNNER_TEMP/weed" ./run_local_secure.sh
|
||||
@@ -3,8 +3,20 @@ name: "test s3 over https using aws-cli"
|
||||
on:
|
||||
push:
|
||||
branches: [master, test-https-s3-awscli]
|
||||
paths:
|
||||
- 'weed/s3api/**'
|
||||
- 'weed/server/**'
|
||||
- 'go.mod'
|
||||
- 'go.sum'
|
||||
- '.github/workflows/test-s3-over-https-using-awscli.yml'
|
||||
pull_request:
|
||||
branches: [master, test-https-s3-awscli]
|
||||
paths:
|
||||
- 'weed/s3api/**'
|
||||
- 'weed/server/**'
|
||||
- 'go.mod'
|
||||
- 'go.sum'
|
||||
- '.github/workflows/test-s3-over-https-using-awscli.yml'
|
||||
|
||||
env:
|
||||
AWS_ACCESS_KEY_ID: some_access_key1
|
||||
@@ -15,16 +27,21 @@ defaults:
|
||||
run:
|
||||
working-directory: weed
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.event.pull_request.number || (github.event_name == 'push' && github.ref) || github.run_id }}
|
||||
cancel-in-progress: ${{ github.event_name == 'pull_request' || (github.event_name == 'push' && !startsWith(github.ref, 'refs/tags/')) }}
|
||||
|
||||
jobs:
|
||||
awscli-tests:
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 10
|
||||
steps:
|
||||
- uses: actions/checkout@v6
|
||||
- uses: actions/checkout@v7
|
||||
|
||||
- uses: actions/setup-go@v6
|
||||
- uses: actions/setup-go@v7
|
||||
with:
|
||||
go-version: ^1.24
|
||||
go-version: ^1.26
|
||||
cache: ${{ runner.environment == 'github-hosted' }}
|
||||
|
||||
- name: Build SeaweedFS
|
||||
run: |
|
||||
@@ -35,7 +52,7 @@ jobs:
|
||||
set -e
|
||||
mkdir -p /tmp/data
|
||||
./weed -v=3 server -s3 -dir=/tmp/data -s3.config=../docker/compose/s3.json -master.peers=none > weed.log 2>&1 &
|
||||
until curl -s http://localhost:8333/ > /dev/null; do sleep 1; done
|
||||
until curl -s http://localhost:8333/healthz > /dev/null; do sleep 1; done
|
||||
|
||||
- name: Setup Caddy
|
||||
run: |
|
||||
@@ -54,7 +71,7 @@ jobs:
|
||||
- name: Start Caddy
|
||||
run: |
|
||||
./caddy start
|
||||
until curl -fsS --insecure https://localhost:8443 > /dev/null; do sleep 1; done
|
||||
until curl -fsS --insecure https://localhost:8443/healthz > /dev/null; do sleep 1; done
|
||||
|
||||
- name: Create Bucket
|
||||
run: |
|
||||
@@ -74,7 +91,11 @@ jobs:
|
||||
set -e
|
||||
dd if=/dev/urandom of=generated bs=1M count=32
|
||||
aws --no-verify-ssl s3 cp --no-progress generated s3://bucket/test-multipart
|
||||
aws --no-verify-ssl s3 cp --no-progress s3://bucket/test-multipart downloaded
|
||||
for i in 1 2 3 4 5; do
|
||||
aws --no-verify-ssl s3 cp --no-progress s3://bucket/test-multipart downloaded && break
|
||||
if [ "$i" -eq 5 ]; then exit 1; fi
|
||||
sleep 2
|
||||
done
|
||||
diff -q generated downloaded
|
||||
rm -f generated downloaded
|
||||
|
||||
@@ -103,7 +124,7 @@ jobs:
|
||||
|
||||
- name: Upload server logs on failure
|
||||
if: failure()
|
||||
uses: actions/upload-artifact@v6
|
||||
uses: actions/upload-artifact@v7
|
||||
with:
|
||||
name: seaweedfs-logs
|
||||
# Note: actions don't use defaults.run.working-directory, so path is relative to workspace root
|
||||
|
||||
@@ -0,0 +1,73 @@
|
||||
name: "TLS Rotation Integration Tests"
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [ master ]
|
||||
paths:
|
||||
- 'weed/**'
|
||||
- 'test/tls_rotation/**'
|
||||
- 'go.mod'
|
||||
- 'go.sum'
|
||||
- '.github/workflows/tls-rotation-tests.yml'
|
||||
pull_request:
|
||||
branches: [ master ]
|
||||
paths:
|
||||
- 'weed/**'
|
||||
- 'test/tls_rotation/**'
|
||||
- 'go.mod'
|
||||
- 'go.sum'
|
||||
- '.github/workflows/tls-rotation-tests.yml'
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.event.pull_request.number || (github.event_name == 'push' && github.ref) || github.run_id }}
|
||||
cancel-in-progress: ${{ github.event_name == 'pull_request' || (github.event_name == 'push' && !startsWith(github.ref, 'refs/tags/')) }}
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
jobs:
|
||||
tls-rotation-tests:
|
||||
name: TLS Rotation Integration Tests
|
||||
runs-on: ubuntu-22.04
|
||||
timeout-minutes: 10
|
||||
steps:
|
||||
- name: Set up Go 1.x
|
||||
uses: actions/setup-go@v7
|
||||
with:
|
||||
cache: ${{ runner.environment == 'github-hosted' }}
|
||||
go-version: ^1.26
|
||||
id: go
|
||||
|
||||
- name: Check out code into the Go module directory
|
||||
uses: actions/checkout@v7
|
||||
|
||||
- name: Build weed binary
|
||||
run: |
|
||||
cd weed && go build -o weed .
|
||||
|
||||
- name: Run TLS Rotation Integration Tests
|
||||
working-directory: test/tls_rotation
|
||||
run: |
|
||||
go test -v -count=1 -timeout 5m
|
||||
|
||||
- name: Collect server logs on failure
|
||||
if: failure()
|
||||
run: |
|
||||
echo "Collecting master logs from temp directories..."
|
||||
mkdir -p /tmp/tls-rotation-test-logs
|
||||
find /tmp -maxdepth 1 -type d -name "TestMasterHTTPS*" 2>/dev/null | while read dir; do
|
||||
if [ -d "$dir" ]; then
|
||||
echo "Found test directory: $dir"
|
||||
cp -r "$dir" /tmp/tls-rotation-test-logs/ 2>/dev/null || true
|
||||
fi
|
||||
done
|
||||
echo "Collected logs:"
|
||||
find /tmp/tls-rotation-test-logs -type f -name "*.log" 2>/dev/null || echo "No logs found"
|
||||
|
||||
- name: Archive logs
|
||||
if: failure()
|
||||
uses: actions/upload-artifact@v7
|
||||
with:
|
||||
name: tls-rotation-test-logs
|
||||
path: /tmp/tls-rotation-test-logs/
|
||||
retention-days: 14
|
||||
@@ -3,20 +3,16 @@ name: "TUS Protocol Tests"
|
||||
on:
|
||||
pull_request:
|
||||
paths:
|
||||
- 'weed/server/filer_server_tus*.go'
|
||||
- 'weed/server/filer_server.go'
|
||||
- 'weed/server/**'
|
||||
- 'weed/filer/**'
|
||||
- 'test/tus/**'
|
||||
- 'go.mod'
|
||||
- 'go.sum'
|
||||
- '.github/workflows/tus-tests.yml'
|
||||
push:
|
||||
branches: [ master, main ]
|
||||
paths:
|
||||
- 'weed/server/filer_server_tus*.go'
|
||||
- 'weed/server/filer_server.go'
|
||||
- 'test/tus/**'
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.head_ref || github.ref }}/tus-tests
|
||||
cancel-in-progress: true
|
||||
group: ${{ github.workflow }}-${{ github.event.pull_request.number || (github.event_name == 'push' && github.ref) || github.run_id }}
|
||||
cancel-in-progress: ${{ github.event_name == 'pull_request' || (github.event_name == 'push' && !startsWith(github.ref, 'refs/tags/')) }}
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
@@ -33,12 +29,16 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Check out code
|
||||
uses: actions/checkout@v6
|
||||
uses: actions/checkout@v7
|
||||
|
||||
- name: Clean up stale FUSE mounts
|
||||
uses: ./.github/actions/cleanup-stale-fuse-mounts
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v6
|
||||
uses: actions/setup-go@v7
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
cache: ${{ runner.environment == 'github-hosted' }}
|
||||
id: go
|
||||
|
||||
- name: Install SeaweedFS
|
||||
@@ -53,7 +53,7 @@ jobs:
|
||||
echo "=== System Information ==="
|
||||
uname -a
|
||||
free -h
|
||||
df -h
|
||||
timeout 15 df -h || true
|
||||
echo "=== Starting TUS Tests ==="
|
||||
|
||||
# Run tests with automatic server management
|
||||
@@ -106,7 +106,7 @@ jobs:
|
||||
|
||||
- name: Upload test logs on failure
|
||||
if: failure()
|
||||
uses: actions/upload-artifact@v6
|
||||
uses: actions/upload-artifact@v7
|
||||
with:
|
||||
name: tus-test-logs
|
||||
path: |
|
||||
|
||||
@@ -0,0 +1,73 @@
|
||||
name: "Vacuum Integration Tests"
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [ master ]
|
||||
paths:
|
||||
- 'weed/**'
|
||||
- 'test/vacuum/**'
|
||||
- 'go.mod'
|
||||
- 'go.sum'
|
||||
- '.github/workflows/vacuum-integration-tests.yml'
|
||||
pull_request:
|
||||
branches: [ master ]
|
||||
paths:
|
||||
- 'weed/**'
|
||||
- 'test/vacuum/**'
|
||||
- 'go.mod'
|
||||
- 'go.sum'
|
||||
- '.github/workflows/vacuum-integration-tests.yml'
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.event.pull_request.number || (github.event_name == 'push' && github.ref) || github.run_id }}
|
||||
cancel-in-progress: ${{ github.event_name == 'pull_request' || (github.event_name == 'push' && !startsWith(github.ref, 'refs/tags/')) }}
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
jobs:
|
||||
vacuum-integration-tests:
|
||||
name: Vacuum Integration Tests
|
||||
runs-on: ubuntu-22.04
|
||||
timeout-minutes: 15
|
||||
steps:
|
||||
- name: Set up Go 1.x
|
||||
uses: actions/setup-go@v7
|
||||
with:
|
||||
cache: ${{ runner.environment == 'github-hosted' }}
|
||||
go-version: ^1.26
|
||||
id: go
|
||||
|
||||
- name: Check out code into the Go module directory
|
||||
uses: actions/checkout@v7
|
||||
|
||||
- name: Build weed binary
|
||||
run: |
|
||||
cd weed && go build -o weed .
|
||||
|
||||
- name: Run Vacuum Integration Tests
|
||||
working-directory: test/vacuum
|
||||
run: |
|
||||
go test -v -timeout 10m
|
||||
|
||||
- name: Collect server logs on failure
|
||||
if: failure()
|
||||
run: |
|
||||
echo "Collecting server logs from temp directories..."
|
||||
mkdir -p /tmp/vacuum-test-logs
|
||||
find /tmp -maxdepth 1 -type d -name "TestVacuum*" 2>/dev/null | while read dir; do
|
||||
if [ -d "$dir" ]; then
|
||||
echo "Found test directory: $dir"
|
||||
cp -r "$dir" /tmp/vacuum-test-logs/ 2>/dev/null || true
|
||||
fi
|
||||
done
|
||||
echo "Collected logs:"
|
||||
find /tmp/vacuum-test-logs -type f -name "*.log" 2>/dev/null || echo "No logs found"
|
||||
|
||||
- name: Archive logs
|
||||
if: failure()
|
||||
uses: actions/upload-artifact@v7
|
||||
with:
|
||||
name: vacuum-integration-test-logs
|
||||
path: /tmp/vacuum-test-logs/
|
||||
retention-days: 14
|
||||
@@ -23,35 +23,32 @@ on:
|
||||
- '.github/workflows/volume-server-integration-tests.yml'
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.head_ref || github.ref }}/volume-server-integration-tests
|
||||
cancel-in-progress: true
|
||||
group: ${{ github.workflow }}-${{ github.event.pull_request.number || (github.event_name == 'push' && github.ref) || github.run_id }}
|
||||
cancel-in-progress: ${{ github.event_name == 'pull_request' || (github.event_name == 'push' && !startsWith(github.ref, 'refs/tags/')) }}
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
env:
|
||||
GO_VERSION: '1.24'
|
||||
TEST_TIMEOUT: '30m'
|
||||
# Keep in step with the length of matrix.shard below.
|
||||
SHARD_COUNT: 3
|
||||
|
||||
jobs:
|
||||
volume-server-integration-tests:
|
||||
name: Volume Server Integration Tests (${{ matrix.test-type }} - Shard ${{ matrix.shard }})
|
||||
build-weed:
|
||||
name: Build SeaweedFS Binary
|
||||
runs-on: ubuntu-22.04
|
||||
timeout-minutes: 45
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
test-type: [grpc, http]
|
||||
shard: [1, 2, 3]
|
||||
timeout-minutes: 15
|
||||
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v6
|
||||
uses: actions/checkout@v7
|
||||
|
||||
- name: Set up Go ${{ env.GO_VERSION }}
|
||||
uses: actions/setup-go@v6
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v7
|
||||
with:
|
||||
go-version: ${{ env.GO_VERSION }}
|
||||
go-version-file: 'go.mod'
|
||||
cache: ${{ runner.environment == 'github-hosted' }}
|
||||
|
||||
- name: Build SeaweedFS binary
|
||||
run: |
|
||||
@@ -60,28 +57,69 @@ jobs:
|
||||
chmod +x weed
|
||||
./weed version
|
||||
|
||||
- name: Package binary
|
||||
run: tar cf weed-binary.tar weed/weed
|
||||
|
||||
- name: Upload binary
|
||||
uses: actions/upload-artifact@v7
|
||||
with:
|
||||
name: weed-binary
|
||||
if-no-files-found: error
|
||||
retention-days: 1
|
||||
path: weed-binary.tar
|
||||
|
||||
volume-server-integration-tests:
|
||||
name: Volume Server Integration Tests (${{ matrix.test-type }} - Shard ${{ matrix.shard }})
|
||||
runs-on: ubuntu-22.04
|
||||
timeout-minutes: 45
|
||||
needs: [build-weed]
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
test-type: [grpc, http]
|
||||
shard: [1, 2, 3]
|
||||
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v7
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v7
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
cache: ${{ runner.environment == 'github-hosted' }}
|
||||
|
||||
- name: Download binary
|
||||
uses: actions/download-artifact@v7
|
||||
with:
|
||||
name: weed-binary
|
||||
|
||||
- name: Restore binary
|
||||
run: |
|
||||
tar xf weed-binary.tar
|
||||
chmod +x weed/weed
|
||||
./weed/weed version
|
||||
|
||||
# Dealing the listed tests out one by one keeps the shards even. Bucketing
|
||||
# them by first letter did not: names cluster, so ^Test[I-S] drew 50 of
|
||||
# the 114 grpc tests and ran nearly twice as long as the other two shards.
|
||||
- name: Select this shard's tests
|
||||
env:
|
||||
TEST_TYPE: ${{ matrix.test-type }}
|
||||
SHARD: ${{ matrix.shard }}
|
||||
run: |
|
||||
tests=$(go test ./test/volume_server/"$TEST_TYPE"/... -list '.*' | grep '^Test' | sort -u)
|
||||
# An empty list would make -run match nothing and the shard pass vacuously.
|
||||
[ -n "$tests" ] || { echo "listed no tests in test/volume_server/$TEST_TYPE"; exit 1; }
|
||||
selected=$(echo "$tests" | awk -v n="$SHARD_COUNT" -v i="$SHARD" 'NR % n == i - 1')
|
||||
echo "shard $SHARD of $SHARD_COUNT runs $(echo "$selected" | wc -l) of $(echo "$tests" | wc -l) tests"
|
||||
echo "TEST_PATTERN=^($(echo "$selected" | paste -sd'|' -))\$" >> "$GITHUB_ENV"
|
||||
|
||||
- name: Run volume server integration tests
|
||||
env:
|
||||
WEED_BINARY: ${{ github.workspace }}/weed/weed
|
||||
run: |
|
||||
if [ "${{ matrix.test-type }}" == "grpc" ]; then
|
||||
if [ "${{ matrix.shard }}" == "1" ]; then
|
||||
TEST_PATTERN="^Test[A-H]"
|
||||
elif [ "${{ matrix.shard }}" == "2" ]; then
|
||||
TEST_PATTERN="^Test[I-S]"
|
||||
else
|
||||
TEST_PATTERN="^Test[T-Z]"
|
||||
fi
|
||||
else
|
||||
if [ "${{ matrix.shard }}" == "1" ]; then
|
||||
TEST_PATTERN="^Test[A-G]"
|
||||
elif [ "${{ matrix.shard }}" == "2" ]; then
|
||||
TEST_PATTERN="^Test[H-R]"
|
||||
else
|
||||
TEST_PATTERN="^Test[S-Z]"
|
||||
fi
|
||||
fi
|
||||
echo "Running volume server integration tests for ${{ matrix.test-type }} (Shard ${{ matrix.shard }}, pattern: ${TEST_PATTERN})..."
|
||||
echo "Running volume server integration tests for ${{ matrix.test-type }} (Shard ${{ matrix.shard }} of ${SHARD_COUNT})..."
|
||||
go test -v -count=1 -timeout=${{ env.TEST_TIMEOUT }} ./test/volume_server/${{ matrix.test-type }}/... -run "${TEST_PATTERN}"
|
||||
|
||||
- name: Collect logs on failure
|
||||
@@ -92,7 +130,7 @@ jobs:
|
||||
|
||||
- name: Archive logs on failure
|
||||
if: failure()
|
||||
uses: actions/upload-artifact@v6
|
||||
uses: actions/upload-artifact@v7
|
||||
with:
|
||||
name: volume-server-integration-test-logs
|
||||
path: /tmp/volume-server-it-logs/
|
||||
@@ -102,26 +140,9 @@ jobs:
|
||||
- name: Test summary
|
||||
if: always()
|
||||
run: |
|
||||
if [ "${{ matrix.test-type }}" == "grpc" ]; then
|
||||
if [ "${{ matrix.shard }}" == "1" ]; then
|
||||
TEST_PATTERN="^Test[A-H]"
|
||||
elif [ "${{ matrix.shard }}" == "2" ]; then
|
||||
TEST_PATTERN="^Test[I-S]"
|
||||
else
|
||||
TEST_PATTERN="^Test[T-Z]"
|
||||
fi
|
||||
else
|
||||
if [ "${{ matrix.shard }}" == "1" ]; then
|
||||
TEST_PATTERN="^Test[A-G]"
|
||||
elif [ "${{ matrix.shard }}" == "2" ]; then
|
||||
TEST_PATTERN="^Test[H-R]"
|
||||
else
|
||||
TEST_PATTERN="^Test[S-Z]"
|
||||
fi
|
||||
fi
|
||||
echo "## Volume Server Integration Test Summary (${{ matrix.test-type }} - Shard ${{ matrix.shard }})" >> "$GITHUB_STEP_SUMMARY"
|
||||
echo "- Suite: test/volume_server/${{ matrix.test-type }} (Pattern: ${TEST_PATTERN})" >> "$GITHUB_STEP_SUMMARY"
|
||||
echo "- Command: go test -v -count=1 -timeout=${{ env.TEST_TIMEOUT }} ./test/volume_server/${{ matrix.test-type }}/... -run \"${TEST_PATTERN}\"" >> "$GITHUB_STEP_SUMMARY"
|
||||
echo "- Suite: test/volume_server/${{ matrix.test-type }} (shard ${{ matrix.shard }} of ${SHARD_COUNT}, see 'Select this shard's tests' for the split)" >> "$GITHUB_STEP_SUMMARY"
|
||||
echo "- Command: go test -v -count=1 -timeout=${{ env.TEST_TIMEOUT }} ./test/volume_server/${{ matrix.test-type }}/... -run \"\${TEST_PATTERN}\"" >> "$GITHUB_STEP_SUMMARY"
|
||||
|
||||
volume-server-rust-smoke:
|
||||
name: Volume Server Rust Smoke (${{ matrix.mode }})
|
||||
@@ -151,10 +172,16 @@ jobs:
|
||||
chmod +x weed
|
||||
./weed version
|
||||
|
||||
- name: Build Rust volume launcher
|
||||
run: |
|
||||
cd rust/volume_server
|
||||
cargo build --release
|
||||
|
||||
- name: Run Rust-mode HTTP smoke tests
|
||||
env:
|
||||
WEED_BINARY: ${{ github.workspace }}/weed/weed
|
||||
VOLUME_SERVER_IMPL: rust
|
||||
RUST_VOLUME_BINARY: ${{ github.workspace }}/rust/volume_server/target/release/weed-volume-rs
|
||||
VOLUME_SERVER_RUST_MODE: ${{ matrix.mode }}
|
||||
run: |
|
||||
go test -v -count=1 -timeout=25m ./test/volume_server/http/... -run "TestAdminStatusAndHealthz|TestUploadReadRangeHeadDeleteRoundTrip"
|
||||
@@ -163,6 +190,7 @@ jobs:
|
||||
env:
|
||||
WEED_BINARY: ${{ github.workspace }}/weed/weed
|
||||
VOLUME_SERVER_IMPL: rust
|
||||
RUST_VOLUME_BINARY: ${{ github.workspace }}/rust/volume_server/target/release/weed-volume-rs
|
||||
VOLUME_SERVER_RUST_MODE: ${{ matrix.mode }}
|
||||
run: |
|
||||
go test -v -count=1 -timeout=25m ./test/volume_server/grpc/... -run "TestStateAndStatusRPCs|TestVolumeSyncStatusAndReadVolumeFileStatus"
|
||||
@@ -175,7 +203,7 @@ jobs:
|
||||
|
||||
- name: Archive logs on failure
|
||||
if: failure()
|
||||
uses: actions/upload-artifact@v6
|
||||
uses: actions/upload-artifact@v7
|
||||
with:
|
||||
name: volume-server-rust-smoke-logs-${{ matrix.mode }}
|
||||
path: /tmp/volume-server-rust-smoke-logs/
|
||||
|
||||
Reference in new issue
Block a user