diff --git a/.github/actions/cleanup-stale-fuse-mounts/action.yml b/.github/actions/cleanup-stale-fuse-mounts/action.yml new file mode 100644 index 0000000000..7e6ddab7bd --- /dev/null +++ b/.github/actions/cleanup-stale-fuse-mounts/action.yml @@ -0,0 +1,11 @@ +name: Clean up stale SeaweedFS test mounts +description: Detach disconnected FUSE test mounts left by earlier jobs on a shared runner. +runs: + using: composite + steps: + - name: Unmount disconnected SeaweedFS test mounts + if: runner.os == 'Linux' && runner.environment == 'self-hosted' + shell: bash + env: + CLEANUP_SCRIPT: ${{ github.action_path }}/cleanup.py + run: python3 "$CLEANUP_SCRIPT" diff --git a/.github/actions/cleanup-stale-fuse-mounts/cleanup.py b/.github/actions/cleanup-stale-fuse-mounts/cleanup.py new file mode 100644 index 0000000000..ad17a767fa --- /dev/null +++ b/.github/actions/cleanup-stale-fuse-mounts/cleanup.py @@ -0,0 +1,89 @@ +#!/usr/bin/env python3 +"""Detach disconnected SeaweedFS integration-test mounts without touching live ones.""" + +import errno +import os +from pathlib import Path +import re +import shutil +import subprocess +import sys +import tempfile + + +def test_mounts(mountinfo): + """Read the kernel table directly: traversing a stale mount may block.""" + roots = {tempfile.gettempdir(), "/tmp", "/var/tmp"} + for line in mountinfo.splitlines(): + fields, filesystem = line.split(" - ", 1) + if filesystem.split()[0].split(".")[0] != "fuse": + continue + fields = fields.split() + # mountinfo escapes whitespace and backslashes with octal sequences. + target = re.sub(r"\\([0-7]{3})", lambda m: chr(int(m[1], 8)), fields[4]) + path = Path(target) + # Test mounts are a mount* leaf under a seaweedfs_fuse_* scratch dir + # (seaweedfs_fuse_test_*/mount, seaweedfs_fuse_failover_*/mount0, ...) + # or the posix-compliance workflow's fixed /tmp/seaweedfs/mount. + if (not path.name.startswith("mount") + or str(path.parent.parent) not in roots + or not (path.parent.name.startswith("seaweedfs_fuse") + or path.parent.name == "seaweedfs")): + continue + yield fields[0], target + + +def mounted(mount_id, target): + return (mount_id, target) in test_mounts(Path("/proc/self/mountinfo").read_text()) + + +def cleanup(): + failed = False + for mount_id, target in test_mounts(Path("/proc/self/mountinfo").read_text()): + try: + # A separate, bounded probe prevents a wedged FUSE daemon from + # hanging cleanup. Only ENOTCONN proves the daemon disconnected; + # a timeout or another error is not enough to detach a live mount. + probe = subprocess.run( + [sys.executable, __file__, "--probe", target], timeout=5, + capture_output=True, check=False, + ) + except subprocess.TimeoutExpired: + print(f"::warning::FUSE probe timed out; leaving {target} mounted") + continue + if probe.returncode != errno.ENOTCONN or not mounted(mount_id, target): + continue + + print(f"Unmounting disconnected FUSE test mount: {target}", flush=True) + # Lazy unmount detaches a dead mount even when an old process still + # holds it open. Prefer the unprivileged FUSE helper; sudo is fallback. + for command in ( + ["fusermount3", "-uz", "--", target], + ["fusermount", "-uz", "--", target], + ["sudo", "-n", "umount", "-l", "--", target], + ): + try: + subprocess.run(command, timeout=10, check=False) + except (OSError, subprocess.TimeoutExpired) as error: + print(f"{command[0]}: {error}", file=sys.stderr) + if not mounted(mount_id, target): + break + else: + print(f"::error::Could not unmount disconnected FUSE test mount: {target}") + failed = True + continue + # The detached mount's parent still holds the crashed run's data and + # logs; test_mounts only yields targets under a SeaweedFS test dir in + # a temp root, so the parent is safe to reclaim here. + shutil.rmtree(Path(target).parent, ignore_errors=True) + return int(failed) + + +if __name__ == "__main__": + if len(sys.argv) == 3 and sys.argv[1] == "--probe": + try: + os.stat(sys.argv[2]) + except OSError as error: + sys.exit(error.errno) + sys.exit(0) + sys.exit(cleanup()) diff --git a/.github/actions/cleanup-stale-fuse-mounts/test_cleanup.py b/.github/actions/cleanup-stale-fuse-mounts/test_cleanup.py new file mode 100644 index 0000000000..b8ee3fa028 --- /dev/null +++ b/.github/actions/cleanup-stale-fuse-mounts/test_cleanup.py @@ -0,0 +1,137 @@ +import errno +import importlib.util +from pathlib import Path +import subprocess +import unittest +from unittest.mock import patch + +spec = importlib.util.spec_from_file_location("cleanup", Path(__file__).with_name("cleanup.py")) +cleanup = importlib.util.module_from_spec(spec) +spec.loader.exec_module(cleanup) + +TARGET = "/tmp/seaweedfs_fuse_test_123/mount" +MOUNTINFO = f"42 1 0:9 / {TARGET} rw - fuse.seaweedfs seaweedfs rw\n" + + +class CleanupTests(unittest.TestCase): + def run_cleanup(self, results, mounted=True): + with patch.object(cleanup.Path, "read_text", return_value=MOUNTINFO), \ + patch.object(cleanup, "mounted", side_effect=mounted if isinstance(mounted, list) else None, + return_value=mounted) as check, \ + patch.object(cleanup.shutil, "rmtree") as rmtree, \ + patch.object(cleanup.subprocess, "run", side_effect=results) as run: + status = cleanup.cleanup() + return status, run, check, rmtree + + def result(self, status): + return subprocess.CompletedProcess([], status) + + def test_only_fuse_integration_test_mounts_are_candidates(self): + rows = MOUNTINFO + rows += MOUNTINFO.replace("fuse.seaweedfs", "ext4") + rows += MOUNTINFO.replace(TARGET, "/mnt/production") + rows += MOUNTINFO.replace(TARGET, "/tmp/seaweedfs_fuse_test_123/data") + rows += MOUNTINFO.replace(TARGET, "/home/user/seaweedfs_fuse_test_123/mount") + rows += MOUNTINFO.replace(TARGET, "/tmp/seaweedfs/data") + self.assertEqual(list(cleanup.test_mounts(rows)), [("42", TARGET)]) + + def test_all_test_mount_layouts_are_candidates(self): + targets = ( + "/tmp/seaweedfs_fuse_test_123/mount", + "/tmp/seaweedfs_fuse_failover_456/mount0", + "/tmp/seaweedfs_fuse_failover_456/mount1", + "/var/tmp/seaweedfs_fuse_p2p_test_789/mount2", + "/tmp/seaweedfs/mount", + ) + rows = "".join( + MOUNTINFO.replace("42", str(40 + i)).replace(TARGET, target) + for i, target in enumerate(targets) + ) + self.assertEqual( + list(cleanup.test_mounts(rows)), + [(str(40 + i), target) for i, target in enumerate(targets)], + ) + + def test_mountinfo_paths_are_unescaped_once(self): + target = "/tmp/seaweedfs_fuse_test_space and\\slash/mount" + encoded = target.replace("\\", r"\134").replace(" ", r"\040") + rows = MOUNTINFO.replace(TARGET, encoded) + self.assertEqual(list(cleanup.test_mounts(rows)), [("42", target)]) + + def test_no_mounts_is_a_noop(self): + with patch.object(cleanup.Path, "read_text", return_value=""), \ + patch.object(cleanup.subprocess, "run") as run: + self.assertEqual(cleanup.cleanup(), 0) + run.assert_not_called() + + def test_healthy_mount_is_untouched(self): + status, run, _, _ = self.run_cleanup([self.result(0)]) + self.assertEqual(status, 0) + self.assertEqual(run.call_count, 1) + + def test_other_probe_errors_are_untouched(self): + for error in (errno.EACCES, errno.ENOENT, errno.EIO): + with self.subTest(error=error): + status, run, _, _ = self.run_cleanup([self.result(error)]) + self.assertEqual(status, 0) + self.assertEqual(run.call_count, 1) + + def test_probe_timeout_does_not_detach_a_possibly_live_mount(self): + status, run, _, _ = self.run_cleanup([subprocess.TimeoutExpired("probe", 5)]) + self.assertEqual(status, 0) + self.assertEqual(run.call_count, 1) + + def test_disconnected_mount_is_lazily_unmounted(self): + status, run, _, _ = self.run_cleanup([self.result(errno.ENOTCONN), self.result(0)], [True, False]) + self.assertEqual(status, 0) + self.assertEqual(run.call_args_list[1].args[0], ["fusermount3", "-uz", "--", TARGET]) + self.assertEqual(run.call_args_list[0].kwargs["timeout"], 5) + self.assertEqual(run.call_args_list[1].kwargs["timeout"], 10) + + def test_replaced_or_removed_mount_is_untouched(self): + status, run, _, _ = self.run_cleanup([self.result(errno.ENOTCONN)], False) + self.assertEqual(status, 0) + self.assertEqual(run.call_count, 1) + + def test_mount_identity_is_checked_without_stat(self): + with patch.object(cleanup.Path, "read_text", return_value=MOUNTINFO): + self.assertTrue(cleanup.mounted("42", TARGET)) + self.assertFalse(cleanup.mounted("43", TARGET)) + + def test_missing_or_timed_out_helper_uses_fallback(self): + for error in (FileNotFoundError(), subprocess.TimeoutExpired("fusermount3", 10)): + with self.subTest(error=error): + status, run, _, _ = self.run_cleanup( + [self.result(errno.ENOTCONN), error, self.result(0)], [True, True, False]) + self.assertEqual(status, 0) + self.assertEqual(run.call_args.args[0], ["fusermount", "-uz", "--", TARGET]) + + def test_failed_helpers_fall_back_to_sudo(self): + status, run, _, _ = self.run_cleanup( + [self.result(errno.ENOTCONN), self.result(1), self.result(1), self.result(0)], + [True, True, True, False]) + self.assertEqual(status, 0) + self.assertEqual(run.call_args.args[0], ["sudo", "-n", "umount", "-l", "--", TARGET]) + + def test_cleanup_fails_when_stale_mount_remains(self): + status, run, _, rmtree = self.run_cleanup( + [self.result(errno.ENOTCONN), self.result(1), self.result(1), self.result(1)]) + self.assertEqual(status, 1) + self.assertEqual(run.call_count, 4) + rmtree.assert_not_called() + + def test_unmounted_test_dir_is_reclaimed(self): + status, _, _, rmtree = self.run_cleanup( + [self.result(errno.ENOTCONN), self.result(0)], [True, False]) + self.assertEqual(status, 0) + rmtree.assert_called_once_with(Path(TARGET).parent, ignore_errors=True) + + def test_probe_process_reports_errno(self): + for target, expected in ((str(Path(__file__)), 0), (str(Path(__file__) / "absent"), errno.ENOTDIR)): + with self.subTest(target=target): + result = subprocess.run([cleanup.sys.executable, cleanup.__file__, "--probe", target], check=False) + self.assertEqual(result.returncode, expected) + + +if __name__ == "__main__": + unittest.main() diff --git a/.github/actions/fix-fusermount-setuid/action.yml b/.github/actions/fix-fusermount-setuid/action.yml index 7aad5bf482..5b0428c713 100644 --- a/.github/actions/fix-fusermount-setuid/action.yml +++ b/.github/actions/fix-fusermount-setuid/action.yml @@ -23,8 +23,14 @@ runs: bin=$(command -v fusermount3 || true) if [ -z "$bin" ]; then - echo "no fusermount3 in PATH" >&2 - exit 1 + # Self-hosted runners are not guaranteed to carry fuse3. + sudo apt-get update + sudo apt-get install -y fuse3 + bin=$(command -v fusermount3 || true) + if [ -z "$bin" ]; then + echo "no fusermount3 in PATH after installing fuse3" >&2 + exit 1 + fi fi if can_mount "$bin"; then ls -l "$bin" diff --git a/.github/workflows/action-tests.yml b/.github/workflows/action-tests.yml new file mode 100644 index 0000000000..88f33ed93c --- /dev/null +++ b/.github/workflows/action-tests.yml @@ -0,0 +1,30 @@ +name: "actions: unit tests" + +on: + push: + branches: [ master ] + paths: + - '.github/actions/**' + - '.github/workflows/action-tests.yml' + pull_request: + branches: [ master ] + paths: + - '.github/actions/**' + - '.github/workflows/action-tests.yml' + +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || (github.event_name == 'push' && github.ref) || github.run_id }} + cancel-in-progress: ${{ github.event_name == 'pull_request' || (github.event_name == 'push' && !startsWith(github.ref, 'refs/tags/')) }} + +permissions: + contents: read + +jobs: + cleanup-stale-fuse-mounts: + runs-on: ubuntu-latest + steps: + - name: Check out code + uses: actions/checkout@v7 + + - name: Run cleanup action unit tests + run: python3 .github/actions/cleanup-stale-fuse-mounts/test_cleanup.py diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml index d8545097d0..79adf73c98 100644 --- a/.github/workflows/codeql.yml +++ b/.github/workflows/codeql.yml @@ -9,8 +9,8 @@ on: - '.github/workflows/codeql.yml' concurrency: - group: ${{ github.head_ref }}/codeql - cancel-in-progress: true + group: ${{ github.workflow }}-${{ github.event.pull_request.number || (github.event_name == 'push' && github.ref) || github.run_id }} + cancel-in-progress: ${{ github.event_name == 'pull_request' || (github.event_name == 'push' && !startsWith(github.ref, 'refs/tags/')) }} jobs: CodeQL-Build: @@ -25,6 +25,12 @@ jobs: - name: Checkout repository uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 + - name: Set up Go + uses: actions/setup-go@v7 + with: + go-version-file: 'go.mod' + cache: ${{ runner.environment == 'github-hosted' }} + # Initializes the CodeQL tools for scanning. - name: Initialize CodeQL uses: github/codeql-action/init@v4.39.0 diff --git a/.github/workflows/codespell.yml b/.github/workflows/codespell.yml index 9c4d99a79e..a7be2ead3b 100644 --- a/.github/workflows/codespell.yml +++ b/.github/workflows/codespell.yml @@ -8,6 +8,10 @@ on: pull_request: branches: [master] +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || (github.event_name == 'push' && github.ref) || github.run_id }} + cancel-in-progress: ${{ github.event_name == 'pull_request' || (github.event_name == 'push' && !startsWith(github.ref, 'refs/tags/')) }} + permissions: contents: read diff --git a/.github/workflows/container_dev.yml b/.github/workflows/container_dev.yml index f20820a59f..2c30f861a2 100644 --- a/.github/workflows/container_dev.yml +++ b/.github/workflows/container_dev.yml @@ -13,6 +13,10 @@ on: - 'go.sum' - '.github/workflows/container_dev.yml' +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || (github.event_name == 'push' && github.ref) || github.run_id }} + cancel-in-progress: ${{ github.event_name == 'pull_request' || (github.event_name == 'push' && !startsWith(github.ref, 'refs/tags/')) }} + permissions: contents: read @@ -51,6 +55,7 @@ jobs: echo "CFLAGS_aarch64_unknown_linux_musl=-U_FORTIFY_SOURCE" >> "$GITHUB_ENV" - name: Cache cargo registry and target + if: runner.environment == 'github-hosted' uses: actions/cache@v6 with: path: | @@ -193,6 +198,12 @@ jobs: platforms: linux/amd64, linux/arm64 tags: ${{ steps.docker_meta.outputs.tags }} labels: ${{ steps.docker_meta.outputs.labels }} + cache-from: type=gha + cache-to: type=gha,mode=max + # Key the checkout layer on the pushed sha so cached layers can't + # serve stale code; the Dockerfile fetch fallback picks it up. + build-args: | + BRANCH=${{ github.sha }} - name: Sign if: github.event_name != 'pull_request' diff --git a/.github/workflows/container_foundationdb_version.yml b/.github/workflows/container_foundationdb_version.yml index 6fa0149b76..b976a9b245 100644 --- a/.github/workflows/container_foundationdb_version.yml +++ b/.github/workflows/container_foundationdb_version.yml @@ -24,6 +24,10 @@ on: required: false default: '' +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || (github.event_name == 'push' && github.ref) || github.event.inputs.image_tag || format('foundationdb_{0}_seaweedfs_{1}', github.event.inputs.fdb_version || '7.4.5', github.event.inputs.seaweedfs_ref || github.ref_name) }} + cancel-in-progress: ${{ github.event_name == 'pull_request' || (github.event_name == 'push' && !startsWith(github.ref, 'refs/tags/')) }} + permissions: contents: read @@ -64,6 +68,7 @@ jobs: uses: actions/setup-go@v7 with: go-version-file: go.mod + cache: ${{ runner.environment == 'github-hosted' }} - name: Run FoundationDB tagged tests env: diff --git a/.github/workflows/container_latest.yml b/.github/workflows/container_latest.yml index 7a47628824..8eccfc89d4 100644 --- a/.github/workflows/container_latest.yml +++ b/.github/workflows/container_latest.yml @@ -30,6 +30,11 @@ on: type: boolean default: false +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || (github.event_name == 'push' && github.ref) || github.event.inputs.image_tag || github.run_id }} + cancel-in-progress: false + queue: max + permissions: contents: read security-events: write @@ -94,6 +99,7 @@ jobs: echo "CFLAGS_aarch64_unknown_linux_musl=-U_FORTIFY_SOURCE" >> "$GITHUB_ENV" - name: Cache cargo registry and target + if: runner.environment == 'github-hosted' uses: actions/cache@v6 with: path: | @@ -164,10 +170,13 @@ jobs: uses: actions/checkout@v7 with: ref: ${{ github.event_name == 'workflow_dispatch' && github.event.inputs.source_ref || github.ref }} + - name: Clean up stale FUSE mounts + uses: ./.github/actions/cleanup-stale-fuse-mounts + - name: Free Disk Space run: | echo "Available disk space before cleanup:" - df -h + timeout 15 df -h || true # Remove pre-installed tools sudo rm -rf /usr/share/dotnet /usr/local/lib/android /opt/ghc /opt/hostedtoolcache/CodeQL # Clean package managers @@ -179,7 +188,7 @@ jobs: [ -d ~/.cache/go-build ] && rm -rf ~/.cache/go-build || true [ -d /go/pkg ] && rm -rf /go/pkg || true echo "Available disk space after cleanup:" - df -h + timeout 15 df -h || true - name: Configure variant id: config diff --git a/.github/workflows/container_release_foundationdb.yml b/.github/workflows/container_release_foundationdb.yml index bf1270b0a8..2050d99604 100644 --- a/.github/workflows/container_release_foundationdb.yml +++ b/.github/workflows/container_release_foundationdb.yml @@ -11,6 +11,11 @@ on: required: true default: '' +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || (github.event_name == 'push' && github.ref) || format('refs/tags/{0}', github.event.inputs.release_tag) || github.run_id }} + cancel-in-progress: false + queue: max + permissions: contents: read diff --git a/.github/workflows/container_release_unified.yml b/.github/workflows/container_release_unified.yml index 095524ce34..17d976509e 100644 --- a/.github/workflows/container_release_unified.yml +++ b/.github/workflows/container_release_unified.yml @@ -25,7 +25,7 @@ on: rocksdb_version: description: 'RocksDB git tag to use when variant=rocksdb' required: false - default: 'v10.10.1' + default: 'v11.1.2' permissions: contents: read @@ -78,6 +78,7 @@ jobs: echo "CFLAGS_aarch64_unknown_linux_musl=-U_FORTIFY_SOURCE" >> "$GITHUB_ENV" - name: Cache cargo registry and target + if: runner.environment == 'github-hosted' uses: actions/cache@v6 with: path: | @@ -179,6 +180,10 @@ jobs: if: github.event_name != 'workflow_dispatch' || github.event.inputs.variant == 'all' || github.event.inputs.variant == matrix.variant uses: actions/checkout@v7 + - name: Clean up stale FUSE mounts + if: github.event_name != 'workflow_dispatch' || github.event.inputs.variant == 'all' || github.event.inputs.variant == matrix.variant + uses: ./.github/actions/cleanup-stale-fuse-mounts + - name: Download pre-built Rust binaries if: github.event_name != 'workflow_dispatch' || github.event.inputs.variant == 'all' || github.event.inputs.variant == matrix.variant uses: actions/download-artifact@v8 @@ -218,7 +223,7 @@ jobs: sudo docker system prune -af --volumes [ -d ~/.cache/go-build ] && rm -rf ~/.cache/go-build || true [ -d /go/pkg ] && rm -rf /go/pkg || true - df -h + timeout 15 df -h || true - name: Docker meta if: github.event_name != 'workflow_dispatch' || github.event.inputs.variant == 'all' || github.event.inputs.variant == matrix.variant @@ -270,8 +275,11 @@ jobs: file: ${{ matrix.dockerfile }} platforms: ${{ matrix.platform }} labels: ${{ steps.docker_meta.outputs.labels }} - # Flat single-platform manifest so imagetools create assembles cleanly. - provenance: false + # mode=max keeps resolvedDependencies complete and records the + # builder identity; attestations ride inside each pushed index and + # survive the imagetools merge into the multi-arch manifest. + provenance: mode=max + sbom: true outputs: type=image,name=${{ env.IMAGE }},push-by-digest=true,name-canonical=true,push=true cache-from: type=gha,scope=${{ matrix.variant }}-${{ matrix.arch }} # max only for rocksdb: its RocksDB compile is sha-independent and worth @@ -281,7 +289,7 @@ jobs: ${{ matrix.build_args }} BUILDKIT_INLINE_CACHE=1 BRANCH=${{ github.sha }} - ${{ matrix.variant == 'rocksdb' && format('ROCKSDB_VERSION={0}', github.event.inputs.rocksdb_version || 'v10.10.1') || '' }} + ${{ matrix.variant == 'rocksdb' && format('ROCKSDB_VERSION={0}', github.event.inputs.rocksdb_version || 'v11.1.2') || '' }} - name: Export digest if: github.event_name != 'workflow_dispatch' || github.event.inputs.variant == 'all' || github.event.inputs.variant == matrix.variant @@ -307,6 +315,8 @@ jobs: permissions: contents: read id-token: write + attestations: write + packages: write strategy: fail-fast: false matrix: @@ -414,6 +424,45 @@ jobs: ${{ env.IMAGE }}@${{ steps.manifest.outputs.digest }} chrislusf/seaweedfs@${{ steps.manifest.outputs.digest }} + # Attest the merged multi-arch digest itself, so digest-pinning consumers + # can verify provenance against the exact index they pull. Both steps + # reuse the GHCR login above and push as OCI referrers. + - name: Attest build provenance for ${{ matrix.variant }} + if: github.event_name != 'workflow_dispatch' || github.event.inputs.variant == 'all' || github.event.inputs.variant == matrix.variant + uses: actions/attest-build-provenance@v4 + with: + subject-name: ${{ env.IMAGE }} + subject-digest: ${{ steps.manifest.outputs.digest }} + push-to-registry: true + + # SBOMs are generated and attested per platform manifest: an index-wide + # scan only describes syft's default platform, while the manifests it + # merges differ (some ship the real binary, others placeholders), so the + # SBOM has to be bound to the platform digest a consumer actually pulls. + - name: Install syft + if: github.event_name != 'workflow_dispatch' || github.event.inputs.variant == 'all' || github.event.inputs.variant == matrix.variant + uses: anchore/sbom-action/download-syft@v0 + + - name: Attest per-platform SBOMs for ${{ matrix.variant }} + if: github.event_name != 'workflow_dispatch' || github.event.inputs.variant == 'all' || github.event.inputs.variant == matrix.variant + working-directory: /tmp/digests + run: | + set -euo pipefail + for f in *; do + # Each saved digest is an OCI index (image + attestation manifests); + # scan and attest the image manifest a same-platform pull resolves. + ref="${{ env.IMAGE }}@sha256:${f}" + image=$(crane manifest "$ref" | jq -r ' + [.manifests[]? | select((.annotations["vnd.docker.reference.type"] // "") != "attestation-manifest" + and (.platform.os // "unknown") != "unknown") | .digest][0] // empty') + ref="${{ env.IMAGE }}@${image:-sha256:${f}}" + syft "registry:${ref}" -o "spdx-json=/tmp/sbom-${f}.spdx.json" + cosign attest --yes \ + --predicate "/tmp/sbom-${f}.spdx.json" \ + --type spdxjson \ + "$ref" + done + # Report-only trivy scan: uploads fixable HIGH/CRITICAL findings to GitHub # Security for visibility, but never blocks the release. Releases (including # `latest`) ship regardless — vulnerabilities are tracked, not gated, since @@ -452,7 +501,9 @@ jobs: scanners: vuln vuln-type: os,library severity: HIGH,CRITICAL - ignore-unfixed: true + # Keep unfixed findings in the report so digest-pinning consumers see + # the complete picture, not just what a patch could resolve. + ignore-unfixed: false limit-severities-for-sarif: true format: sarif output: trivy-results.sarif diff --git a/.github/workflows/container_rocksdb_version.yml b/.github/workflows/container_rocksdb_version.yml index 89b4e82049..0d3b5f2b8f 100644 --- a/.github/workflows/container_rocksdb_version.yml +++ b/.github/workflows/container_rocksdb_version.yml @@ -4,9 +4,9 @@ on: workflow_dispatch: inputs: rocksdb_version: - description: 'RocksDB git tag or branch to build (e.g. v10.10.1)' + description: 'RocksDB git tag or branch to build (e.g. v11.1.2)' required: true - default: 'v10.10.1' + default: 'v11.1.2' seaweedfs_ref: description: 'SeaweedFS git tag, branch, or commit to build' required: true @@ -16,6 +16,11 @@ on: required: false default: '' +concurrency: + group: ${{ github.workflow }}-${{ github.event.inputs.image_tag || format('rocksdb_{0}_seaweedfs_{1}', github.event.inputs.rocksdb_version, github.event.inputs.seaweedfs_ref) }} + cancel-in-progress: false + queue: max + permissions: contents: read diff --git a/.github/workflows/deploy_telemetry.yml b/.github/workflows/deploy_telemetry.yml index e42cdea9ed..2f45c64176 100644 --- a/.github/workflows/deploy_telemetry.yml +++ b/.github/workflows/deploy_telemetry.yml @@ -27,6 +27,7 @@ jobs: uses: actions/setup-go@v7 with: go-version-file: 'telemetry/server/go.mod' + cache: ${{ runner.environment == 'github-hosted' }} - name: Build Telemetry Server if: github.event_name == 'workflow_dispatch' && inputs.deploy diff --git a/.github/workflows/depsreview.yml b/.github/workflows/depsreview.yml index e4338c240a..0ba5dbcd2c 100644 --- a/.github/workflows/depsreview.yml +++ b/.github/workflows/depsreview.yml @@ -1,6 +1,10 @@ name: 'Dependency Review' on: [pull_request] +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || (github.event_name == 'push' && github.ref) || github.run_id }} + cancel-in-progress: ${{ github.event_name == 'pull_request' || (github.event_name == 'push' && !startsWith(github.ref, 'refs/tags/')) }} + permissions: contents: read diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml index 0e58880119..363627df27 100644 --- a/.github/workflows/e2e.yml +++ b/.github/workflows/e2e.yml @@ -19,8 +19,8 @@ on: - '.github/workflows/e2e.yml' concurrency: - group: ${{ github.head_ref }}/e2e - cancel-in-progress: true + group: ${{ github.workflow }}-${{ github.event.pull_request.number || (github.event_name == 'push' && github.ref) || github.run_id }} + cancel-in-progress: ${{ github.event_name == 'pull_request' || (github.event_name == 'push' && !startsWith(github.ref, 'refs/tags/')) }} permissions: contents: read @@ -42,6 +42,7 @@ jobs: uses: actions/setup-go@v7 with: go-version-file: 'go.mod' + cache: ${{ runner.environment == 'github-hosted' }} - name: Configure Docker Hub mirror run: | @@ -52,6 +53,7 @@ jobs: uses: docker/setup-buildx-action@v4 - name: Cache Docker layers + if: runner.environment == 'github-hosted' uses: actions/cache@v6 with: path: /tmp/.buildx-cache diff --git a/.github/workflows/ec-integration-tests.yml b/.github/workflows/ec-integration-tests.yml index e078e01e6c..a1ddfa8e65 100644 --- a/.github/workflows/ec-integration-tests.yml +++ b/.github/workflows/ec-integration-tests.yml @@ -18,6 +18,10 @@ on: - 'go.sum' - '.github/workflows/ec-integration-tests.yml' +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || (github.event_name == 'push' && github.ref) || github.run_id }} + cancel-in-progress: ${{ github.event_name == 'pull_request' || (github.event_name == 'push' && !startsWith(github.ref, 'refs/tags/')) }} + permissions: contents: read @@ -30,6 +34,7 @@ jobs: - name: Set up Go 1.x uses: actions/setup-go@v7 with: + cache: ${{ runner.environment == 'github-hosted' }} go-version: ^1.26 id: go diff --git a/.github/workflows/fuse-dlm-integration.yml b/.github/workflows/fuse-dlm-integration.yml index 47a27cc199..869eae40ad 100644 --- a/.github/workflows/fuse-dlm-integration.yml +++ b/.github/workflows/fuse-dlm-integration.yml @@ -19,8 +19,8 @@ on: - '.github/actions/fix-fusermount-setuid/**' concurrency: - group: ${{ github.head_ref || github.ref }}/fuse-dlm-integration - cancel-in-progress: true + group: ${{ github.workflow }}-${{ github.event.pull_request.number || (github.event_name == 'push' && github.ref) || github.run_id }} + cancel-in-progress: ${{ github.event_name == 'pull_request' || (github.event_name == 'push' && !startsWith(github.ref, 'refs/tags/')) }} permissions: contents: read @@ -39,6 +39,7 @@ jobs: uses: actions/setup-go@v7 with: go-version-file: 'go.mod' + cache: ${{ runner.environment == 'github-hosted' }} - name: Configure FUSE run: | diff --git a/.github/workflows/fuse-failover.yml b/.github/workflows/fuse-failover.yml index c11ca3e2f1..2c03cf9369 100644 --- a/.github/workflows/fuse-failover.yml +++ b/.github/workflows/fuse-failover.yml @@ -24,8 +24,8 @@ on: - '.github/actions/fix-fusermount-setuid/**' concurrency: - group: ${{ github.head_ref || github.ref }}/fuse-failover - cancel-in-progress: true + group: ${{ github.workflow }}-${{ github.event.pull_request.number || (github.event_name == 'push' && github.ref) || github.run_id }} + cancel-in-progress: ${{ github.event_name == 'pull_request' || (github.event_name == 'push' && !startsWith(github.ref, 'refs/tags/')) }} permissions: contents: read @@ -42,10 +42,14 @@ jobs: with: persist-credentials: false + - name: Clean up stale FUSE mounts + uses: ./.github/actions/cleanup-stale-fuse-mounts + - name: Set up Go uses: actions/setup-go@v7 with: go-version-file: 'go.mod' + cache: ${{ runner.environment == 'github-hosted' }} - name: Configure FUSE run: | diff --git a/.github/workflows/fuse-integration.yml b/.github/workflows/fuse-integration.yml index 3b23cb2228..f45e3ed0a1 100644 --- a/.github/workflows/fuse-integration.yml +++ b/.github/workflows/fuse-integration.yml @@ -17,8 +17,8 @@ on: - '.github/actions/fix-fusermount-setuid/**' concurrency: - group: ${{ github.head_ref }}/fuse-integration - cancel-in-progress: true + group: ${{ github.workflow }}-${{ github.event.pull_request.number || (github.event_name == 'push' && github.ref) || github.run_id }} + cancel-in-progress: ${{ github.event_name == 'pull_request' || (github.event_name == 'push' && !startsWith(github.ref, 'refs/tags/')) }} permissions: contents: read @@ -37,6 +37,7 @@ jobs: uses: actions/setup-go@v7 with: go-version-file: 'go.mod' + cache: ${{ runner.environment == 'github-hosted' }} - name: Configure FUSE run: | diff --git a/.github/workflows/fuse-p2p-integration.yml b/.github/workflows/fuse-p2p-integration.yml index 28771e11a9..a65f5ddbc2 100644 --- a/.github/workflows/fuse-p2p-integration.yml +++ b/.github/workflows/fuse-p2p-integration.yml @@ -25,8 +25,8 @@ on: - '.github/actions/fix-fusermount-setuid/**' concurrency: - group: ${{ github.head_ref || github.ref }}/fuse-p2p-integration - cancel-in-progress: true + group: ${{ github.workflow }}-${{ github.event.pull_request.number || (github.event_name == 'push' && github.ref) || github.run_id }} + cancel-in-progress: ${{ github.event_name == 'pull_request' || (github.event_name == 'push' && !startsWith(github.ref, 'refs/tags/')) }} permissions: contents: read @@ -41,10 +41,14 @@ jobs: - name: Check out code uses: actions/checkout@v7 + - name: Clean up stale FUSE mounts + uses: ./.github/actions/cleanup-stale-fuse-mounts + - name: Set up Go uses: actions/setup-go@v7 with: go-version-file: 'go.mod' + cache: ${{ runner.environment == 'github-hosted' }} - name: Configure FUSE run: | diff --git a/.github/workflows/go.yml b/.github/workflows/go.yml index b2e5b52979..0a06fbf6be 100644 --- a/.github/workflows/go.yml +++ b/.github/workflows/go.yml @@ -17,8 +17,8 @@ on: - '.github/workflows/go.yml' concurrency: - group: ${{ github.head_ref }}/go - cancel-in-progress: true + group: ${{ github.workflow }}-${{ github.event.pull_request.number || (github.event_name == 'push' && github.ref) || github.run_id }} + cancel-in-progress: ${{ github.event_name == 'pull_request' || (github.event_name == 'push' && !startsWith(github.ref, 'refs/tags/')) }} permissions: contents: read @@ -35,6 +35,7 @@ jobs: uses: actions/setup-go@v7 with: go-version-file: 'go.mod' + cache: ${{ runner.environment == 'github-hosted' }} - name: Get dependencies run: | cd weed; go get -v -t -d ./... @@ -57,6 +58,7 @@ jobs: uses: actions/setup-go@v7 with: go-version-file: 'go.mod' + cache: ${{ runner.environment == 'github-hosted' }} - name: Go Vet linux/386 (type-checks code and tests for 32-bit int overflows) run: | GOOS=linux GOARCH=386 go vet ./... 2>&1 | grep -v "MessageState contains sync.Mutex" | grep -v "IdentityAccessManagement contains sync.RWMutex" | tee vet-32bit-output.txt @@ -72,6 +74,7 @@ jobs: uses: actions/setup-go@v7 with: go-version-file: 'go.mod' + cache: ${{ runner.environment == 'github-hosted' }} - name: Build run: cd weed; go build -tags "elastic gocdk sqlite ydb tarantool tikv rclone" -v . @@ -94,6 +97,7 @@ jobs: uses: actions/setup-go@v7 with: go-version-file: 'go.mod' + cache: ${{ runner.environment == 'github-hosted' }} # Nothing else on a PR compiles these, so per-OS syscall constants creep # back into shared files unnoticed and only a release build catches it. - name: Cross-compile @@ -128,6 +132,7 @@ jobs: uses: actions/setup-go@v7 with: go-version-file: 'go.mod' + cache: ${{ runner.environment == 'github-hosted' }} - name: Test env: RUN_REDIS_TESTS: "1" @@ -143,6 +148,7 @@ jobs: uses: actions/setup-go@v7 with: go-version-file: 'go.mod' + cache: ${{ runner.environment == 'github-hosted' }} # 386 test binaries run natively on the amd64 runner. This catches what vet # can't: unaligned 64-bit atomics and arithmetic that wraps at runtime. # -short skips the e2e suites already covered on amd64. @@ -159,6 +165,7 @@ jobs: uses: actions/setup-go@v7 with: go-version-file: 'go.mod' + cache: ${{ runner.environment == 'github-hosted' }} # weed/kms/azure is excluded from every default build, so without this # nothing compiles it and tag-gated code silently rots. - name: Build and test with azurekms diff --git a/.github/workflows/helm_ci.yml b/.github/workflows/helm_ci.yml index ce0ecf916b..0a75617d79 100644 --- a/.github/workflows/helm_ci.yml +++ b/.github/workflows/helm_ci.yml @@ -8,6 +8,10 @@ on: branches: [ master ] paths: ['k8s/**', '.github/workflows/helm_ci.yml', '.github/scripts/helm_jwt_keys.py'] +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || (github.event_name == 'push' && github.ref) || github.run_id }} + cancel-in-progress: ${{ github.event_name == 'pull_request' || (github.event_name == 'push' && !startsWith(github.ref, 'refs/tags/')) }} + permissions: contents: read diff --git a/.github/workflows/java_integration_tests.yml b/.github/workflows/java_integration_tests.yml index 075df5f857..cdb61b3faf 100644 --- a/.github/workflows/java_integration_tests.yml +++ b/.github/workflows/java_integration_tests.yml @@ -14,6 +14,10 @@ on: - 'weed/**' - '.github/workflows/java_integration_tests.yml' +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || (github.event_name == 'push' && github.ref) || github.run_id }} + cancel-in-progress: ${{ github.event_name == 'pull_request' || (github.event_name == 'push' && !startsWith(github.ref, 'refs/tags/')) }} + jobs: test: name: Java Integration Tests @@ -31,6 +35,7 @@ jobs: uses: actions/setup-go@v7 with: go-version-file: 'go.mod' + cache: ${{ runner.environment == 'github-hosted' }} id: go - name: Set up Java @@ -38,7 +43,7 @@ jobs: with: java-version: ${{ matrix.java }} distribution: 'temurin' - cache: 'maven' + cache: ${{ runner.environment == 'github-hosted' && 'maven' || '' }} - name: Build SeaweedFS run: | @@ -55,6 +60,7 @@ jobs: # Start SeaweedFS with optimized settings for CI # Include S3 API for s3copier integration tests weed server -dir="$WEED_DATA_DIR" \ + -ip=127.0.0.1 -ip.bind=127.0.0.1 \ -master.raftHashicorp \ -master.electionTimeout=1s \ -master.volumeSizeLimitMB=100 \ diff --git a/.github/workflows/java_release.yml b/.github/workflows/java_release.yml index 282c56245b..3cde5af868 100644 --- a/.github/workflows/java_release.yml +++ b/.github/workflows/java_release.yml @@ -77,7 +77,7 @@ jobs: with: java-version: '17' distribution: 'temurin' - cache: 'maven' + cache: ${{ runner.environment == 'github-hosted' && 'maven' || '' }} server-id: central server-username: MAVEN_CENTRAL_USERNAME server-password: MAVEN_CENTRAL_PASSWORD diff --git a/.github/workflows/java_unit_tests.yml b/.github/workflows/java_unit_tests.yml index 753d1708e8..ed29f8dab3 100644 --- a/.github/workflows/java_unit_tests.yml +++ b/.github/workflows/java_unit_tests.yml @@ -12,6 +12,10 @@ on: - 'other/java/**' - '.github/workflows/java_unit_tests.yml' +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || (github.event_name == 'push' && github.ref) || github.run_id }} + cancel-in-progress: ${{ github.event_name == 'pull_request' || (github.event_name == 'push' && !startsWith(github.ref, 'refs/tags/')) }} + jobs: test: name: Java Unit Tests @@ -30,7 +34,7 @@ jobs: with: java-version: ${{ matrix.java }} distribution: 'temurin' - cache: 'maven' + cache: ${{ runner.environment == 'github-hosted' && 'maven' || '' }} - name: Build and Install SeaweedFS Client working-directory: other/java/client diff --git a/.github/workflows/kafka-quicktest.yml b/.github/workflows/kafka-quicktest.yml index c18a87048b..f48c5487df 100644 --- a/.github/workflows/kafka-quicktest.yml +++ b/.github/workflows/kafka-quicktest.yml @@ -24,8 +24,8 @@ on: workflow_dispatch: # Allow manual trigger concurrency: - group: ${{ github.head_ref }}/kafka-quicktest - cancel-in-progress: true + group: ${{ github.workflow }}-${{ github.event.pull_request.number || (github.event_name == 'push' && github.ref) || github.run_id }} + cancel-in-progress: ${{ github.event_name == 'pull_request' || (github.event_name == 'push' && !startsWith(github.ref, 'refs/tags/')) }} permissions: contents: read @@ -43,7 +43,7 @@ jobs: uses: actions/setup-go@v7 with: go-version: ^1.26 - cache: true + cache: ${{ runner.environment == 'github-hosted' }} cache-dependency-path: | **/go.sum id: go diff --git a/.github/workflows/kafka-tests.yml b/.github/workflows/kafka-tests.yml index 7a40ee3583..4a59cb6724 100644 --- a/.github/workflows/kafka-tests.yml +++ b/.github/workflows/kafka-tests.yml @@ -23,8 +23,8 @@ on: - '.github/workflows/kafka-tests.yml' concurrency: - group: ${{ github.head_ref }}/kafka-tests - cancel-in-progress: true + group: ${{ github.workflow }}-${{ github.event.pull_request.number || (github.event_name == 'push' && github.ref) || github.run_id }} + cancel-in-progress: ${{ github.event_name == 'pull_request' || (github.event_name == 'push' && !startsWith(github.ref, 'refs/tags/')) }} # Force different runners for better isolation env: @@ -53,6 +53,7 @@ jobs: - name: Set up Go 1.x uses: actions/setup-go@v7 with: + cache: ${{ runner.environment == 'github-hosted' }} go-version: ^1.26 id: go @@ -98,6 +99,7 @@ jobs: - name: Set up Go 1.x uses: actions/setup-go@v7 with: + cache: ${{ runner.environment == 'github-hosted' }} go-version: ^1.26 id: go @@ -149,7 +151,7 @@ jobs: uses: actions/setup-go@v7 with: go-version: ^1.26 - cache: true + cache: ${{ runner.environment == 'github-hosted' }} cache-dependency-path: | **/go.sum id: go @@ -328,7 +330,7 @@ jobs: uses: actions/setup-go@v7 with: go-version: ^1.26 - cache: true + cache: ${{ runner.environment == 'github-hosted' }} cache-dependency-path: | **/go.sum id: go @@ -490,7 +492,7 @@ jobs: uses: actions/setup-go@v7 with: go-version: ^1.26 - cache: true + cache: ${{ runner.environment == 'github-hosted' }} cache-dependency-path: | **/go.sum id: go @@ -648,7 +650,7 @@ jobs: uses: actions/setup-go@v7 with: go-version: ^1.26 - cache: true + cache: ${{ runner.environment == 'github-hosted' }} cache-dependency-path: | **/go.sum id: go @@ -805,6 +807,7 @@ jobs: - name: Set up Go 1.x uses: actions/setup-go@v7 with: + cache: ${{ runner.environment == 'github-hosted' }} go-version: ^1.26 id: go diff --git a/.github/workflows/kms-tests.yml b/.github/workflows/kms-tests.yml index 3c997fd523..6b77a896f1 100644 --- a/.github/workflows/kms-tests.yml +++ b/.github/workflows/kms-tests.yml @@ -19,8 +19,8 @@ on: - 'test/kms/**' concurrency: - group: ${{ github.head_ref || github.ref }}-kms-tests - cancel-in-progress: true + group: ${{ github.workflow }}-${{ github.event.pull_request.number || (github.event_name == 'push' && github.ref) || github.run_id }} + cancel-in-progress: ${{ github.event_name == 'pull_request' || (github.event_name == 'push' && !startsWith(github.ref, 'refs/tags/')) }} permissions: contents: read @@ -43,6 +43,7 @@ jobs: uses: actions/setup-go@v7 with: go-version-file: 'go.mod' + cache: ${{ runner.environment == 'github-hosted' }} id: go - name: Install SeaweedFS @@ -88,6 +89,7 @@ jobs: uses: actions/setup-go@v7 with: go-version-file: 'go.mod' + cache: ${{ runner.environment == 'github-hosted' }} id: go - name: Install SeaweedFS diff --git a/.github/workflows/master-cold-start-tests.yml b/.github/workflows/master-cold-start-tests.yml index bd0ed72243..bc96b833bf 100644 --- a/.github/workflows/master-cold-start-tests.yml +++ b/.github/workflows/master-cold-start-tests.yml @@ -21,8 +21,8 @@ on: - '.github/workflows/master-cold-start-tests.yml' concurrency: - group: ${{ github.head_ref || github.ref }}/master-cold-start-tests - cancel-in-progress: true + group: ${{ github.workflow }}-${{ github.event.pull_request.number || (github.event_name == 'push' && github.ref) || github.run_id }} + cancel-in-progress: ${{ github.event_name == 'pull_request' || (github.event_name == 'push' && !startsWith(github.ref, 'refs/tags/')) }} permissions: contents: read @@ -42,6 +42,7 @@ jobs: uses: actions/setup-go@v7 with: go-version-file: 'go.mod' + cache: ${{ runner.environment == 'github-hosted' }} - name: Build weed binary run: | diff --git a/.github/workflows/metadata-subscribe-tests.yml b/.github/workflows/metadata-subscribe-tests.yml index b47c16b9cd..b0bb96bc25 100644 --- a/.github/workflows/metadata-subscribe-tests.yml +++ b/.github/workflows/metadata-subscribe-tests.yml @@ -25,8 +25,8 @@ on: - '.github/workflows/metadata-subscribe-tests.yml' concurrency: - group: ${{ github.head_ref }}/metadata-subscribe-tests - cancel-in-progress: true + group: ${{ github.workflow }}-${{ github.event.pull_request.number || (github.event_name == 'push' && github.ref) || github.run_id }} + cancel-in-progress: ${{ github.event_name == 'pull_request' || (github.event_name == 'push' && !startsWith(github.ref, 'refs/tags/')) }} permissions: contents: read @@ -48,6 +48,7 @@ jobs: uses: actions/setup-go@v7 with: go-version-file: 'go.mod' + cache: ${{ runner.environment == 'github-hosted' }} - name: Build SeaweedFS run: | diff --git a/.github/workflows/mount-benchmark.yml b/.github/workflows/mount-benchmark.yml index 90310ee6cf..206e082243 100644 --- a/.github/workflows/mount-benchmark.yml +++ b/.github/workflows/mount-benchmark.yml @@ -10,8 +10,8 @@ on: branches: [ 'winfsp-bench**' ] concurrency: - group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} - cancel-in-progress: true + group: ${{ github.workflow }}-${{ github.event.pull_request.number || (github.event_name == 'push' && github.ref) || github.run_id }} + cancel-in-progress: ${{ github.event_name == 'pull_request' || (github.event_name == 'push' && !startsWith(github.ref, 'refs/tags/')) }} permissions: contents: read @@ -32,6 +32,7 @@ jobs: - uses: actions/setup-go@v7 with: go-version-file: 'go.mod' + cache: ${{ runner.environment == 'github-hosted' }} - name: Install WinFsp and rclone run: choco install winfsp rclone -y --no-progress @@ -150,6 +151,7 @@ jobs: - uses: actions/setup-go@v7 with: go-version-file: 'go.mod' + cache: ${{ runner.environment == 'github-hosted' }} - name: Repair the fusermount3 setuid bit uses: ./.github/actions/fix-fusermount-setuid diff --git a/.github/workflows/mount-windows-conformance.yml b/.github/workflows/mount-windows-conformance.yml index 6d490c5d8a..c58d063575 100644 --- a/.github/workflows/mount-windows-conformance.yml +++ b/.github/workflows/mount-windows-conformance.yml @@ -24,8 +24,8 @@ on: - '.github/workflows/mount-windows-conformance.yml' concurrency: - group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} - cancel-in-progress: true + group: ${{ github.workflow }}-${{ github.event.pull_request.number || (github.event_name == 'push' && github.ref) || github.run_id }} + cancel-in-progress: ${{ github.event_name == 'pull_request' || (github.event_name == 'push' && !startsWith(github.ref, 'refs/tags/')) }} permissions: contents: read @@ -49,6 +49,7 @@ jobs: - uses: actions/setup-go@v7 with: go-version-file: 'go.mod' + cache: ${{ runner.environment == 'github-hosted' }} # cgofuse loads winfsp-x64.dll at run time, so WinFsp is needed here but # not to build. diff --git a/.github/workflows/mount-windows.yml b/.github/workflows/mount-windows.yml index 91f61d8093..7954176ce7 100644 --- a/.github/workflows/mount-windows.yml +++ b/.github/workflows/mount-windows.yml @@ -24,8 +24,8 @@ on: - '.github/workflows/mount-windows.yml' concurrency: - group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} - cancel-in-progress: true + group: ${{ github.workflow }}-${{ github.event.pull_request.number || (github.event_name == 'push' && github.ref) || github.run_id }} + cancel-in-progress: ${{ github.event_name == 'pull_request' || (github.event_name == 'push' && !startsWith(github.ref, 'refs/tags/')) }} permissions: contents: read @@ -49,6 +49,7 @@ jobs: - uses: actions/setup-go@v7 with: go-version-file: 'go.mod' + cache: ${{ runner.environment == 'github-hosted' }} # cgofuse loads winfsp-x64.dll at run time, so WinFsp is needed here but # not to build. diff --git a/.github/workflows/multi-master-tests.yml b/.github/workflows/multi-master-tests.yml index 2e0cfe8d95..af282120b8 100644 --- a/.github/workflows/multi-master-tests.yml +++ b/.github/workflows/multi-master-tests.yml @@ -21,8 +21,8 @@ on: - '.github/workflows/multi-master-tests.yml' concurrency: - group: ${{ github.head_ref || github.ref }}/multi-master-tests - cancel-in-progress: true + group: ${{ github.workflow }}-${{ github.event.pull_request.number || (github.event_name == 'push' && github.ref) || github.run_id }} + cancel-in-progress: ${{ github.event_name == 'pull_request' || (github.event_name == 'push' && !startsWith(github.ref, 'refs/tags/')) }} permissions: contents: read @@ -40,6 +40,7 @@ jobs: uses: actions/setup-go@v7 with: go-version-file: 'go.mod' + cache: ${{ runner.environment == 'github-hosted' }} - name: Install SeaweedFS run: | diff --git a/.github/workflows/performance.yml b/.github/workflows/performance.yml index 95750f6891..0d22915b22 100644 --- a/.github/workflows/performance.yml +++ b/.github/workflows/performance.yml @@ -45,8 +45,8 @@ on: type: string concurrency: - group: ${{ github.head_ref || github.ref }}/performance - cancel-in-progress: true + group: ${{ github.workflow }}-${{ github.event.pull_request.number || (github.event_name == 'push' && github.ref) || github.run_id }} + cancel-in-progress: ${{ github.event_name == 'pull_request' || (github.event_name == 'push' && !startsWith(github.ref, 'refs/tags/')) }} permissions: contents: read @@ -67,6 +67,7 @@ jobs: uses: actions/setup-go@v7 with: go-version-file: 'go.mod' + cache: ${{ runner.environment == 'github-hosted' }} - name: Build weed run: go build -o weed_bin ./weed @@ -175,13 +176,14 @@ jobs: uses: actions/setup-go@v7 with: go-version-file: 'go.mod' + cache: ${{ runner.environment == 'github-hosted' }} - name: Install Rust toolchain if: matrix.impl == 'rust' uses: dtolnay/rust-toolchain@stable - name: Cache cargo registry and target - if: matrix.impl == 'rust' + if: matrix.impl == 'rust' && runner.environment == 'github-hosted' uses: actions/cache@v6 with: path: | @@ -327,13 +329,14 @@ jobs: uses: actions/setup-go@v7 with: go-version-file: 'go.mod' + cache: ${{ runner.environment == 'github-hosted' }} - name: Install Rust toolchain if: matrix.impl == 'rust' uses: dtolnay/rust-toolchain@stable - name: Cache cargo registry and target - if: matrix.impl == 'rust' + if: matrix.impl == 'rust' && runner.environment == 'github-hosted' uses: actions/cache@v6 with: path: | diff --git a/.github/workflows/pjdfstest.yml b/.github/workflows/pjdfstest.yml index 80845380ca..a88d19d48a 100644 --- a/.github/workflows/pjdfstest.yml +++ b/.github/workflows/pjdfstest.yml @@ -38,15 +38,23 @@ jobs: uses: actions/setup-go@v7 with: go-version-file: 'go.mod' + cache: ${{ runner.environment == 'github-hosted' }} - name: Start local Docker registry - run: docker run -d --restart=always -p 5000:5000 --name registry registry:2 + run: | + if ! docker start registry 2>/dev/null; then + docker rm -f registry >/dev/null 2>&1 || true + docker run -d --restart=always -p 5000:5000 --name registry registry:2 + fi - name: Set up Docker Buildx uses: docker/setup-buildx-action@v4 with: driver-opts: network=host + - name: Set up Docker Compose + uses: docker/setup-compose-action@v2 + - name: Build weed race binary run: | cd docker diff --git a/.github/workflows/plugin-workers.yml b/.github/workflows/plugin-workers.yml index b3f10551c4..55d3f56478 100644 --- a/.github/workflows/plugin-workers.yml +++ b/.github/workflows/plugin-workers.yml @@ -18,6 +18,10 @@ on: - 'go.sum' - '.github/workflows/plugin-workers.yml' +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || (github.event_name == 'push' && github.ref) || github.run_id }} + cancel-in-progress: ${{ github.event_name == 'pull_request' || (github.event_name == 'push' && !startsWith(github.ref, 'refs/tags/')) }} + permissions: contents: read @@ -41,6 +45,7 @@ jobs: - name: Set up Go 1.x uses: actions/setup-go@v7 with: + cache: ${{ runner.environment == 'github-hosted' }} go-version: ^1.26 id: go diff --git a/.github/workflows/posix-compliance.yml b/.github/workflows/posix-compliance.yml index fc93694466..66efc0de44 100644 --- a/.github/workflows/posix-compliance.yml +++ b/.github/workflows/posix-compliance.yml @@ -58,11 +58,14 @@ jobs: with: fetch-depth: 1 + - name: Clean up stale FUSE mounts + uses: ./.github/actions/cleanup-stale-fuse-mounts + - name: Set up Go ${{ env.GO_VERSION }} uses: actions/setup-go@v7 with: go-version: ${{ env.GO_VERSION }} - cache: true + cache: ${{ runner.environment == 'github-hosted' }} cache-dependency-path: | go.sum test/fuse_integration/go.sum @@ -98,7 +101,11 @@ jobs: # Set permissions on FUSE device sudo chmod 666 /dev/fuse - + + # Allow non-root mounts to pass -o allow_other (weed -allowOthers) + echo 'user_allow_other' | sudo tee -a /etc/fuse.conf + sudo chmod 644 /etc/fuse.conf + # Ensure fuse module is loaded sudo modprobe fuse || true @@ -389,7 +396,7 @@ jobs: echo "" >> reports/debug/system_info.txt echo "Disk Space" >> reports/debug/system_info.txt - df -h >> reports/debug/system_info.txt + timeout 15 df -h >> reports/debug/system_info.txt || true echo "" >> reports/debug/system_info.txt echo "Memory Information" >> reports/debug/system_info.txt @@ -564,7 +571,7 @@ jobs: uses: actions/setup-go@v7 with: go-version: ${{ env.GO_VERSION }} - cache: true + cache: ${{ runner.environment == 'github-hosted' }} - name: Install macFUSE run: | @@ -617,6 +624,7 @@ jobs: uses: actions/setup-go@v7 with: go-version: ${{ env.GO_VERSION }} + cache: ${{ runner.environment == 'github-hosted' }} - name: Install security analysis tools run: | @@ -673,6 +681,7 @@ jobs: uses: actions/setup-go@v7 with: go-version: ${{ env.GO_VERSION }} + cache: ${{ runner.environment == 'github-hosted' }} - name: Install dependencies run: | @@ -680,6 +689,8 @@ jobs: sudo apt-get install -y fuse libfuse-dev fio sudo usermod -a -G fuse $USER sudo chmod 666 /dev/fuse + # The posix suites mount with -allowOthers; allow_other needs this + echo 'user_allow_other' | sudo tee -a /etc/fuse.conf - name: Build SeaweedFS run: make @@ -723,12 +734,15 @@ jobs: uses: actions/setup-go@v7 with: go-version: ${{ env.GO_VERSION }} + cache: ${{ runner.environment == 'github-hosted' }} - name: Install dependencies run: | sudo apt-get update sudo apt-get install -y fuse libfuse-dev pip3 install --user nfstest + # The posix suites mount with -allowOthers; allow_other needs this + echo 'user_allow_other' | sudo tee -a /etc/fuse.conf - name: Build SeaweedFS run: make diff --git a/.github/workflows/postgres-tests.yml b/.github/workflows/postgres-tests.yml index a0369b134e..bbcef063bf 100644 --- a/.github/workflows/postgres-tests.yml +++ b/.github/workflows/postgres-tests.yml @@ -23,8 +23,8 @@ on: - '.github/workflows/postgres-tests.yml' concurrency: - group: ${{ github.head_ref }}/postgres-tests - cancel-in-progress: true + group: ${{ github.workflow }}-${{ github.event.pull_request.number || (github.event_name == 'push' && github.ref) || github.run_id }} + cancel-in-progress: ${{ github.event_name == 'pull_request' || (github.event_name == 'push' && !startsWith(github.ref, 'refs/tags/')) }} permissions: contents: read @@ -41,6 +41,7 @@ jobs: - name: Set up Go 1.x uses: actions/setup-go@v7 with: + cache: ${{ runner.environment == 'github-hosted' }} go-version: ^1.26 id: go @@ -56,6 +57,7 @@ jobs: uses: docker/setup-buildx-action@v4 - name: Cache Docker layers + if: runner.environment == 'github-hosted' uses: actions/cache@v6 with: path: /tmp/.buildx-cache diff --git a/.github/workflows/rust-volume-server-tests.yml b/.github/workflows/rust-volume-server-tests.yml index fd68db1a5d..e6223aef29 100644 --- a/.github/workflows/rust-volume-server-tests.yml +++ b/.github/workflows/rust-volume-server-tests.yml @@ -21,8 +21,8 @@ on: - '.github/workflows/rust-volume-server-tests.yml' concurrency: - group: ${{ github.workflow }}-${{ github.head_ref || github.ref }} - cancel-in-progress: true + group: ${{ github.workflow }}-${{ github.event.pull_request.number || (github.event_name == 'push' && github.ref) || github.run_id }} + cancel-in-progress: ${{ github.event_name == 'pull_request' || (github.event_name == 'push' && !startsWith(github.ref, 'refs/tags/')) }} permissions: contents: read @@ -52,6 +52,65 @@ jobs: - 'seaweed-volume/**' - '.github/workflows/rust-volume-server-tests.yml' + build-test-binaries: + name: Build Test Binaries + runs-on: ubuntu-22.04 + timeout-minutes: 30 + + steps: + - name: Checkout code + uses: actions/checkout@v7 + + - name: Set up Go + uses: actions/setup-go@v7 + with: + go-version-file: 'go.mod' + cache: ${{ runner.environment == 'github-hosted' }} + + - name: Install Rust toolchain + uses: dtolnay/rust-toolchain@stable + + # cargo tracks its own inputs but not the runner's C toolchain, so a cached + # target/ can carry C objects built against a different glibc than we link against. + - name: Fingerprint build toolchain + id: toolchain + run: echo "fingerprint=$(getconf GNU_LIBC_VERSION | tr ' ' '-')-rustc-$(rustc -V | awk '{print $2}')" >> "$GITHUB_OUTPUT" + + - name: Cache cargo registry and target + if: runner.environment == 'github-hosted' + uses: actions/cache@v6 + with: + path: | + ~/.cargo/registry + ~/.cargo/git + seaweed-volume/target + key: rust-${{ steps.toolchain.outputs.fingerprint }}-${{ hashFiles('seaweed-volume/Cargo.lock', 'seaweed-common/Cargo.lock') }} + restore-keys: | + rust-${{ steps.toolchain.outputs.fingerprint }}- + + - name: Build Go weed binary + run: | + cd weed + go build -tags 5BytesOffset -o weed . + chmod +x weed + ./weed version + + - name: Build Rust volume binary + run: cd seaweed-volume && cargo build --release + + - name: Package binaries + run: tar cf rust-test-binaries.tar weed/weed seaweed-volume/target/release/weed-volume + + - name: Upload test binaries + uses: actions/upload-artifact@v7 + with: + # archive layout mirrors the workspace: weed/weed and + # seaweed-volume/target/release/weed-volume + name: rust-test-binaries + if-no-files-found: error + retention-days: 1 + path: rust-test-binaries.tar + rust-unit-tests: name: Rust Unit Tests runs-on: ubuntu-22.04 @@ -71,6 +130,7 @@ jobs: run: echo "fingerprint=$(getconf GNU_LIBC_VERSION | tr ' ' '-')-rustc-$(rustc -V | awk '{print $2}')" >> "$GITHUB_OUTPUT" - name: Cache cargo registry and target + if: runner.environment == 'github-hosted' uses: actions/cache@v6 with: path: | @@ -134,6 +194,7 @@ jobs: run: echo "fingerprint=windows-rustc-$(rustc -V | awk '{print $2}')" >> "$GITHUB_OUTPUT" - name: Cache cargo registry and target + if: runner.environment == 'github-hosted' uses: actions/cache@v6 with: path: | @@ -154,6 +215,7 @@ jobs: name: Rust Integration Tests runs-on: ubuntu-22.04 timeout-minutes: 30 + needs: [build-test-binaries] steps: - name: Checkout code @@ -163,36 +225,18 @@ jobs: uses: actions/setup-go@v7 with: go-version-file: 'go.mod' + cache: ${{ runner.environment == 'github-hosted' }} - - name: Install Rust toolchain - uses: dtolnay/rust-toolchain@stable - - # cargo tracks its own inputs but not the runner's C toolchain, so a cached - # target/ can carry C objects built against a different glibc than we link against. - - name: Fingerprint build toolchain - id: toolchain - run: echo "fingerprint=$(getconf GNU_LIBC_VERSION | tr ' ' '-')-rustc-$(rustc -V | awk '{print $2}')" >> "$GITHUB_OUTPUT" - - - name: Cache cargo registry and target - uses: actions/cache@v6 + - name: Download test binaries + uses: actions/download-artifact@v7 with: - path: | - ~/.cargo/registry - ~/.cargo/git - seaweed-volume/target - key: rust-${{ steps.toolchain.outputs.fingerprint }}-${{ hashFiles('seaweed-volume/Cargo.lock', 'seaweed-common/Cargo.lock') }} - restore-keys: | - rust-${{ steps.toolchain.outputs.fingerprint }}- + name: rust-test-binaries - - name: Build Go weed binary + - name: Restore binaries run: | - cd weed - go build -tags 5BytesOffset -o weed . - chmod +x weed - ./weed version - - - name: Build Rust volume binary - run: cd seaweed-volume && cargo build --release + tar xf rust-test-binaries.tar + chmod +x weed/weed seaweed-volume/target/release/weed-volume + ./weed/weed version - name: Run integration tests env: @@ -228,6 +272,7 @@ jobs: name: Go Tests with Rust Volume (${{ matrix.test-type }} - Shard ${{ matrix.shard }}) runs-on: ubuntu-22.04 timeout-minutes: 45 + needs: [build-test-binaries] env: # Keep in step with the length of matrix.shard below. SHARD_COUNT: 3 @@ -245,36 +290,19 @@ jobs: uses: actions/setup-go@v7 with: go-version-file: 'go.mod' + cache: ${{ runner.environment == 'github-hosted' }} - - name: Install Rust toolchain - uses: dtolnay/rust-toolchain@stable - - # cargo tracks its own inputs but not the runner's C toolchain, so a cached - # target/ can carry C objects built against a different glibc than we link against. - - name: Fingerprint build toolchain - id: toolchain - run: echo "fingerprint=$(getconf GNU_LIBC_VERSION | tr ' ' '-')-rustc-$(rustc -V | awk '{print $2}')" >> "$GITHUB_OUTPUT" - - - name: Cache cargo registry and target - uses: actions/cache@v6 + - name: Download test binaries + uses: actions/download-artifact@v7 with: - path: | - ~/.cargo/registry - ~/.cargo/git - seaweed-volume/target - key: rust-${{ steps.toolchain.outputs.fingerprint }}-${{ hashFiles('seaweed-volume/Cargo.lock', 'seaweed-common/Cargo.lock') }} - restore-keys: | - rust-${{ steps.toolchain.outputs.fingerprint }}- + name: rust-test-binaries - - name: Build Go weed binary + - name: Restore binaries run: | - cd weed - go build -tags 5BytesOffset -o weed . - chmod +x weed - ./weed version - - - name: Build Rust volume binary - run: cd seaweed-volume && cargo build --release + tar xf rust-test-binaries.tar + chmod +x weed/weed seaweed-volume/target/release/weed-volume + ./weed/weed version + ./seaweed-volume/target/release/weed-volume --help >/dev/null || true # Dealing the listed tests out one by one keeps the shards even. Bucketing # them by first letter did not: names cluster, so ^Test[I-S] drew 50 of diff --git a/.github/workflows/rust-worker-tests.yml b/.github/workflows/rust-worker-tests.yml index d2fde3ed0d..7bfaecb695 100644 --- a/.github/workflows/rust-worker-tests.yml +++ b/.github/workflows/rust-worker-tests.yml @@ -17,8 +17,8 @@ on: - '.github/workflows/rust-worker-tests.yml' concurrency: - group: ${{ github.workflow }}-${{ github.head_ref || github.ref }} - cancel-in-progress: true + group: ${{ github.workflow }}-${{ github.event.pull_request.number || (github.event_name == 'push' && github.ref) || github.run_id }} + cancel-in-progress: ${{ github.event_name == 'pull_request' || (github.event_name == 'push' && !startsWith(github.ref, 'refs/tags/')) }} permissions: contents: read @@ -45,6 +45,7 @@ jobs: run: echo "fingerprint=$(getconf GNU_LIBC_VERSION | tr ' ' '-')-rustc-$(rustc -V | awk '{print $2}')" >> "$GITHUB_OUTPUT" - name: Cache cargo registry and target + if: runner.environment == 'github-hosted' uses: actions/cache@v6 with: path: | diff --git a/.github/workflows/rust_binaries_dev.yml b/.github/workflows/rust_binaries_dev.yml index 563022d028..e49abdbbd8 100644 --- a/.github/workflows/rust_binaries_dev.yml +++ b/.github/workflows/rust_binaries_dev.yml @@ -46,6 +46,7 @@ jobs: uses: dtolnay/rust-toolchain@stable - name: Cache cargo registry and target + if: runner.environment == 'github-hosted' uses: actions/cache@v6 with: path: | @@ -114,6 +115,7 @@ jobs: targets: ${{ matrix.target }} - name: Cache cargo registry and target + if: runner.environment == 'github-hosted' uses: actions/cache@v6 with: path: | diff --git a/.github/workflows/rust_binaries_release.yml b/.github/workflows/rust_binaries_release.yml index a5a8b61455..f9446a917a 100644 --- a/.github/workflows/rust_binaries_release.yml +++ b/.github/workflows/rust_binaries_release.yml @@ -48,6 +48,7 @@ jobs: echo "OPENSSL_LIB_DIR=/usr/lib/aarch64-linux-gnu" >> "$GITHUB_ENV" - name: Cache cargo registry and target + if: runner.environment == 'github-hosted' uses: actions/cache@v6 with: path: | @@ -153,6 +154,7 @@ jobs: echo "CARGO_TARGET_AARCH64_UNKNOWN_LINUX_GNU_LINKER=aarch64-linux-gnu-gcc" >> "$GITHUB_ENV" - name: Cache cargo registry and target + if: runner.environment == 'github-hosted' uses: actions/cache@v6 with: path: | @@ -230,6 +232,7 @@ jobs: targets: ${{ matrix.target }} - name: Cache cargo registry and target + if: runner.environment == 'github-hosted' uses: actions/cache@v6 with: path: | @@ -305,6 +308,7 @@ jobs: uses: dtolnay/rust-toolchain@stable - name: Cache cargo registry and target + if: runner.environment == 'github-hosted' uses: actions/cache@v6 with: path: | diff --git a/.github/workflows/s3-etag-acl-tests.yml b/.github/workflows/s3-etag-acl-tests.yml index 12425f6f70..5f8754a372 100644 --- a/.github/workflows/s3-etag-acl-tests.yml +++ b/.github/workflows/s3-etag-acl-tests.yml @@ -21,8 +21,8 @@ on: - '.github/workflows/s3-etag-acl-tests.yml' concurrency: - group: ${{ github.head_ref || github.ref }}/s3-etag-acl-tests - cancel-in-progress: true + group: ${{ github.workflow }}-${{ github.event.pull_request.number || (github.event_name == 'push' && github.ref) || github.run_id }} + cancel-in-progress: ${{ github.event_name == 'pull_request' || (github.event_name == 'push' && !startsWith(github.ref, 'refs/tags/')) }} permissions: contents: read @@ -40,6 +40,7 @@ jobs: uses: actions/setup-go@v7 with: go-version-file: 'go.mod' + cache: ${{ runner.environment == 'github-hosted' }} - name: Install SeaweedFS run: | diff --git a/.github/workflows/s3-example-integration-tests.yml b/.github/workflows/s3-example-integration-tests.yml index 4e39c3bc07..d0855c65ca 100644 --- a/.github/workflows/s3-example-integration-tests.yml +++ b/.github/workflows/s3-example-integration-tests.yml @@ -13,8 +13,8 @@ on: - '.github/workflows/s3-example-integration-tests.yml' concurrency: - group: ${{ github.head_ref }}/s3-integration-tests - cancel-in-progress: true + group: ${{ github.workflow }}-${{ github.event.pull_request.number || (github.event_name == 'push' && github.ref) || github.run_id }} + cancel-in-progress: ${{ github.event_name == 'pull_request' || (github.event_name == 'push' && !startsWith(github.ref, 'refs/tags/')) }} permissions: contents: read @@ -33,6 +33,7 @@ jobs: uses: actions/setup-go@v7 with: go-version-file: 'go.mod' + cache: ${{ runner.environment == 'github-hosted' }} id: go - name: Build SeaweedFS diff --git a/.github/workflows/s3-filer-group-tests.yml b/.github/workflows/s3-filer-group-tests.yml index bda1ac2bca..d714ca98e3 100644 --- a/.github/workflows/s3-filer-group-tests.yml +++ b/.github/workflows/s3-filer-group-tests.yml @@ -12,8 +12,8 @@ on: - '.github/workflows/s3-filer-group-tests.yml' concurrency: - group: ${{ github.head_ref }}/s3-filer-group-tests - cancel-in-progress: true + group: ${{ github.workflow }}-${{ github.event.pull_request.number || (github.event_name == 'push' && github.ref) || github.run_id }} + cancel-in-progress: ${{ github.event_name == 'pull_request' || (github.event_name == 'push' && !startsWith(github.ref, 'refs/tags/')) }} permissions: contents: read @@ -32,10 +32,14 @@ jobs: - name: Check out code uses: actions/checkout@v7 + - name: Clean up stale FUSE mounts + uses: ./.github/actions/cleanup-stale-fuse-mounts + - name: Set up Go uses: actions/setup-go@v7 with: go-version-file: 'go.mod' + cache: ${{ runner.environment == 'github-hosted' }} id: go - name: Install SeaweedFS @@ -50,7 +54,7 @@ jobs: echo "=== System Information ===" uname -a free -h - df -h + timeout 15 df -h || true echo "=== Starting Filer Group Tests ===" # Run tests with automatic server management diff --git a/.github/workflows/s3-go-tests.yml b/.github/workflows/s3-go-tests.yml index 19944e9eb1..1260e5af41 100644 --- a/.github/workflows/s3-go-tests.yml +++ b/.github/workflows/s3-go-tests.yml @@ -12,8 +12,8 @@ on: - '.github/workflows/s3-go-tests.yml' concurrency: - group: ${{ github.head_ref }}/s3-go-tests - cancel-in-progress: true + group: ${{ github.workflow }}-${{ github.event.pull_request.number || (github.event_name == 'push' && github.ref) || github.run_id }} + cancel-in-progress: ${{ github.event_name == 'pull_request' || (github.event_name == 'push' && !startsWith(github.ref, 'refs/tags/')) }} permissions: contents: read @@ -35,10 +35,14 @@ jobs: - name: Check out code uses: actions/checkout@v7 + - name: Clean up stale FUSE mounts + uses: ./.github/actions/cleanup-stale-fuse-mounts + - name: Set up Go uses: actions/setup-go@v7 with: go-version-file: 'go.mod' + cache: ${{ runner.environment == 'github-hosted' }} id: go - name: Install SeaweedFS @@ -53,7 +57,7 @@ jobs: echo "=== System Information ===" uname -a free -h - df -h + timeout 15 df -h || true echo "=== Starting Tests ===" # Run tests with automatic server management @@ -103,6 +107,7 @@ jobs: uses: actions/setup-go@v7 with: go-version-file: 'go.mod' + cache: ${{ runner.environment == 'github-hosted' }} id: go - name: Install SeaweedFS @@ -151,6 +156,7 @@ jobs: uses: actions/setup-go@v7 with: go-version-file: 'go.mod' + cache: ${{ runner.environment == 'github-hosted' }} id: go - name: Install SeaweedFS @@ -198,10 +204,14 @@ jobs: - name: Check out code uses: actions/checkout@v7 + - name: Clean up stale FUSE mounts + uses: ./.github/actions/cleanup-stale-fuse-mounts + - name: Set up Go uses: actions/setup-go@v7 with: go-version-file: 'go.mod' + cache: ${{ runner.environment == 'github-hosted' }} id: go - name: Install SeaweedFS @@ -216,7 +226,7 @@ jobs: echo "=== System Information ===" uname -a free -h - df -h + timeout 15 df -h || true echo "=== Starting Tests ===" # Run tests with automatic server management @@ -283,6 +293,7 @@ jobs: - TestLifecycleSuspendedVersioningExpiration - TestLifecycleTagFilter - TestLifecycleVersionedBucketCreatesDeleteMarker + - TestLifecycleVersionedBucketExpiresOnSecondRun steps: - name: Check out code @@ -292,6 +303,7 @@ jobs: uses: actions/setup-go@v7 with: go-version-file: 'go.mod' + cache: ${{ runner.environment == 'github-hosted' }} id: go - name: Install SeaweedFS @@ -333,10 +345,14 @@ jobs: - name: Check out code uses: actions/checkout@v7 + - name: Clean up stale FUSE mounts + uses: ./.github/actions/cleanup-stale-fuse-mounts + - name: Set up Go uses: actions/setup-go@v7 with: go-version-file: 'go.mod' + cache: ${{ runner.environment == 'github-hosted' }} id: go - name: Install SeaweedFS @@ -351,7 +367,7 @@ jobs: echo "=== System Information ===" uname -a free -h - df -h + timeout 15 df -h || true echo "=== Starting Tests ===" make test-with-server @@ -391,10 +407,14 @@ jobs: - name: Check out code uses: actions/checkout@v7 + - name: Clean up stale FUSE mounts + uses: ./.github/actions/cleanup-stale-fuse-mounts + - name: Set up Go uses: actions/setup-go@v7 with: go-version-file: 'go.mod' + cache: ${{ runner.environment == 'github-hosted' }} id: go - name: Install SeaweedFS @@ -409,7 +429,7 @@ jobs: echo "=== System Information ===" uname -a free -h - df -h + timeout 15 df -h || true echo "=== Starting Tests ===" # Run tests with automatic server management @@ -459,6 +479,7 @@ jobs: uses: actions/setup-go@v7 with: go-version-file: 'go.mod' + cache: ${{ runner.environment == 'github-hosted' }} id: go - name: Install SeaweedFS @@ -508,6 +529,7 @@ jobs: uses: actions/setup-go@v7 with: go-version-file: 'go.mod' + cache: ${{ runner.environment == 'github-hosted' }} id: go - name: Install SeaweedFS @@ -576,6 +598,7 @@ jobs: uses: actions/setup-go@v7 with: go-version-file: 'go.mod' + cache: ${{ runner.environment == 'github-hosted' }} id: go - name: Install SeaweedFS @@ -631,6 +654,7 @@ jobs: uses: actions/setup-go@v7 with: go-version-file: 'go.mod' + cache: ${{ runner.environment == 'github-hosted' }} id: go - name: Install SeaweedFS diff --git a/.github/workflows/s3-iam-tests.yml b/.github/workflows/s3-iam-tests.yml index 121f835083..9d57baf978 100644 --- a/.github/workflows/s3-iam-tests.yml +++ b/.github/workflows/s3-iam-tests.yml @@ -20,8 +20,8 @@ on: - '.github/workflows/s3-iam-tests.yml' concurrency: - group: ${{ github.head_ref }}/s3-iam-tests - cancel-in-progress: true + group: ${{ github.workflow }}-${{ github.event.pull_request.number || (github.event_name == 'push' && github.ref) || github.run_id }} + cancel-in-progress: ${{ github.event_name == 'pull_request' || (github.event_name == 'push' && !startsWith(github.ref, 'refs/tags/')) }} permissions: contents: read @@ -45,6 +45,7 @@ jobs: uses: actions/setup-go@v7 with: go-version-file: 'go.mod' + cache: ${{ runner.environment == 'github-hosted' }} id: go - name: Get dependencies @@ -90,10 +91,14 @@ jobs: - name: Check out code uses: actions/checkout@v7 + - name: Clean up stale FUSE mounts + uses: ./.github/actions/cleanup-stale-fuse-mounts + - name: Set up Go uses: actions/setup-go@v7 with: go-version-file: 'go.mod' + cache: ${{ runner.environment == 'github-hosted' }} id: go - name: Install SeaweedFS @@ -109,7 +114,7 @@ jobs: echo "=== System Information ===" uname -a free -h - df -h + timeout 15 df -h || true echo "=== Starting S3 IAM Integration Tests (${{ matrix.test-type }}) ===" # Set WEED_BINARY to use the installed version @@ -208,6 +213,7 @@ jobs: uses: actions/setup-go@v7 with: go-version-file: 'go.mod' + cache: ${{ runner.environment == 'github-hosted' }} id: go - name: Install SeaweedFS @@ -268,6 +274,7 @@ jobs: uses: actions/setup-go@v7 with: go-version-file: 'go.mod' + cache: ${{ runner.environment == 'github-hosted' }} id: go - name: Install SeaweedFS diff --git a/.github/workflows/s3-keycloak-tests.yml b/.github/workflows/s3-keycloak-tests.yml index d2677431c1..f3955ca3fc 100644 --- a/.github/workflows/s3-keycloak-tests.yml +++ b/.github/workflows/s3-keycloak-tests.yml @@ -16,8 +16,8 @@ on: - '.github/workflows/s3-keycloak-tests.yml' concurrency: - group: ${{ github.head_ref }}/s3-keycloak-tests - cancel-in-progress: true + group: ${{ github.workflow }}-${{ github.event.pull_request.number || (github.event_name == 'push' && github.ref) || github.run_id }} + cancel-in-progress: ${{ github.event_name == 'pull_request' || (github.event_name == 'push' && !startsWith(github.ref, 'refs/tags/')) }} permissions: contents: read @@ -37,10 +37,14 @@ jobs: - name: Check out code uses: actions/checkout@v7 + - name: Clean up stale FUSE mounts + uses: ./.github/actions/cleanup-stale-fuse-mounts + - name: Set up Go uses: actions/setup-go@v7 with: go-version-file: 'go.mod' + cache: ${{ runner.environment == 'github-hosted' }} id: go - name: Install SeaweedFS @@ -56,7 +60,7 @@ jobs: echo "=== System Information ===" uname -a free -h - df -h + timeout 15 df -h || true echo "=== Starting S3 Keycloak Integration Tests ===" # Set WEED_BINARY to use the installed version diff --git a/.github/workflows/s3-mutation-regression-tests.yml b/.github/workflows/s3-mutation-regression-tests.yml index acf09a8f27..d352215176 100644 --- a/.github/workflows/s3-mutation-regression-tests.yml +++ b/.github/workflows/s3-mutation-regression-tests.yml @@ -12,8 +12,8 @@ on: - '.github/workflows/s3-mutation-regression-tests.yml' concurrency: - group: ${{ github.head_ref }}/s3-mutation-regression-tests - cancel-in-progress: true + group: ${{ github.workflow }}-${{ github.event.pull_request.number || (github.event_name == 'push' && github.ref) || github.run_id }} + cancel-in-progress: ${{ github.event_name == 'pull_request' || (github.event_name == 'push' && !startsWith(github.ref, 'refs/tags/')) }} permissions: contents: read @@ -32,6 +32,7 @@ jobs: uses: actions/setup-go@v7 with: go-version-file: 'go.mod' + cache: ${{ runner.environment == 'github-hosted' }} - name: Run S3 versioning regression tests timeout-minutes: 20 @@ -85,6 +86,7 @@ jobs: uses: actions/setup-go@v7 with: go-version-file: 'go.mod' + cache: ${{ runner.environment == 'github-hosted' }} - name: Run S3 delete regression tests timeout-minutes: 15 @@ -123,6 +125,7 @@ jobs: uses: actions/setup-go@v7 with: go-version-file: 'go.mod' + cache: ${{ runner.environment == 'github-hosted' }} - name: Build SeaweedFS run: | diff --git a/.github/workflows/s3-parquet-tests.yml b/.github/workflows/s3-parquet-tests.yml index b86c3869e4..d0a99b04be 100644 --- a/.github/workflows/s3-parquet-tests.yml +++ b/.github/workflows/s3-parquet-tests.yml @@ -23,6 +23,10 @@ env: S3_ENDPOINT_URL: http://localhost:8333 BUCKET_NAME: test-parquet-bucket +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || (github.event_name == 'push' && github.ref) || github.run_id }} + cancel-in-progress: ${{ github.event_name == 'pull_request' || (github.event_name == 'push' && !startsWith(github.ref, 'refs/tags/')) }} + jobs: parquet-integration-tests: name: PyArrow Parquet Tests (Python ${{ matrix.python-version }}) @@ -42,13 +46,13 @@ jobs: uses: actions/setup-go@v7 with: go-version: ^1.26 - cache: true + cache: ${{ runner.environment == 'github-hosted' }} - name: Set up Python ${{ matrix.python-version }} uses: actions/setup-python@v7 with: python-version: ${{ matrix.python-version }} - cache: 'pip' + cache: ${{ runner.environment == 'github-hosted' && 'pip' || '' }} cache-dependency-path: 'test/s3/parquet/requirements.txt' - name: Install system dependencies @@ -150,7 +154,7 @@ jobs: uses: actions/setup-go@v7 with: go-version: ^1.26 - cache: true + cache: ${{ runner.environment == 'github-hosted' }} - name: Run Go unit tests run: | diff --git a/.github/workflows/s3-policy-tests.yml b/.github/workflows/s3-policy-tests.yml index 7bb900c150..2fbeff3474 100644 --- a/.github/workflows/s3-policy-tests.yml +++ b/.github/workflows/s3-policy-tests.yml @@ -24,8 +24,8 @@ on: - '.github/workflows/s3-policy-tests.yml' concurrency: - group: ${{ github.head_ref }}/s3-policy-tests - cancel-in-progress: true + group: ${{ github.workflow }}-${{ github.event.pull_request.number || (github.event_name == 'push' && github.ref) || github.run_id }} + cancel-in-progress: ${{ github.event_name == 'pull_request' || (github.event_name == 'push' && !startsWith(github.ref, 'refs/tags/')) }} permissions: contents: read @@ -49,6 +49,7 @@ jobs: uses: actions/setup-go@v7 with: go-version-file: 'go.mod' + cache: ${{ runner.environment == 'github-hosted' }} id: go - name: Get dependencies @@ -88,10 +89,14 @@ jobs: - name: Check out code uses: actions/checkout@v7 + - name: Clean up stale FUSE mounts + uses: ./.github/actions/cleanup-stale-fuse-mounts + - name: Set up Go uses: actions/setup-go@v7 with: go-version-file: 'go.mod' + cache: ${{ runner.environment == 'github-hosted' }} id: go - name: Install SeaweedFS @@ -106,7 +111,7 @@ jobs: echo "=== System Information ===" uname -a free -h - df -h + timeout 15 df -h || true echo "=== Starting S3 Policy Variables Integration Tests ===" @@ -197,10 +202,14 @@ jobs: - name: Check out code uses: actions/checkout@v7 + - name: Clean up stale FUSE mounts + uses: ./.github/actions/cleanup-stale-fuse-mounts + - name: Set up Go uses: actions/setup-go@v7 with: go-version-file: 'go.mod' + cache: ${{ runner.environment == 'github-hosted' }} id: go - name: Install SeaweedFS @@ -215,7 +224,7 @@ jobs: echo "=== System Information ===" uname -a free -h - df -h + timeout 15 df -h || true echo "=== Starting S3 Policy Enforcement Tests (${{ matrix.test-case }}) ===" @@ -319,6 +328,7 @@ jobs: uses: actions/setup-go@v7 with: go-version-file: 'go.mod' + cache: ${{ runner.environment == 'github-hosted' }} id: go - name: Install SeaweedFS @@ -406,6 +416,7 @@ jobs: uses: actions/setup-go@v7 with: go-version-file: 'go.mod' + cache: ${{ runner.environment == 'github-hosted' }} id: go - name: Install SeaweedFS diff --git a/.github/workflows/s3-proxy-signature-tests.yml b/.github/workflows/s3-proxy-signature-tests.yml index a8cc6c3679..ded2068737 100644 --- a/.github/workflows/s3-proxy-signature-tests.yml +++ b/.github/workflows/s3-proxy-signature-tests.yml @@ -21,8 +21,8 @@ on: - '.github/workflows/s3-proxy-signature-tests.yml' concurrency: - group: ${{ github.head_ref || github.ref }}/s3-proxy-signature-tests - cancel-in-progress: true + group: ${{ github.workflow }}-${{ github.event.pull_request.number || (github.event_name == 'push' && github.ref) || github.run_id }} + cancel-in-progress: ${{ github.event_name == 'pull_request' || (github.event_name == 'push' && !startsWith(github.ref, 'refs/tags/')) }} permissions: contents: read @@ -40,6 +40,7 @@ jobs: uses: actions/setup-go@v7 with: go-version-file: 'go.mod' + cache: ${{ runner.environment == 'github-hosted' }} id: go - name: Configure Docker Hub mirror diff --git a/.github/workflows/s3-sdk-v2-routing-tests.yml b/.github/workflows/s3-sdk-v2-routing-tests.yml index e63528e145..65aac2004a 100644 --- a/.github/workflows/s3-sdk-v2-routing-tests.yml +++ b/.github/workflows/s3-sdk-v2-routing-tests.yml @@ -15,8 +15,8 @@ on: - '.github/workflows/s3-sdk-v2-routing-tests.yml' concurrency: - group: ${{ github.head_ref || github.ref }}/s3-sdk-v2-routing-tests - cancel-in-progress: true + group: ${{ github.workflow }}-${{ github.event.pull_request.number || (github.event_name == 'push' && github.ref) || github.run_id }} + cancel-in-progress: ${{ github.event_name == 'pull_request' || (github.event_name == 'push' && !startsWith(github.ref, 'refs/tags/')) }} permissions: contents: read @@ -34,6 +34,7 @@ jobs: uses: actions/setup-go@v7 with: go-version-file: 'go.mod' + cache: ${{ runner.environment == 'github-hosted' }} - name: Install SeaweedFS run: | diff --git a/.github/workflows/s3-snowflake-tests.yml b/.github/workflows/s3-snowflake-tests.yml index ec3ec6a32a..94163e36cd 100644 --- a/.github/workflows/s3-snowflake-tests.yml +++ b/.github/workflows/s3-snowflake-tests.yml @@ -60,6 +60,7 @@ jobs: uses: actions/setup-go@v7 with: go-version-file: 'go.mod' + cache: ${{ runner.environment == 'github-hosted' }} id: go - name: Set up Java @@ -67,7 +68,7 @@ jobs: with: java-version: '17' distribution: 'temurin' - cache: 'maven' + cache: ${{ runner.environment == 'github-hosted' && 'maven' || '' }} - name: Install SeaweedFS run: | diff --git a/.github/workflows/s3-spark-tests.yml b/.github/workflows/s3-spark-tests.yml index c8682992de..59fae691a8 100644 --- a/.github/workflows/s3-spark-tests.yml +++ b/.github/workflows/s3-spark-tests.yml @@ -11,8 +11,8 @@ on: workflow_dispatch: concurrency: - group: ${{ github.head_ref }}/s3-spark-tests - cancel-in-progress: true + group: ${{ github.workflow }}-${{ github.event.pull_request.number || (github.event_name == 'push' && github.ref) || github.run_id }} + cancel-in-progress: ${{ github.event_name == 'pull_request' || (github.event_name == 'push' && !startsWith(github.ref, 'refs/tags/')) }} permissions: contents: read @@ -27,10 +27,14 @@ jobs: - name: Check out code uses: actions/checkout@v7 + - name: Clean up stale FUSE mounts + uses: ./.github/actions/cleanup-stale-fuse-mounts + - name: Set up Go uses: actions/setup-go@v7 with: go-version-file: 'go.mod' + cache: ${{ runner.environment == 'github-hosted' }} id: go - name: Configure Docker Hub mirror @@ -56,7 +60,7 @@ jobs: echo "=== System Information ===" uname -a free -h - df -h + timeout 15 df -h || true echo "=== Starting S3 Spark Integration Tests ===" go test -v -timeout 30m . 2>&1 | tee test-output.log || { diff --git a/.github/workflows/s3-sse-tests.yml b/.github/workflows/s3-sse-tests.yml index d1e2102203..2ea0531f0a 100644 --- a/.github/workflows/s3-sse-tests.yml +++ b/.github/workflows/s3-sse-tests.yml @@ -23,8 +23,8 @@ on: - 'test/s3/sse/**' concurrency: - group: ${{ github.head_ref }}/s3-sse-tests - cancel-in-progress: true + group: ${{ github.workflow }}-${{ github.event.pull_request.number || (github.event_name == 'push' && github.ref) || github.run_id }} + cancel-in-progress: ${{ github.event_name == 'pull_request' || (github.event_name == 'push' && !startsWith(github.ref, 'refs/tags/')) }} permissions: contents: read @@ -46,10 +46,14 @@ jobs: - name: Check out code uses: actions/checkout@v7 + - name: Clean up stale FUSE mounts + uses: ./.github/actions/cleanup-stale-fuse-mounts + - name: Set up Go uses: actions/setup-go@v7 with: go-version-file: 'go.mod' + cache: ${{ runner.environment == 'github-hosted' }} id: go - name: Install SeaweedFS @@ -64,7 +68,7 @@ jobs: echo "=== System Information ===" uname -a free -h - df -h + timeout 15 df -h || true echo "=== Starting SSE Tests ===" # Run tests with automatic server management @@ -118,6 +122,7 @@ jobs: uses: actions/setup-go@v7 with: go-version-file: 'go.mod' + cache: ${{ runner.environment == 'github-hosted' }} id: go - name: Install SeaweedFS @@ -166,6 +171,7 @@ jobs: uses: actions/setup-go@v7 with: go-version-file: 'go.mod' + cache: ${{ runner.environment == 'github-hosted' }} id: go - name: Install SeaweedFS @@ -215,6 +221,7 @@ jobs: uses: actions/setup-go@v7 with: go-version-file: 'go.mod' + cache: ${{ runner.environment == 'github-hosted' }} id: go - name: Install SeaweedFS @@ -264,6 +271,7 @@ jobs: uses: actions/setup-go@v7 with: go-version-file: 'go.mod' + cache: ${{ runner.environment == 'github-hosted' }} id: go - name: Install SeaweedFS @@ -315,6 +323,7 @@ jobs: uses: actions/setup-go@v7 with: go-version-file: 'go.mod' + cache: ${{ runner.environment == 'github-hosted' }} id: go - name: Install SeaweedFS @@ -363,6 +372,7 @@ jobs: uses: actions/setup-go@v7 with: go-version-file: 'go.mod' + cache: ${{ runner.environment == 'github-hosted' }} id: go - name: Install SeaweedFS diff --git a/.github/workflows/s3-tables-tests.yml b/.github/workflows/s3-tables-tests.yml index 19c1395fe6..19674a3325 100644 --- a/.github/workflows/s3-tables-tests.yml +++ b/.github/workflows/s3-tables-tests.yml @@ -12,6 +12,10 @@ on: - 'go.sum' - '.github/workflows/s3-tables-tests.yml' +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || (github.event_name == 'push' && github.ref) || github.run_id }} + cancel-in-progress: ${{ github.event_name == 'pull_request' || (github.event_name == 'push' && !startsWith(github.ref, 'refs/tags/')) }} + permissions: contents: read @@ -26,10 +30,14 @@ jobs: - name: Check out code uses: actions/checkout@v7 + - name: Clean up stale FUSE mounts + uses: ./.github/actions/cleanup-stale-fuse-mounts + - name: Set up Go uses: actions/setup-go@v7 with: go-version-file: 'go.mod' + cache: ${{ runner.environment == 'github-hosted' }} id: go - name: Run go mod tidy @@ -48,7 +56,8 @@ jobs: echo "=== System Information ===" uname -a free -h - df -h + # A stale FUSE mount can hang or fail df; keep diagnostics best effort. + timeout 15 df -h || true echo "=== Starting S3 Tables Tests ===" # Run S3 Tables integration tests @@ -86,10 +95,14 @@ jobs: - name: Check out code uses: actions/checkout@v7 + - name: Clean up stale FUSE mounts + uses: ./.github/actions/cleanup-stale-fuse-mounts + - name: Set up Go uses: actions/setup-go@v7 with: go-version-file: 'go.mod' + cache: ${{ runner.environment == 'github-hosted' }} id: go - name: Configure Docker Hub mirror @@ -109,7 +122,7 @@ jobs: echo "=== System Information ===" uname -a free -h - df -h + timeout 15 df -h || true echo "=== Starting Iceberg Catalog Tests ===" # Run Iceberg catalog integration tests using Makefile (handles build) @@ -147,10 +160,14 @@ jobs: - name: Check out code uses: actions/checkout@v7 + - name: Clean up stale FUSE mounts + uses: ./.github/actions/cleanup-stale-fuse-mounts + - name: Set up Go uses: actions/setup-go@v7 with: go-version-file: 'go.mod' + cache: ${{ runner.environment == 'github-hosted' }} id: go - name: Configure Docker Hub mirror @@ -179,7 +196,7 @@ jobs: echo "=== System Information ===" uname -a free -h - df -h + timeout 15 df -h || true echo "=== Starting Trino Iceberg Catalog Tests ===" # Run Trino + Iceberg catalog integration tests @@ -217,10 +234,14 @@ jobs: - name: Check out code uses: actions/checkout@v7 + - name: Clean up stale FUSE mounts + uses: ./.github/actions/cleanup-stale-fuse-mounts + - name: Set up Go uses: actions/setup-go@v7 with: go-version-file: 'go.mod' + cache: ${{ runner.environment == 'github-hosted' }} id: go - name: Configure Docker Hub mirror @@ -250,7 +271,7 @@ jobs: echo "=== System Information ===" uname -a free -h - df -h + timeout 15 df -h || true docker info echo "=== Starting Dremio Iceberg Catalog Tests ===" @@ -290,10 +311,14 @@ jobs: - name: Check out code uses: actions/checkout@v7 + - name: Clean up stale FUSE mounts + uses: ./.github/actions/cleanup-stale-fuse-mounts + - name: Set up Go uses: actions/setup-go@v7 with: go-version-file: 'go.mod' + cache: ${{ runner.environment == 'github-hosted' }} id: go - name: Configure Docker Hub mirror @@ -323,7 +348,7 @@ jobs: echo "=== System Information ===" uname -a free -h - df -h + timeout 15 df -h || true docker info echo "=== Starting Doris Iceberg Catalog Tests ===" @@ -373,10 +398,14 @@ jobs: - name: Check out code uses: actions/checkout@v7 + - name: Clean up stale FUSE mounts + uses: ./.github/actions/cleanup-stale-fuse-mounts + - name: Set up Go uses: actions/setup-go@v7 with: go-version-file: 'go.mod' + cache: ${{ runner.environment == 'github-hosted' }} id: go - name: Configure Docker Hub mirror @@ -408,7 +437,7 @@ jobs: echo "=== System Information ===" uname -a free -h - df -h + timeout 15 df -h || true docker info echo "=== Starting ClickHouse Iceberg Catalog Tests ===" @@ -460,10 +489,14 @@ jobs: - name: Check out code uses: actions/checkout@v7 + - name: Clean up stale FUSE mounts + uses: ./.github/actions/cleanup-stale-fuse-mounts + - name: Set up Go uses: actions/setup-go@v7 with: go-version-file: 'go.mod' + cache: ${{ runner.environment == 'github-hosted' }} id: go - name: Configure Docker Hub mirror @@ -496,7 +529,7 @@ jobs: echo "=== System Information ===" uname -a free -h - df -h + timeout 15 df -h || true docker info echo "=== Starting OLake Iceberg Catalog Tests ===" @@ -559,10 +592,14 @@ jobs: - name: Check out code uses: actions/checkout@v7 + - name: Clean up stale FUSE mounts + uses: ./.github/actions/cleanup-stale-fuse-mounts + - name: Set up Go uses: actions/setup-go@v7 with: go-version-file: 'go.mod' + cache: ${{ runner.environment == 'github-hosted' }} id: go - name: Run go mod tidy @@ -590,7 +627,7 @@ jobs: echo "=== System Information ===" uname -a free -h - df -h + timeout 15 df -h || true echo "=== Starting Polaris Tests ===" go test -v -timeout 20m ./test/s3tables/polaris 2>&1 | tee test/s3tables/polaris/test-output.log || { @@ -627,10 +664,14 @@ jobs: - name: Check out code uses: actions/checkout@v7 + - name: Clean up stale FUSE mounts + uses: ./.github/actions/cleanup-stale-fuse-mounts + - name: Set up Go uses: actions/setup-go@v7 with: go-version-file: 'go.mod' + cache: ${{ runner.environment == 'github-hosted' }} id: go - name: Configure Docker Hub mirror @@ -659,7 +700,7 @@ jobs: echo "=== System Information ===" uname -a free -h - df -h + timeout 15 df -h || true echo "=== Starting Spark Iceberg Catalog Tests ===" # Run Spark + Iceberg catalog integration tests @@ -697,10 +738,14 @@ jobs: - name: Check out code uses: actions/checkout@v7 + - name: Clean up stale FUSE mounts + uses: ./.github/actions/cleanup-stale-fuse-mounts + - name: Set up Go uses: actions/setup-go@v7 with: go-version-file: 'go.mod' + cache: ${{ runner.environment == 'github-hosted' }} id: go - name: Configure Docker Hub mirror @@ -730,7 +775,7 @@ jobs: echo "=== System Information ===" uname -a free -h - df -h + timeout 15 df -h || true echo "=== Starting RisingWave Catalog Tests ===" # Run RisingWave catalog integration tests @@ -768,10 +813,14 @@ jobs: - name: Check out code uses: actions/checkout@v7 + - name: Clean up stale FUSE mounts + uses: ./.github/actions/cleanup-stale-fuse-mounts + - name: Set up Go uses: actions/setup-go@v7 with: go-version-file: 'go.mod' + cache: ${{ runner.environment == 'github-hosted' }} id: go - name: Configure Docker Hub mirror @@ -785,6 +834,7 @@ jobs: - name: Restore python:3 image cache id: python-image + if: runner.environment == 'github-hosted' uses: actions/cache@v6 with: path: /tmp/python3-image.tar @@ -824,7 +874,7 @@ jobs: echo "=== System Information ===" uname -a free -h - df -h + timeout 15 df -h || true echo "=== Starting STS Integration Tests ===" # Run STS integration tests @@ -862,10 +912,14 @@ jobs: - name: Check out code uses: actions/checkout@v7 + - name: Clean up stale FUSE mounts + uses: ./.github/actions/cleanup-stale-fuse-mounts + - name: Set up Go uses: actions/setup-go@v7 with: go-version-file: 'go.mod' + cache: ${{ runner.environment == 'github-hosted' }} id: go - name: Run go mod tidy @@ -884,7 +938,7 @@ jobs: echo "=== System Information ===" uname -a free -h - df -h + timeout 15 df -h || true echo "=== Starting Lakekeeper Integration Tests ===" # Run Lakekeeper integration tests @@ -922,10 +976,14 @@ jobs: - name: Check out code uses: actions/checkout@v7 + - name: Clean up stale FUSE mounts + uses: ./.github/actions/cleanup-stale-fuse-mounts + - name: Set up Go uses: actions/setup-go@v7 with: go-version-file: 'go.mod' + cache: ${{ runner.environment == 'github-hosted' }} id: go - name: Configure Docker Hub mirror @@ -955,7 +1013,7 @@ jobs: echo "=== System Information ===" uname -a free -h - df -h + timeout 15 df -h || true docker info echo "=== Starting Unity Catalog Tests ===" @@ -995,10 +1053,14 @@ jobs: - name: Check out code uses: actions/checkout@v7 + - name: Clean up stale FUSE mounts + uses: ./.github/actions/cleanup-stale-fuse-mounts + - name: Set up Go uses: actions/setup-go@v7 with: go-version-file: 'go.mod' + cache: ${{ runner.environment == 'github-hosted' }} id: go - name: Configure Docker Hub mirror @@ -1027,7 +1089,7 @@ jobs: echo "=== System Information ===" uname -a free -h - df -h + timeout 15 df -h || true docker info echo "=== Starting LanceDB Namespace Tests ===" @@ -1069,6 +1131,7 @@ jobs: uses: actions/setup-go@v7 with: go-version-file: 'go.mod' + cache: ${{ runner.environment == 'github-hosted' }} id: go - name: Configure Docker Hub mirror @@ -1137,10 +1200,14 @@ jobs: # so do not leave a token in the checkout for it to pick up. persist-credentials: false + - name: Clean up stale FUSE mounts + uses: ./.github/actions/cleanup-stale-fuse-mounts + - name: Set up Go uses: actions/setup-go@v7 with: go-version-file: 'go.mod' + cache: ${{ runner.environment == 'github-hosted' }} id: go - name: Configure Docker Hub mirror @@ -1170,7 +1237,7 @@ jobs: echo "=== System Information ===" uname -a free -h - df -h + timeout 15 df -h || true docker info echo "=== Starting DuckDB Lance Tests ===" @@ -1212,10 +1279,14 @@ jobs: # so do not leave a token in the checkout for it to pick up. persist-credentials: false + - name: Clean up stale FUSE mounts + uses: ./.github/actions/cleanup-stale-fuse-mounts + - name: Set up Go uses: actions/setup-go@v7 with: go-version-file: 'go.mod' + cache: ${{ runner.environment == 'github-hosted' }} id: go - name: Configure Docker Hub mirror @@ -1244,7 +1315,7 @@ jobs: echo "=== System Information ===" uname -a free -h - df -h + timeout 15 df -h || true docker info echo "=== Starting Spark Lance Namespace Tests ===" @@ -1286,6 +1357,7 @@ jobs: uses: actions/setup-go@v7 with: go-version-file: 'go.mod' + cache: ${{ runner.environment == 'github-hosted' }} id: go - name: Verify S3 Tables Package Builds @@ -1351,6 +1423,7 @@ jobs: uses: actions/setup-go@v7 with: go-version-file: 'go.mod' + cache: ${{ runner.environment == 'github-hosted' }} id: go - name: Check Go Format @@ -1402,6 +1475,7 @@ jobs: uses: actions/setup-go@v7 with: go-version-file: 'go.mod' + cache: ${{ runner.environment == 'github-hosted' }} id: go - name: Run Go Vet diff --git a/.github/workflows/s3tests.yml b/.github/workflows/s3tests.yml index 7d481a3f63..971cd033d5 100644 --- a/.github/workflows/s3tests.yml +++ b/.github/workflows/s3tests.yml @@ -25,8 +25,8 @@ on: - '.github/workflows/s3tests.yml' concurrency: - group: ${{ github.head_ref }}/s3tests - cancel-in-progress: true + group: ${{ github.workflow }}-${{ github.event.pull_request.number || (github.event_name == 'push' && github.ref) || github.run_id }} + cancel-in-progress: ${{ github.event_name == 'pull_request' || (github.event_name == 'push' && !startsWith(github.ref, 'refs/tags/')) }} permissions: contents: read @@ -44,6 +44,7 @@ jobs: uses: actions/setup-go@v7 with: go-version-file: 'go.mod' + cache: ${{ runner.environment == 'github-hosted' }} id: go - name: Set up Python @@ -57,9 +58,9 @@ jobs: cd s3-tests sudo apt-get update -qq sudo apt-get install -y -qq libxml2-dev libxslt1-dev zlib1g-dev - pip install -r requirements.txt - pip install tox - pip install -e . + python3 -m pip install -r requirements.txt + python3 -m pip install tox + python3 -m pip install -e . - name: Fix S3 tests bucket creation conflicts run: | @@ -396,6 +397,7 @@ jobs: uses: actions/setup-go@v7 with: go-version-file: 'go.mod' + cache: ${{ runner.environment == 'github-hosted' }} id: go - name: Set up Python @@ -409,9 +411,9 @@ jobs: cd s3-tests sudo apt-get update -qq sudo apt-get install -y -qq libxml2-dev libxslt1-dev zlib1g-dev - pip install -r requirements.txt - pip install tox - pip install -e . + python3 -m pip install -r requirements.txt + python3 -m pip install tox + python3 -m pip install -e . - name: Fix S3 tests bucket creation conflicts run: | @@ -566,6 +568,7 @@ jobs: uses: actions/setup-go@v7 with: go-version-file: 'go.mod' + cache: ${{ runner.environment == 'github-hosted' }} id: go - name: Set up Python @@ -579,9 +582,9 @@ jobs: cd s3-tests sudo apt-get update -qq sudo apt-get install -y -qq libxml2-dev libxslt1-dev zlib1g-dev - pip install -r requirements.txt - pip install tox - pip install -e . + python3 -m pip install -r requirements.txt + python3 -m pip install tox + python3 -m pip install -e . - name: Run S3 CORS tests timeout-minutes: 10 @@ -691,6 +694,7 @@ jobs: uses: actions/setup-go@v7 with: go-version-file: 'go.mod' + cache: ${{ runner.environment == 'github-hosted' }} id: go - name: Run SeaweedFS Custom S3 Copy tests @@ -793,6 +797,7 @@ jobs: uses: actions/setup-go@v7 with: go-version-file: 'go.mod' + cache: ${{ runner.environment == 'github-hosted' }} id: go - name: Set up Python @@ -806,9 +811,9 @@ jobs: cd s3-tests sudo apt-get update -qq sudo apt-get install -y -qq libxml2-dev libxslt1-dev zlib1g-dev - pip install -r requirements.txt - pip install tox - pip install -e . + python3 -m pip install -r requirements.txt + python3 -m pip install tox + python3 -m pip install -e . - name: Run Ceph S3 tests with SQL store timeout-minutes: 15 diff --git a/.github/workflows/samba-integration.yml b/.github/workflows/samba-integration.yml index 082ec5a6df..94b9d670d3 100644 --- a/.github/workflows/samba-integration.yml +++ b/.github/workflows/samba-integration.yml @@ -40,15 +40,23 @@ jobs: uses: actions/setup-go@v7 with: go-version-file: 'go.mod' + cache: ${{ runner.environment == 'github-hosted' }} - name: Start local Docker registry - run: docker run -d --restart=always -p 5000:5000 --name registry registry:2 + run: | + if ! docker start registry 2>/dev/null; then + docker rm -f registry >/dev/null 2>&1 || true + docker run -d --restart=always -p 5000:5000 --name registry registry:2 + fi - name: Set up Docker Buildx uses: docker/setup-buildx-action@v4 with: driver-opts: network=host + - name: Set up Docker Compose + uses: docker/setup-compose-action@v2 + - name: Build weed race binary run: | cd docker diff --git a/.github/workflows/sftp-tests.yml b/.github/workflows/sftp-tests.yml index 377608dcf3..0930b06b92 100644 --- a/.github/workflows/sftp-tests.yml +++ b/.github/workflows/sftp-tests.yml @@ -17,8 +17,8 @@ on: - '.github/workflows/sftp-tests.yml' concurrency: - group: ${{ github.head_ref }}/sftp-tests - cancel-in-progress: true + group: ${{ github.workflow }}-${{ github.event.pull_request.number || (github.event_name == 'push' && github.ref) || github.run_id }} + cancel-in-progress: ${{ github.event_name == 'pull_request' || (github.event_name == 'push' && !startsWith(github.ref, 'refs/tags/')) }} permissions: contents: read @@ -40,6 +40,7 @@ jobs: uses: actions/setup-go@v7 with: go-version-file: 'go.mod' + cache: ${{ runner.environment == 'github-hosted' }} - name: Install dependencies run: | diff --git a/.github/workflows/spark-integration-tests.yml b/.github/workflows/spark-integration-tests.yml index 9a18898e2b..8aec50f8bd 100644 --- a/.github/workflows/spark-integration-tests.yml +++ b/.github/workflows/spark-integration-tests.yml @@ -15,6 +15,10 @@ on: - '.github/workflows/spark-integration-tests.yml' workflow_dispatch: +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || (github.event_name == 'push' && github.ref) || github.run_id }} + cancel-in-progress: ${{ github.event_name == 'pull_request' || (github.event_name == 'push' && !startsWith(github.ref, 'refs/tags/')) }} + permissions: contents: read checks: write @@ -38,12 +42,13 @@ jobs: with: java-version: '11' distribution: 'temurin' - cache: maven + cache: ${{ runner.environment == 'github-hosted' && 'maven' || '' }} - name: Set up Go uses: actions/setup-go@v7 with: go-version-file: 'go.mod' + cache: ${{ runner.environment == 'github-hosted' }} - name: Build SeaweedFS binary run: | diff --git a/.github/workflows/star_history.yml b/.github/workflows/star_history.yml index 0734b27352..016dd86c63 100644 --- a/.github/workflows/star_history.yml +++ b/.github/workflows/star_history.yml @@ -10,12 +10,11 @@ permissions: contents: write concurrency: - # Only one chart regeneration per branch at a time; a newer run on the same - # branch cancels an in-flight one so overlapping runs never conflict on - # note/star_history.svg during rebase. Scoped by ref so a manual run on - # another branch can't cancel the daily master update. + # Only one chart regeneration per branch at a time so overlapping runs never + # conflict on note/star_history.svg during rebase. Scoped by ref so a manual + # run on another branch can't cancel the daily master update. group: star-history-${{ github.ref }} - cancel-in-progress: true + cancel-in-progress: ${{ github.event_name == 'pull_request' || (github.event_name == 'push' && !startsWith(github.ref, 'refs/tags/')) }} jobs: render: @@ -33,7 +32,7 @@ jobs: uses: actions/setup-python@v7 with: python-version: "3.x" - cache: pip + cache: ${{ runner.environment == 'github-hosted' && 'pip' || '' }} cache-dependency-path: .github/scripts/star_history.py - name: Install matplotlib diff --git a/.github/workflows/telemetry-integration.yml b/.github/workflows/telemetry-integration.yml index 68dd2d1b79..b7aa690fb6 100644 --- a/.github/workflows/telemetry-integration.yml +++ b/.github/workflows/telemetry-integration.yml @@ -14,6 +14,10 @@ on: - 'weed/telemetry/**' - '.github/workflows/telemetry-integration.yml' +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || (github.event_name == 'push' && github.ref) || github.run_id }} + cancel-in-progress: ${{ github.event_name == 'pull_request' || (github.event_name == 'push' && !startsWith(github.ref, 'refs/tags/')) }} + permissions: contents: read @@ -30,6 +34,7 @@ jobs: uses: actions/setup-go@v7 with: go-version-file: 'go.mod' + cache: ${{ runner.environment == 'github-hosted' }} - name: Build telemetry server run: cd telemetry/server && go build -o telemetry-server . diff --git a/.github/workflows/terraform_ci.yml b/.github/workflows/terraform_ci.yml index 1e905e6f0e..b2bcca7500 100644 --- a/.github/workflows/terraform_ci.yml +++ b/.github/workflows/terraform_ci.yml @@ -8,6 +8,10 @@ on: branches: [ master ] paths: ['terraform/**', '.github/workflows/terraform_ci.yml'] +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || (github.event_name == 'push' && github.ref) || github.run_id }} + cancel-in-progress: ${{ github.event_name == 'pull_request' || (github.event_name == 'push' && !startsWith(github.ref, 'refs/tags/')) }} + permissions: contents: read @@ -62,6 +66,7 @@ jobs: uses: actions/setup-go@v7 with: go-version-file: go.mod + cache: ${{ runner.environment == 'github-hosted' }} - name: Build weed run: go build -o "$RUNNER_TEMP/weed" ./weed diff --git a/.github/workflows/test-s3-over-https-using-awscli.yml b/.github/workflows/test-s3-over-https-using-awscli.yml index 903c3bf912..8ccb35eeb7 100644 --- a/.github/workflows/test-s3-over-https-using-awscli.yml +++ b/.github/workflows/test-s3-over-https-using-awscli.yml @@ -27,6 +27,10 @@ defaults: run: working-directory: weed +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || (github.event_name == 'push' && github.ref) || github.run_id }} + cancel-in-progress: ${{ github.event_name == 'pull_request' || (github.event_name == 'push' && !startsWith(github.ref, 'refs/tags/')) }} + jobs: awscli-tests: runs-on: ubuntu-latest @@ -37,6 +41,7 @@ jobs: - uses: actions/setup-go@v7 with: go-version: ^1.26 + cache: ${{ runner.environment == 'github-hosted' }} - name: Build SeaweedFS run: | diff --git a/.github/workflows/tls-rotation-tests.yml b/.github/workflows/tls-rotation-tests.yml index ce4ecc99d2..669ca4794e 100644 --- a/.github/workflows/tls-rotation-tests.yml +++ b/.github/workflows/tls-rotation-tests.yml @@ -18,6 +18,10 @@ on: - 'go.sum' - '.github/workflows/tls-rotation-tests.yml' +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || (github.event_name == 'push' && github.ref) || github.run_id }} + cancel-in-progress: ${{ github.event_name == 'pull_request' || (github.event_name == 'push' && !startsWith(github.ref, 'refs/tags/')) }} + permissions: contents: read @@ -30,6 +34,7 @@ jobs: - name: Set up Go 1.x uses: actions/setup-go@v7 with: + cache: ${{ runner.environment == 'github-hosted' }} go-version: ^1.26 id: go diff --git a/.github/workflows/tus-tests.yml b/.github/workflows/tus-tests.yml index 9e7f4e1c1f..1140c7f505 100644 --- a/.github/workflows/tus-tests.yml +++ b/.github/workflows/tus-tests.yml @@ -11,8 +11,8 @@ on: - '.github/workflows/tus-tests.yml' concurrency: - group: ${{ github.head_ref || github.ref }}/tus-tests - cancel-in-progress: true + group: ${{ github.workflow }}-${{ github.event.pull_request.number || (github.event_name == 'push' && github.ref) || github.run_id }} + cancel-in-progress: ${{ github.event_name == 'pull_request' || (github.event_name == 'push' && !startsWith(github.ref, 'refs/tags/')) }} permissions: contents: read @@ -31,10 +31,14 @@ jobs: - name: Check out code uses: actions/checkout@v7 + - name: Clean up stale FUSE mounts + uses: ./.github/actions/cleanup-stale-fuse-mounts + - name: Set up Go uses: actions/setup-go@v7 with: go-version-file: 'go.mod' + cache: ${{ runner.environment == 'github-hosted' }} id: go - name: Install SeaweedFS @@ -49,7 +53,7 @@ jobs: echo "=== System Information ===" uname -a free -h - df -h + timeout 15 df -h || true echo "=== Starting TUS Tests ===" # Run tests with automatic server management diff --git a/.github/workflows/vacuum-integration-tests.yml b/.github/workflows/vacuum-integration-tests.yml index e9f2a4ce13..93eefec26e 100644 --- a/.github/workflows/vacuum-integration-tests.yml +++ b/.github/workflows/vacuum-integration-tests.yml @@ -18,6 +18,10 @@ on: - 'go.sum' - '.github/workflows/vacuum-integration-tests.yml' +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || (github.event_name == 'push' && github.ref) || github.run_id }} + cancel-in-progress: ${{ github.event_name == 'pull_request' || (github.event_name == 'push' && !startsWith(github.ref, 'refs/tags/')) }} + permissions: contents: read @@ -30,6 +34,7 @@ jobs: - name: Set up Go 1.x uses: actions/setup-go@v7 with: + cache: ${{ runner.environment == 'github-hosted' }} go-version: ^1.26 id: go diff --git a/.github/workflows/volume-server-integration-tests.yml b/.github/workflows/volume-server-integration-tests.yml index c1874fa198..b1a4569111 100644 --- a/.github/workflows/volume-server-integration-tests.yml +++ b/.github/workflows/volume-server-integration-tests.yml @@ -21,8 +21,8 @@ on: - '.github/workflows/volume-server-integration-tests.yml' concurrency: - group: ${{ github.head_ref || github.ref }}/volume-server-integration-tests - cancel-in-progress: true + group: ${{ github.workflow }}-${{ github.event.pull_request.number || (github.event_name == 'push' && github.ref) || github.run_id }} + cancel-in-progress: ${{ github.event_name == 'pull_request' || (github.event_name == 'push' && !startsWith(github.ref, 'refs/tags/')) }} permissions: contents: read @@ -33,10 +33,44 @@ env: SHARD_COUNT: 3 jobs: + build-weed: + name: Build SeaweedFS Binary + runs-on: ubuntu-22.04 + timeout-minutes: 15 + + steps: + - name: Checkout code + uses: actions/checkout@v7 + + - name: Set up Go + uses: actions/setup-go@v7 + with: + go-version-file: 'go.mod' + cache: ${{ runner.environment == 'github-hosted' }} + + - name: Build SeaweedFS binary + run: | + cd weed + go build -o weed . + chmod +x weed + ./weed version + + - name: Package binary + run: tar cf weed-binary.tar weed/weed + + - name: Upload binary + uses: actions/upload-artifact@v7 + with: + name: weed-binary + if-no-files-found: error + retention-days: 1 + path: weed-binary.tar + volume-server-integration-tests: name: Volume Server Integration Tests (${{ matrix.test-type }} - Shard ${{ matrix.shard }}) runs-on: ubuntu-22.04 timeout-minutes: 45 + needs: [build-weed] strategy: fail-fast: false matrix: @@ -51,13 +85,18 @@ jobs: uses: actions/setup-go@v7 with: go-version-file: 'go.mod' + cache: ${{ runner.environment == 'github-hosted' }} - - name: Build SeaweedFS binary + - name: Download binary + uses: actions/download-artifact@v7 + with: + name: weed-binary + + - name: Restore binary run: | - cd weed - go build -o weed . - chmod +x weed - ./weed version + tar xf weed-binary.tar + chmod +x weed/weed + ./weed/weed version # Dealing the listed tests out one by one keeps the shards even. Bucketing # them by first letter did not: names cluster, so ^Test[I-S] drew 50 of diff --git a/.gitignore b/.gitignore index 9a3bf80cd3..c383623df9 100644 --- a/.gitignore +++ b/.gitignore @@ -144,3 +144,4 @@ weed_bin telemetry/server/telemetry-server .aider* /seaweed-volume/docs +__pycache__/ diff --git a/docker/Dockerfile.rocksdb_dev_env b/docker/Dockerfile.rocksdb_dev_env index 95154c3737..2b6b0431b1 100644 --- a/docker/Dockerfile.rocksdb_dev_env +++ b/docker/Dockerfile.rocksdb_dev_env @@ -3,7 +3,7 @@ FROM golang:1.26 AS builder RUN apt-get update RUN apt-get install -y build-essential libsnappy-dev zlib1g-dev libbz2-dev libgflags-dev liblz4-dev libzstd-dev -ARG ROCKSDB_VERSION=v10.10.1 +ARG ROCKSDB_VERSION=v11.1.2 ENV ROCKSDB_VERSION=${ROCKSDB_VERSION} # build RocksDB diff --git a/docker/Dockerfile.rocksdb_large b/docker/Dockerfile.rocksdb_large index d902c544e9..757af7e64d 100644 --- a/docker/Dockerfile.rocksdb_large +++ b/docker/Dockerfile.rocksdb_large @@ -3,7 +3,7 @@ FROM golang:1.26 AS builder RUN apt-get update RUN apt-get install -y build-essential libsnappy-dev zlib1g-dev libbz2-dev libgflags-dev liblz4-dev libzstd-dev -ARG ROCKSDB_VERSION=v10.10.1 +ARG ROCKSDB_VERSION=v11.1.2 ENV ROCKSDB_VERSION=${ROCKSDB_VERSION} # build RocksDB diff --git a/go.mod b/go.mod index 73251d4c3c..ccfaa9ab5d 100644 --- a/go.mod +++ b/go.mod @@ -362,7 +362,7 @@ require ( github.com/flynn/noise v1.1.0 // indirect github.com/gabriel-vasile/mimetype v1.4.13 // indirect github.com/geoffgarside/ber v1.2.0 // indirect - github.com/go-chi/chi/v5 v5.3.1 // indirect + github.com/go-chi/chi/v5 v5.4.0 // indirect github.com/go-darwin/apfs v0.0.0-20211011131704-f84b94dbf348 // indirect github.com/go-jose/go-jose/v4 v4.1.5 // indirect github.com/go-logr/logr v1.4.4 // indirect diff --git a/go.sum b/go.sum index 4c1630efe7..a9f7da602a 100644 --- a/go.sum +++ b/go.sum @@ -483,8 +483,8 @@ github.com/gin-gonic/gin v1.11.0 h1:OW/6PLjyusp2PPXtyxKHU0RbX6I/l28FTdDlae5ueWk= github.com/gin-gonic/gin v1.11.0/go.mod h1:+iq/FyxlGzII0KHiBGjuNn4UNENUlKbGlNmc+W50Dls= github.com/go-asn1-ber/asn1-ber v1.5.8 h1:H9AZkK22UOmfX8J84ubyaZxKJZ3FMHVwn8swoMML7iQ= github.com/go-asn1-ber/asn1-ber v1.5.8/go.mod h1:hEBeB/ic+5LoWskz+yKT7vGhhPYkProFKoKdwZRWMe0= -github.com/go-chi/chi/v5 v5.3.1 h1:3j4HZLGZQ3JpMCrPJF/Jl3mYJfWLKBfNJ6quurUGCf8= -github.com/go-chi/chi/v5 v5.3.1/go.mod h1:R+tYY2hNuVUUjxoPtqUdgBqevM9s9njzkTLutVsOCto= +github.com/go-chi/chi/v5 v5.4.0 h1:+ufzvFTGKyin26Or5DLrlIDq4ygIDBVcj7oVNeeaYFE= +github.com/go-chi/chi/v5 v5.4.0/go.mod h1:d1IG7GtKPSjPyBP365PH8sAGVHk3qfB/IhjFNRH1wWM= github.com/go-darwin/apfs v0.0.0-20211011131704-f84b94dbf348 h1:JnrjqG5iR07/8k7NqrLNilRsl3s1EPRQEGvbPyOce68= github.com/go-darwin/apfs v0.0.0-20211011131704-f84b94dbf348/go.mod h1:Czxo/d1g948LtrALAZdL04TL/HnkopquAjxYUuI02bo= github.com/go-errors/errors v1.5.1 h1:ZwEMSLRCapFLflTpT7NKaAc7ukJ8ZPEjzlxt8rPN8bk= diff --git a/k8s/charts/seaweedfs/templates/shared/post-install-bucket-hook.yaml b/k8s/charts/seaweedfs/templates/shared/post-install-bucket-hook.yaml index 11ea7feff6..3952c73fbf 100644 --- a/k8s/charts/seaweedfs/templates/shared/post-install-bucket-hook.yaml +++ b/k8s/charts/seaweedfs/templates/shared/post-install-bucket-hook.yaml @@ -172,7 +172,8 @@ spec: {{- $bucketName := .name }} {{- $bucketLock := or .lock .objectLock .withLock }} bucket_list=$(/bin/echo 's3.bucket.list' | /usr/bin/weed shell) || { echo "Error listing s3 buckets"; exit 1; } - if echo "$bucket_list" | awk '{print $1}' | grep -Fxq "{{ $bucketName }}"; then + {{- /* One awk: grep -q exits early, awk takes SIGPIPE, pipefail fails the test. */}} + if printf '%s\n' "$bucket_list" | awk -v bucket="{{ $bucketName }}" '$1 "" == bucket { found = 1 } END { exit !found }'; then echo "Bucket '{{ $bucketName }}' already exists, skipping creation." else echo "Creating bucket '{{ $bucketName }}'..." diff --git a/note/star_history.svg b/note/star_history.svg index ad8995f239..064e5b443b 100644 --- a/note/star_history.svg +++ b/note/star_history.svg @@ -6,7 +6,7 @@ - 2026-10-23T02:28:24.716056 + 2026-10-24T02:56:41.212930 image/svg+xml @@ -40,23 +40,23 @@ z - + - - + - + - + - + - + - + - + - + - + - + - + @@ -301,18 +301,18 @@ L 353.430232 26.215385 - + - + - + @@ -325,18 +325,18 @@ L 449.355659 26.215385 - + - + - + @@ -349,18 +349,18 @@ L 545.149861 26.215385 - + - + - + @@ -484,41 +484,41 @@ z - + - - + - + - + - + - + - + - + - + @@ -577,18 +577,18 @@ L 709.2 267.748071 - + - + - + @@ -599,18 +599,18 @@ L 709.2 223.798916 - + - + - + @@ -621,18 +621,18 @@ L 709.2 179.849762 - + - + - + @@ -643,18 +643,18 @@ L 709.2 135.900608 - + - + - + - + - + - + @@ -813,1023 +813,1009 @@ z +" clip-path="url(#p16b1f4eee4)" style="fill: none; stroke: #0969da; stroke-width: 1.6; stroke-linecap: square"/> + diff --git a/other/java/client/src/main/proto/filer.proto b/other/java/client/src/main/proto/filer.proto index 9a9813b956..6888e895da 100644 --- a/other/java/client/src/main/proto/filer.proto +++ b/other/java/client/src/main/proto/filer.proto @@ -771,7 +771,8 @@ message FilerConf { string replication = 3; string ttl = 4; string disk_type = 5; - bool fsync = 6; + // unset inherits from the enclosing path rule, set overrides it + optional bool fsync = 6; uint32 volume_growth_count = 7; bool read_only = 8; string data_center = 9; diff --git a/seaweed-volume/Cargo.toml b/seaweed-volume/Cargo.toml index 1e4a58f2a2..29c37d4ff3 100644 --- a/seaweed-volume/Cargo.toml +++ b/seaweed-volume/Cargo.toml @@ -157,6 +157,8 @@ windows-sys = { version = "0.61", features = ["Win32_Storage_FileSystem"] } [dev-dependencies] tempfile = "3" +# Paused-time tests drive the write-queue batch window deterministically. +tokio = { version = "1", features = ["full", "test-util"] } # Already a transitive dependency of aws-sdk-s3 at a single locked version; # needed directly only for the canned HttpClient in remote_storage::s3 tests. aws-smithy-runtime-api = "1" diff --git a/seaweed-volume/src/config.rs b/seaweed-volume/src/config.rs index a50e52e093..cf73731cc1 100644 --- a/seaweed-volume/src/config.rs +++ b/seaweed-volume/src/config.rs @@ -233,6 +233,25 @@ pub struct Cli { #[arg(long = "volume.allowUntrustedRemoteEndpoints", default_value_t = false)] pub allow_untrusted_remote_endpoints: bool, + /// Minimum bytes an fsync batch gathers before a drained queue flushes + /// without waiting. + #[arg(long = "fsyncMinBatchBytes", default_value_t = 1024 * 1024)] + pub fsync_min_batch_bytes: u64, + + /// Minimum requests an fsync batch gathers before a drained queue flushes + /// without waiting. + #[arg(long = "fsyncMinBatchRequests", default_value_t = 128)] + pub fsync_min_batch_requests: usize, + + /// How long a below-minimum fsync batch waits for more requests once the + /// queue drains; 0 flushes immediately. + #[arg( + long = "fsyncBatchWait", + default_value = "500us", + value_parser = parse_go_duration + )] + pub fsync_batch_wait: std::time::Duration, + /// A file of command line options, each line in optionName=optionValue format. #[arg(long = "options", default_value = "")] pub options: String, @@ -308,6 +327,14 @@ pub struct VolumeServerConfig { /// If true, FetchAndWriteNeedle skips remote S3 endpoint validation /// (allows loopback / link-local / metadata hosts). pub allow_untrusted_remote_endpoints: bool, + /// Minimum bytes an fsync batch gathers before a drained queue flushes + /// without waiting. + pub fsync_min_batch_bytes: u64, + /// Minimum requests an fsync batch gathers before a drained queue flushes + /// without waiting. + pub fsync_min_batch_requests: usize, + /// How long a below-minimum fsync batch waits for stragglers. + pub fsync_batch_wait: std::time::Duration, } pub use crate::storage::needle_map::NeedleMapKind; @@ -938,6 +965,9 @@ fn resolve_config_with_env(cli: Cli, env: EnvLookup<'_>) -> VolumeServerConfig { concurrent_download_limit: cli.concurrent_download_limit_mb as i64 * 1024 * 1024, inflight_upload_data_timeout: cli.inflight_upload_data_timeout, inflight_download_data_timeout: cli.inflight_download_data_timeout, + fsync_min_batch_bytes: cli.fsync_min_batch_bytes, + fsync_min_batch_requests: cli.fsync_min_batch_requests, + fsync_batch_wait: cli.fsync_batch_wait, has_slow_read: cli.has_slow_read, read_buffer_size_mb: cli.read_buffer_size_mb, ldb_timeout: cli.ldb_timeout, diff --git a/seaweed-volume/src/main.rs b/seaweed-volume/src/main.rs index c57a92e3b8..46fe45289d 100644 --- a/seaweed-volume/src/main.rs +++ b/seaweed-volume/src/main.rs @@ -23,7 +23,7 @@ use seaweed_volume::server::request_id::GrpcRequestIdLayer; use seaweed_volume::server::volume_server::{ RuntimeMetricsConfig, VolumeServerState, build_metrics_router, }; -use seaweed_volume::server::write_queue::WriteQueue; +use seaweed_volume::server::write_queue::{self, WriteQueue}; use seaweed_volume::storage::store::Store; use seaweed_volume::storage::types::DiskType; @@ -436,7 +436,17 @@ async fn run( // The write queue is always on: durable (fsync) uploads go through it so // a batch shares one .dat/.idx sync, the way Go's per-volume async writer // group-commits. Non-fsync writes stay on the synchronous path. - let wq = WriteQueue::new(state.clone(), 128); + let min_bytes = usize::try_from(config.fsync_min_batch_bytes) + .map_err(|_| "fsyncMinBatchBytes exceeds this platform's maximum usize value")?; + let wq = WriteQueue::new( + state.clone(), + 128, + write_queue::SyncBatchOption { + min_bytes, + min_requests: config.fsync_min_batch_requests, + wait: config.fsync_batch_wait, + }, + ); let _ = state.write_queue.set(wq); // Set initial metric gauges for concurrent limits and max volumes diff --git a/seaweed-volume/src/server/grpc_client.rs b/seaweed-volume/src/server/grpc_client.rs index 45a0170744..9567e43658 100644 --- a/seaweed-volume/src/server/grpc_client.rs +++ b/seaweed-volume/src/server/grpc_client.rs @@ -393,6 +393,9 @@ mod tests { fix_jpg_orientation: false, read_mode: ReadMode::Local, allow_untrusted_remote_endpoints: false, + fsync_min_batch_bytes: 1024 * 1024, + fsync_min_batch_requests: 128, + fsync_batch_wait: std::time::Duration::from_micros(500), cpu_profile: String::new(), mem_profile: String::new(), compaction_byte_per_second: 0, diff --git a/seaweed-volume/src/server/handlers.rs b/seaweed-volume/src/server/handlers.rs index 4eb390f314..1310793564 100644 --- a/seaweed-volume/src/server/handlers.rs +++ b/seaweed-volume/src/server/handlers.rs @@ -207,9 +207,15 @@ fn read_needle_for_get( if request.try_meta_first(plan.size()) { plan.read_meta(&mut n)?; if n.cookie != cookie { + // The meta read answered the request. + plan.note_meta_only_success(); return Ok(None); } if request.served_from_source(&n) { + if request.is_head { + // HEAD ends here; no payload read will record the success. + plan.note_meta_only_success(); + } let info = plan.into_stream_info(&n); return Ok(Some((n, Some(info)))); } @@ -237,6 +243,9 @@ struct StreamingBody { hold_read_lock_for_stream: bool, _held_read_lease: Option, io_unavailable: Arc, + /// Chunk read failures feed the volume's io-error streak, and finishing + /// the stream clears it — Go readNeedleDataInto parity. + io_errors: Arc, /// Pending chunk read; it hands `buf` back with the result. pending: Option)>>, /// Chunk buffer, reclaimed once the previous frame has been written out. @@ -298,12 +307,18 @@ impl http_body::Body for StreamingBody { ), ))); } + // Hyper may not poll again once Content-Length is + // satisfied, so the read's success is recorded here. + self.io_errors.check_read_write_error(None); } return std::task::Poll::Ready(Some(Ok(http_body::Frame::data( chunk, )))); } - Ok(Err(e)) => return std::task::Poll::Ready(Some(Err(e))), + Ok(Err(e)) => { + self.io_errors.check_read_write_error(Some(&e)); + return std::task::Poll::Ready(Some(Err(e))); + } Err(e) => { return std::task::Poll::Ready(Some(Err(std::io::Error::other(e)))); } @@ -314,6 +329,7 @@ impl http_body::Body for StreamingBody { let total = self.data_size as usize; if self.pos >= total { + self.io_errors.check_read_write_error(None); return std::task::Poll::Ready(None); } @@ -1105,6 +1121,10 @@ async fn get_or_head_handler_inner( let (etag, last_modified_str) = etag_and_last_modified(&n); if let Some(resp) = not_modified_response(&n, &headers, &etag, &last_modified_str) { + // A request that ends here was answered by the meta read alone. + if let Some(io_errors) = strategy.io_errors() { + io_errors.check_read_write_error(None); + } return resp; } @@ -1198,6 +1218,19 @@ struct ReadPlan { strategy: ReadStrategy, } +impl ReadStrategy { + /// The tracker a terminal meta read reports success to, when the reply + /// is served from the data file. + fn io_errors(&self) -> Option<&Arc> { + match self { + ReadStrategy::Stream(info) + | ReadStrategy::HeadFromMeta(info) + | ReadStrategy::RangeFromSource(info) => Some(&info.io_errors), + ReadStrategy::Buffered => None, + } + } +} + /// The 401 reply for a bad read JWT; Go checks it before NewVolumeId, so a /// bad path with JWT enabled is a 401, not a 400. fn reject_read_jwt( @@ -1486,41 +1519,78 @@ fn not_modified_response( etag: &str, last_modified_str: &Option, ) -> Option { - // If-Modified-Since first (Go order), then If-None-Match. + let not_modified = || { + let mut resp = StatusCode::NOT_MODIFIED.into_response(); + resp.headers_mut() + .insert(header::ETAG, etag.parse().unwrap()); + if let Some(lm) = last_modified_str { + resp.headers_mut() + .insert(header::LAST_MODIFIED, lm.parse().unwrap()); + } + resp + }; + + // If-Modified-Since applies only when there is no If-None-Match (RFC 9110 13.2.2). + if headers.contains_key(header::IF_NONE_MATCH) { + let inm = headers + .get_all(header::IF_NONE_MATCH) + .iter() + .map(|v| v.to_str().unwrap_or("")) + .collect::>() + .join(","); + return etag_list_matches(&inm, etag).then(not_modified); + } + if n.last_modified > 0 - && let Some(ims_header) = headers.get(header::IF_MODIFIED_SINCE) - && let Ok(ims_str) = ims_header.to_str() + && let Some(ims_str) = headers + .get(header::IF_MODIFIED_SINCE) + .and_then(|v| v.to_str().ok()) { // Parse HTTP date format: "Mon, 02 Jan 2006 15:04:05 GMT" if let Ok(ims_time) = chrono::NaiveDateTime::parse_from_str(ims_str, "%a, %d %b %Y %H:%M:%S GMT") && (n.last_modified as i64) <= ims_time.and_utc().timestamp() { - let mut resp = StatusCode::NOT_MODIFIED.into_response(); - if let Some(lm) = last_modified_str { - resp.headers_mut() - .insert(header::LAST_MODIFIED, lm.parse().unwrap()); - } - // Go sets ETag after the 304 paths, so a 304 has no ETag. - return Some(resp); + return Some(not_modified()); } } - - if let Some(if_none_match) = headers.get(header::IF_NONE_MATCH) - && let Ok(inm) = if_none_match.to_str() - && inm == etag - { - let mut resp = StatusCode::NOT_MODIFIED.into_response(); - if let Some(lm) = last_modified_str { - resp.headers_mut() - .insert(header::LAST_MODIFIED, lm.parse().unwrap()); - } - // Go sets ETag after the 304 paths, so a 304 has no ETag. - return Some(resp); - } None } +/// Weak comparison against "*" or a comma-separated entity-tag list (RFC 9110 13.1.2). +fn etag_list_matches(header: &str, etag: &str) -> bool { + if header.trim() == "*" { + return true; + } + let etag = etag.trim_matches('"'); + split_etag_list(header).into_iter().any(|tag| { + let tag = tag.trim(); + let tag = tag.strip_prefix("W/").unwrap_or(tag); + // Empty list elements are ignored, so padding like a trailing + // comma cannot match an empty opaque tag. + !tag.is_empty() && tag.trim_matches('"') == etag + }) +} + +/// Splits an entity-tag list on the commas between tags; a comma inside a +/// quoted opaque tag does not split. +fn split_etag_list(header: &str) -> Vec<&str> { + let mut tags = Vec::new(); + let (mut start, mut quoted) = (0, false); + for (i, b) in header.bytes().enumerate() { + match b { + b'"' => quoted = !quoted, + b',' if !quoted => { + tags.push(&header[start..i]); + start = i + 1; + } + _ => {} + } + } + tags.push(&header[start..]); + tags +} + /// The 200 reply headers, and the extension, which falls back to the stored name's. fn read_response_headers( n: &Needle, @@ -1722,6 +1792,7 @@ fn stream_response( data_file_access_control: info.data_file_access_control, hold_read_lock_for_stream: !state.has_slow_read, io_unavailable: info.io_unavailable, + io_errors: info.io_errors, pending: None, buf: bytes::BytesMut::new(), state: tracking_state, @@ -2121,9 +2192,21 @@ fn handle_range_request_from_source( return Ok(Vec::new()); } let mut buf = vec![0u8; length as usize]; - info.source - .read_exact_at(&mut buf, info.data_file_offset + start as u64)?; - Ok(buf) + match info + .source + .read_exact_at(&mut buf, info.data_file_offset + start as u64) + { + // A finished range read clears the streak a failed meta read + // counted, like readNeedleDataInto's terminal report. + Ok(()) => { + info.io_errors.check_read_write_error(None); + Ok(buf) + } + Err(e) => { + info.io_errors.check_read_write_error(Some(&e)); + Err(e) + } + } }; if ranges.len() == 1 { @@ -5225,6 +5308,7 @@ mod tests { hold_read_lock_for_stream: true, _held_read_lease: None, io_unavailable: Arc::default(), + io_errors: Arc::default(), pending: None, buf: bytes::BytesMut::new(), state: None, @@ -5597,6 +5681,80 @@ mod tests { delete_handler(State(state.clone()), request).await.status() } + #[test] + fn test_not_modified_checks_if_none_match_first() { + let n = Needle { + last_modified: 1700000000, + ..Needle::default() + }; + let (etag, last_modified_str) = etag_and_last_modified(&n); + let bare = n.etag(); + let before = "Tue, 14 Nov 2023 22:12:20 GMT".to_string(); + let after = "Tue, 14 Nov 2023 22:14:20 GMT".to_string(); + + let cases: Vec<(Vec<(axum::http::HeaderName, String)>, bool)> = vec![ + (vec![(header::IF_NONE_MATCH, format!("\"{bare}\""))], true), + (vec![(header::IF_NONE_MATCH, format!("W/\"{bare}\""))], true), + ( + vec![(header::IF_NONE_MATCH, format!("\"x\", \"{bare}\""))], + true, + ), + (vec![(header::IF_NONE_MATCH, "*".into())], true), + (vec![(header::IF_NONE_MATCH, "\"x\"".into())], false), + ( + vec![ + (header::IF_NONE_MATCH, "\"x\"".into()), + (header::IF_NONE_MATCH, format!("\"{bare}\"")), + ], + true, + ), + ( + vec![(header::IF_NONE_MATCH, format!("\"{bare},old\""))], + false, + ), + ( + vec![(header::IF_NONE_MATCH, format!("\"x\",,\"{bare}\""))], + true, + ), + ( + vec![ + (header::IF_NONE_MATCH, "\"x\"".into()), + (header::IF_MODIFIED_SINCE, after.clone()), + ], + false, + ), + ( + vec![ + (header::IF_NONE_MATCH, "".into()), + (header::IF_MODIFIED_SINCE, after.clone()), + ], + false, + ), + (vec![(header::IF_MODIFIED_SINCE, after)], true), + (vec![(header::IF_MODIFIED_SINCE, before)], false), + ]; + for (pairs, want) in cases { + let mut headers = HeaderMap::new(); + for (name, value) in &pairs { + headers.append(name, value.parse().unwrap()); + } + assert_eq!( + not_modified_response(&n, &headers, &etag, &last_modified_str).is_some(), + want, + "{pairs:?}" + ); + } + + let mut headers = HeaderMap::new(); + headers.insert( + header::IF_NONE_MATCH, + format!("\"{bare}\"").parse().unwrap(), + ); + let resp = not_modified_response(&n, &headers, &etag, &last_modified_str).unwrap(); + assert!(resp.headers().contains_key(header::ETAG)); + assert!(resp.headers().contains_key(header::LAST_MODIFIED)); + } + /// A GET whose needle read is parked must not hold the store lock: a /// writer, here a real append to the same volume, must get through. #[tokio::test(flavor = "multi_thread", worker_threads = 2)] @@ -5771,6 +5929,44 @@ mod tests { assert_eq!(status, StatusCode::NOT_FOUND); } + /// A ranged read served from the data file reports its outcome to the + /// io-error streak, like readNeedleDataInto: a finished range clears the + /// count a failed meta read left. + #[test] + fn test_range_from_source_reports_to_io_error_streak() { + use std::io::Write; + + let tmp = tempfile::TempDir::new().unwrap(); + let file = std::fs::OpenOptions::new() + .read(true) + .write(true) + .create(true) + .open(tmp.path().join("1.dat")) + .unwrap(); + let data: Vec = (0..256u32).map(|i| i as u8).collect(); + (&file).write_all(&data).unwrap(); + + let io_errors = Arc::new(crate::storage::io_error::IoErrorTracker::default()); + let eio = crate::storage::volume::media_error_for_test(); + io_errors.check_read_write_error(Some(&eio)); + io_errors.check_read_write_error(Some(&eio)); + assert_eq!(io_errors.get_io_error_state().1, 2); + + let info = crate::storage::volume::NeedleStreamInfo { + source: crate::storage::volume::NeedleStreamSource::Local(file), + data_file_offset: 0, + data_size: data.len() as u32, + data_file_access_control: Arc::default(), + io_unavailable: Arc::default(), + io_errors: io_errors.clone(), + needle_id: NeedleId(1), + checksum: 0, + }; + let resp = handle_range_request_from_source("bytes=10-19", info, HeaderMap::new(), None); + assert_eq!(resp.status(), StatusCode::PARTIAL_CONTENT); + assert_eq!(io_errors.get_io_error_state().1, 0); + } + /// Range is read as Go reads it: a byte its parseRange cannot parse is a /// 416, Unicode whitespace around a range is trimmed, an empty value is no /// range, and HEAD ignores it. @@ -7223,6 +7419,7 @@ mod tests { .set(crate::server::write_queue::WriteQueue::new( state.clone(), 128, + crate::server::write_queue::SyncBatchOption::default(), )) .is_ok() ); diff --git a/seaweed-volume/src/server/profiling.rs b/seaweed-volume/src/server/profiling.rs index 9d5398b22c..5622e9d428 100644 --- a/seaweed-volume/src/server/profiling.rs +++ b/seaweed-volume/src/server/profiling.rs @@ -132,6 +132,9 @@ mod tests { fix_jpg_orientation: false, read_mode: ReadMode::Local, allow_untrusted_remote_endpoints: false, + fsync_min_batch_bytes: 1024 * 1024, + fsync_min_batch_requests: 128, + fsync_batch_wait: std::time::Duration::from_micros(500), cpu_profile: String::new(), mem_profile: String::new(), compaction_byte_per_second: 0, diff --git a/seaweed-volume/src/server/write_queue.rs b/seaweed-volume/src/server/write_queue.rs index 01c4f69e72..dc649e0d72 100644 --- a/seaweed-volume/src/server/write_queue.rs +++ b/seaweed-volume/src/server/write_queue.rs @@ -56,6 +56,27 @@ const MAX_CONCURRENT_VOLUME_WRITES: usize = 8; /// retirement is the volume leaving the store. const IDLE_WORKER_POLL: std::time::Duration = std::time::Duration::from_secs(60); +/// Tunes the per-volume durable-write worker. Once the queue drains with the +/// batch still below min_bytes and min_requests, the batch stays open up to +/// wait for stragglers instead of paying an fsync per request. Mirrors Go's +/// storage.SyncBatchOption. +#[derive(Clone, Copy)] +pub struct SyncBatchOption { + pub min_bytes: usize, + pub min_requests: usize, + pub wait: std::time::Duration, +} + +impl Default for SyncBatchOption { + fn default() -> Self { + SyncBatchOption { + min_bytes: 1024 * 1024, + min_requests: MAX_BATCH_SIZE, + wait: std::time::Duration::from_micros(500), + } + } +} + /// Handle for submitting write requests to the per-volume workers. #[derive(Clone)] pub struct WriteQueue { @@ -67,23 +88,35 @@ pub struct WriteQueue { /// alone would multiply the old single queue's capacity by volume count. queue_permits: Arc, workers: Arc>>>, + sync_batch: SyncBatchOption, /// Submissions accepted, for tests asserting which writes route here. #[cfg(test)] submits: Arc, + /// Times a worker armed the straggler window, for tests that must land a + /// write inside an already-open batch. + #[cfg(test)] + windows_armed: Arc, } impl WriteQueue { /// Create a new write queue. Per-volume workers are spawned lazily on the /// first submission for that volume. - pub fn new(state: Arc, capacity: usize) -> Self { + pub fn new( + state: Arc, + capacity: usize, + sync_batch: SyncBatchOption, + ) -> Self { WriteQueue { state, capacity, io_permits: Arc::new(Semaphore::new(MAX_CONCURRENT_VOLUME_WRITES)), queue_permits: Arc::new(Semaphore::new(capacity)), workers: Arc::new(Mutex::new(HashMap::new())), + sync_batch, #[cfg(test)] submits: Arc::new(std::sync::atomic::AtomicU64::new(0)), + #[cfg(test)] + windows_armed: Arc::new(std::sync::atomic::AtomicU64::new(0)), } } @@ -93,6 +126,12 @@ impl WriteQueue { self.submits.load(std::sync::atomic::Ordering::Relaxed) } + /// How many batch windows workers have armed so far. + #[cfg(test)] + fn windows_armed_for_test(&self) -> u64 { + self.windows_armed.load(std::sync::atomic::Ordering::Relaxed) + } + /// Sender for the volume's worker, spawning one on first use. fn worker_for(&self, volume_id: VolumeId) -> mpsc::Sender { let mut workers = self.workers.lock().unwrap(); @@ -107,6 +146,9 @@ impl WriteQueue { state: self.state.clone(), io_permits: self.io_permits.clone(), workers: self.workers.clone(), + sync_batch: self.sync_batch, + #[cfg(test)] + windows_armed: self.windows_armed.clone(), } .run(), ); @@ -191,6 +233,9 @@ struct VolumeWriteWorker { state: Arc, io_permits: Arc, workers: Arc>>>, + sync_batch: SyncBatchOption, + #[cfg(test)] + windows_armed: Arc, } impl VolumeWriteWorker { @@ -238,14 +283,40 @@ impl VolumeWriteWorker { let mut batch = Vec::with_capacity(MAX_BATCH_SIZE); let mut batch_bytes: usize = first.needle.data.len(); batch.push(first); + // Armed once the queue first drains below the minimum batch; holds the + // batch open for stragglers so the wait bound is per batch, not per + // lull (Go's batchWait). + let mut window_deadline = None; while batch.len() < MAX_BATCH_SIZE && batch_bytes < MAX_BATCH_BYTES { - match self.rx.try_recv() { - Ok(req) => { - batch_bytes += req.needle.data.len(); - batch.push(req); - } - Err(_) => break, + if window_deadline.is_some_and(|d| tokio::time::Instant::now() >= d) { + break; } + let req = match self.rx.try_recv() { + Ok(req) => req, + Err(mpsc::error::TryRecvError::Empty) => { + let sync = self.sync_batch; + if sync.wait.is_zero() + || batch_bytes >= sync.min_bytes + || batch.len() >= sync.min_requests + { + // a drained queue flushes right away once it holds enough + break; + } + let deadline = *window_deadline.get_or_insert_with(|| { + #[cfg(test)] + self.windows_armed + .fetch_add(1, std::sync::atomic::Ordering::Relaxed); + tokio::time::Instant::now() + sync.wait + }); + match tokio::time::timeout_at(deadline, self.rx.recv()).await { + Ok(Some(req)) => req, + Ok(None) | Err(_) => break, + } + } + Err(mpsc::error::TryRecvError::Disconnected) => break, + }; + batch_bytes += req.needle.data.len(); + batch.push(req); } debug!("processing write batch of {} requests", batch.len()); @@ -380,7 +451,7 @@ mod tests { async fn test_write_queue_submit_no_volume() { // Submit a write to a non-existent volume -- should return VolumeError::NotFound. let state = make_test_state(); - let queue = WriteQueue::new(state, MAX_BATCH_SIZE); + let queue = WriteQueue::new(state, MAX_BATCH_SIZE, SyncBatchOption::default()); let needle = Needle { id: 1.into(), @@ -402,7 +473,7 @@ mod tests { async fn test_write_queue_concurrent_submissions() { // Submit multiple concurrent writes -- all should complete (with errors since no volume). let state = make_test_state(); - let queue = WriteQueue::new(state, MAX_BATCH_SIZE); + let queue = WriteQueue::new(state, MAX_BATCH_SIZE, SyncBatchOption::default()); let mut handles = Vec::new(); for i in 0..10u64 { @@ -430,7 +501,7 @@ mod tests { async fn test_write_queue_batching() { // Verify that many concurrent writes get processed (testing the batching path). let state = make_test_state(); - let queue = WriteQueue::new(state, MAX_BATCH_SIZE); + let queue = WriteQueue::new(state, MAX_BATCH_SIZE, SyncBatchOption::default()); // Submit MAX_BATCH_SIZE requests concurrently let mut handles = Vec::new(); @@ -522,7 +593,7 @@ mod tests { async fn test_write_queue_dropped_sender() { // When the queue is dropped, subsequent submits should fail gracefully. let state = make_test_state(); - let queue = WriteQueue::new(state, 1); + let queue = WriteQueue::new(state, 1, SyncBatchOption::default()); // Clone then drop the original -- the worker keeps running via its rx handle. let queue2 = queue.clone(); @@ -665,7 +736,7 @@ mod tests { .unwrap(); } } - let queue = WriteQueue::new(state, MAX_BATCH_SIZE); + let queue = WriteQueue::new(state, MAX_BATCH_SIZE, SyncBatchOption::default()); let in_append = Arc::new(Barrier::new(2)); let release = Arc::new(Barrier::new(2)); @@ -734,4 +805,85 @@ mod tests { assert!(a1.await.unwrap().is_ok()); assert!(a2.await.unwrap().is_ok()); } + + /// A durable write below the minimum batch rides out its window so a + /// straggler joins the same sync instead of each paying an fsync alone — + /// Go's batchWait. Paused time makes the hold observable: a lone write can + /// only complete once the runtime auto-advances to the window's deadline. + #[tokio::test(start_paused = true)] + async fn test_batch_window_defers_flush_for_stragglers() { + use crate::config::MinFreeSpace; + use crate::storage::types::{DiskType, NeedleId}; + use crate::storage::volume::VolumeSpec; + use std::time::Duration; + + let tmp = tempfile::TempDir::new().unwrap(); + let dir = tmp.path().to_str().unwrap(); + let state = make_test_state(); + { + let mut store = state.store.write().unwrap(); + store + .add_location( + dir, + dir, + 10, + DiskType::HardDrive, + MinFreeSpace::Percent(1.0), + Vec::new(), + ) + .unwrap(); + store + .add_volume(VolumeId(1), DiskType::HardDrive, &VolumeSpec::default()) + .unwrap(); + } + + let queue = WriteQueue::new( + state, + MAX_BATCH_SIZE, + SyncBatchOption { + min_bytes: usize::MAX, + min_requests: MAX_BATCH_SIZE, + wait: Duration::from_secs(30), + }, + ); + + let needle = |id: u64| Needle { + id: NeedleId(id), + cookie: 0x1111.into(), + data: vec![id as u8; 8], + data_size: 8, + ..Needle::default() + }; + + let start = tokio::time::Instant::now(); + queue.submit(VolumeId(1), needle(1), true).await.unwrap(); + assert!(start.elapsed() >= Duration::from_secs(30)); + + // Wait for the worker to take the second write and open its batch + // window, so the straggler provably lands inside an open batch + // instead of arriving during the initial drain. + let armed = queue.windows_armed_for_test(); + let second = tokio::spawn({ + let queue = queue.clone(); + async move { queue.submit(VolumeId(1), needle(2), true).await } + }); + for _ in 0..10_000 { + if queue.windows_armed_for_test() > armed { + break; + } + tokio::task::yield_now().await; + } + assert!(queue.windows_armed_for_test() > armed); + let third = tokio::spawn({ + let queue = queue.clone(); + async move { queue.submit(VolumeId(1), needle(3), true).await } + }); + let (second, third) = tokio::join!(second, third); + second.unwrap().unwrap(); + third.unwrap().unwrap(); + // One shared window ends both writes ~30s after their batch armed; + // back-to-back windows would keep the third out to ~90s. + let elapsed = start.elapsed(); + assert!(elapsed >= Duration::from_secs(60) && elapsed < Duration::from_secs(90)); + } } diff --git a/seaweed-volume/src/storage/volume.rs b/seaweed-volume/src/storage/volume.rs index 83bc73093d..7fc7be57de 100644 --- a/seaweed-volume/src/storage/volume.rs +++ b/seaweed-volume/src/storage/volume.rs @@ -836,7 +836,7 @@ impl NeedleReadPlan { pub(crate) fn read_meta(&mut self, n: &mut Needle) -> Result<(), VolumeError> { let (source, size, version) = (&self.source, self.size, self.version); let read_at = |buf: &mut [u8], at: u64| Ok(source.read_exact_at(buf, at)?); - with_offset_retry(&mut self.offset, |off| { + let result = with_offset_retry(&mut self.offset, |off| { if version == VERSION_1 { // A V1 body is all data, so its "meta" is the whole record. let mut buf = vec![0u8; get_actual_size(size, version) as usize]; @@ -847,13 +847,26 @@ impl NeedleReadPlan { } else { Volume::read_needle_meta_blob_and_parse(read_at, n, off, size, version) } - })?; + }); + // Report failures only: a successful meta read precedes the payload + // read, so clearing the streak here would reset the count before + // every failing download and the volume could never reach quarantine. + if let Err(VolumeError::Io(e)) = &result { + self.io_errors.check_read_write_error(Some(e)); + } + result?; if needle_expired(n) { return Err(VolumeError::NotFound); } Ok(()) } + /// A successful meta read ended the request (e.g. HEAD), so it may reset + /// the streak — unlike a meta read that precedes a payload read. + pub(crate) fn note_meta_only_success(&self) { + self.io_errors.check_read_write_error(None); + } + /// For a reply served from the data file, after `read_meta` filled `n`. pub(crate) fn into_stream_info(self, n: &Needle) -> NeedleStreamInfo { // V1 data starts right after the header, V2/V3 after the DataSize field. @@ -868,6 +881,7 @@ impl NeedleReadPlan { data_size: n.data_size, data_file_access_control: self.data_file_access_control, io_unavailable: self.io_unavailable, + io_errors: self.io_errors, needle_id: self.needle_id, checksum: n.checksum.0, } @@ -1144,6 +1158,9 @@ pub struct NeedleStreamInfo { pub data_file_access_control: Arc, /// Checked before each chunk: the volume can become unavailable mid-stream. pub(crate) io_unavailable: Arc, + /// Streamed chunks report read failures here and clear the streak when + /// the whole body has been emitted, mirroring readNeedleDataInto. + pub(crate) io_errors: Arc, pub needle_id: NeedleId, /// Checksum stored in the needle tail, verified once the last chunk has /// been read — before that frame is emitted. @@ -2918,18 +2935,52 @@ impl Volume { } // Cookie validation for existing needle (matches Go: check whenever nm.Get returns ok) - if let Some(nm) = &io.nm - && let Some(nv) = nm.get(n.id)? + let indexed = match io.nm.as_ref() { + Some(nm) => nm.get(n.id)?, + None => None, + }; + if let Some(nv) = indexed + && !nv.offset.is_zero() { let mut existing = Needle::default(); // Read only the header to check cookie self.read_needle_header_unlocked(io, &mut existing, nv.offset.to_actual_offset())?; - if n.cookie.0 == 0 && !check_cookie { - n.cookie = existing.cookie; + if existing.id != n.id && existing.size != nv.size { + // a 4-byte index entry can index past the 32 GiB wrap: the + // low offset may hold another needle while the real record + // sits at the wrapped offset, where reads already retry + #[cfg(not(feature = "5bytes"))] + if nv.offset.to_actual_offset() < MAX_POSSIBLE_VOLUME_SIZE as i64 { + let mut wrapped = Needle::default(); + match self.read_needle_header_unlocked( + io, + &mut wrapped, + nv.offset.to_actual_offset() + MAX_POSSIBLE_VOLUME_SIZE as i64, + ) { + Ok(()) => existing = wrapped, + Err(VolumeError::Io(e)) if e.kind() == io::ErrorKind::UnexpectedEof => {} + Err(e) => return Err(e), + } + } } - if existing.cookie != n.cookie { - return Err(VolumeError::CookieMismatch(n.cookie.0)); + + if existing.id != n.id && existing.size != nv.size { + // the entry resolves to a record that is neither the indexed + // key nor the indexed size, so it is stale rather than a + // duplicate — skip the cookie check and let publish_write + // overwrite it once the append commits + warn!( + "volume {} index entry for needle {} resolves to needle {}; overwriting stale index entry", + self.id.0, n.id.0, existing.id.0 + ); + } else { + if n.cookie.0 == 0 && !check_cookie { + n.cookie = existing.cookie; + } + if existing.cookie != n.cookie { + return Err(VolumeError::CookieMismatch(n.cookie.0)); + } } } Ok(false) @@ -6067,7 +6118,7 @@ fn preallocate_file(file: &File, size: u64) { /// An OS error the platform reports for failing storage media, which is /// what counts toward the I/O error streak. #[cfg(test)] -fn media_error_for_test() -> io::Error { +pub(crate) fn media_error_for_test() -> io::Error { #[cfg(unix)] { io::Error::from_raw_os_error(libc::EIO) @@ -9044,6 +9095,107 @@ mod tests { assert!(matches!(err, VolumeError::CookieMismatch(_))); } + /// A damaged index entry resolving to a different needle must miss on + /// reads and be dropped by the next write for the key, not rejected as a + /// cookie conflict. + #[test] + fn test_stale_index_entry_heals_on_write() { + let tmp = TempDir::new().unwrap(); + let dir = tmp.path().to_str().unwrap(); + let v = make_test_volume(dir); + + let mut first = Needle { + id: NeedleId(2), + cookie: Cookie(0x2222), + data: b"second".to_vec(), + data_size: 6, + ..Needle::default() + }; + v.write_needle(&mut first, true, false).unwrap(); + + // point key 9 at needle 2's record with a size that does not match + // the record, as a corrupt index entry would look + { + let mut io = v.io.write().unwrap(); + let nm = io.nm.as_mut().unwrap(); + let nv = nm.get(NeedleId(2)).unwrap().unwrap(); + nm.put(NeedleId(9), nv.offset, Size(nv.size.0 + 1)).unwrap(); + } + + let mut probe = Needle { + id: NeedleId(9), + ..Needle::default() + }; + assert!(v.read_needle(&mut probe).is_err()); + + let mut n = Needle { + id: NeedleId(9), + cookie: Cookie(0xabcd), + data: b"ninth".to_vec(), + data_size: 5, + ..Needle::default() + }; + v.write_needle(&mut n, true, false).unwrap(); + + let mut back = Needle { + id: NeedleId(9), + ..Needle::default() + }; + v.read_needle(&mut back).unwrap(); + assert_eq!(back.data, b"ninth"); + assert_eq!(back.cookie, Cookie(0xabcd)); + } + + /// A stale index entry is only replaced once the new record commits: a + /// durable write rolled back by a failed fsync leaves the index exactly + /// as it found it. + #[test] + fn test_stale_index_entry_survives_failed_write() { + let tmp = TempDir::new().unwrap(); + let dir = tmp.path().to_str().unwrap(); + let v = make_test_volume(dir); + + let mut existing = Needle { + id: NeedleId(2), + cookie: Cookie(0x2222), + data: b"second".to_vec(), + data_size: 6, + ..Needle::default() + }; + v.write_needle(&mut existing, true, false).unwrap(); + + let stale_nv = { + let mut io = v.io.write().unwrap(); + let nm = io.nm.as_mut().unwrap(); + let nv = nm.get(NeedleId(2)).unwrap().unwrap(); + nm.put(NeedleId(9), nv.offset, Size(nv.size.0 + 1)).unwrap(); + nm.get(NeedleId(9)).unwrap().unwrap() + }; + + v.fail_next_fsync_for_test(true); + let mut replacement = Needle { + id: NeedleId(9), + cookie: Cookie(0xabcd), + data: b"ninth".to_vec(), + data_size: 5, + ..Needle::default() + }; + assert!(v.write_needle(&mut replacement, true, true).is_err()); + v.fail_next_fsync_for_test(false); + + let now = { + let io = v.io.read().unwrap(); + io.nm + .as_ref() + .unwrap() + .get(NeedleId(9)) + .unwrap() + .expect("a failed write must not drop the index entry it meant to replace") + }; + assert_eq!(now.offset, stale_nv.offset); + assert_eq!(now.size, stale_nv.size); + } + // A size disagreeing with the blob's own header indexes the needle at the wrong // length and, on v3, stamps the append timestamp into the middle of the needle. #[test] diff --git a/seaweed-volume/tests/http_integration.rs b/seaweed-volume/tests/http_integration.rs index 7622fab739..b90234a5ad 100644 --- a/seaweed-volume/tests/http_integration.rs +++ b/seaweed-volume/tests/http_integration.rs @@ -1340,11 +1340,11 @@ async fn non_ascii_fid_and_ttl_are_rejected_not_panicked() { #[tokio::test] async fn write_queue_upload_returns_same_etag_as_direct_write() { - use seaweed_volume::server::write_queue::WriteQueue; + use seaweed_volume::server::write_queue::{SyncBatchOption, WriteQueue}; let (direct_state, _direct_tmp) = test_state(); let (queued_state, _queued_tmp) = test_state(); - let wq = WriteQueue::new(queued_state.clone(), 128); + let wq = WriteQueue::new(queued_state.clone(), 128, SyncBatchOption::default()); let _ = queued_state.write_queue.set(wq); let compressible = "seaweedfs ".repeat(200).into_bytes(); diff --git a/test/erasure_coding/ec_integration_test.go b/test/erasure_coding/ec_integration_test.go index f3a1461336..33be5a46b8 100644 --- a/test/erasure_coding/ec_integration_test.go +++ b/test/erasure_coding/ec_integration_test.go @@ -3,9 +3,12 @@ package erasure_coding import ( "bytes" "context" + "encoding/json" "fmt" "io" "math" + "net" + "net/http" "os" "os/exec" "path/filepath" @@ -13,6 +16,7 @@ import ( "testing" "time" + "github.com/seaweedfs/seaweedfs/test/testutil" "github.com/seaweedfs/seaweedfs/weed/operation" "github.com/seaweedfs/seaweedfs/weed/pb" "github.com/seaweedfs/seaweedfs/weed/pb/volume_server_pb" @@ -47,17 +51,15 @@ func TestECEncodingVolumeLocationTimingBug(t *testing.T) { defer cluster.Stop() // Wait for servers to be ready - require.NoError(t, waitForServer("127.0.0.1:9333", 30*time.Second)) - require.NoError(t, waitForServer("127.0.0.1:8080", 30*time.Second)) - require.NoError(t, waitForServer("127.0.0.1:8081", 30*time.Second)) - require.NoError(t, waitForServer("127.0.0.1:8082", 30*time.Second)) - require.NoError(t, waitForServer("127.0.0.1:8083", 30*time.Second)) - require.NoError(t, waitForServer("127.0.0.1:8084", 30*time.Second)) - require.NoError(t, waitForServer("127.0.0.1:8085", 30*time.Second)) + require.NoError(t, waitForServer(cluster.masterAddr, 30*time.Second)) + for _, addr := range cluster.volumeAddrs { + require.NoError(t, waitForServer(addr, 30*time.Second)) + } + require.NoError(t, waitForMasterLeader(cluster.masterAddr, 30*time.Second)) // Create command environment options := &shell.ShellOptions{ - Masters: stringPtr("127.0.0.1:9333"), + Masters: stringPtr(cluster.masterAddr), GrpcDialOption: grpc.WithInsecure(), FilerGroup: stringPtr("default"), } @@ -71,7 +73,7 @@ func TestECEncodingVolumeLocationTimingBug(t *testing.T) { // Upload some test data to create volumes testData := []byte("This is test data for EC encoding integration test") - volumeId, err := uploadTestData(testData, "127.0.0.1:9333") + volumeId, err := uploadTestData(testData, cluster.masterAddr) require.NoError(t, err) t.Logf("Created volume %d with test data", volumeId) @@ -257,12 +259,13 @@ func TestECEncodingMasterTimingRaceCondition(t *testing.T) { defer cluster.Stop() // Wait for servers to be ready - require.NoError(t, waitForServer("127.0.0.1:9333", 30*time.Second)) - require.NoError(t, waitForServer("127.0.0.1:8080", 30*time.Second)) + require.NoError(t, waitForServer(cluster.masterAddr, 30*time.Second)) + require.NoError(t, waitForServer(cluster.volumeAddrs[0], 30*time.Second)) + require.NoError(t, waitForMasterLeader(cluster.masterAddr, 30*time.Second)) // Create command environment options := &shell.ShellOptions{ - Masters: stringPtr("127.0.0.1:9333"), + Masters: stringPtr(cluster.masterAddr), GrpcDialOption: grpc.WithInsecure(), FilerGroup: stringPtr("default"), } @@ -276,7 +279,7 @@ func TestECEncodingMasterTimingRaceCondition(t *testing.T) { // Upload test data testData := []byte("Race condition test data") - volumeId, err := uploadTestData(testData, "127.0.0.1:9333") + volumeId, err := uploadTestData(testData, cluster.masterAddr) require.NoError(t, err) t.Logf("Created volume %d for race condition test", volumeId) @@ -366,6 +369,8 @@ func TestECEncodingMasterTimingRaceCondition(t *testing.T) { type TestCluster struct { masterCmd *exec.Cmd volumeServers []*exec.Cmd + masterAddr string + volumeAddrs []string } func (c *TestCluster) Stop() { @@ -402,9 +407,23 @@ func startSeaweedFSCluster(ctx context.Context, dataDir string) (*TestCluster, e securityToml := filepath.Join(dataDir, "security.toml") os.WriteFile(securityToml, []byte("# Empty security config for testing\n"), 0644) + // Allocate ports instead of using the defaults: on a shared or reused + // runner a stale weed process can still hold 9333/808x, which kills the + // freshly started master at bind time and leaves the test talking to a + // raft follower that answers every assign with "not current leader". + // Mini-safe ports guarantee each HTTP port's derived gRPC port + // (port + 10000) is free as well. + ports, err := testutil.AllocateMiniPorts(7) + if err != nil { + return nil, fmt.Errorf("failed to allocate ports: %v", err) + } + masterPort := ports[0] + masterAddr := fmt.Sprintf("127.0.0.1:%d", masterPort) + cluster.masterAddr = masterAddr + // Start master server masterCmd := exec.CommandContext(ctx, weedBinary, "master", - "-port", "9333", + "-port", fmt.Sprintf("%d", masterPort), "-mdir", masterDir, "-volumeSizeLimitMB", "10", // Small volumes for testing "-ip", "127.0.0.1", @@ -432,13 +451,14 @@ func startSeaweedFSCluster(ctx context.Context, dataDir string) (*TestCluster, e volumeDir := filepath.Join(dataDir, fmt.Sprintf("volume%d", i)) os.MkdirAll(volumeDir, 0755) - port := fmt.Sprintf("808%d", i) + port := fmt.Sprintf("%d", ports[1+i]) + cluster.volumeAddrs = append(cluster.volumeAddrs, fmt.Sprintf("127.0.0.1:%d", ports[1+i])) rack := fmt.Sprintf("rack%d", i) volumeCmd := exec.CommandContext(ctx, weedBinary, "volume", "-port", port, "-dir", volumeDir, "-max", "10", - "-master", "127.0.0.1:9333", + "-master", masterAddr, "-volume.allowUntrustedRemoteEndpoints", "-ip", "127.0.0.1", "-dataCenter", "dc1", @@ -498,7 +518,8 @@ func findWeedBinary() string { func waitForServer(address string, timeout time.Duration) error { start := time.Now() for time.Since(start) < timeout { - if conn, err := grpc.NewClient(address, grpc.WithInsecure()); err == nil { + conn, err := net.DialTimeout("tcp", address, time.Second) + if err == nil { conn.Close() return nil } @@ -507,6 +528,29 @@ func waitForServer(address string, timeout time.Duration) error { return fmt.Errorf("timeout waiting for server %s", address) } +// waitForMasterLeader polls the master's /cluster/status endpoint until it +// reports itself as the raft leader. A goraft master can take up to the +// configured election timeout (10s by default) before it is ready to serve +// raft-gated requests like volume assign, so a bare port check is not enough. +func waitForMasterLeader(masterAddr string, timeout time.Duration) error { + start := time.Now() + for time.Since(start) < timeout { + resp, err := http.Get(fmt.Sprintf("http://%s/cluster/status", masterAddr)) + if err == nil { + var status struct { + IsLeader bool `json:"IsLeader"` + } + err = json.NewDecoder(resp.Body).Decode(&status) + resp.Body.Close() + if err == nil && status.IsLeader { + return nil + } + } + time.Sleep(500 * time.Millisecond) + } + return fmt.Errorf("timeout waiting for master %s to become raft leader", masterAddr) +} + func uploadTestData(data []byte, masterAddress string) (needle.VolumeId, error) { // Upload data to get a file ID assignResult, err := operation.Assign(context.Background(), func(ctx context.Context) pb.ServerAddress { @@ -2209,14 +2253,15 @@ func TestECEncodeReplicatedVolumeSync(t *testing.T) { defer cluster.Stop() // Wait for servers to be ready - require.NoError(t, waitForServer("127.0.0.1:9333", 30*time.Second)) - for i := 0; i < 6; i++ { - require.NoError(t, waitForServer(fmt.Sprintf("127.0.0.1:%d", 8080+i), 30*time.Second)) + require.NoError(t, waitForServer(cluster.masterAddr, 30*time.Second)) + for _, addr := range cluster.volumeAddrs { + require.NoError(t, waitForServer(addr, 30*time.Second)) } + require.NoError(t, waitForMasterLeader(cluster.masterAddr, 30*time.Second)) // Create command environment options := &shell.ShellOptions{ - Masters: stringPtr("127.0.0.1:9333"), + Masters: stringPtr(cluster.masterAddr), GrpcDialOption: grpc.WithInsecure(), FilerGroup: stringPtr("default"), } @@ -2240,7 +2285,7 @@ func TestECEncodeReplicatedVolumeSync(t *testing.T) { var volumeId needle.VolumeId var uploadErr error for retry := 0; retry < 5; retry++ { - volumeId, uploadErr = uploadTestDataWithReplication(t, "127.0.0.1:9333", testCollection, testReplication) + volumeId, uploadErr = uploadTestDataWithReplication(t, cluster.masterAddr, testCollection, testReplication) if uploadErr == nil { break } @@ -2290,7 +2335,7 @@ func TestECEncodeReplicatedVolumeSync(t *testing.T) { var volumeId needle.VolumeId var uploadErr error for retry := 0; retry < 5; retry++ { - volumeId, uploadErr = uploadTestDataWithReplication(t, "127.0.0.1:9333", testCollection, testReplication) + volumeId, uploadErr = uploadTestDataWithReplication(t, cluster.masterAddr, testCollection, testReplication) if uploadErr == nil { break } diff --git a/test/fuse_integration/framework.go b/test/fuse_integration/framework.go index 8424fb1d79..9fe62fcd71 100644 --- a/test/fuse_integration/framework.go +++ b/test/fuse_integration/framework.go @@ -4,9 +4,12 @@ import ( "fmt" "io/fs" "net" + "net/http" "os" "os/exec" "path/filepath" + "strconv" + "strings" "syscall" "testing" "time" @@ -14,19 +17,19 @@ import ( "github.com/stretchr/testify/require" ) -// FuseTestFramework provides utilities for FUSE integration testing +// FuseTestFramework provides utilities for FUSE integration testing. +// It starts a single "weed mini" process (master+volume+filer in one) +// and a separate "weed mount" process for the FUSE filesystem. type FuseTestFramework struct { t *testing.T tempDir string mountPoint string dataDir string - masterProcess *os.Process - volumeProcess *os.Process - filerProcess *os.Process - mountProcess *os.Process - masterAddr string - volumeAddr string + logDir string + miniProcess *managedProcess + mountProcess *managedProcess filerAddr string + filerPort int weedBinary string isSetup bool config *TestConfig @@ -35,14 +38,18 @@ type FuseTestFramework struct { // TestConfig holds configuration for FUSE tests type TestConfig struct { - Collection string - Replication string - ChunkSizeMB int - CacheSizeMB int - NumVolumes int - EnableDebug bool - MountOptions []string - SkipCleanup bool // for debugging failed tests + Collection string + Replication string + ChunkSizeMB int + CacheSizeMB int + NumVolumes int + EnableDebug bool + // MountGlobalOptions are glog flags (-v, -logtostderr, ...) registered on + // weed's global flagset; they must precede the subcommand name or the + // mount process dies at flag parsing. + MountGlobalOptions []string + MountOptions []string + SkipCleanup bool // for debugging failed tests } // DefaultTestConfig returns a default configuration for FUSE tests @@ -59,7 +66,7 @@ func DefaultTestConfig() *TestConfig { } } -// NewFuseTestFramework creates a new FUSE testing framework +// NewFuseTestFramework creates a new FUSE testing framework. func NewFuseTestFramework(t *testing.T, config *TestConfig) *FuseTestFramework { if config == nil { config = DefaultTestConfig() @@ -68,20 +75,63 @@ func NewFuseTestFramework(t *testing.T, config *TestConfig) *FuseTestFramework { tempDir, err := os.MkdirTemp("", "seaweedfs_fuse_test_") require.NoError(t, err) + filerPort := freePort(t) + return &FuseTestFramework{ t: t, tempDir: tempDir, mountPoint: filepath.Join(tempDir, "mount"), dataDir: filepath.Join(tempDir, "data"), - masterAddr: "127.0.0.1:19333", - volumeAddr: "127.0.0.1:18080", - filerAddr: "127.0.0.1:18888", + logDir: filepath.Join(tempDir, "logs"), + filerPort: filerPort, + filerAddr: fmt.Sprintf("127.0.0.1:%d", filerPort), weedBinary: findWeedBinary(), isSetup: false, config: config, } } +// grpcPortOffset mirrors weed's HTTP->gRPC port convention (gRPC = HTTP + offset); +// "weed mount" derives the filer gRPC port from the filer HTTP address the same way. +const grpcPortOffset = 10000 + +// freePort returns a free filer HTTP port whose gRPC sibling (port+grpcPortOffset) +// is also free. Both must stay below the Linux ephemeral floor (32768): a gRPC port +// above it can be transiently grabbed by an outbound connection, forcing mini to +// relocate its filer gRPC port while "weed mount" keeps dialing HTTP+10000 — the +// mount then never connects and the test times out. Capping HTTP at 22000 keeps the +// gRPC port at or below 32000. +func freePort(t *testing.T) int { + t.Helper() + const ( + minServicePort = 20000 + maxServicePort = 22000 + ) + + portCount := maxServicePort - minServicePort + 1 + start := minServicePort + int(time.Now().UnixNano()%int64(portCount)) + + for attempt := 0; attempt < portCount; attempt++ { + port := minServicePort + (start-minServicePort+attempt)%portCount + if portIsFree(port) && portIsFree(port+grpcPortOffset) { + return port + } + } + + t.Fatalf("failed to allocate a free HTTP/gRPC port pair in [%d,%d]", minServicePort, maxServicePort) + return 0 +} + +// portIsFree reports whether a TCP port can currently be bound on 127.0.0.1. +func portIsFree(port int) bool { + l, err := net.Listen("tcp", net.JoinHostPort("127.0.0.1", strconv.Itoa(port))) + if err != nil { + return false + } + l.Close() + return true +} + // Setup starts SeaweedFS cluster and mounts FUSE filesystem func (f *FuseTestFramework) Setup(config *TestConfig) error { if f.isSetup { @@ -159,41 +209,22 @@ func (f *FuseTestFramework) Setup(config *TestConfig) error { } // Create directories for full cluster setup - dirs := []string{f.mountPoint, f.dataDir} + dirs := []string{f.mountPoint, f.logDir, f.dataDir} for _, dir := range dirs { if err := os.MkdirAll(dir, 0755); err != nil { return fmt.Errorf("failed to create directory %s: %v", dir, err) } } - // Start master - if err := f.startMaster(config); err != nil { - return fmt.Errorf("failed to start master: %v", err) + // Start weed mini (master + volume + filer in one process) + if err := f.startMini(config); err != nil { + return fmt.Errorf("failed to start weed mini: %v", err) } - // Wait for master to be ready - if err := f.waitForService(f.masterAddr, 30*time.Second); err != nil { - return fmt.Errorf("master not ready: %v", err) - } - - // Start volume servers - if err := f.startVolumeServers(config); err != nil { - return fmt.Errorf("failed to start volume servers: %v", err) - } - - // Wait for volume server to be ready - if err := f.waitForService(f.volumeAddr, 30*time.Second); err != nil { - return fmt.Errorf("volume server not ready: %v", err) - } - - // Start filer - if err := f.startFiler(config); err != nil { - return fmt.Errorf("failed to start filer: %v", err) - } - - // Wait for filer to be ready - if err := f.waitForService(f.filerAddr, 30*time.Second); err != nil { - return fmt.Errorf("filer not ready: %v", err) + // Wait for filer to be ready (mini starts all services on filerPort) + if err := f.waitForService(f.miniProcess, f.filerAddr, 30*time.Second); err != nil { + f.dumpLog("mini") + return fmt.Errorf("weed mini not ready: %v", err) } // Skip rather than fail on a host with no FUSE device to mount through @@ -208,14 +239,24 @@ func (f *FuseTestFramework) Setup(config *TestConfig) error { // Wait for mount to be ready if err := f.waitForMount(30 * time.Second); err != nil { + f.dumpLog("mount") return fmt.Errorf("FUSE mount not ready: %v", err) } + // A mounted filesystem still cannot write until a volume registers and a + // grow round finishes, so gate on a real write instead of letting the + // first test hits ride out the startup window as EIO on close. + if err := f.waitForWritable(90 * time.Second); err != nil { + f.dumpLog("mount") + return fmt.Errorf("FUSE mount not writable: %v", err) + } + f.isSetup = true return nil } -// Cleanup stops all processes and removes temporary files +// Cleanup stops all processes and removes temporary files. +// If the test failed, it dumps logs automatically. func (f *FuseTestFramework) Cleanup() { // An external mount needs no teardown; only this run's work dir is ours. // A TEST_SKIP_CLUSTER_SETUP that fell through to a self-started cluster @@ -228,25 +269,32 @@ func (f *FuseTestFramework) Cleanup() { return } - if f.mountProcess != nil { - f.unmountFuse() + if f.t.Failed() { + f.DumpLogs() } // Stop processes in reverse order - processes := []*os.Process{f.mountProcess, f.filerProcess, f.volumeProcess, f.masterProcess} - for _, proc := range processes { - if proc != nil { - proc.Signal(syscall.SIGTERM) - proc.Wait() - } + f.unmountFuse() + if f.miniProcess != nil { + f.miniProcess.stop() + f.miniProcess = nil } + f.copyLogsForCI() + // Remove temp directory if !f.config.SkipCleanup { os.RemoveAll(f.tempDir) } } +// DumpLogs prints the tail of all SeaweedFS process logs to test output. +func (f *FuseTestFramework) DumpLogs() { + for _, name := range []string{"mini", "mount"} { + f.dumpLog(name) + } +} + // GetMountPoint returns the FUSE mount point path func (f *FuseTestFramework) GetMountPoint() string { return f.mountPoint @@ -257,88 +305,164 @@ func (f *FuseTestFramework) GetFilerAddr() string { return f.filerAddr } -// startMaster starts the SeaweedFS master server -func (f *FuseTestFramework) startMaster(config *TestConfig) error { +// managedProcess is a started weed sub-command whose exit is watched, so a wait +// for it to come up ends the moment it dies instead of burning its full timeout. +type managedProcess struct { + cmd *exec.Cmd + done chan struct{} + err error // exit error, read only after done is closed + logPath string +} + +// exited returns the exit error once the process is gone, nil while it runs. +func (p *managedProcess) exited() error { + select { + case <-p.done: + if p.err != nil { + return p.err + } + return fmt.Errorf("exit status 0") + default: + return nil + } +} + +// stop asks the process to terminate and waits for it to go away. +func (p *managedProcess) stop() { + // Signal fails with os.ErrProcessDone when the child is already gone, which + // is exactly the case the select below handles. + _ = p.cmd.Process.Signal(syscall.SIGTERM) + select { + case <-p.done: + case <-time.After(10 * time.Second): + p.cmd.Process.Kill() + <-p.done + } +} + +// startProcess is a helper that starts a weed sub-command with output captured +// to a log file in f.logDir. +func (f *FuseTestFramework) startProcess(name string, args []string) (*managedProcess, error) { + logPath := filepath.Join(f.logDir, name+".log") + logFile, err := os.Create(logPath) + if err != nil { + return nil, fmt.Errorf("create log file: %v", err) + } + cmd := exec.Command(f.weedBinary, args...) + cmd.Dir = f.tempDir + cmd.Stdout = logFile + cmd.Stderr = logFile + if err := cmd.Start(); err != nil { + logFile.Close() + return nil, err + } + // Close the file handle — the child process inherited it. + logFile.Close() + + p := &managedProcess{cmd: cmd, done: make(chan struct{}), logPath: logPath} + go func() { + p.err = cmd.Wait() + close(p.done) + }() + return p, nil +} + +// logTail returns the last bytes of a captured process log so a startup +// failure message carries the fatal line instead of a bare timeout. +func logTail(path string) string { + data, err := os.ReadFile(path) + if err != nil || len(data) == 0 { + return "" + } + const maxTail = 2 * 1024 + if len(data) > maxTail { + data = data[len(data)-maxTail:] + } + return string(data) +} + +// dumpLog prints the last lines of a process log file to the test output +// for debugging when a service fails to start or a test fails. +func (f *FuseTestFramework) dumpLog(name string) { + data, err := os.ReadFile(filepath.Join(f.logDir, name+".log")) + if err != nil { + f.t.Logf("[%s log] (not available: %v)", name, err) + return + } + // Show last 16KB on failure for meaningful context. + const maxTail = 16 * 1024 + if len(data) > maxTail { + data = data[len(data)-maxTail:] + } + f.t.Logf("[%s log tail (%d bytes)]\n%s", name, len(data), string(data)) +} + +// copyLogsForCI copies SeaweedFS process logs under a per-test directory in +// /tmp/seaweedfs-fuse-logs/ so the CI workflow can upload them as artifacts. +// Each framework instance gets its own directory: every test in the suite +// builds one, and fixed filenames would let a later passing test overwrite +// the logs of an earlier failing one. +func (f *FuseTestFramework) copyLogsForCI() { + ciLogDir := filepath.Join("/tmp/seaweedfs-fuse-logs", strings.NewReplacer("/", "_", " ", "_").Replace(f.t.Name())) + if err := os.MkdirAll(ciLogDir, 0755); err != nil { + return + } + for _, name := range []string{"mini", "mount"} { + src := filepath.Join(f.logDir, name+".log") + data, err := os.ReadFile(src) + if err != nil { + continue + } + os.WriteFile(filepath.Join(ciLogDir, name+".log"), data, 0644) + } +} + +// startMini starts "weed mini" which runs master+volume+filer in one process. +func (f *FuseTestFramework) startMini(config *TestConfig) error { var args []string if config.EnableDebug { // -v is a global flag; before the subcommand or flag parsing dies args = append(args, "-v=4") } - args = append(args, - "master", - "-ip=127.0.0.1", - "-port=19333", - "-mdir=" + filepath.Join(f.dataDir, "master"), - "-raftBootstrap", - ) - - cmd := exec.Command(f.weedBinary, args...) - cmd.Dir = f.tempDir - if err := cmd.Start(); err != nil { - return err - } - f.masterProcess = cmd.Process - return nil -} - -// startVolumeServers starts SeaweedFS volume servers -func (f *FuseTestFramework) startVolumeServers(config *TestConfig) error { - var args []string - if config.EnableDebug { - args = append(args, "-v=4") + adminPort := freePort(f.t) + for adminPort == f.filerPort { + adminPort = freePort(f.t) } args = append(args, - "volume", - "-mserver=" + f.masterAddr, + "mini", + "-dir="+f.dataDir, "-ip=127.0.0.1", - "-port=18080", - "-dir=" + filepath.Join(f.dataDir, "volume"), - fmt.Sprintf("-max=%d", config.NumVolumes), + "-ip.bind=127.0.0.1", + "-filer.port="+strconv.Itoa(f.filerPort), + "-s3=false", + "-webdav=false", + "-admin.ui=false", + "-admin.port="+strconv.Itoa(adminPort), + "-admin.port.grpc="+strconv.Itoa(adminPort+grpcPortOffset), + "-admin.port.legacy=0", ) - cmd := exec.Command(f.weedBinary, args...) - cmd.Dir = f.tempDir - if err := cmd.Start(); err != nil { + proc, err := f.startProcess("mini", args) + if err != nil { return err } - f.volumeProcess = cmd.Process - return nil -} - -// startFiler starts the SeaweedFS filer server -func (f *FuseTestFramework) startFiler(config *TestConfig) error { - var args []string - if config.EnableDebug { - args = append(args, "-v=4") - } - args = append(args, - "filer", - "-master=" + f.masterAddr, - "-ip=127.0.0.1", - "-port=18888", - ) - - cmd := exec.Command(f.weedBinary, args...) - cmd.Dir = f.tempDir - if err := cmd.Start(); err != nil { - return err - } - f.filerProcess = cmd.Process + f.miniProcess = proc return nil } // mountFuse mounts the SeaweedFS FUSE filesystem func (f *FuseTestFramework) mountFuse(config *TestConfig) error { - var args []string + args := append([]string{}, config.MountGlobalOptions...) if config.EnableDebug { args = append(args, "-v=4") } args = append(args, "mount", - "-filer=" + f.filerAddr, - "-dir=" + f.mountPoint, + "-filer="+f.filerAddr, + "-dir="+f.mountPoint, "-filer.path=/", "-dirAutoCreate", + "-allowOthers=false", ) if config.Collection != "" { @@ -351,55 +475,63 @@ func (f *FuseTestFramework) mountFuse(config *TestConfig) error { args = append(args, fmt.Sprintf("-chunkSizeLimitMB=%d", config.ChunkSizeMB)) } if config.CacheSizeMB > 0 { - args = append(args, fmt.Sprintf("-cacheSizeMB=%d", config.CacheSizeMB)) + args = append(args, fmt.Sprintf("-cacheCapacityMB=%d", config.CacheSizeMB)) } args = append(args, config.MountOptions...) - cmd := exec.Command(f.weedBinary, args...) - cmd.Dir = f.tempDir - if err := cmd.Start(); err != nil { + proc, err := f.startProcess("mount", args) + if err != nil { return err } - f.mountProcess = cmd.Process + f.mountProcess = proc return nil } // unmountFuse unmounts the FUSE filesystem func (f *FuseTestFramework) unmountFuse() error { if f.mountProcess != nil { - f.mountProcess.Signal(syscall.SIGTERM) - f.mountProcess.Wait() + f.mountProcess.stop() f.mountProcess = nil } // Also try system unmount as backup + exec.Command("fusermount3", "-u", f.mountPoint).Run() + exec.Command("fusermount", "-u", f.mountPoint).Run() exec.Command("umount", f.mountPoint).Run() return nil } -// waitForService waits for a service to be available -func (f *FuseTestFramework) waitForService(addr string, timeout time.Duration) error { +// waitForService waits until the weed process answers its own /healthz, so a +// foreign listener on the port cannot pass for readiness. +func (f *FuseTestFramework) waitForService(proc *managedProcess, addr string, timeout time.Duration) error { deadline := time.Now().Add(timeout) + client := &http.Client{Timeout: time.Second} for time.Now().Before(deadline) { - conn, err := net.DialTimeout("tcp", addr, 1*time.Second) + resp, err := client.Get("http://" + addr + "/healthz") if err == nil { - conn.Close() - return nil + resp.Body.Close() + if resp.StatusCode == http.StatusOK && proc.exited() == nil { + return nil + } + } + if exitErr := proc.exited(); exitErr != nil { + return fmt.Errorf("process exited (%v) before %s accepted connections: %s", exitErr, addr, logTail(proc.logPath)) } time.Sleep(100 * time.Millisecond) } - return fmt.Errorf("service at %s not ready within timeout", addr) + return fmt.Errorf("service at %s not ready within timeout: %s", addr, logTail(proc.logPath)) } -// waitForMount waits for the FUSE mount to be ready +// waitForMount waits for the FUSE mount to be ready. A stat/ReadDir probe +// alone is not enough: the bare mount point directory passes both before the +// mount process finishes starting, letting tests race ahead and write to the +// local disk underneath the mount. The mount point's device ID differing from +// its parent's confirms a filesystem is actually mounted there. func (f *FuseTestFramework) waitForMount(timeout time.Duration) error { - // The mount point is created before weed mount starts, so Stat/ReadDir - // alone succeed on the plain directory. The device id only differs from - // its parent's once a filesystem is actually mounted on it. parentDev, err := deviceID(filepath.Dir(f.mountPoint)) if err != nil { - return fmt.Errorf("stat mount parent: %w", err) + return fmt.Errorf("stat mount point parent: %w", err) } deadline := time.Now().Add(timeout) for time.Now().Before(deadline) { @@ -408,9 +540,76 @@ func (f *FuseTestFramework) waitForMount(timeout time.Duration) error { return nil } } + // A mount that cannot mount at all (no /dev/fuse, fusermount not setuid) + // dies within a second; reporting that beats waiting out the timeout. + if exitErr := f.mountProcess.exited(); exitErr != nil { + return fmt.Errorf("mount process exited (%v): %s", exitErr, logTail(f.mountProcess.logPath)) + } time.Sleep(100 * time.Millisecond) } - return fmt.Errorf("mount point not ready within timeout") + return fmt.Errorf("mount point not ready within timeout: %s", logTail(f.mountProcess.logPath)) +} + +// waitForWritable probes the mount with a real write+read+remove until one +// succeeds, covering the window where a fresh cluster cannot yet assign a +// file id (volume not registered, grow pending, leader settle). Each attempt +// runs off the loop because a wedged mount can hold a file op open, and one +// attempt stays in flight until it finishes or the overall deadline passes. +func (f *FuseTestFramework) waitForWritable(timeout time.Duration) error { + probe := filepath.Join(f.mountPoint, ".framework_write_probe") + payload := []byte("probe") + deadline := time.Now().Add(timeout) + var lastErr error + for time.Now().Before(deadline) { + if exitErr := f.mountProcess.exited(); exitErr != nil { + return fmt.Errorf("mount process exited (%v): %s", exitErr, logTail(f.mountProcess.logPath)) + } + result := make(chan error, 1) + go func() { + if err := os.WriteFile(probe, payload, 0644); err != nil { + result <- err + return + } + data, err := os.ReadFile(probe) + if err != nil { + result <- err + return + } + if len(data) != len(payload) { + result <- fmt.Errorf("read back %d bytes, want %d", len(data), len(payload)) + return + } + result <- os.Remove(probe) + }() + tick := time.NewTicker(500 * time.Millisecond) + attemptOpen := true + for attemptOpen { + select { + case err := <-result: + attemptOpen = false + if err == nil { + tick.Stop() + return nil + } + lastErr = err + case <-tick.C: + if exitErr := f.mountProcess.exited(); exitErr != nil { + tick.Stop() + return fmt.Errorf("mount process exited (%v): %s", exitErr, logTail(f.mountProcess.logPath)) + } + if !time.Now().Before(deadline) { + tick.Stop() + if lastErr == nil { + return fmt.Errorf("write probe did not complete within timeout") + } + return fmt.Errorf("write probe failed within timeout: %v", lastErr) + } + } + } + tick.Stop() + time.Sleep(500 * time.Millisecond) + } + return fmt.Errorf("write probe failed within timeout: %v", lastErr) } // fuseAvailable reports whether the host has a FUSE device node to mount diff --git a/test/fuse_integration/framework_test.go b/test/fuse_integration/framework_test.go index 23af7e9dd2..a7662e96e0 100644 --- a/test/fuse_integration/framework_test.go +++ b/test/fuse_integration/framework_test.go @@ -8,6 +8,7 @@ import ( "os/exec" "path/filepath" "strconv" + "strings" "syscall" "testing" "time" @@ -152,7 +153,7 @@ func (f *FuseTestFramework) Setup(config *TestConfig) error { } // Wait for filer to be ready (mini starts all services on filerPort) - if err := f.waitForService(f.miniProcess, f.filerAddr, 30*time.Second); err != nil { + if err := f.waitForService(f.miniProcess, f.filerAddr, 90*time.Second); err != nil { f.dumpLog("mini") return fmt.Errorf("weed mini not ready: %v", err) } @@ -163,11 +164,18 @@ func (f *FuseTestFramework) Setup(config *TestConfig) error { } // Wait for mount to be ready - if err := f.waitForMount(30 * time.Second); err != nil { + if err := f.waitForMount(90 * time.Second); err != nil { f.dumpLog("mount") return fmt.Errorf("FUSE mount not ready: %v", err) } + // Gate on a real write: a fresh mini still cannot assign a file id until + // its volume registers and a grow round finishes. + if err := f.waitForWritable(90 * time.Second); err != nil { + f.dumpLog("mount") + return fmt.Errorf("FUSE mount not writable: %v", err) + } + f.isSetup = true return nil } @@ -286,11 +294,16 @@ func (f *FuseTestFramework) dumpLog(name string) { f.t.Logf("[%s log tail (%d bytes)]\n%s", name, len(data), string(data)) } -// copyLogsForCI copies SeaweedFS process logs to /tmp/seaweedfs-fuse-logs/ -// so the CI workflow can upload them as artifacts. +// copyLogsForCI copies SeaweedFS process logs under a per-test directory in +// /tmp/seaweedfs-fuse-logs/ so the CI workflow can upload them as artifacts. +// Each framework instance gets its own directory: every test in the suite +// builds one, and fixed filenames would let a later passing test overwrite +// the logs of an earlier failing one. func (f *FuseTestFramework) copyLogsForCI() { - ciLogDir := "/tmp/seaweedfs-fuse-logs" - os.MkdirAll(ciLogDir, 0755) + ciLogDir := filepath.Join("/tmp/seaweedfs-fuse-logs", strings.NewReplacer("/", "_", " ", "_").Replace(f.t.Name())) + if err := os.MkdirAll(ciLogDir, 0755); err != nil { + return + } for _, name := range []string{"mini", "mount"} { src := filepath.Join(f.logDir, name+".log") data, err := os.ReadFile(src) @@ -307,6 +320,10 @@ func (f *FuseTestFramework) startMini(config *TestConfig) error { if config.EnableDebug { args = append(args, "-v=4") } + adminPort := freePort(f.t) + for adminPort == f.filerPort { + adminPort = freePort(f.t) + } args = append(args, "mini", "-dir="+f.dataDir, @@ -316,6 +333,9 @@ func (f *FuseTestFramework) startMini(config *TestConfig) error { "-s3=false", "-webdav=false", "-admin.ui=false", + "-admin.port="+strconv.Itoa(adminPort), + "-admin.port.grpc="+strconv.Itoa(adminPort+grpcPortOffset), + "-admin.port.legacy=0", ) proc, err := f.startProcess("mini", args) @@ -421,6 +441,68 @@ func (f *FuseTestFramework) waitForMount(timeout time.Duration) error { return fmt.Errorf("mount point not ready within timeout") } +// waitForWritable probes the mount with a real write+read+remove until one +// succeeds, covering the window where a fresh cluster cannot yet assign a +// file id (volume not registered, grow pending, leader settle). Each attempt +// runs off the loop because a wedged mount can hold a file op open, and one +// attempt stays in flight until it finishes or the overall deadline passes. +func (f *FuseTestFramework) waitForWritable(timeout time.Duration) error { + probe := filepath.Join(f.mountPoint, ".framework_write_probe") + payload := []byte("probe") + deadline := time.Now().Add(timeout) + var lastErr error + for time.Now().Before(deadline) { + if exitErr := f.mountProcess.exited(); exitErr != nil { + return fmt.Errorf("mount process exited (%v)", exitErr) + } + result := make(chan error, 1) + go func() { + if err := os.WriteFile(probe, payload, 0644); err != nil { + result <- err + return + } + data, err := os.ReadFile(probe) + if err != nil { + result <- err + return + } + if len(data) != len(payload) { + result <- fmt.Errorf("read back %d bytes, want %d", len(data), len(payload)) + return + } + result <- os.Remove(probe) + }() + tick := time.NewTicker(500 * time.Millisecond) + attemptOpen := true + for attemptOpen { + select { + case err := <-result: + attemptOpen = false + if err == nil { + tick.Stop() + return nil + } + lastErr = err + case <-tick.C: + if exitErr := f.mountProcess.exited(); exitErr != nil { + tick.Stop() + return fmt.Errorf("mount process exited (%v)", exitErr) + } + if !time.Now().Before(deadline) { + tick.Stop() + if lastErr == nil { + return fmt.Errorf("write probe did not complete within timeout") + } + return fmt.Errorf("write probe failed within timeout: %v", lastErr) + } + } + } + tick.Stop() + time.Sleep(500 * time.Millisecond) + } + return fmt.Errorf("write probe failed within timeout: %v", lastErr) +} + // deviceID returns the device ID of the filesystem containing path. func deviceID(path string) (uint64, error) { var st syscall.Stat_t diff --git a/test/fuse_integration/posix_external_test.go b/test/fuse_integration/posix_external_test.go index 2c63fcb929..5a330fa2a4 100644 --- a/test/fuse_integration/posix_external_test.go +++ b/test/fuse_integration/posix_external_test.go @@ -102,10 +102,13 @@ func (s *ExternalPOSIXTestSuite) TestPjdFsTest(t *testing.T) { } } - // Run each test category + // pjdfstest asserts full POSIX semantics the mount does not provide + // yet, so category failures are report-only unless the run asked for + // external tools to gate. + strict := os.Getenv("POSIX_REQUIRE_EXTERNAL_TOOLS") == "true" for _, test := range pjdTests { t.Run(test.name, func(t *testing.T) { - s.runPjdTest(t, pjdDir, test.testPath, testDir, test.critical) + s.runPjdTest(t, pjdDir, test.testPath, testDir, strict && test.critical) }) } } diff --git a/test/kms/Makefile b/test/kms/Makefile index a46aa19b25..47736f6800 100644 --- a/test/kms/Makefile +++ b/test/kms/Makefile @@ -174,12 +174,12 @@ start-openbao-ci: ## Start OpenBao via Docker for CI ghcr.io/openbao/openbao:latest \ bao server -dev -dev-root-token-id=$(OPENBAO_TOKEN) -dev-listen-address=0.0.0.0:8200 @echo "Waiting for OpenBao to be ready..." - @for i in $$(seq 1 30); do \ + @for i in $$(seq 1 90); do \ if curl -s $(OPENBAO_ADDR)/v1/sys/health > /dev/null 2>&1; then \ echo "$(GREEN)OpenBao is ready$(NC)"; \ break; \ fi; \ - if [ $$i -eq 30 ]; then \ + if [ $$i -eq 90 ]; then \ echo "$(RED)Timeout waiting for OpenBao$(NC)"; \ docker logs openbao-ci 2>&1 | tail -20; \ exit 1; \ @@ -213,12 +213,12 @@ start-seaweedfs-ci: check-binary ## Start weed mini with OpenBao KMS config -ip=127.0.0.1 \ > /tmp/seaweedfs-kms-mini.log 2>&1 & echo $$! > /tmp/weed-kms-mini.pid @echo "Checking S3 service is ready..." - @for i in $$(seq 1 30); do \ + @for i in $$(seq 1 90); do \ if curl -s http://127.0.0.1:$(S3_PORT) > /dev/null 2>&1; then \ echo "$(GREEN)S3 service is ready$(NC)"; \ break; \ fi; \ - if [ $$i -eq 30 ]; then \ + if [ $$i -eq 90 ]; then \ echo "$(RED)Timeout waiting for S3 service$(NC)"; \ echo "=== Last 50 lines of server log ==="; \ tail -50 /tmp/seaweedfs-kms-mini.log 2>/dev/null || echo "No server log found"; \ diff --git a/test/s3/copying/Makefile b/test/s3/copying/Makefile index ea399223f0..50bedf2007 100644 --- a/test/s3/copying/Makefile +++ b/test/s3/copying/Makefile @@ -76,12 +76,12 @@ start-seaweedfs: check-binary -ip=127.0.0.1 \ > /tmp/seaweedfs-mini.log 2>&1 & echo $$! > /tmp/weed-mini.pid @echo "$(YELLOW)Waiting for S3 service to be ready...$(NC)" - @for i in $$(seq 1 30); do \ + @for i in $$(seq 1 90); do \ if curl -s -f http://127.0.0.1:$(S3_PORT) > /dev/null 2>&1; then \ echo "$(GREEN)S3 service is ready$(NC)"; \ break; \ fi; \ - echo "Waiting for S3 service... ($$i/30)"; \ + echo "Waiting for S3 service... ($$i/90)"; \ sleep 1; \ done @sleep 2 diff --git a/test/s3/cors/Makefile b/test/s3/cors/Makefile index 1378c7439e..ac56e11bc0 100644 --- a/test/s3/cors/Makefile +++ b/test/s3/cors/Makefile @@ -92,7 +92,7 @@ start-server: check-deps echo $$! > weed-server.pid @echo "Waiting for S3 server to be ready..." - @for i in $$(seq 1 30); do \ + @for i in $$(seq 1 90); do \ if echo | nc -z localhost $(S3_PORT); then \ echo "S3 server is ready!"; \ exit 0; \ diff --git a/test/s3/filer_group/Makefile b/test/s3/filer_group/Makefile index 21d5c050a8..7420b23149 100644 --- a/test/s3/filer_group/Makefile +++ b/test/s3/filer_group/Makefile @@ -79,14 +79,14 @@ start-server: check-deps echo $$! > weed-server.pid @echo "Waiting for S3 server to be ready..." - @for i in $$(seq 1 30); do \ + @for i in $$(seq 1 90); do \ if echo | nc -z localhost $(S3_PORT); then \ echo "S3 server is ready!"; \ exit 0; \ fi; \ sleep 1; \ done; \ - echo "❌ Server failed to start within 30 seconds"; \ + echo "❌ Server failed to start within 90 seconds"; \ if [ -f weed-server.log ]; then \ cat weed-server.log; \ fi; \ diff --git a/test/s3/iam/Makefile b/test/s3/iam/Makefile index 6dbb54299f..70f8972e14 100644 --- a/test/s3/iam/Makefile +++ b/test/s3/iam/Makefile @@ -81,7 +81,7 @@ start-services: ## Start SeaweedFS services for testing wait-for-services: ## Wait for all services to be ready @echo "Waiting for services to be ready..." @echo "Checking S3 API server..." - @for i in $$(seq 1 30); do \ + @for i in $$(seq 1 90); do \ if curl -s http://localhost:$(S3_PORT) > /dev/null 2>&1; then \ echo "OK S3 API server is ready"; \ exit 0; \ diff --git a/test/s3/iam/s3_iam_admin_test.go b/test/s3/iam/s3_iam_admin_test.go index 75556b43f7..70906dd26e 100644 --- a/test/s3/iam/s3_iam_admin_test.go +++ b/test/s3/iam/s3_iam_admin_test.go @@ -295,8 +295,8 @@ func TestIAMPolicyManagement(t *testing.T) { assert.Equal(t, true, aws.BoolValue(listVersionsResp.Versions[0].IsDefaultVersion)) getVersionResp, err := iamClient.GetPolicyVersion(&iam.GetPolicyVersionInput{ - PolicyArn: createResp.Policy.Arn, - VersionId: aws.String("v1"), + PolicyArn: createResp.Policy.Arn, + VersionId: aws.String("v1"), }) require.NoError(t, err) require.NotNil(t, getVersionResp.PolicyVersion) diff --git a/test/s3/iam/s3_sts_get_federation_token_test.go b/test/s3/iam/s3_sts_get_federation_token_test.go index 2a718cba90..69807bb169 100644 --- a/test/s3/iam/s3_sts_get_federation_token_test.go +++ b/test/s3/iam/s3_sts_get_federation_token_test.go @@ -14,8 +14,8 @@ import ( "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/aws/credentials" "github.com/aws/aws-sdk-go/aws/session" - "github.com/aws/aws-sdk-go/service/s3" v4 "github.com/aws/aws-sdk-go/aws/signer/v4" + "github.com/aws/aws-sdk-go/service/s3" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) diff --git a/test/s3/lifecycle/Makefile b/test/s3/lifecycle/Makefile index 98aec8eceb..cf55c424e6 100644 --- a/test/s3/lifecycle/Makefile +++ b/test/s3/lifecycle/Makefile @@ -42,6 +42,8 @@ start-server: build-weed @mkdir -p $(SERVER_DIR) @AWS_ACCESS_KEY_ID=$(ACCESS_KEY) AWS_SECRET_ACCESS_KEY=$(SECRET_KEY) $(WEED_BINARY) mini \ -dir=$(SERVER_DIR) \ + -ip=127.0.0.1 \ + -ip.bind=127.0.0.1 \ -s3.port=$(S3_PORT) \ -filer.port.grpc=$(FILER_GRPC_PORT) \ > weed-test.log 2>&1 & \ diff --git a/test/s3/lifecycle/s3_lifecycle_object_lock_test.go b/test/s3/lifecycle/s3_lifecycle_object_lock_test.go index 7196346de9..e601f85500 100644 --- a/test/s3/lifecycle/s3_lifecycle_object_lock_test.go +++ b/test/s3/lifecycle/s3_lifecycle_object_lock_test.go @@ -116,4 +116,3 @@ func TestLifecycleSkipsObjectLockedObjects(t *testing.T) { }) require.NoError(t, err, "object under GOVERNANCE retention must survive the sweep") } - diff --git a/test/s3/lifecycle/s3_lifecycle_versioning_test.go b/test/s3/lifecycle/s3_lifecycle_versioning_test.go index c21a37e04b..a25fabcdc3 100644 --- a/test/s3/lifecycle/s3_lifecycle_versioning_test.go +++ b/test/s3/lifecycle/s3_lifecycle_versioning_test.go @@ -164,6 +164,55 @@ func TestLifecycleVersionedBucketCreatesDeleteMarker(t *testing.T) { require.True(t, sawMarker, "ListObjectVersions must show a delete marker as latest for %s", key) } +// TestLifecycleVersionedBucketExpiresOnSecondRun covers the reported +// regression where Expiration.Days only fired on the cold-start walk: a +// versioned PUT surfaces in the meta-log as a version-file event, which +// the router used to drop, so once every shard had a saved cursor no +// daily run ever scheduled another delete marker. The first run here +// establishes the cursor; the second must expire an object that arrived +// in between, with no rule change to force a recovery walk. +func TestLifecycleVersionedBucketExpiresOnSecondRun(t *testing.T) { + c := s3Client(t) + fc, fcClose := filerClient(t) + defer fcClose() + + bucket := uniqueBucket("vexp2") + mustCreateBucket(t, c, bucket) + putVersioningEnabled(t, c, bucket) + putExpirationLifecycle(t, c, bucket, "expire/", 1) + + // First pass: any object will do — what matters is the cursor saved + // afterward, which switches later passes from the recovery walk to + // meta-log replay. + putObject(t, c, bucket, "expire/first.txt", "v1") + runLifecycleShard(t) + + const key = "expire/daily.txt" + putObject(t, c, bucket, key, "v1") + headOut, err := c.HeadObject(context.Background(), &s3.HeadObjectInput{ + Bucket: aws.String(bucket), Key: aws.String(key), + }) + require.NoError(t, err) + versionID := aws.ToString(headOut.VersionId) + require.NotEmpty(t, versionID) + backdateVersionedMtime(t, fc, bucket, key, versionID, 30) + + runLifecycleShard(t) + + require.Eventuallyf(t, func() bool { + _, err := c.HeadObject(context.Background(), &s3.HeadObjectInput{ + Bucket: aws.String(bucket), Key: aws.String(key), + }) + return isS3NotFound(err) + }, 30*time.Second, 500*time.Millisecond, "second run must create a delete marker for %s/%s", bucket, key) + + directHead, err := c.HeadObject(context.Background(), &s3.HeadObjectInput{ + Bucket: aws.String(bucket), Key: aws.String(key), VersionId: aws.String(versionID), + }) + require.NoError(t, err, "original version must still exist") + require.NotNil(t, directHead) +} + // TestLifecycleNoncurrentVersionExpiration: NoncurrentVersionExpiration // only fires on noncurrent versions. PUT v1, PUT v2 (so v1 → noncurrent), // backdate v1, run worker. v1 must be removed; v2 stays current. diff --git a/test/s3/normal/s3_integration_test.go b/test/s3/normal/s3_integration_test.go index e9a9ed6908..e37972c4bb 100644 --- a/test/s3/normal/s3_integration_test.go +++ b/test/s3/normal/s3_integration_test.go @@ -117,7 +117,8 @@ func TestS3Integration(t *testing.T) { // Extra flags can be appended via extraArgs. func startMiniCluster(t *testing.T, extraArgs ...string) (*TestCluster, error) { // Allocate non-colliding ports (including gRPC offsets) for weed mini. - ports := testutil.MustFreeMiniPorts(t, []string{"Master", "Volume", "Filer", "S3"}) + ports := testutil.MustFreeMiniPorts(t, []string{"Master", "Volume", "Filer", "S3", "Admin"}) + adminPort := ports[4] masterPort := ports[0] volumePort := ports[1] filerPort := ports[2] @@ -186,13 +187,15 @@ func startMiniCluster(t *testing.T, extraArgs ...string) (*TestCluster, error) { "-volume.port=" + strconv.Itoa(volumePort), "-filer.port=" + strconv.Itoa(filerPort), "-s3.port=" + strconv.Itoa(s3Port), - "-webdav.port=0", // Disable WebDAV + "-webdav=false", // Disable WebDAV "-admin.ui=false", // Disable admin UI "-master.volumeSizeLimitMB=32", // Small volumes for testing "-ip=127.0.0.1", "-master.peers=none", // Faster startup "-s3.iam.readOnly=false", // Enable IAM write operations for tests }, extraArgs...) + os.Args = append(os.Args, testutil.MiniAdminArgs(adminPort)...) + os.Args = append(os.Args, "-s3.port.lance=0", "-s3.port.iceberg=0") // Suppress most logging during tests glog.MaxSize = 1024 * 1024 diff --git a/test/s3/policy/policy_test.go b/test/s3/policy/policy_test.go index 2e3e1364e3..302ddbceb5 100644 --- a/test/s3/policy/policy_test.go +++ b/test/s3/policy/policy_test.go @@ -710,12 +710,12 @@ func startMiniCluster(t *testing.T) (*TestCluster, error) { // mini in a goroutine, but the previous mini's admin goroutine is // still binding 12535 — so the next test's admin can never become // ready and mini.go fatals on its readiness check. - ports := testutil.MustAllocatePorts(t, 10) + ports := testutil.MustAllocatePorts(t, 8) + adminPort := testutil.MustFreeMiniPort(t, "admin") masterPort, masterGrpcPort := ports[0], ports[1] volumePort, volumeGrpcPort := ports[2], ports[3] filerPort, filerGrpcPort := ports[4], ports[5] s3Port, s3GrpcPort := ports[6], ports[7] - adminPort, adminGrpcPort := ports[8], ports[9] // Manually-managed temp dir (not t.TempDir()) so we control removal order: // the dir is removed inside Stop() AFTER the mini goroutine has fully @@ -786,15 +786,16 @@ enabled = true "-filer.port.grpc=" + strconv.Itoa(filerGrpcPort), "-s3.port=" + strconv.Itoa(s3Port), "-s3.port.grpc=" + strconv.Itoa(s3GrpcPort), - "-admin.port=" + strconv.Itoa(adminPort), - "-admin.port.grpc=" + strconv.Itoa(adminGrpcPort), - "-webdav.port=0", + "-webdav=false", + "-s3.port.lance=0", + "-s3.port.iceberg=0", "-admin.ui=false", "-master.volumeSizeLimitMB=32", "-ip=127.0.0.1", "-master.peers=none", "-s3.iam.readOnly=false", } + os.Args = append(os.Args, testutil.MiniAdminArgs(adminPort)...) glog.MaxSize = 1024 * 1024 for _, cmd := range command.Commands { if cmd.Name() == "mini" && cmd.Run != nil { diff --git a/test/s3/remote_cache/remote_cache_evict_test.go b/test/s3/remote_cache/remote_cache_evict_test.go index 49060319f4..8fab9c53b9 100644 --- a/test/s3/remote_cache/remote_cache_evict_test.go +++ b/test/s3/remote_cache/remote_cache_evict_test.go @@ -16,6 +16,7 @@ import ( "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/service/s3" + "github.com/seaweedfs/seaweedfs/test/testutil" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) @@ -48,12 +49,15 @@ func startMini(t *testing.T, dir string, args ...string) { logFile, err := os.Create(logPath) require.NoError(t, err) miniLogPaths = append(miniLogPaths, logPath) - cmd := exec.Command(weedBinary, append([]string{"mini", + miniArgs := append([]string{"mini", "-dir=" + dir, "-s3.config=s3_config.json", "-s3.allowDeleteBucketNotEmpty=true", "-ip=127.0.0.1", "-ip.bind=127.0.0.1", - }, args...)...) + }, args...) + miniArgs = append(miniArgs, testutil.MiniAdminArgs(testutil.MustFreeMiniPort(t, "admin"))...) + miniArgs = append(miniArgs, "-s3.port.lance=0", "-s3.port.iceberg=0") + cmd := exec.Command(weedBinary, miniArgs...) cmd.Stdout = logFile cmd.Stderr = logFile require.NoError(t, cmd.Start()) @@ -95,6 +99,27 @@ func shellOn(t *testing.T, masterPort, command string) string { return stripLogs(string(out)) } +// waitForFilerListed polls the master until it lists a filer. A fresh +// single-filer mini registers its filer only after an ~10 s empty-membership +// confirm, and weed shell finds the filer through that listing, so shell +// commands sent before it fail. +func waitForFilerListed(t *testing.T, masterPort string) { + t.Helper() + deadline := time.Now().Add(60 * time.Second) + for { + cmd := exec.Command(weedBinary, "shell", "-master=localhost:"+masterPort) + cmd.Stdin = strings.NewReader("cluster.ps\nexit\n") + out, err := cmd.CombinedOutput() + if err == nil && strings.Contains(string(out), "* filers ") && !strings.Contains(string(out), "* filers 0\n") { + return + } + if time.Now().After(deadline) { + t.Fatalf("master on %s lists no filer after 60s: %v\n%s", masterPort, err, out) + } + time.Sleep(time.Second) + } +} + func chunkCountOn(t *testing.T, masterPort, path string) string { meta := shellOn(t, masterPort, "fs.meta.cat "+path) idx := strings.LastIndex(meta, "chunks ") @@ -162,6 +187,7 @@ func TestRemoteCacheEvictUnderPressure(t *testing.T) { "-master.volumeSizeLimitMB=32", "-volume.max=2", "-filer.remoteCacheEvictThreshold=0.99") waitForHTTP(t, evictPrimaryS3) + waitForFilerListed(t, evictPrimaryMaster) remote := createS3Client(evictRemoteS3) _, err := remote.CreateBucket(&s3.CreateBucketInput{Bucket: aws.String(evictBucket)}) diff --git a/test/s3/sdk_v2_routing/s3_sdk_v2_routing_test.go b/test/s3/sdk_v2_routing/s3_sdk_v2_routing_test.go index 2345fdf573..9adc2300f2 100644 --- a/test/s3/sdk_v2_routing/s3_sdk_v2_routing_test.go +++ b/test/s3/sdk_v2_routing/s3_sdk_v2_routing_test.go @@ -207,4 +207,3 @@ func TestCreateAndListBuckets_ServiceLevel(t *testing.T) { } require.True(t, found, "bucket %q must appear in service-level ListBuckets", bucket) } - diff --git a/test/s3/spark/setup_test.go b/test/s3/spark/setup_test.go index efdb2c60d5..3b647a2f8e 100644 --- a/test/s3/spark/setup_test.go +++ b/test/s3/spark/setup_test.go @@ -94,7 +94,7 @@ func (env *TestEnvironment) StartSeaweedFS(t *testing.T) { t.Fatalf("failed to create temp directory: %v", err) } - ports := testutil.MustFreeMiniPorts(t, []string{"Master", "Filer", "S3"}) + ports := testutil.MustFreeMiniPorts(t, []string{"Master", "Filer", "S3", "Admin"}) env.masterPort = ports[0] env.filerPort = ports[1] env.s3Port = ports[2] @@ -105,8 +105,7 @@ func (env *TestEnvironment) StartSeaweedFS(t *testing.T) { t.Fatalf("failed to create IAM config: %v", err) } - env.masterProcess = exec.Command( - env.weedBinary, "mini", + miniArgs := append([]string{"mini", "-ip", bindIP, "-ip.bind", "0.0.0.0", "-master.port", fmt.Sprintf("%d", env.masterPort), @@ -114,7 +113,9 @@ func (env *TestEnvironment) StartSeaweedFS(t *testing.T) { "-s3.port", fmt.Sprintf("%d", env.s3Port), "-s3.config", iamConfigPath, "-dir", env.seaweedfsDataDir, - ) + }, testutil.MiniAdminArgs(ports[3])...) + miniArgs = append(miniArgs, "-webdav=false", "-s3.port.lance=0", "-s3.port.iceberg=0") + env.masterProcess = exec.Command(env.weedBinary, miniArgs...) weedLogPath := filepath.Join(env.seaweedfsDataDir, "weed-mini.log") weedLogFile, err := os.Create(weedLogPath) if err != nil { diff --git a/test/s3/sse/Makefile b/test/s3/sse/Makefile index cd7d50e9e8..e5072dbdba 100644 --- a/test/s3/sse/Makefile +++ b/test/s3/sse/Makefile @@ -106,7 +106,7 @@ start-seaweedfs: check-binary > /tmp/seaweedfs-sse-mini.log 2>&1 & echo $$! > /tmp/weed-mini.pid @echo "Checking S3 service is ready..." - @for i in $$(seq 1 30); do \ + @for i in $$(seq 1 90); do \ if curl -s http://127.0.0.1:$(S3_PORT) > /dev/null 2>&1; then \ echo "✅ S3 service is ready"; \ break; \ @@ -347,7 +347,7 @@ start-seaweedfs-ci: check-binary > /tmp/seaweedfs-sse-mini.log 2>&1 & echo $$! > /tmp/weed-mini.pid @echo "Checking S3 service is ready..." - @for i in $$(seq 1 30); do \ + @for i in $$(seq 1 90); do \ if curl -s http://127.0.0.1:$(S3_PORT) > /dev/null 2>&1; then \ echo "✅ S3 service is ready"; \ break; \ @@ -491,7 +491,7 @@ test-volume-encryption: build-weed -ip=127.0.0.1 \ > /tmp/seaweedfs-sse-mini.log 2>&1 & echo $$! > /tmp/weed-mini.pid @echo "Checking S3 service is ready..." - @for i in $$(seq 1 30); do \ + @for i in $$(seq 1 90); do \ if curl -s http://127.0.0.1:$(S3_PORT) > /dev/null 2>&1; then \ echo "✅ S3 service is ready"; \ break; \ diff --git a/test/s3/versioning/Makefile b/test/s3/versioning/Makefile index 621543bbc8..26ec27704f 100644 --- a/test/s3/versioning/Makefile +++ b/test/s3/versioning/Makefile @@ -94,6 +94,7 @@ start-server: check-deps @echo "🔍 DEBUG: Launching SeaweedFS server in background..." @echo "🔍 DEBUG: Command: $(WEED_BINARY) mini -dir=./test-volume-data -s3.port=$(S3_PORT) -s3.config=../../../docker/compose/s3.json" @$(WEED_BINARY) mini \ + -ip=127.0.0.1 -ip.bind=127.0.0.1 \ -dir=./test-volume-data \ -s3.port=$(S3_PORT) \ -s3.config=../../../docker/compose/s3.json \ @@ -229,13 +230,13 @@ test-with-server: start-server test-versioning-with-configs: check-deps @echo "Testing with different S3 configurations..." @echo "Testing with delete bucket not empty allowed..." - @$(WEED_BINARY) mini -s3.port=$(S3_PORT) -s3.allowDeleteBucketNotEmpty=true > weed-test-config1.log 2>&1 & echo $$! > weed-config1.pid + @$(WEED_BINARY) mini -ip=127.0.0.1 -ip.bind=127.0.0.1 -s3.port=$(S3_PORT) -s3.allowDeleteBucketNotEmpty=true > weed-test-config1.log 2>&1 & echo $$! > weed-config1.pid @sleep 5 @go test -v -timeout=5m -run "TestVersioningBasicWorkflow" . || true @if [ -f weed-config1.pid ]; then kill -TERM $$(cat weed-config1.pid) 2>/dev/null || true; rm -f weed-config1.pid; fi @sleep 2 @echo "Testing with delete bucket not empty disabled..." - @$(WEED_BINARY) mini -s3.port=$(S3_PORT) -s3.allowDeleteBucketNotEmpty=false > weed-test-config2.log 2>&1 & echo $$! > weed-config2.pid + @$(WEED_BINARY) mini -ip=127.0.0.1 -ip.bind=127.0.0.1 -s3.port=$(S3_PORT) -s3.allowDeleteBucketNotEmpty=false > weed-test-config2.log 2>&1 & echo $$! > weed-config2.pid @sleep 5 @go test -v -timeout=5m -run "TestVersioningBasicWorkflow" . || true @if [ -f weed-config2.pid ]; then kill -TERM $$(cat weed-config2.pid) 2>/dev/null || true; rm -f weed-config2.pid; fi @@ -279,6 +280,7 @@ debug-server: @$(MAKE) stop-server @mkdir -p ./test-volume-data @$(WEED_BINARY) mini \ + -ip=127.0.0.1 -ip.bind=127.0.0.1 \ -debug \ -s3.port=$(S3_PORT) \ -s3.allowDeleteBucketNotEmpty=true \ @@ -323,6 +325,7 @@ health-check: start-server-simple: check-deps @echo "Starting SeaweedFS server (simple mode)..." @$(WEED_BINARY) mini \ + -ip=127.0.0.1 -ip.bind=127.0.0.1 \ -dir=$(SERVER_DIR) \ -s3 \ -s3.port=$(S3_PORT) \ @@ -331,7 +334,7 @@ start-server-simple: check-deps echo $$! > weed-server.pid @echo "Waiting for S3 server to be ready..." - @for i in $$(seq 1 30); do \ + @for i in $$(seq 1 90); do \ if echo | nc -z localhost $(S3_PORT); then \ echo "S3 server is ready!"; \ exit 0; \ diff --git a/test/s3tables/admin/admin_iceberg_pages_test.go b/test/s3tables/admin/admin_iceberg_pages_test.go index 4b6b2e4102..e914c3d2a7 100644 --- a/test/s3tables/admin/admin_iceberg_pages_test.go +++ b/test/s3tables/admin/admin_iceberg_pages_test.go @@ -91,7 +91,11 @@ func TestMain(m *testing.M) { } func startCluster(testDir string) (*testCluster, error) { - ports, err := testutil.AllocatePorts(10) + ports, err := testutil.AllocatePorts(9) + if err != nil { + return nil, err + } + adminPorts, err := testutil.AllocateMiniPorts(1) if err != nil { return nil, err } @@ -99,7 +103,8 @@ func startCluster(testDir string) (*testCluster, error) { volumePort, volumeGrpc := ports[2], ports[3] filerPort, filerGrpc := ports[4], ports[5] s3Port, s3Grpc := ports[6], ports[7] - adminPort, adminGrpc := ports[8], ports[9] + adminPort := adminPorts[0] + icebergPort := ports[8] // Empty security.toml disables JWT auth. if err := os.WriteFile(filepath.Join(testDir, "security.toml"), []byte("# test\n"), 0644); err != nil { @@ -150,14 +155,15 @@ func startCluster(testDir string) (*testCluster, error) { "-filer.port.grpc=" + strconv.Itoa(filerGrpc), "-s3.port=" + strconv.Itoa(s3Port), "-s3.port.grpc=" + strconv.Itoa(s3Grpc), - "-admin.port=" + strconv.Itoa(adminPort), - "-admin.port.grpc=" + strconv.Itoa(adminGrpc), - "-webdav.port=0", + "-webdav=false", // Disable WebDAV + "-s3.port.lance=0", // Disable Lance + "-s3.port.iceberg=" + strconv.Itoa(icebergPort), "-master.volumeSizeLimitMB=32", "-ip=127.0.0.1", "-master.peers=none", "-s3.iam.readOnly=false", } + args = append(args, testutil.MiniAdminArgs(adminPort)...) os.Args = append([]string{"weed"}, args...) glog.MaxSize = 1024 * 1024 for _, cmd := range command.Commands { diff --git a/test/s3tables/catalog/duckdb_deterministic_location_test.go b/test/s3tables/catalog/duckdb_deterministic_location_test.go index 8cccdddadc..256efb68c7 100644 --- a/test/s3tables/catalog/duckdb_deterministic_location_test.go +++ b/test/s3tables/catalog/duckdb_deterministic_location_test.go @@ -115,6 +115,9 @@ SELECT 'duckdb deterministic-location describe ok' AS marker; t.Logf("DuckDB output:\n%s", outputStr) if err != nil { + if testutil.IsIllegalInstruction(err) { + t.Skipf("DuckDB image cannot run on this CPU: %v", err) + } // Tolerate environments where the Iceberg extension/version is too // limited to exercise this code path. The intent is to gain // coverage where it's available, not to require it everywhere. diff --git a/test/s3tables/catalog/duckdb_oauth_test.go b/test/s3tables/catalog/duckdb_oauth_test.go index 865a7ee8d5..aa151f48e3 100644 --- a/test/s3tables/catalog/duckdb_oauth_test.go +++ b/test/s3tables/catalog/duckdb_oauth_test.go @@ -135,6 +135,8 @@ func (env *oauthTestEnv) start(t *testing.T) { if env.s3ExternalURL != "" { args = append(args, "-s3.externalUrl", env.s3ExternalURL) } + args = append(args, testutil.MiniAdminArgs(testutil.MustFreeMiniPort(t, "admin"))...) + args = append(args, "-s3.port.lance=0") cmd := exec.CommandContext(ctx, env.weedBinary, args...) cmd.Dir = env.dataDir @@ -349,6 +351,9 @@ SELECT 'DuckDB Iceberg OAuth secret created successfully' as result; t.Logf("DuckDB output:\n%s", outputStr) if err != nil { + if testutil.IsIllegalInstruction(err) { + t.Skipf("DuckDB image cannot run on this CPU: %v", err) + } if strings.Contains(outputStr, "iceberg extension is not available") || strings.Contains(outputStr, "Failed to load") { t.Skip("Skipping: Iceberg extension not available in DuckDB Docker image") @@ -377,6 +382,9 @@ SELECT 'DuckDB Iceberg OAuth secret created successfully' as result; t.Logf("DuckDB fallback output:\n%s", fallbackStr) if fallbackErr != nil { + if testutil.IsIllegalInstruction(fallbackErr) { + t.Skipf("DuckDB image cannot run on this CPU: %v", fallbackErr) + } if strings.Contains(fallbackStr, "NotFound_404") && strings.Contains(fallbackStr, "/v1/oauth/tokens") { t.Fatal("OAuth token endpoint returned 404 - the fix is not working") } diff --git a/test/s3tables/catalog/iceberg_catalog_test.go b/test/s3tables/catalog/iceberg_catalog_test.go index b45f3714ad..d8490fa629 100644 --- a/test/s3tables/catalog/iceberg_catalog_test.go +++ b/test/s3tables/catalog/iceberg_catalog_test.go @@ -63,6 +63,7 @@ type TestEnvironment struct { s3GrpcPort int icebergPort int lancePort int + adminPort int masterPort int masterGrpcPort int filerPort int @@ -117,6 +118,10 @@ func newTestEnvironmentForMain() (*TestEnvironment, error) { if err != nil { return nil, fmt.Errorf("allocate ports: %w", err) } + adminPorts, err := testutil.AllocateMiniPorts(1) + if err != nil { + return nil, fmt.Errorf("allocate admin port: %w", err) + } return &TestEnvironment{ seaweedDir: seaweedDir, @@ -132,6 +137,7 @@ func newTestEnvironmentForMain() (*TestEnvironment, error) { volumePort: ports[7], volumeGrpcPort: ports[8], lancePort: ports[9], + adminPort: adminPorts[0], dockerAvailable: testutil.HasDocker(), }, nil } @@ -152,7 +158,7 @@ func (env *TestEnvironment) startSeaweedFSForMain() error { } } - cmd := exec.CommandContext(ctx, env.weedBinary, "mini", + miniArgs := append([]string{"mini", "-master.port", fmt.Sprintf("%d", env.masterPort), "-master.port.grpc", fmt.Sprintf("%d", env.masterGrpcPort), "-volume.port", fmt.Sprintf("%d", env.volumePort), @@ -165,7 +171,9 @@ func (env *TestEnvironment) startSeaweedFSForMain() error { "-s3.port.lance", fmt.Sprintf("%d", env.lancePort), "-ip.bind", "0.0.0.0", "-dir", env.dataDir, - ) + }, testutil.MiniAdminArgs(env.adminPort)...) + miniArgs = append(miniArgs, "-webdav=false") + cmd := exec.CommandContext(ctx, env.weedBinary, miniArgs...) cmd.Stdout = os.Stdout cmd.Stderr = os.Stderr @@ -566,6 +574,9 @@ SELECT 'Iceberg extension loaded successfully' as result; output, err := cmd.CombinedOutput() if err != nil { t.Logf("DuckDB output: %s", output) + if testutil.IsIllegalInstruction(err) { + t.Skipf("DuckDB image cannot run on this CPU: %v", err) + } // Check for expected errors in certain CI environments outputStr := string(output) if strings.Contains(outputStr, "iceberg extension is not available") || diff --git a/test/s3tables/catalog/issue_9103_test.go b/test/s3tables/catalog/issue_9103_test.go index be3dddbd13..0d2f86b360 100644 --- a/test/s3tables/catalog/issue_9103_test.go +++ b/test/s3tables/catalog/issue_9103_test.go @@ -218,6 +218,9 @@ SELECT * FROM test_catalog.%s.%s LIMIT 0; } if runErr != nil { + if testutil.IsIllegalInstruction(runErr) { + t.Skipf("DuckDB image cannot run on this CPU: %v", runErr) + } if strings.Contains(outStr, "does not exist because schema") && strings.Contains(outStr, namespace) { t.Fatalf("reproduced issue #9103: DuckDB cannot see namespace %q after ATTACH 's3://%s/'; see output above", diff --git a/test/s3tables/catalog/lance_client_test.go b/test/s3tables/catalog/lance_client_test.go index e44d032060..a9abac6f42 100644 --- a/test/s3tables/catalog/lance_client_test.go +++ b/test/s3tables/catalog/lance_client_test.go @@ -2,11 +2,13 @@ package catalog import ( "fmt" - "github.com/seaweedfs/seaweedfs/weed/s3api/s3tables" "os" "os/exec" "path/filepath" "testing" + + "github.com/seaweedfs/seaweedfs/test/testutil" + "github.com/seaweedfs/seaweedfs/weed/s3api/s3tables" ) // TestLanceNamespaceRealClient drives the namespace with the Lance client rather @@ -58,6 +60,9 @@ func TestLanceNamespaceRealClient(t *testing.T) { t.Logf("Running Lance client test against %s", namespaceURL) if err := cmd.Run(); err != nil { + if testutil.IsIllegalInstruction(err) { + t.Skipf("Lance client image cannot run on this CPU: %v", err) + } t.Errorf("Lance client test failed: %v", err) } } diff --git a/test/s3tables/catalog_clickhouse/clickhouse_catalog_test.go b/test/s3tables/catalog_clickhouse/clickhouse_catalog_test.go index d12173398c..b85629fed4 100644 --- a/test/s3tables/catalog_clickhouse/clickhouse_catalog_test.go +++ b/test/s3tables/catalog_clickhouse/clickhouse_catalog_test.go @@ -364,7 +364,7 @@ func (env *TestEnvironment) StartSeaweedFS(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) env.weedCancel = cancel - cmd := exec.CommandContext(ctx, env.weedBinary, "mini", + miniArgs := append([]string{"mini", "-master.port", fmt.Sprintf("%d", env.masterPort), "-master.port.grpc", fmt.Sprintf("%d", env.masterGrpcPort), "-volume.port", fmt.Sprintf("%d", env.volumePort), @@ -378,7 +378,9 @@ func (env *TestEnvironment) StartSeaweedFS(t *testing.T) { "-ip", env.bindIP, "-ip.bind", "0.0.0.0", "-dir", env.dataDir, - ) + }, testutil.MiniAdminArgs(testutil.MustFreeMiniPort(t, "admin"))...) + miniArgs = append(miniArgs, "-webdav=false", "-s3.port.lance=0") + cmd := exec.CommandContext(ctx, env.weedBinary, miniArgs...) cmd.Dir = env.dataDir cmd.Stdout = os.Stdout cmd.Stderr = os.Stderr @@ -869,6 +871,11 @@ func requireClickHouseRuntime(t *testing.T) { if !hasDocker() { t.Skip("Docker not available, skipping ClickHouse integration test") } + // ClickHouse builds for x86-64-v3 (AVX2) since 26.5; on a CPU without it + // the binary dies on SIGILL inside the entrypoint before the server can + // come up. + testutil.RequireImageRunnable(t, + "--entrypoint", "clickhouse", clickhouseImage(), "local", "--query", "SELECT 1") } // hasDocker reports whether `docker version` can run, which we treat as a diff --git a/test/s3tables/catalog_doris/doris_catalog_test.go b/test/s3tables/catalog_doris/doris_catalog_test.go index b7b071b3f2..3a8aaf9e22 100644 --- a/test/s3tables/catalog_doris/doris_catalog_test.go +++ b/test/s3tables/catalog_doris/doris_catalog_test.go @@ -303,7 +303,7 @@ func (env *TestEnvironment) StartSeaweedFS(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) env.weedCancel = cancel - cmd := exec.CommandContext(ctx, env.weedBinary, "mini", + miniArgs := append([]string{"mini", "-master.port", fmt.Sprintf("%d", env.masterPort), "-master.port.grpc", fmt.Sprintf("%d", env.masterGrpcPort), "-volume.port", fmt.Sprintf("%d", env.volumePort), @@ -317,7 +317,9 @@ func (env *TestEnvironment) StartSeaweedFS(t *testing.T) { "-ip", env.bindIP, "-ip.bind", "0.0.0.0", "-dir", env.dataDir, - ) + }, testutil.MiniAdminArgs(testutil.MustFreeMiniPort(t, "admin"))...) + miniArgs = append(miniArgs, "-webdav=false", "-s3.port.lance=0") + cmd := exec.CommandContext(ctx, env.weedBinary, miniArgs...) cmd.Dir = env.dataDir cmd.Stdout = os.Stdout cmd.Stderr = os.Stderr diff --git a/test/s3tables/catalog_dremio/dremio_catalog_test.go b/test/s3tables/catalog_dremio/dremio_catalog_test.go index 09eed0055e..35fedb00f1 100644 --- a/test/s3tables/catalog_dremio/dremio_catalog_test.go +++ b/test/s3tables/catalog_dremio/dremio_catalog_test.go @@ -289,7 +289,7 @@ func (env *TestEnvironment) StartSeaweedFS(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) env.weedCancel = cancel - cmd := exec.CommandContext(ctx, env.weedBinary, "mini", + miniArgs := append([]string{"mini", "-master.port", fmt.Sprintf("%d", env.masterPort), "-master.port.grpc", fmt.Sprintf("%d", env.masterGrpcPort), "-volume.port", fmt.Sprintf("%d", env.volumePort), @@ -303,7 +303,9 @@ func (env *TestEnvironment) StartSeaweedFS(t *testing.T) { "-ip", env.bindIP, "-ip.bind", "0.0.0.0", "-dir", env.dataDir, - ) + }, testutil.MiniAdminArgs(testutil.MustFreeMiniPort(t, "admin"))...) + miniArgs = append(miniArgs, "-webdav=false", "-s3.port.lance=0") + cmd := exec.CommandContext(ctx, env.weedBinary, miniArgs...) cmd.Dir = env.dataDir cmd.Stdout = os.Stdout cmd.Stderr = os.Stderr diff --git a/test/s3tables/catalog_duckdb_lance/duckdb_lance_test.go b/test/s3tables/catalog_duckdb_lance/duckdb_lance_test.go index 997cab2601..5cefd12059 100644 --- a/test/s3tables/catalog_duckdb_lance/duckdb_lance_test.go +++ b/test/s3tables/catalog_duckdb_lance/duckdb_lance_test.go @@ -142,7 +142,7 @@ func (env *environment) start(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) env.weedCancel = cancel - cmd := exec.CommandContext(ctx, env.weedBinary, "mini", + miniArgs := append([]string{"mini", "-master.port", fmt.Sprintf("%d", env.masterPort), "-master.port.grpc", fmt.Sprintf("%d", env.masterGrpcPort), "-volume.port", fmt.Sprintf("%d", env.volumePort), @@ -156,7 +156,9 @@ func (env *environment) start(t *testing.T) { "-ip", env.bindIP, "-ip.bind", "0.0.0.0", "-dir", env.dataDir, - ) + }, testutil.MiniAdminArgs(testutil.MustFreeMiniPort(t, "admin"))...) + miniArgs = append(miniArgs, "-webdav=false", "-s3.port.iceberg=0") + cmd := exec.CommandContext(ctx, env.weedBinary, miniArgs...) cmd.Dir = env.dataDir cmd.Stdout = os.Stdout cmd.Stderr = os.Stderr diff --git a/test/s3tables/catalog_lancedb/lancedb_catalog_test.go b/test/s3tables/catalog_lancedb/lancedb_catalog_test.go index 85a31861f6..be45fd7f18 100644 --- a/test/s3tables/catalog_lancedb/lancedb_catalog_test.go +++ b/test/s3tables/catalog_lancedb/lancedb_catalog_test.go @@ -137,7 +137,7 @@ func (env *environment) start(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) env.weedCancel = cancel - cmd := exec.CommandContext(ctx, env.weedBinary, "mini", + miniArgs := append([]string{"mini", "-master.port", fmt.Sprintf("%d", env.masterPort), "-master.port.grpc", fmt.Sprintf("%d", env.masterGrpcPort), "-volume.port", fmt.Sprintf("%d", env.volumePort), @@ -151,7 +151,9 @@ func (env *environment) start(t *testing.T) { "-ip", env.bindIP, "-ip.bind", "0.0.0.0", "-dir", env.dataDir, - ) + }, testutil.MiniAdminArgs(testutil.MustFreeMiniPort(t, "admin"))...) + miniArgs = append(miniArgs, "-webdav=false", "-s3.port.iceberg=0") + cmd := exec.CommandContext(ctx, env.weedBinary, miniArgs...) cmd.Dir = env.dataDir cmd.Stdout = os.Stdout cmd.Stderr = os.Stderr @@ -270,6 +272,9 @@ func (env *environment) runClient(t *testing.T, bucket string) { out, err := cmd.CombinedOutput() t.Logf("LanceDB client output:\n%s", out) if err != nil { + if testutil.IsIllegalInstruction(err) { + t.Skipf("LanceDB client image cannot run on this CPU: %v", err) + } t.Fatalf("the LanceDB client failed: %v", err) } if !strings.Contains(string(out), "PASS") { diff --git a/test/s3tables/catalog_olake/olake_catalog_test.go b/test/s3tables/catalog_olake/olake_catalog_test.go index eafd815e11..bbed0c2a3d 100644 --- a/test/s3tables/catalog_olake/olake_catalog_test.go +++ b/test/s3tables/catalog_olake/olake_catalog_test.go @@ -275,7 +275,7 @@ func (env *TestEnvironment) StartSeaweedFS(t *testing.T) { t.Fatalf("Failed to create IAM config: %v", err) } - cmd := exec.Command(env.weedBinary, "mini", + miniArgs := append([]string{"mini", "-master.port", fmt.Sprintf("%d", env.masterPort), "-master.port.grpc", fmt.Sprintf("%d", env.masterGrpcPort), "-volume.port", fmt.Sprintf("%d", env.volumePort), @@ -292,7 +292,9 @@ func (env *TestEnvironment) StartSeaweedFS(t *testing.T) { "-ip", env.bindIP, "-ip.bind", "0.0.0.0", "-dir", env.dataDir, - ) + }, testutil.MiniAdminArgs(testutil.MustFreeMiniPort(t, "admin"))...) + miniArgs = append(miniArgs, "-webdav=false", "-s3.port.lance=0") + cmd := exec.Command(env.weedBinary, miniArgs...) cmd.Dir = env.dataDir cmd.Stdout = os.Stdout cmd.Stderr = os.Stderr diff --git a/test/s3tables/catalog_risingwave/setup_test.go b/test/s3tables/catalog_risingwave/setup_test.go index b893c1d91a..f2adf83433 100644 --- a/test/s3tables/catalog_risingwave/setup_test.go +++ b/test/s3tables/catalog_risingwave/setup_test.go @@ -113,7 +113,7 @@ func (env *TestEnvironment) StartSeaweedFS(t *testing.T) { t.Fatalf("failed to create temp directory: %v", err) } - ports := testutil.MustFreeMiniPorts(t, []string{"Master", "Filer", "S3", "Iceberg"}) + ports := testutil.MustFreeMiniPorts(t, []string{"Master", "Filer", "S3", "Iceberg", "Admin"}) env.masterPort = ports[0] env.filerPort = ports[1] env.s3Port = ports[2] @@ -134,8 +134,7 @@ func (env *TestEnvironment) StartSeaweedFS(t *testing.T) { env.logFile = logFile // Start SeaweedFS using weed mini (all-in-one including Iceberg REST) - env.masterProcess = exec.Command( - "weed", "mini", + miniArgs := append([]string{"mini", "-ip", env.bindIP, "-ip.bind", "0.0.0.0", "-master.port", fmt.Sprintf("%d", env.masterPort), @@ -144,7 +143,9 @@ func (env *TestEnvironment) StartSeaweedFS(t *testing.T) { "-s3.port.iceberg", fmt.Sprintf("%d", env.icebergRestPort), "-s3.config", iamConfigPath, "-dir", env.seaweedfsDataDir, - ) + }, testutil.MiniAdminArgs(ports[4])...) + miniArgs = append(miniArgs, "-webdav=false", "-s3.port.lance=0") + env.masterProcess = exec.Command("weed", miniArgs...) env.masterProcess.Stdout = logFile env.masterProcess.Stderr = logFile env.masterProcess.Env = append(os.Environ(), @@ -193,6 +194,7 @@ func (env *TestEnvironment) StartRisingWave(t *testing.T) { if output, err := cmd.CombinedOutput(); err != nil { t.Fatalf("failed to start RisingWave container: %v\n%s", err, string(output)) } + env.checkRisingWaveContainer(t) // Start a sidecar postgres container for running psql commands sidecarName := "seaweed-risingwave-sidecar-" + randomString(8) @@ -204,6 +206,7 @@ func (env *TestEnvironment) StartRisingWave(t *testing.T) { "sleep", "infinity", ) if output, err := sidecarCmd.CombinedOutput(); err != nil { + env.checkRisingWaveContainer(t) t.Fatalf("failed to start postgres sidecar: %v\n%s", err, string(output)) } @@ -219,6 +222,7 @@ func (env *TestEnvironment) waitForRisingWave(timeout time.Duration) bool { deadline := time.Now().Add(timeout) env.t.Logf(">>> Waiting for RisingWave to be ready (timeout %v)...\n", timeout) for time.Now().Before(deadline) { + env.checkRisingWaveContainer(env.t) if output, err := runPostgresClientSQL(env.postgresSidecar, "SELECT 1;"); err == nil { env.t.Logf(">>> RisingWave is ready.\n") return true @@ -230,6 +234,26 @@ func (env *TestEnvironment) waitForRisingWave(timeout time.Duration) bool { return false } +func (env *TestEnvironment) checkRisingWaveContainer(t *testing.T) { + t.Helper() + out, err := exec.Command("docker", "inspect", "--format", "{{.State.Running}} {{.State.ExitCode}} {{.State.OOMKilled}}", env.risingwaveContainer).CombinedOutput() + if err != nil { + t.Fatalf("inspect RisingWave container: %v\n%s", err, out) + } + state := strings.Fields(string(out)) + if len(state) != 3 { + t.Fatalf("unexpected RisingWave container state: %s", out) + } + if state[0] == "true" { + return + } + if state[1] == "132" && state[2] == "false" { + t.Skip("RisingWave image exited with illegal instruction on this CPU") + } + logs, _ := exec.Command("docker", "logs", env.risingwaveContainer).CombinedOutput() + t.Fatalf("RisingWave container exited before it became ready (code %s, OOM killed: %s):\n%s", state[1], state[2], logs) +} + func runPostgresClientSQL(containerName, sql string) ([]byte, error) { ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) defer cancel() diff --git a/test/s3tables/catalog_spark/setup_test.go b/test/s3tables/catalog_spark/setup_test.go index 9091b201c8..7d64dfd6eb 100644 --- a/test/s3tables/catalog_spark/setup_test.go +++ b/test/s3tables/catalog_spark/setup_test.go @@ -90,7 +90,7 @@ func (env *TestEnvironment) StartSeaweedFS(t *testing.T) { t.Fatalf("failed to create temp directory: %v", err) } - ports := testutil.MustFreeMiniPorts(t, []string{"Master", "Filer", "S3", "Iceberg"}) + ports := testutil.MustFreeMiniPorts(t, []string{"Master", "Filer", "S3", "Iceberg", "Admin"}) env.masterPort = ports[0] env.filerPort = ports[1] env.s3Port = ports[2] @@ -104,8 +104,7 @@ func (env *TestEnvironment) StartSeaweedFS(t *testing.T) { } // Start SeaweedFS using weed mini (all-in-one including Iceberg REST) - env.masterProcess = exec.Command( - "weed", "mini", + miniArgs := append([]string{"mini", "-ip", bindIP, "-ip.bind", "0.0.0.0", "-master.port", fmt.Sprintf("%d", env.masterPort), @@ -114,7 +113,9 @@ func (env *TestEnvironment) StartSeaweedFS(t *testing.T) { "-s3.port.iceberg", fmt.Sprintf("%d", env.icebergRestPort), "-s3.config", iamConfigPath, "-dir", env.seaweedfsDataDir, - ) + }, testutil.MiniAdminArgs(ports[4])...) + miniArgs = append(miniArgs, "-webdav=false", "-s3.port.lance=0") + env.masterProcess = exec.Command("weed", miniArgs...) env.masterProcess.Env = append(os.Environ(), "AWS_ACCESS_KEY_ID="+env.accessKey, "AWS_SECRET_ACCESS_KEY="+env.secretKey, diff --git a/test/s3tables/catalog_spark_lance/spark_lance_test.go b/test/s3tables/catalog_spark_lance/spark_lance_test.go index d5d808096d..5bf1a0359f 100644 --- a/test/s3tables/catalog_spark_lance/spark_lance_test.go +++ b/test/s3tables/catalog_spark_lance/spark_lance_test.go @@ -137,7 +137,7 @@ func (env *environment) start(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) env.weedCancel = cancel - cmd := exec.CommandContext(ctx, env.weedBinary, "mini", + miniArgs := append([]string{"mini", "-master.port", fmt.Sprintf("%d", env.masterPort), "-master.port.grpc", fmt.Sprintf("%d", env.masterGrpcPort), "-volume.port", fmt.Sprintf("%d", env.volumePort), @@ -151,7 +151,9 @@ func (env *environment) start(t *testing.T) { "-ip", env.bindIP, "-ip.bind", "0.0.0.0", "-dir", env.dataDir, - ) + }, testutil.MiniAdminArgs(testutil.MustFreeMiniPort(t, "admin"))...) + miniArgs = append(miniArgs, "-webdav=false", "-s3.port.iceberg=0") + cmd := exec.CommandContext(ctx, env.weedBinary, miniArgs...) cmd.Dir = env.dataDir cmd.Stdout = os.Stdout cmd.Stderr = os.Stderr diff --git a/test/s3tables/catalog_trino/trino_catalog_test.go b/test/s3tables/catalog_trino/trino_catalog_test.go index 7645dd2a8a..ddfc60fdeb 100644 --- a/test/s3tables/catalog_trino/trino_catalog_test.go +++ b/test/s3tables/catalog_trino/trino_catalog_test.go @@ -256,7 +256,7 @@ func (env *TestEnvironment) StartSeaweedFS(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) env.weedCancel = cancel - cmd := exec.CommandContext(ctx, env.weedBinary, "mini", + miniArgs := append([]string{"mini", "-master.port", fmt.Sprintf("%d", env.masterPort), "-master.port.grpc", fmt.Sprintf("%d", env.masterGrpcPort), "-volume.port", fmt.Sprintf("%d", env.volumePort), @@ -270,7 +270,9 @@ func (env *TestEnvironment) StartSeaweedFS(t *testing.T) { "-ip", env.bindIP, "-ip.bind", "0.0.0.0", "-dir", env.dataDir, - ) + }, testutil.MiniAdminArgs(testutil.MustFreeMiniPort(t, "admin"))...) + miniArgs = append(miniArgs, "-webdav=false", "-s3.port.lance=0") + cmd := exec.CommandContext(ctx, env.weedBinary, miniArgs...) cmd.Dir = env.dataDir cmd.Stdout = os.Stdout cmd.Stderr = os.Stderr diff --git a/test/s3tables/lakekeeper/lakekeeper_test.go b/test/s3tables/lakekeeper/lakekeeper_test.go index 7076cf187a..d3a23c76c4 100644 --- a/test/s3tables/lakekeeper/lakekeeper_test.go +++ b/test/s3tables/lakekeeper/lakekeeper_test.go @@ -205,7 +205,7 @@ func (env *TestEnvironment) StartSeaweedFS(t *testing.T) { env.weedCancel = cancel // Start weed mini with both S3 config (standard IAM) and IAM config (advanced IAM/STS) - cmd := exec.CommandContext(ctx, env.weedBinary, "-v", "4", "mini", + miniArgs := append([]string{"-v", "4", "mini", "-master.port", fmt.Sprintf("%d", env.masterPort), "-master.port.grpc", fmt.Sprintf("%d", env.masterGrpcPort), "-volume.port", fmt.Sprintf("%d", env.volumePort), @@ -220,7 +220,9 @@ func (env *TestEnvironment) StartSeaweedFS(t *testing.T) { "-ip", env.bindIP, "-ip.bind", "0.0.0.0", "-dir", env.dataDir, - ) + }, testutil.MiniAdminArgs(testutil.MustFreeMiniPort(t, "admin"))...) + miniArgs = append(miniArgs, "-webdav=false", "-s3.port.lance=0", "-s3.port.iceberg=0") + cmd := exec.CommandContext(ctx, env.weedBinary, miniArgs...) cmd.Dir = env.dataDir cmd.Stdout = os.Stdout cmd.Stderr = os.Stderr diff --git a/test/s3tables/lifecycle/cluster_test.go b/test/s3tables/lifecycle/cluster_test.go index 2ab04812e3..ce5037f6b5 100644 --- a/test/s3tables/lifecycle/cluster_test.go +++ b/test/s3tables/lifecycle/cluster_test.go @@ -99,6 +99,7 @@ type environment struct { s3GrpcPort int icebergPort int lancePort int + adminPort int weedCancel context.CancelFunc weedCmd *exec.Cmd @@ -138,6 +139,10 @@ func newEnvironment() (*environment, error) { if err != nil { return nil, fmt.Errorf("allocate ports: %w", err) } + adminPorts, err := testutil.AllocateMiniPorts(1) + if err != nil { + return nil, fmt.Errorf("allocate admin port: %w", err) + } return &environment{ weedBinary: weedBinary, rootDir: seaweedDir, @@ -153,6 +158,7 @@ func newEnvironment() (*environment, error) { s3GrpcPort: ports[7], icebergPort: ports[8], lancePort: ports[9], + adminPort: adminPorts[0], }, nil } @@ -166,7 +172,7 @@ func (env *environment) start() error { ctx, cancel := context.WithCancel(context.Background()) env.weedCancel = cancel - cmd := exec.CommandContext(ctx, env.weedBinary, "mini", + miniArgs := append([]string{"mini", "-master.port", fmt.Sprintf("%d", env.masterPort), "-master.port.grpc", fmt.Sprintf("%d", env.masterGrpcPort), "-volume.port", fmt.Sprintf("%d", env.volumePort), @@ -180,7 +186,9 @@ func (env *environment) start() error { "-s3.config", iamConfigPath, "-ip.bind", "0.0.0.0", "-dir", env.dataDir, - ) + }, testutil.MiniAdminArgs(env.adminPort)...) + miniArgs = append(miniArgs, "-webdav=false") + cmd := exec.CommandContext(ctx, env.weedBinary, miniArgs...) cmd.Dir = env.dataDir cmd.Stdout = os.Stdout cmd.Stderr = os.Stderr diff --git a/test/s3tables/lifecycle/iceberg_lifecycle_test.go b/test/s3tables/lifecycle/iceberg_lifecycle_test.go index 2c34d7b5ce..0eae42a575 100644 --- a/test/s3tables/lifecycle/iceberg_lifecycle_test.go +++ b/test/s3tables/lifecycle/iceberg_lifecycle_test.go @@ -70,6 +70,13 @@ func TestIcebergTableLifecycle(t *testing.T) { func testIcebergLifecycleWithDuckDB(t *testing.T) { env := shared + + // The write phase's first act is INSTALLing the iceberg extension, whose + // binary can itself target a newer x86-64 baseline than the host CPU has. + testutil.RequireImageRunnable(t, + "--entrypoint", "duckdb", duckdbImage, + "-c", "INSTALL iceberg; LOAD iceberg; SELECT 1;") + bucket := "lifecycle-duckdb-" + randomSuffix() namespace, table := "sales", "events" env.createTableBucket(t, bucket, "ICEBERG") @@ -109,6 +116,9 @@ func testIcebergLifecycleWithPyIceberg(t *testing.T) { namespace, table := "sales", "events" env.createTableBucket(t, bucket, "ICEBERG") buildClientImage(t, pyicebergImage, "Dockerfile.pyiceberg") + testutil.RequireImageRunnable(t, + "--entrypoint", "python3", pyicebergImage, + "-c", "import pyarrow; from pyiceberg.catalog import load_catalog") before := env.pyiceberg(t, "write", bucket, namespace, table) assertSeeded(t, before) diff --git a/test/s3tables/lifecycle/lance_lifecycle_test.go b/test/s3tables/lifecycle/lance_lifecycle_test.go index ea076eef55..31b5b6c173 100644 --- a/test/s3tables/lifecycle/lance_lifecycle_test.go +++ b/test/s3tables/lifecycle/lance_lifecycle_test.go @@ -35,6 +35,9 @@ func TestLanceTableLifecycle(t *testing.T) { namespace, table := "ml", "events" env.createTableBucket(t, bucket, "LANCE") buildClientImage(t, lanceImage, "Dockerfile.lance") + testutil.RequireImageRunnable(t, + "--entrypoint", "python3", lanceImage, + "-c", "import lance; import lance_namespace; import pyarrow") before := env.lance(t, "write", bucket, namespace, table) assertSeeded(t, before) diff --git a/test/s3tables/maintenance/maintenance_integration_test.go b/test/s3tables/maintenance/maintenance_integration_test.go index 4a5b763497..ddbfb69172 100644 --- a/test/s3tables/maintenance/maintenance_integration_test.go +++ b/test/s3tables/maintenance/maintenance_integration_test.go @@ -92,7 +92,11 @@ func TestMain(m *testing.M) { } func startCluster(testDir string, extraArgs []string) (*testCluster, error) { - ports, err := testutil.AllocatePorts(10) + ports, err := testutil.AllocatePorts(9) + if err != nil { + return nil, err + } + adminPorts, err := testutil.AllocateMiniPorts(1) if err != nil { return nil, err } @@ -100,7 +104,8 @@ func startCluster(testDir string, extraArgs []string) (*testCluster, error) { volumePort, volumeGrpc := ports[2], ports[3] filerPort, filerGrpc := ports[4], ports[5] s3Port, s3Grpc := ports[6], ports[7] - adminPort, adminGrpc := ports[8], ports[9] + adminPort := adminPorts[0] + icebergPort := ports[8] _ = os.Remove(filepath.Join(testDir, "mini.options")) @@ -146,15 +151,16 @@ func startCluster(testDir string, extraArgs []string) (*testCluster, error) { "-filer.port.grpc=" + strconv.Itoa(filerGrpc), "-s3.port=" + strconv.Itoa(s3Port), "-s3.port.grpc=" + strconv.Itoa(s3Grpc), - "-admin.port=" + strconv.Itoa(adminPort), - "-admin.port.grpc=" + strconv.Itoa(adminGrpc), - "-webdav.port=0", + "-webdav=false", // Disable WebDAV + "-s3.port.lance=0", // Disable Lance + "-s3.port.iceberg=" + strconv.Itoa(icebergPort), "-admin.ui=false", "-master.volumeSizeLimitMB=32", "-ip=127.0.0.1", "-master.peers=none", "-s3.iam.readOnly=false", } + args = append(args, testutil.MiniAdminArgs(adminPort)...) args = append(args, extraArgs...) os.Args = append([]string{"weed"}, args...) glog.MaxSize = 1024 * 1024 diff --git a/test/s3tables/polaris/polaris_env_test.go b/test/s3tables/polaris/polaris_env_test.go index fe286e18ab..8a0c58a505 100644 --- a/test/s3tables/polaris/polaris_env_test.go +++ b/test/s3tables/polaris/polaris_env_test.go @@ -182,7 +182,7 @@ func (env *TestEnvironment) StartSeaweedFS(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) env.weedCancel = cancel - cmd := exec.CommandContext(ctx, env.weedBinary, "-v", "4", "mini", + miniArgs := append([]string{"-v", "4", "mini", "-master.port", fmt.Sprintf("%d", env.masterPort), "-master.port.grpc", fmt.Sprintf("%d", env.masterGrpcPort), "-volume.port", fmt.Sprintf("%d", env.volumePort), @@ -197,7 +197,9 @@ func (env *TestEnvironment) StartSeaweedFS(t *testing.T) { "-ip", env.bindIP, "-ip.bind", "0.0.0.0", "-dir", env.dataDir, - ) + }, testutil.MiniAdminArgs(testutil.MustFreeMiniPort(t, "admin"))...) + miniArgs = append(miniArgs, "-webdav=false", "-s3.port.lance=0", "-s3.port.iceberg=0") + cmd := exec.CommandContext(ctx, env.weedBinary, miniArgs...) cmd.Dir = env.dataDir cmd.Stdout = os.Stdout cmd.Stderr = os.Stderr diff --git a/test/s3tables/sts_integration/sts_integration_test.go b/test/s3tables/sts_integration/sts_integration_test.go index 672043cab3..524713fe47 100644 --- a/test/s3tables/sts_integration/sts_integration_test.go +++ b/test/s3tables/sts_integration/sts_integration_test.go @@ -242,7 +242,7 @@ func (env *TestEnvironment) StartSeaweedFS(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) env.weedCancel = cancel - cmd := exec.CommandContext(ctx, env.weedBinary, "mini", + miniArgs := append([]string{"mini", "-master.port", fmt.Sprintf("%d", env.masterPort), "-master.port.grpc", fmt.Sprintf("%d", env.masterGrpcPort), "-volume.port", fmt.Sprintf("%d", env.volumePort), @@ -257,7 +257,9 @@ func (env *TestEnvironment) StartSeaweedFS(t *testing.T) { "-ip", env.bindIP, "-ip.bind", "0.0.0.0", "-dir", env.dataDir, - ) + }, testutil.MiniAdminArgs(testutil.MustFreeMiniPort(t, "admin"))...) + miniArgs = append(miniArgs, "-webdav=false", "-s3.port.lance=0", "-s3.port.iceberg=0") + cmd := exec.CommandContext(ctx, env.weedBinary, miniArgs...) cmd.Dir = env.dataDir cmd.Stdout = os.Stdout cmd.Stderr = os.Stderr diff --git a/test/s3tables/table-buckets/s3tables_integration_test.go b/test/s3tables/table-buckets/s3tables_integration_test.go index 80b711e478..70d7979df1 100644 --- a/test/s3tables/table-buckets/s3tables_integration_test.go +++ b/test/s3tables/table-buckets/s3tables_integration_test.go @@ -594,11 +594,16 @@ func testTargetOperations(t *testing.T, client *S3TablesClient) { // directory. It does not require a *testing.T so it can be called from TestMain. // extraArgs are appended to the default mini command flags. func startMiniClusterInDir(testDir string, extraArgs []string) (*TestCluster, error) { - // We need 10 unique ports: Master(2), Volume(2), Filer(2), S3(2), Admin(2) - ports, err := testutil.AllocatePorts(10) + // We need 9 unique ports: Master(2), Volume(2), Filer(2), S3(2), Iceberg(1); + // the admin pair comes from the mini-safe range. + ports, err := testutil.AllocatePorts(9) if err != nil { return nil, fmt.Errorf("failed to find available ports: %v", err) } + adminPorts, err := testutil.AllocateMiniPorts(1) + if err != nil { + return nil, fmt.Errorf("failed to allocate admin port: %v", err) + } masterPort := ports[0] masterGrpcPort := ports[1] @@ -608,8 +613,8 @@ func startMiniClusterInDir(testDir string, extraArgs []string) (*TestCluster, er filerGrpcPort := ports[5] s3Port := ports[6] s3GrpcPort := ports[7] - adminPort := ports[8] - adminGrpcPort := ports[9] + adminPort := adminPorts[0] + icebergPort := ports[8] // Ensure no configuration file from previous runs configFile := filepath.Join(testDir, "mini.options") @@ -674,15 +679,16 @@ func startMiniClusterInDir(testDir string, extraArgs []string) (*TestCluster, er "-filer.port.grpc=" + strconv.Itoa(filerGrpcPort), "-s3.port=" + strconv.Itoa(s3Port), "-s3.port.grpc=" + strconv.Itoa(s3GrpcPort), - "-admin.port=" + strconv.Itoa(adminPort), - "-admin.port.grpc=" + strconv.Itoa(adminGrpcPort), - "-webdav.port=0", // Disable WebDAV + "-webdav=false", // Disable WebDAV + "-s3.port.lance=0", // Disable Lance + "-s3.port.iceberg=" + strconv.Itoa(icebergPort), "-admin.ui=false", // Disable admin UI "-master.volumeSizeLimitMB=32", // Small volumes for testing "-ip=127.0.0.1", "-master.peers=none", // Faster startup "-s3.iam.readOnly=false", // Enable IAM write operations for tests } + baseArgs = append(baseArgs, testutil.MiniAdminArgs(adminPort)...) if len(extraArgs) > 0 { baseArgs = append(baseArgs, extraArgs...) } diff --git a/test/s3tables/unity_catalog/setup_test.go b/test/s3tables/unity_catalog/setup_test.go index b56044cb40..3309aff254 100644 --- a/test/s3tables/unity_catalog/setup_test.go +++ b/test/s3tables/unity_catalog/setup_test.go @@ -183,6 +183,8 @@ func (env *testEnv) startSeaweedFS(t *testing.T, iamJSON string) { if enableSTS { weedArgs = append(weedArgs, "-s3.iam.config", iamConfigPath, "-s3.iam.readOnly=false") } + weedArgs = append(weedArgs, testutil.MiniAdminArgs(testutil.MustFreeMiniPort(t, "admin"))...) + weedArgs = append(weedArgs, "-webdav=false", "-s3.port.lance=0", "-s3.port.iceberg=0") cmd := exec.CommandContext(ctx, env.weedBinary, weedArgs...) cmd.Dir = env.dataDir diff --git a/test/testutil/helpers.go b/test/testutil/helpers.go index 268b35b29f..e0eafb33f3 100644 --- a/test/testutil/helpers.go +++ b/test/testutil/helpers.go @@ -2,12 +2,15 @@ package testutil import ( "context" + "errors" "fmt" "net" "net/http" "os" "os/exec" "path/filepath" + "strings" + "testing" "time" ) @@ -20,6 +23,39 @@ func HasDocker() bool { return cmd.Run() == nil } +// RequireImageRunnable runs a throwaway container with the given `docker run` +// arguments and skips the test when its process dies on SIGILL. Upstream +// images increasingly target x86-64-v3 (AVX2) - ClickHouse since 26.5 - so a +// host CPU without those instructions cannot run them at all, and the failure +// says nothing about the code under test. Any other outcome, whether a clean +// run, an ordinary non-zero exit, or a pull failure, is left for the test +// itself to report: the probe can only skip on the one failure the code +// cannot cause. +func RequireImageRunnable(t *testing.T, args ...string) { + t.Helper() + + ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute) + defer cancel() + cmd := exec.CommandContext(ctx, "docker", append([]string{"run", "--rm"}, args...)...) + out, err := cmd.CombinedOutput() + if err == nil { + return + } + var exitErr *exec.ExitError + // 132 = 128 + SIGILL: docker propagates the container process's death. + if errors.As(err, &exitErr) && + (exitErr.ExitCode() == 132 || + strings.Contains(string(out), "Illegal instruction")) { + t.Skipf("image cannot execute on this CPU (illegal instruction): docker run %s\n%s", + strings.Join(args, " "), out) + } +} + +func IsIllegalInstruction(err error) bool { + var exitErr *exec.ExitError + return errors.As(err, &exitErr) && exitErr.ExitCode() == 132 +} + func FindBindIP() string { addrs, err := net.InterfaceAddrs() if err != nil { diff --git a/test/testutil/helpers_test.go b/test/testutil/helpers_test.go new file mode 100644 index 0000000000..efc302fc6c --- /dev/null +++ b/test/testutil/helpers_test.go @@ -0,0 +1,39 @@ +package testutil + +import ( + "errors" + "fmt" + "os" + "os/exec" + "strconv" + "testing" +) + +func TestIsIllegalInstruction(t *testing.T) { + if code := os.Getenv("SEAWEEDFS_TEST_EXIT_CODE"); code != "" { + status, _ := strconv.Atoi(code) + os.Exit(status) + } + + for _, tc := range []struct { + name string + code string + want bool + }{ + {"illegal instruction", "132", true}, + {"ordinary failure", "1", false}, + {"out of memory", "137", false}, + } { + t.Run(tc.name, func(t *testing.T) { + cmd := exec.Command(os.Args[0], "-test.run=^TestIsIllegalInstruction$") + cmd.Env = append(os.Environ(), "SEAWEEDFS_TEST_EXIT_CODE="+tc.code) + err := cmd.Run() + if got := IsIllegalInstruction(fmt.Errorf("docker run: %w", err)); got != tc.want { + t.Fatalf("IsIllegalInstruction(%v) = %v, want %v", err, got, tc.want) + } + }) + } + if IsIllegalInstruction(nil) || IsIllegalInstruction(errors.New("exit status 132")) { + t.Fatal("non-process errors must not be treated as illegal instructions") + } +} diff --git a/test/testutil/ports.go b/test/testutil/ports.go index 0950cf5209..5a1ce76d46 100644 --- a/test/testutil/ports.go +++ b/test/testutil/ports.go @@ -167,6 +167,22 @@ func MustFreeMiniPort(t *testing.T, name string) int { return MustFreeMiniPorts(t, []string{name})[0] } +// MiniAdminArgs returns weed mini flags pinning the admin server to +// adminPort and adminPort+GrpcPortOffset and disabling the legacy compat +// listener. Unconfigured minis all bind the fixed default admin port 12535 +// and the legacy port 23646 (+33646 gRPC), so sequential or concurrent +// minis on one host collide on them. Allocate adminPort with +// MustFreeMiniPort(s) — the admin binds late in mini startup, after the +// cluster's own outbound connections may steal an ephemeral port, so it +// must stay below the Linux ephemeral port floor. +func MiniAdminArgs(adminPort int) []string { + return []string{ + fmt.Sprintf("-admin.port=%d", adminPort), + fmt.Sprintf("-admin.port.grpc=%d", adminPort+GrpcPortOffset), + "-admin.port.legacy=0", + } +} + // AllocatePortSet atomically allocates miniCount master-style port pairs (each // with port and port+GrpcPortOffset reserved) plus regularCount additional // distinct ports. All listeners are held open until the entire batch is diff --git a/test/tus/Makefile b/test/tus/Makefile index 172b3af506..63f2ed50cc 100644 --- a/test/tus/Makefile +++ b/test/tus/Makefile @@ -106,17 +106,17 @@ start-seaweedfs: check-binary # Wait for filer to be ready @echo "$(YELLOW)Waiting for filer to be ready...$(NC)" - @for i in $$(seq 1 30); do \ + @for i in $$(seq 1 90); do \ if curl -s -f http://127.0.0.1:$(FILER_PORT)/ > /dev/null 2>&1; then \ echo "$(GREEN)Filer is ready$(NC)"; \ break; \ fi; \ - if [ $$i -eq 30 ]; then \ - echo "$(RED)Filer failed to start within 30 seconds$(NC)"; \ + if [ $$i -eq 90 ]; then \ + echo "$(RED)Filer failed to start within 90 seconds$(NC)"; \ $(MAKE) debug-logs; \ exit 1; \ fi; \ - echo "Waiting for filer... ($$i/30)"; \ + echo "Waiting for filer... ($$i/90)"; \ sleep 1; \ done diff --git a/test/volume_server/grpc/fetch_remote_s3_test.go b/test/volume_server/grpc/fetch_remote_s3_test.go index d3c9aa673e..856b712357 100644 --- a/test/volume_server/grpc/fetch_remote_s3_test.go +++ b/test/volume_server/grpc/fetch_remote_s3_test.go @@ -37,23 +37,28 @@ func startWeedMini(t *testing.T) (s3Endpoint string, cleanup func()) { } } - miniPorts := testutil.MustFreeMiniPorts(t, []string{"Master", "Volume", "Filer", "S3"}) + miniPorts := testutil.MustFreeMiniPorts(t, []string{"Master", "Volume", "Filer", "S3", "Admin"}) miniMasterPort := miniPorts[0] miniVolumePort := miniPorts[1] miniFilerPort := miniPorts[2] miniS3Port := miniPorts[3] + miniAdminPort := miniPorts[4] miniDir := t.TempDir() os.WriteFile(filepath.Join(miniDir, "security.toml"), []byte("# empty\n"), 0644) ctx, cancel := context.WithCancel(context.Background()) - miniCmd := exec.CommandContext(ctx, weedBin, "mini", + miniArgs := append([]string{"mini", fmt.Sprintf("-dir=%s", miniDir), fmt.Sprintf("-master.port=%d", miniMasterPort), fmt.Sprintf("-volume.port=%d", miniVolumePort), fmt.Sprintf("-filer.port=%d", miniFilerPort), fmt.Sprintf("-s3.port=%d", miniS3Port), - ) + "-webdav=false", + "-s3.port.lance=0", + "-s3.port.iceberg=0", + }, testutil.MiniAdminArgs(miniAdminPort)...) + miniCmd := exec.CommandContext(ctx, weedBin, miniArgs...) miniCmd.Env = append(os.Environ(), "AWS_ACCESS_KEY_ID=admin", "AWS_SECRET_ACCESS_KEY=admin") miniCmd.Dir = miniDir logFile, _ := os.CreateTemp("", "weed-mini-*.log") diff --git a/weed/admin/dash/config_toml.go b/weed/admin/dash/config_toml.go index 92532963ef..bdc00a7328 100644 --- a/weed/admin/dash/config_toml.go +++ b/weed/admin/dash/config_toml.go @@ -102,6 +102,15 @@ func (cp *ConfigPersistence) ApplyMaintenanceConfigFromToml(v TomlConfig) error ecConf.StrictPlacement = v.GetBool(k) ecChanged = true } + if k := "maintenance.erasure_coding.free_slots_deadband"; v.IsSet(k) { + ecConf.FreeSlotsDeadband = v.GetInt(k) + if ecConf.FreeSlotsDeadband < 0 { + ecConf.FreeSlotsDeadband = 0 + } else if ecConf.FreeSlotsDeadband > 100000 { + ecConf.FreeSlotsDeadband = 100000 + } + ecChanged = true + } if !maintenanceChanged && !vacuumChanged && !balanceChanged && !ecChanged { return nil @@ -223,12 +232,13 @@ var pluginConfigSections = []pluginConfigSection{ jobType: "erasure_coding", prefix: "maintenance.erasure_coding", workerKeys: map[string]func(v TomlConfig, key string) *plugin_pb.ConfigValue{ - "fullness_ratio": doubleValue, - "quiet_for_seconds": int64Value, - "min_size_mb": int64Value, - "preferred_tags": stringListValue, - "replica_placement": stringValue, - "strict_placement": boolValue, + "fullness_ratio": doubleValue, + "quiet_for_seconds": int64Value, + "min_size_mb": int64Value, + "preferred_tags": stringListValue, + "replica_placement": stringValue, + "strict_placement": boolValue, + "free_slots_deadband": int64Value, }, // workers read collection_filter from the admin values, not the worker values adminKeys: map[string]func(v TomlConfig, key string) *plugin_pb.ConfigValue{ diff --git a/weed/command/filer.go b/weed/command/filer.go index 245182117a..5be17a26af 100644 --- a/weed/command/filer.go +++ b/weed/command/filer.go @@ -5,7 +5,6 @@ import ( "crypto/tls" "crypto/x509" "fmt" - "net" "net/http" "os" "runtime" @@ -522,22 +521,17 @@ func (fo *FilerOptions) startFiler() { } stopGrpcServer := func() { gracefulStopGrpc(grpcS, gracefulTimeout) } - var socketServer *http.Server if runtime.GOOS != "windows" { localSocket := *fo.localSocket - if localSocket == "" { + explicit := localSocket != "" + if !explicit { localSocket = fmt.Sprintf("/tmp/seaweedfs-filer-%d.sock", *fo.port) } - if err := os.Remove(localSocket); err != nil && !os.IsNotExist(err) { - glog.Fatalf("Failed to remove %s, error: %s", localSocket, err.Error()) + if filerSocketListener := pb.ListenOnLocalSocket(localSocket); filerSocketListener != nil { + socketServer := newHttpServer(defaultHandler, nil) + httpServers = append(httpServers, socketServer) + go socketServer.Serve(filerSocketListener) } - filerSocketListener, err := net.Listen("unix", localSocket) - if err != nil { - glog.Fatalf("Failed to listen on %s: %v", localSocket, err) - } - socketServer = newHttpServer(defaultHandler, nil) - httpServers = append(httpServers, socketServer) - go socketServer.Serve(filerSocketListener) } if viper.GetString("https.filer.key") != "" { diff --git a/weed/command/filer_backup.go b/weed/command/filer_backup.go index 96b363cf55..f184dd23dc 100644 --- a/weed/command/filer_backup.go +++ b/weed/command/filer_backup.go @@ -42,6 +42,7 @@ type FilerBackupOptions struct { timeAgo *time.Duration retentionDays *int initialSnapshot *bool + concurrency *int } var ( @@ -65,6 +66,7 @@ func init() { filerBackupOptions.disableErrorRetry = cmdFilerBackup.Flag.Bool("disableErrorRetry", false, "disables errors retry, only logs will print") filerBackupOptions.ignore404Error = cmdFilerBackup.Flag.Bool("ignore404Error", true, "ignore 404 errors from filer") filerBackupOptions.initialSnapshot = cmdFilerBackup.Flag.Bool("initialSnapshot", false, "before subscribing to metadata updates, walk the live filer tree under -filerPath and seed the destination, then subscribe from the walk-start timestamp so concurrent changes are still captured. Runs on every start when -timeAgo is 0 and overwrites the saved checkpoint, so it also re-seeds a reinitialized destination. Remove it once the backup is caught up, otherwise it re-walks the whole tree on each restart.") + filerBackupOptions.concurrency = cmdFilerBackup.Flag.Int("concurrency", DefaultConcurrencyLimit, "the maximum number of files that will be backed up concurrently.") } var cmdFilerBackup = &Command{ @@ -105,10 +107,12 @@ func runFilerBackup(cmd *Command, args []string) bool { clientId := util.RandomInt32() var clientEpoch int32 + var resumeTsNs int64 for { clientEpoch++ - err := doFilerBackup(grpcDialOption, &filerBackupOptions, reExcludeFileName, excludeFileNames, excludePathPatterns, clientId, clientEpoch) + var err error + resumeTsNs, err = doFilerBackup(grpcDialOption, &filerBackupOptions, reExcludeFileName, excludeFileNames, excludePathPatterns, clientId, clientEpoch, resumeTsNs) if err != nil { glog.Errorf("backup from %s: %v", *filerBackupOptions.filer, err) time.Sleep(1747 * time.Millisecond) @@ -120,12 +124,12 @@ const ( BackupKeyPrefix = "backup." ) -func doFilerBackup(grpcDialOption grpc.DialOption, backupOption *FilerBackupOptions, reExcludeFileName *regexp.Regexp, excludeFileNames []*wildcard.WildcardMatcher, excludePathPatterns []*wildcard.WildcardMatcher, clientId int32, clientEpoch int32) error { +func doFilerBackup(grpcDialOption grpc.DialOption, backupOption *FilerBackupOptions, reExcludeFileName *regexp.Regexp, excludeFileNames []*wildcard.WildcardMatcher, excludePathPatterns []*wildcard.WildcardMatcher, clientId int32, clientEpoch int32, resumeTsNs int64) (int64, error) { // find data sink dataSink := findSink(util.GetViper()) if dataSink == nil { - return fmt.Errorf("no data sink configured in replication.toml") + return 0, fmt.Errorf("no data sink configured in replication.toml") } sourceFiler := pb.ServerAddress(*backupOption.filer) @@ -139,14 +143,20 @@ func doFilerBackup(grpcDialOption grpc.DialOption, backupOption *FilerBackupOpti startFrom := time.Unix(0, 0) sinkId, legacySinkId := backupCheckpointIds(sourcePath, dataSink) runSnapshot := *backupOption.initialSnapshot && timeAgo == 0 - if timeAgo == 0 { + if resumeTsNs > 0 { + // The previous attempt pinned where it stopped — a failed event's + // timestamp or the processed watermark — which can sit behind both the + // persisted checkpoint and a recomputed -timeAgo window. + startFrom = time.Unix(0, resumeTsNs) + glog.V(0).Infof("resuming previous attempt at %v", startFrom) + } else if timeAgo == 0 { if runSnapshot { // snapshot below sets the start point; no checkpoint read needed glog.V(0).Infof("initialSnapshot requested — walking live tree before subscribing") } else { lastOffsetTsNs, err := getOffsetWithFallback(grpcDialOption, sourceFiler, BackupKeyPrefix, sinkId, BackupKeyPrefix, legacySinkId) if err != nil { - glog.V(0).Infof("starting from %v (offset read failed: %v)", startFrom, err) + return 0, fmt.Errorf("read backup offset: %w", err) } else if lastOffsetTsNs > 0 { startFrom = time.Unix(0, lastOffsetTsNs) glog.V(0).Infof("resuming from %v", startFrom) @@ -166,11 +176,11 @@ func doFilerBackup(grpcDialOption grpc.DialOption, backupOption *FilerBackupOpti sourceFiler.ToGrpcAddress(), sourcePath, *backupOption.proxyByFiler); err != nil { - return fmt.Errorf("filersource initialization failed: %v", err) + return resumeTsNs, fmt.Errorf("filersource initialization failed: %v", err) } if err := repl_util.InitializeSSEForReplication(filerSource); err != nil { - return fmt.Errorf("SSE initialization failed: %v", err) + return resumeTsNs, fmt.Errorf("SSE initialization failed: %v", err) } dataSink.SetSourceFiler(filerSource) @@ -181,14 +191,14 @@ func doFilerBackup(grpcDialOption grpc.DialOption, backupOption *FilerBackupOpti if runSnapshot { snapshotTsNs, err := runInitialSnapshot(sourceFiler.ToGrpcAddress(), filerSource, sourcePath, targetPath, excludePaths, reExcludeFileName, excludeFileNames, excludePathPatterns, dataSink, *backupOption.ignore404Error) if err != nil { - return fmt.Errorf("initial snapshot: %w", err) + return 0, fmt.Errorf("initial snapshot: %w", err) } // The walk can take hours on large trees; retry the tiny KV write a // handful of times before giving up so a flaky filer KV doesn't force // the whole walk to repeat on the next retry loop iteration. if err := persistSnapshotOffset(grpcDialOption, sourceFiler, sinkId, snapshotTsNs); err != nil { glog.Errorf("initialSnapshot: FAILED to persist offset %d for sinkId %d after retries: %v — the next retry will redo the full walk", snapshotTsNs, sinkId, err) - return fmt.Errorf("persist initial snapshot offset: %w", err) + return 0, fmt.Errorf("persist initial snapshot offset: %w", err) } startFrom = time.Unix(0, snapshotTsNs) // walk once per process; retries resume from the persisted checkpoint @@ -218,9 +228,74 @@ func doFilerBackup(grpcDialOption grpc.DialOption, backupOption *FilerBackupOpti processEventFn = genProcessFunction(sourcePath, targetPath, excludePaths, reExcludeFileName, excludeFileNames, excludePathPatterns, dataSink, *backupOption.doDeleteFiles, debug) } - processEventFnWithOffset := pb.AddOffsetFunc(processEventFn, 3*time.Second, func(counter int64, lastTsNs int64) error { - glog.V(0).Infof("backup %s progressed to %v %0.2f/sec", sourceFiler, time.Unix(0, lastTsNs), float64(counter)/float64(3)) - return setOffset(grpcDialOption, sourceFiler, BackupKeyPrefix, sinkId, lastTsNs) + if *backupOption.disableErrorRetry { + // TrivialOnError semantics: log the failure and advance. Left to the + // processor a permanent failure would pin the checkpoint and replay + // forever, which is the opposite of what this flag asks for. + inner := processEventFn + processEventFn = func(resp *filer_pb.SubscribeMetadataResponse) error { + if err := inner(resp); err != nil { + glog.Errorf("process %v: %v", resp, err) + } + return nil + } + } + + // Serial event handling caps throughput at one fetch+write at a time, so a + // busy source keeps outrunning the backup. The metadata processor fans + // events out while serializing same-path and subtree-barrier conflicts, + // and the checkpoint only advances on the contiguous processed watermark. + concurrency := *backupOption.concurrency + if concurrency < 1 || concurrency > 1024 { + glog.Warningf("invalid concurrency value %d, using default: %d", concurrency, DefaultConcurrencyLimit) + concurrency = DefaultConcurrencyLimit + } + processor := NewMetadataProcessor(processEventFn, concurrency, startFrom.UnixNano()) + + // AddOffsetFunc only checkpoints while events arrive; persist the + // completed watermark on a ticker too so work finishing on a quiet + // stream does not sit uncheckpointed until the next event. + var lastPersistedTsNs atomic.Int64 + lastPersistedTsNs.Store(-1) + stopPersist := make(chan struct{}) + persistDone := make(chan struct{}) + go func() { + defer close(persistDone) + ticker := time.NewTicker(3 * time.Second) + defer ticker.Stop() + persist := func() { + if ts := resumeTsNsFor(processor); ts != lastPersistedTsNs.Load() { + if err := setOffset(grpcDialOption, sourceFiler, BackupKeyPrefix, sinkId, ts); err == nil { + lastPersistedTsNs.Store(ts) + } else { + glog.Warningf("persist backup offset: %v", err) + } + } + } + for { + select { + case <-stopPersist: + persist() + return + case <-ticker.C: + persist() + } + } + }() + + var lastLogTsNs = time.Now().UnixNano() + processEventFnWithOffset := pb.AddOffsetFunc(func(resp *filer_pb.SubscribeMetadataResponse) error { + processor.AddSyncJob(resp) + return nil + }, 3*time.Second, func(counter int64, lastTsNs int64) error { + offsetTsNs := processor.processedTsWatermark.Load() + if offsetTsNs == 0 { + return nil + } + now := time.Now().UnixNano() + glog.V(0).Infof("backup %s received %0.2f events/sec, processed through %v", sourceFiler, float64(counter)/(float64(now-lastLogTsNs)/1e9), time.Unix(0, offsetTsNs)) + lastLogTsNs = now + return nil }) if dataSink.IsIncremental() && *filerBackupOptions.retentionDays > 0 { @@ -256,9 +331,29 @@ func doFilerBackup(grpcDialOption grpc.DialOption, backupOption *FilerBackupOpti StartTsNs: startFrom.UnixNano(), StopTsNs: 0, EventErrorType: eventErrorType, + GetResumeTsNs: func() int64 { + return resumeTsNsFor(processor) + }, + Resubscribe: processor.ResubscribeCh(), } - return pb.FollowMetadata(sourceFiler, grpcDialOption, metadataFollowOption, processEventFnWithOffset) + err := pb.FollowMetadata(sourceFiler, grpcDialOption, metadataFollowOption, processEventFnWithOffset) + // Jobs admitted before the stream ended keep running; drain them so the + // retry loop's replay cannot race an older write past a newer one. + processor.Drain() + close(stopPersist) + // setOffset has no context, so a stalled checkpoint RPC must not hold the + // retry loop: bound the wait and let a late write land in the background. + select { + case <-persistDone: + case <-time.After(30 * time.Second): + glog.Warningf("backup %s offset checkpoint did not finish in 30s; reconnecting anyway", sourceFiler) + } + // Resume the next attempt at the floored watermark — never recompute a + // position that skips events. The floor sits one tick below the earliest + // failure and stays there even when a redelivery clears the pin, so + // events dropped while the processor was stopped still replay. + return resumeTsNsFor(processor), err } diff --git a/weed/command/filer_backup_offset_test.go b/weed/command/filer_backup_offset_test.go index 7ae18a3c36..e42efdaa2b 100644 --- a/weed/command/filer_backup_offset_test.go +++ b/weed/command/filer_backup_offset_test.go @@ -3,11 +3,14 @@ package command import ( "context" "net" + "strings" "sync" + "sync/atomic" "testing" "github.com/seaweedfs/seaweedfs/weed/pb" "github.com/seaweedfs/seaweedfs/weed/pb/filer_pb" + "github.com/seaweedfs/seaweedfs/weed/util" "google.golang.org/grpc" "google.golang.org/grpc/credentials/insecure" ) @@ -34,14 +37,35 @@ func (s *kvFilerServer) KvPut(_ context.Context, req *filer_pb.KvPutRequest) (*f return &filer_pb.KvPutResponse{}, nil } +type failingKvFilerServer struct { + filer_pb.UnimplementedSeaweedFilerServer + calls atomic.Int32 +} + +func (s *failingKvFilerServer) KvGet(context.Context, *filer_pb.KvGetRequest) (*filer_pb.KvGetResponse, error) { + if s.calls.Add(1) == 1 { + return &filer_pb.KvGetResponse{Error: "kv store unavailable"}, nil + } + return &filer_pb.KvGetResponse{}, nil +} + +// a miss reads as ErrNotFound so the retried attempt skips the KEK retry loop +func (s *failingKvFilerServer) LookupDirectoryEntry(context.Context, *filer_pb.LookupDirectoryEntryRequest) (*filer_pb.LookupDirectoryEntryResponse, error) { + return &filer_pb.LookupDirectoryEntryResponse{}, nil +} + func startKvFiler(t *testing.T) (pb.ServerAddress, grpc.DialOption) { + return startFiler(t, &kvFilerServer{kv: map[string][]byte{}}) +} + +func startFiler(t *testing.T, server filer_pb.SeaweedFilerServer) (pb.ServerAddress, grpc.DialOption) { t.Helper() listener, err := net.Listen("tcp", "127.0.0.1:0") if err != nil { t.Fatalf("listen: %v", err) } grpcServer := grpc.NewServer() - filer_pb.RegisterSeaweedFilerServer(grpcServer, &kvFilerServer{kv: map[string][]byte{}}) + filer_pb.RegisterSeaweedFilerServer(grpcServer, server) go func() { _ = grpcServer.Serve(listener) }() t.Cleanup(func() { grpcServer.Stop() }) grpcPort := listener.Addr().(*net.TCPAddr).Port @@ -110,3 +134,38 @@ func TestBackupOffset_DistinctDestinationsDoNotShare(t *testing.T) { t.Fatalf("B reads (%d, %v), want (2000, nil)", got, err) } } + +func TestFilerBackup_OffsetReadFailureFailsAttempt(t *testing.T) { + server := &failingKvFilerServer{} + filerAddr, dial := startFiler(t, server) + + config := util.GetViper() + config.Set("sink.local.enabled", true) + config.Set("sink.local.directory", t.TempDir()) + t.Cleanup(func() { + config.Set("sink.local.enabled", false) + config.Set("sink.local.directory", "") + }) + + options := filerBackupOptions + address := string(filerAddr) + options.filer = &address + + resumeTsNs, err := doFilerBackup(dial, &options, nil, nil, nil, 1, 1, 0) + if err == nil || !strings.Contains(err.Error(), "kv store unavailable") { + t.Fatalf("doFilerBackup error = %v, want the offset read failure", err) + } + if resumeTsNs != 0 { + t.Fatalf("resume position = %d, want 0", resumeTsNs) + } + + // the next attempt re-reads the checkpoint and gets past it once the KV + // recovers, so it must fail later, not on the offset read + _, err = doFilerBackup(dial, &options, nil, nil, nil, 1, 2, resumeTsNs) + if err == nil || strings.Contains(err.Error(), "kv store unavailable") { + t.Fatalf("retried doFilerBackup error = %v, want a later-stage failure", err) + } + if got := server.calls.Load(); got < 2 { + t.Fatalf("checkpoint reads across attempts = %d, want a re-read", got) + } +} diff --git a/weed/command/filer_remote_gateway_buckets.go b/weed/command/filer_remote_gateway_buckets.go index 569ef62430..7ef16cd284 100644 --- a/weed/command/filer_remote_gateway_buckets.go +++ b/weed/command/filer_remote_gateway_buckets.go @@ -48,7 +48,7 @@ func (option *RemoteGatewayOptions) followBucketUpdatesAndUploadToRemote(filerSo now := time.Now().UnixNano() glog.V(0).Infof("remote sync %s progressed to %v %0.2f/sec", *option.filerAddress, time.Unix(0, offsetTsNs), float64(counter)/(float64(now-lastLogTsNs)/1e9)) lastLogTsNs = now - return remote_storage.SetSyncOffset(option.grpcDialOption, pb.ServerAddress(*option.filerAddress), option.bucketsDir, offsetTsNs) + return remote_storage.SetSyncOffset(option.grpcDialOption, pb.ServerAddress(*option.filerAddress), option.bucketsDir, resumeTsNsFor(processor)) }) option.clientEpoch++ @@ -65,7 +65,7 @@ func (option *RemoteGatewayOptions) followBucketUpdatesAndUploadToRemote(filerSo StopTsNs: 0, EventErrorType: pb.RetryForeverOnError, GetResumeTsNs: func() int64 { - return processor.processedTsWatermark.Load() + return resumeTsNsFor(processor) }, Resubscribe: processor.ResubscribeCh(), } diff --git a/weed/command/filer_remote_sync_dir.go b/weed/command/filer_remote_sync_dir.go index d36147191f..38261cac68 100644 --- a/weed/command/filer_remote_sync_dir.go +++ b/weed/command/filer_remote_sync_dir.go @@ -55,7 +55,7 @@ func followUpdatesAndUploadToRemote(option *RemoteSyncOptions, filerSource *sour now := time.Now().UnixNano() glog.V(0).Infof("remote sync %s progressed to %v %0.2f/sec", *option.filerAddress, time.Unix(0, offsetTsNs), float64(counter)/(float64(now-lastLogTsNs)/1e9)) lastLogTsNs = now - return remote_storage.SetSyncOffset(option.grpcDialOption, pb.ServerAddress(*option.filerAddress), mountedDir, offsetTsNs) + return remote_storage.SetSyncOffset(option.grpcDialOption, pb.ServerAddress(*option.filerAddress), mountedDir, resumeTsNsFor(processor)) }) option.clientEpoch++ @@ -77,7 +77,7 @@ func followUpdatesAndUploadToRemote(option *RemoteSyncOptions, filerSource *sour StopTsNs: 0, EventErrorType: pb.RetryForeverOnError, GetResumeTsNs: func() int64 { - return processor.processedTsWatermark.Load() + return resumeTsNsFor(processor) }, Resubscribe: processor.ResubscribeCh(), } @@ -140,63 +140,24 @@ func (option *RemoteSyncOptions) makeEventProcessor(remoteStorage *remote_pb.Rem if filer_pb.IsEmpty(resp) { return nil } + mustSend := false + var renameSrc util.FullPath + if message.OldEntry != nil && message.NewEntry != nil { + reshaped, inMount := reshapeCrossMountRename(mountedDir, resp) + if !inMount { + return nil + } + // a create reshaped from a rename into the mount can carry a + // RemoteEntry earned at the old path: it stamps the old key, so + // it cannot vouch for this destination + mustSend = filer_pb.IsCreate(reshaped) + if mustSend { + renameSrc = util.NewFullPath(resp.Directory, message.OldEntry.Name) + } + resp, message = reshaped, reshaped.EventNotification + } if filer_pb.IsCreate(resp) { - if isMultipartUploadFile(message.NewParentPath, message.NewEntry.Name) { - return nil - } - if isSystemPath(message.NewParentPath, message.NewEntry.Name) { - glog.V(2).Infof("skipping create of system path: %s/%s", message.NewParentPath, message.NewEntry.Name) - return nil - } - // Propagate delete markers as deletions on the remote. - // Delete markers are zero-content version entries, so they - // would be filtered out by the HasData check below. - if isDeleteMarker(message.NewEntry) { - if newParent, newName, ok := rewriteVersionedSourcePath(message.NewParentPath, message.NewEntry.Name); ok { - dest := toRemoteStorageLocation(util.FullPath(mountedDir), util.NewFullPath(newParent, newName), remoteStorageMountLocation) - return syncDeleteMarker(client, option, message, dest) - } - return nil - } - if !filer.HasData(message.NewEntry) { - return nil - } - glog.V(2).Infof("create: %+v", resp) - if !shouldSendToRemote(message.NewEntry) { - glog.V(2).Infof("skipping creating: %+v", resp) - return nil - } - // Rewrite internal versioning paths to the original S3 key - // to prevent double-versioning when central also has versioning enabled - parentPath, entryName := message.NewParentPath, message.NewEntry.Name - isRewrittenVersion := false - if newParent, newName, ok := rewriteVersionedSourcePath(parentPath, entryName); ok { - glog.V(0).Infof("rewrite versioned path %s/%s -> %s/%s", parentPath, entryName, newParent, newName) - parentPath, entryName = newParent, newName - isRewrittenVersion = true - } - dest := toRemoteStorageLocation(util.FullPath(mountedDir), util.NewFullPath(parentPath, entryName), remoteStorageMountLocation) - if message.NewEntry.IsDirectory { - glog.V(0).Infof("mkdir %s", remote_storage.FormatLocation(dest)) - return client.WriteDirectory(dest, remoteWriteEntry(message.NewEntry, *option.storageClass)) - } - remoteEntry, writeErr := retriedWriteFile(client, filerSource, message.NewParentPath, remoteWriteEntry(message.NewEntry, *option.storageClass), dest) - if errors.Is(writeErr, errSuperseded) { - glog.Errorf("skipping %s: %v", remote_storage.FormatLocation(dest), writeErr) - return nil - } - if writeErr != nil { - return writeErr - } - // Skip updateLocalEntry for versioned rewrites: the logical - // object (e.g. file.xml) has no filer entry in versioned - // buckets, and stamping the internal v_* entry with a - // RemoteEntry for the logical key is semantically wrong. - // Replay is safe because S3 PutObject is idempotent. - if isRewrittenVersion { - return nil - } - return updateLocalEntry(option, message.NewParentPath, message.NewEntry, remoteEntry) + return processCreateEvent(option, filerSource, *option.storageClass, client, mountedDir, remoteStorageMountLocation, resp, mustSend, renameSrc) } if filer_pb.IsDelete(resp) { return processDeleteEvent(filerSource, client, mountedDir, remoteStorageMountLocation, resp) @@ -210,6 +171,136 @@ func (option *RemoteSyncOptions) makeEventProcessor(remoteStorage *remote_pb.Rem return eachEntryFunc, nil } +// reshapeCrossMountRename maps an update event onto the side of it the mount +// covers. The subscription delivers a rename when either of its paths lies +// under the mounted directory, and only a path under it maps to a remote +// key: toRemoteStorageLocation slices the mount prefix off, which panics on +// a shorter path and names a key in the wrong tree on a longer one. A rename +// into the mount is the creation of the new entry, a rename out of it the +// deletion of the old entry's object, unless that entry is remote-only and +// the object holds its only copy. An update with both paths inside is +// returned as is; one with neither is not the mount's (false). +func reshapeCrossMountRename(mountedDir string, resp *filer_pb.SubscribeMetadataResponse) (*filer_pb.SubscribeMetadataResponse, bool) { + message := resp.EventNotification + oldInMount := isUnderMount(mountedDir, resp.Directory, message.OldEntry.Name) + newInMount := isUnderMount(mountedDir, message.NewParentPath, message.NewEntry.Name) + switch { + case oldInMount && newInMount: + return resp, true + case newInMount: + glog.V(0).Infof("rename into mount %s: %s/%s -> %s/%s, creating", mountedDir, resp.Directory, message.OldEntry.Name, message.NewParentPath, message.NewEntry.Name) + return &filer_pb.SubscribeMetadataResponse{ + Directory: message.NewParentPath, + EventNotification: &filer_pb.EventNotification{NewEntry: message.NewEntry, NewParentPath: message.NewParentPath, Signatures: message.Signatures}, + TsNs: resp.TsNs, + }, true + case oldInMount: + if isRemoteOnly(message.OldEntry) { + glog.V(0).Infof("rename out of mount %s: %s/%s -> %s/%s, keeping the remote-only object", mountedDir, resp.Directory, message.OldEntry.Name, message.NewParentPath, message.NewEntry.Name) + return nil, false + } + glog.V(0).Infof("rename out of mount %s: %s/%s -> %s/%s, deleting", mountedDir, resp.Directory, message.OldEntry.Name, message.NewParentPath, message.NewEntry.Name) + return &filer_pb.SubscribeMetadataResponse{ + Directory: resp.Directory, + EventNotification: &filer_pb.EventNotification{OldEntry: message.OldEntry, Signatures: message.Signatures}, + TsNs: resp.TsNs, + }, true + default: + return nil, false + } +} + +// isUnderMount reports whether dir/name is the mounted directory or lies +// below it; a trailing slash on the mount flag does not change the answer. +func isUnderMount(mountedDir string, dir string, name string) bool { + mount := strings.TrimSuffix(mountedDir, "/") + path := string(util.NewFullPath(dir, name)) + return path == mount || strings.HasPrefix(path, mount+"/") +} + +// processCreateEvent uploads a created entry to the remote key its path maps +// to. mustSend skips the sync-stamp check: a create reshaped out of a rename +// into the mount inherits a RemoteEntry stamped at the old path's key, which +// says nothing about whether this destination holds the content. renameSrc is +// the path that rename came from, and is empty for a genuine create. +func processCreateEvent( + filerClient filer_pb.FilerClient, + filerSource filer_pb.FilerClient, + storageClass string, + client remote_storage.RemoteStorageClient, + mountedDir string, + remoteStorageMountLocation *remote_pb.RemoteStorageLocation, + resp *filer_pb.SubscribeMetadataResponse, + mustSend bool, + renameSrc util.FullPath, +) error { + message := resp.EventNotification + if isMultipartUploadFile(message.NewParentPath, message.NewEntry.Name) { + return nil + } + if isSystemPath(message.NewParentPath, message.NewEntry.Name) { + glog.V(2).Infof("skipping create of system path: %s/%s", message.NewParentPath, message.NewEntry.Name) + return nil + } + // Propagate delete markers as deletions on the remote. + // Delete markers are zero-content version entries, so they + // would be filtered out by the HasData check below. + if isDeleteMarker(message.NewEntry) { + if newParent, newName, ok := rewriteVersionedSourcePath(message.NewParentPath, message.NewEntry.Name); ok { + dest := toRemoteStorageLocation(util.FullPath(mountedDir), util.NewFullPath(newParent, newName), remoteStorageMountLocation) + return syncDeleteMarker(client, filerClient, message, dest) + } + return nil + } + // A create reshaped from a rename into the mount (mustSend) writes an empty + // entry as a zero-byte object; a dropped local copy still skips since the + // rename already copied its object to the new key. A stamped entry whose + // size diverged since (a truncate) is not a dropped copy and sends its + // new content. + if !filer.HasData(message.NewEntry) && !(mustSend && !isDroppedLocalCopy(message.NewEntry)) { + return nil + } + glog.V(2).Infof("create: %+v", resp) + if !mustSend && !shouldSendToRemote(message.NewEntry) { + glog.V(2).Infof("skipping creating: %+v", resp) + return nil + } + // Rewrite internal versioning paths to the original S3 key + // to prevent double-versioning when central also has versioning enabled + parentPath, entryName := message.NewParentPath, message.NewEntry.Name + isRewrittenVersion := false + if newParent, newName, ok := rewriteVersionedSourcePath(parentPath, entryName); ok { + glog.V(0).Infof("rewrite versioned path %s/%s -> %s/%s", parentPath, entryName, newParent, newName) + parentPath, entryName = newParent, newName + isRewrittenVersion = true + } + dest := toRemoteStorageLocation(util.FullPath(mountedDir), util.NewFullPath(parentPath, entryName), remoteStorageMountLocation) + if message.NewEntry.IsDirectory { + glog.V(0).Infof("mkdir %s", remote_storage.FormatLocation(dest)) + return client.WriteDirectory(dest, remoteWriteEntry(message.NewEntry, storageClass)) + } + remoteEntry, writeErr := retriedWriteFile(client, filerSource, message.NewParentPath, remoteWriteEntry(message.NewEntry, storageClass), dest) + if errors.Is(writeErr, errSuperseded) { + glog.Errorf("skipping %s: %v", remote_storage.FormatLocation(dest), writeErr) + if mustSend { + return recoverSupersededCreate(filerClient, filerSource, client, mountedDir, remoteStorageMountLocation, message.NewParentPath, message.NewEntry.Name, renameSrc, dest, storageClass) + } + return nil + } + if writeErr != nil { + return writeErr + } + // Skip updateLocalEntry for versioned rewrites: the logical + // object (e.g. file.xml) has no filer entry in versioned + // buckets, and stamping the internal v_* entry with a + // RemoteEntry for the logical key is semantically wrong. + // Replay is safe because S3 PutObject is idempotent. + if isRewrittenVersion { + return nil + } + return updateLocalEntry(filerClient, message.NewParentPath, message.NewEntry, remoteEntry) +} + // processDeleteEvent removes the remote object an entry mapped to. A remote // object that is already absent counts as deleted: the entry was never // uploaded (created and deleted faster than the sync ran, or a replay of an @@ -651,6 +742,176 @@ func uploadCurrentEntry(filerClient filer_pb.FilerClient, filerSource filer_pb.F return updateLocalEntry(filerClient, dir, current, remoteEntry) } +// recoverSupersededCreate finishes a create reshaped from a rename into the +// mount when the filer has moved past the entry the event described. A plain +// superseded create skips because the superseding event follows in the log; a +// reshaped one has no later event for this destination. remote.uncache accepts +// the rename's inherited stamp and drops the chunks, and the update it logs +// skips the upload through isDroppedLocalCopy, so the content would exist +// nowhere. What the filer holds now decides: local content uploads here only +// when the superseding event will not (the stamp-edge case uploadCurrentEntry +// covers), and remote-only content copies from the object its stamp +// describes, the way completeRemoteOnlyRename finishes a rename. +func recoverSupersededCreate( + filerClient filer_pb.FilerClient, + filerSource filer_pb.FilerClient, + client remote_storage.RemoteStorageClient, + mountedDir string, + remoteStorageMountLocation *remote_pb.RemoteStorageLocation, + dir, name string, + renameSrc util.FullPath, + dest *remote_pb.RemoteStorageLocation, + storageClass string, +) error { + current, err := currentEntry(filerSource, dir, name) + if err != nil { + return err + } + if current == nil || current.IsDirectory { + return nil + } + if isRemoteOnly(current) { + return copyRenameInSource(filerClient, client, mountedDir, remoteStorageMountLocation, dir, current, renameSrc, dest, storageClass) + } + if shouldSendToRemote(current) { + glog.V(0).Infof("leaving %s to the event that superseded the rename", remote_storage.FormatLocation(dest)) + return nil + } + glog.V(0).Infof("uploading the current %s in place of the superseded rename", remote_storage.FormatLocation(dest)) + remoteEntry, writeErr := retriedWriteFile(client, filerSource, dir, remoteWriteEntry(current, storageClass), dest) + if errors.Is(writeErr, errSuperseded) { + glog.Errorf("skipping %s: %v", remote_storage.FormatLocation(dest), writeErr) + return nil + } + if writeErr != nil { + return writeErr + } + return updateLocalEntry(filerClient, dir, current, remoteEntry) +} + +// copyRenameInSource gives the destination key the content of a remote-only +// entry a rename carried into the mount. The object sits at the key the +// renamed-from path maps to on its own mount's remote, or where a +// pending-rename marker under this mount points. The source object is left +// alone: the source side removes it through its own delete event. When the +// destination already holds what the stamp describes (an earlier run copied +// it), only the stamp is refreshed. With no object on either side the event +// fails and holds the offset. +func copyRenameInSource( + filerClient filer_pb.FilerClient, + client remote_storage.RemoteStorageClient, + mountedDir string, + remoteStorageMountLocation *remote_pb.RemoteStorageLocation, + dir string, + current *filer_pb.Entry, + renameSrc util.FullPath, + dest *remote_pb.RemoteStorageLocation, + storageClass string, +) error { + srcClient, srcLoc, err := renameInSource(filerClient, client, mountedDir, remoteStorageMountLocation, current, renameSrc) + if err != nil { + return err + } + if existing, err := client.StatFile(dest); err == nil { + if remote_storage.Describes(current.RemoteEntry, existing) { + glog.V(0).Infof("%s already holds the renamed content", remote_storage.FormatLocation(dest)) + return updateLocalEntry(filerClient, dir, current, existing) + } + glog.V(0).Infof("%s holds an object the entry does not describe (size %d, want %d); replacing it from %s", remote_storage.FormatLocation(dest), existing.RemoteSize, current.RemoteEntry.RemoteSize, remote_storage.FormatLocation(srcLoc)) + } else if !errors.Is(err, remote_storage.ErrRemoteObjectNotFound) { + return err + } + stat, err := srcClient.StatFile(srcLoc) + if errors.Is(err, remote_storage.ErrRemoteObjectNotFound) { + return fmt.Errorf("%s: content is on neither %s nor %s", util.NewFullPath(dir, current.Name), remote_storage.FormatLocation(srcLoc), remote_storage.FormatLocation(dest)) + } + if err != nil { + return err + } + reader, err := openRemoteObject(srcClient, srcLoc, stat.RemoteSize) + if err != nil { + return err + } + defer reader.Close() + glog.V(0).Infof("copy %s -> %s", remote_storage.FormatLocation(srcLoc), remote_storage.FormatLocation(dest)) + remoteEntry, err := client.WriteFile(dest, remoteWriteEntry(current, storageClass), reader) + if err != nil { + return err + } + return updateLocalEntry(filerClient, dir, current, remoteEntry) +} + +// renameInSource locates the object holding a remote-only rename-in entry's +// content and the client that serves it. A pending-rename marker under this +// mount names a key on this remote; otherwise the renamed-from path is +// resolved through the filer's mount mappings to its own mount's remote +// location, served by this client when the source shares this storage. +func renameInSource( + filerClient filer_pb.FilerClient, + client remote_storage.RemoteStorageClient, + mountedDir string, + remoteStorageMountLocation *remote_pb.RemoteStorageLocation, + current *filer_pb.Entry, + renameSrc util.FullPath, +) (remote_storage.RemoteStorageClient, *remote_pb.RemoteStorageLocation, error) { + if src := util.FullPath(current.Extended[filer.ExtRemoteRenameSourceKey]); strings.HasPrefix(string(src), strings.TrimSuffix(mountedDir, "/")+"/") { + return client, toRemoteStorageLocation(util.FullPath(mountedDir), src, remoteStorageMountLocation), nil + } + mappings, err := readMountMappings(filerClient) + if err != nil { + return nil, nil, err + } + srcMount, srcRemoteLoc, err := filer.FindMountedRemoteMapping(mappings, string(renameSrc)) + if err != nil { + return nil, nil, fmt.Errorf("rename source %s: %w", renameSrc, err) + } + srcLoc := filer.MapFullPathToRemoteStorageLocation(util.FullPath(srcMount), srcRemoteLoc, renameSrc) + if srcRemoteLoc.Name == remoteStorageMountLocation.Name { + return client, srcLoc, nil + } + conf, err := readRemoteStorageConf(filerClient, srcRemoteLoc.Name) + if err != nil { + return nil, nil, fmt.Errorf("rename source %s: %w", renameSrc, err) + } + srcClient, err := remote_storage.GetRemoteStorage(conf) + if err != nil { + return nil, nil, fmt.Errorf("rename source %s: %w", renameSrc, err) + } + return srcClient, srcLoc, nil +} + +// readMountMappings reads the filer's remote mount table, the file +// DetectMountInfo reads at startup, through the event's filer client. +func readMountMappings(filerClient filer_pb.FilerClient) (mappings *remote_pb.RemoteStorageMapping, err error) { + var content []byte + err = filerClient.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error { + var readErr error + content, readErr = filer.ReadInsideFiler(context.Background(), client, filer.DirectoryEtcRemote, filer.REMOTE_STORAGE_MOUNT_FILE) + return readErr + }) + if err != nil && !errors.Is(err, filer_pb.ErrNotFound) { + return nil, err + } + return filer.UnmarshalRemoteStorageMappings(content) +} + +// readRemoteStorageConf reads a remote storage's configuration entry. +func readRemoteStorageConf(filerClient filer_pb.FilerClient, storageName string) (*remote_pb.RemoteConf, error) { + var content []byte + if err := filerClient.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error { + var readErr error + content, readErr = filer.ReadInsideFiler(context.Background(), client, filer.DirectoryEtcRemote, storageName+filer.REMOTE_STORAGE_CONF_SUFFIX) + return readErr + }); err != nil { + return nil, err + } + conf := &remote_pb.RemoteConf{} + if err := proto.Unmarshal(content, conf); err != nil { + return nil, fmt.Errorf("unmarshal %s/%s: %w", filer.DirectoryEtcRemote, storageName+filer.REMOTE_STORAGE_CONF_SUFFIX, err) + } + return conf, nil +} + // isSuperseded reports whether the filer has moved past the entry an event // described: it is deleted, or it no longer references every chunk the event // named. Those are the chunks the filer deletes when an entry is updated, so diff --git a/weed/command/filer_remote_sync_dir_test.go b/weed/command/filer_remote_sync_dir_test.go index 93c8340b47..5b004bd068 100644 --- a/weed/command/filer_remote_sync_dir_test.go +++ b/weed/command/filer_remote_sync_dir_test.go @@ -413,10 +413,12 @@ func TestIsMetadataOnlyUpdate(t *testing.T) { } // stubFilerClient serves one entry; nil means not found. A non-nil err fails -// every lookup instead. +// every lookup instead. entries, when set, answers lookups by "dir/name" +// before entry is consulted. type stubFilerClient struct { filer_pb.SeaweedFilerClient entry *filer_pb.Entry + entries map[string]*filer_pb.Entry err error updateErr error lookups int @@ -424,11 +426,14 @@ type stubFilerClient struct { updated *filer_pb.UpdateEntryRequest } -func (c *stubFilerClient) LookupDirectoryEntry(context.Context, *filer_pb.LookupDirectoryEntryRequest, ...grpc.CallOption) (*filer_pb.LookupDirectoryEntryResponse, error) { +func (c *stubFilerClient) LookupDirectoryEntry(_ context.Context, req *filer_pb.LookupDirectoryEntryRequest, _ ...grpc.CallOption) (*filer_pb.LookupDirectoryEntryResponse, error) { c.lookups++ if c.err != nil { return nil, c.err } + if c.entries != nil { + return &filer_pb.LookupDirectoryEntryResponse{Entry: c.entries[req.Directory+"/"+req.Name]}, nil + } return &filer_pb.LookupDirectoryEntryResponse{Entry: c.entry}, nil } @@ -2017,6 +2022,354 @@ func TestMetadataUpdateKeepsTheStoredStampOnNilResult(t *testing.T) { } } +// TestReshapeCrossMountRename covers an update event with a path outside +// the mount. The subscription delivers a rename when either path is under +// the mount, and an update of the /etc/remote directory entry itself through +// the daemon's /etc/remote prefix; processUpdateEvent then sliced the mount +// prefix off both paths, which panics on a path shorter than the mount (seen +// on a replay: "slice bounds out of range [20:11]" on the 11-byte +// /etc/remote, every restart dying on the same event) and names a key in the +// wrong tree on a longer one. +func TestReshapeCrossMountRename(t *testing.T) { + const mount = "/buckets/mount" + chunked := func(name string) *filer_pb.Entry { + return &filer_pb.Entry{Name: name, Attributes: &filer_pb.FuseAttributes{Mtime: 1786096669, FileSize: 7}, Chunks: []*filer_pb.FileChunk{{FileId: "3,01", Size: 7}}} + } + event := func(oldDir, oldName, newDir, newName string, oldEntry *filer_pb.Entry) *filer_pb.SubscribeMetadataResponse { + if oldEntry == nil { + oldEntry = chunked(oldName) + } + return &filer_pb.SubscribeMetadataResponse{ + Directory: oldDir, + TsNs: 42, + EventNotification: &filer_pb.EventNotification{OldEntry: oldEntry, NewParentPath: newDir, NewEntry: chunked(newName), Signatures: []int32{7}}, + } + } + + t.Run("into the mount from a shorter path is a create of the new entry", func(t *testing.T) { + resp := event("/plain", "in.txt", mount+"/lock", "in.txt", nil) + got, inMount := reshapeCrossMountRename(mount, resp) + if !inMount || !filer_pb.IsCreate(got) { + t.Fatalf("got %+v inMount=%v, want a create", got, inMount) + } + if got.Directory != mount+"/lock" || got.EventNotification.NewParentPath != mount+"/lock" || got.EventNotification.NewEntry.Name != "in.txt" { + t.Errorf("create at %s/%s (dir %s), want %s/in.txt", got.EventNotification.NewParentPath, got.EventNotification.NewEntry.Name, got.Directory, mount+"/lock") + } + if got.TsNs != 42 || len(got.EventNotification.Signatures) != 1 { + t.Errorf("timestamp and signatures must carry over: %+v", got) + } + }) + + t.Run("into the mount from a longer path outside it is a create, not a key in the wrong tree", func(t *testing.T) { + resp := event("/buckets/other/deeper/tree", "x.txt", mount, "x.txt", nil) + got, inMount := reshapeCrossMountRename(mount, resp) + if !inMount || !filer_pb.IsCreate(got) || got.EventNotification.NewParentPath != mount { + t.Fatalf("got %+v inMount=%v, want a create under %s", got, inMount, mount) + } + }) + + t.Run("out of the mount is a delete of the old entry", func(t *testing.T) { + resp := event(mount+"/lock", "out.txt", "/plain", "out.txt", nil) + got, inMount := reshapeCrossMountRename(mount, resp) + if !inMount || !filer_pb.IsDelete(got) { + t.Fatalf("got %+v inMount=%v, want a delete", got, inMount) + } + if got.Directory != mount+"/lock" || got.EventNotification.OldEntry.Name != "out.txt" { + t.Errorf("delete of %s/%s, want %s/out.txt", got.Directory, got.EventNotification.OldEntry.Name, mount+"/lock") + } + }) + + t.Run("out of the mount keeps the object of a remote-only entry", func(t *testing.T) { + remoteOnly := &filer_pb.Entry{Name: "out.txt", Attributes: &filer_pb.FuseAttributes{Mtime: 1786096669, FileSize: 7}, RemoteEntry: &filer_pb.RemoteEntry{StorageName: "b2", RemoteSize: 7, RemoteMtime: 1786096669}} + resp := event(mount+"/lock", "out.txt", "/plain", "out.txt", remoteOnly) + if got, inMount := reshapeCrossMountRename(mount, resp); inMount || got != nil { + t.Fatalf("got %+v inMount=%v, want the event dropped", got, inMount) + } + }) + + t.Run("both paths inside is the update itself", func(t *testing.T) { + resp := event(mount+"/a", "x.txt", mount+"/b", "y.txt", nil) + if got, inMount := reshapeCrossMountRename(mount, resp); !inMount || got != resp { + t.Fatalf("got %+v inMount=%v, want the same event", got, inMount) + } + }) + + t.Run("an update of the /etc/remote directory entry itself is not the mount's event", func(t *testing.T) { + dirEntry := &filer_pb.Entry{Name: "remote", IsDirectory: true, Attributes: &filer_pb.FuseAttributes{Mtime: 1786096669}} + resp := &filer_pb.SubscribeMetadataResponse{ + Directory: "/etc", + EventNotification: &filer_pb.EventNotification{OldEntry: dirEntry, NewParentPath: "/etc", NewEntry: dirEntry}, + } + if got, inMount := reshapeCrossMountRename("/buckets/c4x-storage", resp); inMount || got != nil { + t.Fatalf("got %+v inMount=%v, want the event dropped", got, inMount) + } + }) + + t.Run("neither path inside is not the mount's event", func(t *testing.T) { + resp := event("/plain", "x.txt", "/buckets/mounted-not", "x.txt", nil) + if got, inMount := reshapeCrossMountRename(mount, resp); inMount || got != nil { + t.Fatalf("got %+v inMount=%v, want the event dropped", got, inMount) + } + }) + + t.Run("a trailing slash on the mount flag changes nothing", func(t *testing.T) { + resp := event("/plain", "in.txt", mount, "in.txt", nil) + if got, inMount := reshapeCrossMountRename(mount+"/", resp); !inMount || !filer_pb.IsCreate(got) { + t.Fatalf("got %+v inMount=%v, want a create", got, inMount) + } + }) +} + +// TestRenameIntoMountUploadsDespiteTheOldStamp covers a file synced at a path +// under the mount, renamed out (which deleted its remote object), and renamed +// back in: the entry still carries the RemoteEntry the first sync stamped, so +// shouldSendToRemote would skip the create the reshape made, leaving the new +// key without an object. A reshaped create must send unconditionally. +func TestRenameIntoMountUploadsDespiteTheOldStamp(t *testing.T) { + const mountedDir = "/buckets/mount" + mountLoc := &remote_pb.RemoteStorageLocation{Name: "b2", Bucket: "bucket", Path: "/"} + + stamp := &filer_pb.RemoteEntry{StorageName: "b2", RemoteETag: "abc", RemoteSize: 7, RemoteMtime: 1786096669, LastLocalSyncTsNs: 1786096669*1e9 + 1} + newEntry := &filer_pb.Entry{ + Name: "back.txt", + Content: []byte("payload"), + Attributes: &filer_pb.FuseAttributes{Mtime: 1786096669}, + RemoteEntry: stamp, + } + resp, inMount := reshapeCrossMountRename(mountedDir, &filer_pb.SubscribeMetadataResponse{ + Directory: "/plain", + EventNotification: &filer_pb.EventNotification{ + OldEntry: &filer_pb.Entry{Name: "back.txt", Attributes: &filer_pb.FuseAttributes{Mtime: 1786096669}, RemoteEntry: stamp}, + NewParentPath: mountedDir + "/lock", + NewEntry: newEntry, + }, + }) + if !inMount || !filer_pb.IsCreate(resp) { + t.Fatalf("reshape = %+v inMount=%v, want a create", resp, inMount) + } + if shouldSendToRemote(resp.EventNotification.NewEntry) { + t.Fatal("precondition: the stamp the rename carried should fail shouldSendToRemote") + } + + remote := &recordingRemote{} + filerClient := &stubFilerClient{entry: newEntry} + if err := processCreateEvent(filerClient, filerClient, "", remote, mountedDir, mountLoc, resp, true, "/plain/back.txt"); err != nil { + t.Fatal(err) + } + wantWrite := &remote_pb.RemoteStorageLocation{Name: "b2", Bucket: "bucket", Path: "/lock/back.txt"} + if len(remote.writes) != 1 || !proto.Equal(remote.writes[0], wantWrite) { + t.Fatalf("writes = %+v, want the new key %s written", remote.writes, remote_storage.FormatLocation(wantWrite)) + } + + // a plain create keeps trusting the stamp + remote = &recordingRemote{} + if err := processCreateEvent(filerClient, filerClient, "", remote, mountedDir, mountLoc, resp, false, ""); err != nil { + t.Fatal(err) + } + if len(remote.writes) != 0 { + t.Errorf("writes = %+v, want the stamped create skipped", remote.writes) + } +} + +// TestRenameIntoMountWritesAnEmptyFile: a rename into the mount is reshaped +// to a create with mustSend, and an empty entry has no data for the HasData +// gate; the new key still takes a zero-byte object. A remote-only entry holds +// nothing local to send and is skipped as before. +func TestRenameIntoMountWritesAnEmptyFile(t *testing.T) { + const mountedDir = "/buckets/mount" + mountLoc := &remote_pb.RemoteStorageLocation{Name: "b2", Bucket: "bucket", Path: "/"} + renameIn := func(entry *filer_pb.Entry) *filer_pb.SubscribeMetadataResponse { + resp, inMount := reshapeCrossMountRename(mountedDir, &filer_pb.SubscribeMetadataResponse{ + Directory: "/plain", + EventNotification: &filer_pb.EventNotification{OldEntry: entry, NewParentPath: mountedDir + "/lock", NewEntry: entry}, + }) + if !inMount || !filer_pb.IsCreate(resp) { + t.Fatalf("reshape = %+v inMount=%v, want a create", resp, inMount) + } + return resp + } + wantWrite := &remote_pb.RemoteStorageLocation{Name: "b2", Bucket: "bucket", Path: "/lock/empty.txt"} + + t.Run("an empty entry writes a zero-byte object", func(t *testing.T) { + entry := &filer_pb.Entry{Name: "empty.txt", Attributes: &filer_pb.FuseAttributes{Mtime: 1786096669}} + if filer.HasData(entry) { + t.Fatal("precondition: an empty entry carries no data") + } + remote := &recordingRemote{} + filerClient := &stubFilerClient{entry: entry} + if err := processCreateEvent(filerClient, filerClient, "", remote, mountedDir, mountLoc, renameIn(entry), true, "/plain/empty.txt"); err != nil { + t.Fatal(err) + } + if len(remote.writes) != 1 || !proto.Equal(remote.writes[0], wantWrite) { + t.Errorf("writes = %+v, want the new key %s written", remote.writes, remote_storage.FormatLocation(wantWrite)) + } + }) + + t.Run("a remote-only entry still skips", func(t *testing.T) { + stamp := &filer_pb.RemoteEntry{StorageName: "b2", RemoteETag: "abc", RemoteSize: 7, RemoteMtime: 1786096669, LastLocalSyncTsNs: 1786096669*1e9 + 1} + entry := &filer_pb.Entry{Name: "empty.txt", Attributes: &filer_pb.FuseAttributes{Mtime: 1786096669, FileSize: 7}, RemoteEntry: stamp} + if !isRemoteOnly(entry) { + t.Fatal("precondition: the entry should be remote-only") + } + remote := &recordingRemote{} + filerClient := &stubFilerClient{entry: entry} + if err := processCreateEvent(filerClient, filerClient, "", remote, mountedDir, mountLoc, renameIn(entry), true, "/plain/empty.txt"); err != nil { + t.Fatal(err) + } + if len(remote.writes) != 0 { + t.Errorf("writes = %+v, want none for a remote-only entry", remote.writes) + } + }) + + // a stamped entry truncated to zero is not remote-only content: its stamp + // describes bytes the file no longer holds, so the new key takes a + // zero-byte object + t.Run("a truncated stamped entry writes a zero-byte object", func(t *testing.T) { + stamp := &filer_pb.RemoteEntry{StorageName: "b2", RemoteETag: "abc", RemoteSize: 7, RemoteMtime: 1786096669, LastLocalSyncTsNs: 1786096669*1e9 + 1} + entry := &filer_pb.Entry{Name: "empty.txt", Attributes: &filer_pb.FuseAttributes{Mtime: 1786096669}, RemoteEntry: stamp} + if !isRemoteOnly(entry) || isDroppedLocalCopy(entry) { + t.Fatal("precondition: a truncated stamped entry reports remote-only but is not a dropped copy") + } + remote := &recordingRemote{} + filerClient := &stubFilerClient{entry: entry} + if err := processCreateEvent(filerClient, filerClient, "", remote, mountedDir, mountLoc, renameIn(entry), true, "/plain/empty.txt"); err != nil { + t.Fatal(err) + } + if len(remote.writes) != 1 || !proto.Equal(remote.writes[0], wantWrite) { + t.Errorf("writes = %+v, want the new key %s written", remote.writes, remote_storage.FormatLocation(wantWrite)) + } + }) + + // an ordinary create of an empty entry is unchanged: no object + t.Run("a plain empty create still writes nothing", func(t *testing.T) { + entry := &filer_pb.Entry{Name: "empty.txt", Attributes: &filer_pb.FuseAttributes{Mtime: 1786096669}} + remote := &recordingRemote{} + filerClient := &stubFilerClient{entry: entry} + if err := processCreateEvent(filerClient, filerClient, "", remote, mountedDir, mountLoc, renameIn(entry), false, ""); err != nil { + t.Fatal(err) + } + if len(remote.writes) != 0 { + t.Errorf("writes = %+v, want the empty create skipped", remote.writes) + } + }) +} + +// TestSupersededRenameInRecoversTheCurrentEntry covers a rename into the +// mount whose create event the sync reaches only after remote.uncache dropped +// the moved file's chunks: the upload is superseded, and skipping it leaves +// the destination object missing once the source side's delete runs. +func TestSupersededRenameInRecoversTheCurrentEntry(t *testing.T) { + const mountedDir = "/buckets/mount" + mountLoc := &remote_pb.RemoteStorageLocation{Name: "b2", Bucket: "bucket", Path: "/mnt"} + srcMountLoc := &remote_pb.RemoteStorageLocation{Name: "b2", Bucket: "bucket", Path: "/plain-remote"} + + stamp := &filer_pb.RemoteEntry{StorageName: "b2", RemoteETag: "abc", RemoteSize: 4, RemoteMtime: 1786096669, LastLocalSyncTsNs: 1786096669*1e9 + 1} + // the entry as the rename logged it: local chunks plus the stamp earned + // under the source mount + eventEntry := &filer_pb.Entry{ + Name: "a.txt", + Attributes: &filer_pb.FuseAttributes{Mtime: 1786096669, FileSize: 4}, + Chunks: []*filer_pb.FileChunk{{FileId: "1,01111111", Size: 4}}, + RemoteEntry: stamp, + } + // the entry as remote.uncache left it: the chunks are gone and the stamp + // still describes the source mount's object + uncached := &filer_pb.Entry{ + Name: "a.txt", + Attributes: &filer_pb.FuseAttributes{Mtime: 1786096669, FileSize: 4}, + RemoteEntry: stamp, + } + mappingContent, err := proto.Marshal(&remote_pb.RemoteStorageMapping{Mappings: map[string]*remote_pb.RemoteStorageLocation{ + mountedDir: mountLoc, + "/plain": srcMountLoc, + }}) + if err != nil { + t.Fatal(err) + } + mappingEntry := &filer_pb.Entry{Name: filer.REMOTE_STORAGE_MOUNT_FILE, Content: mappingContent} + renameIn := &filer_pb.SubscribeMetadataResponse{ + Directory: mountedDir + "/lock", + EventNotification: &filer_pb.EventNotification{NewParentPath: mountedDir + "/lock", NewEntry: eventEntry}, + } + dest := &remote_pb.RemoteStorageLocation{Name: "b2", Bucket: "bucket", Path: "/mnt/lock/a.txt"} + + newFilerClient := func(entry *filer_pb.Entry) *stubFilerClient { + return &stubFilerClient{entries: map[string]*filer_pb.Entry{ + mountedDir + "/lock/a.txt": proto.Clone(entry).(*filer_pb.Entry), + filer.DirectoryEtcRemote + "/" + filer.REMOTE_STORAGE_MOUNT_FILE: mappingEntry, + }} + } + + t.Run("uncached meanwhile: the source object is copied and the entry stamped", func(t *testing.T) { + remote := &recordingRemote{objects: map[string][]byte{"/plain-remote/a.txt": []byte("data")}} + filerClient := newFilerClient(uncached) + if err := processCreateEvent(filerClient, filerClient, "", remote, mountedDir, mountLoc, renameIn, true, "/plain/a.txt"); err != nil { + t.Fatal(err) + } + if len(remote.writes) != 1 || !proto.Equal(remote.writes[0], dest) { + t.Fatalf("writes = %+v, want the source copied to %s", remote.writes, remote_storage.FormatLocation(dest)) + } + if string(remote.written[0]) != "data" { + t.Errorf("written = %q, want the source object's bytes", remote.written[0]) + } + if filerClient.updates != 1 { + t.Errorf("updates = %d, want the entry stamped with the copy", filerClient.updates) + } + }) + + t.Run("the destination already holds it: only a stamp", func(t *testing.T) { + remote := &recordingRemote{objects: map[string][]byte{"/mnt/lock/a.txt": []byte("data")}, statETags: map[string]string{"/mnt/lock/a.txt": "abc"}} + filerClient := newFilerClient(uncached) + if err := processCreateEvent(filerClient, filerClient, "", remote, mountedDir, mountLoc, renameIn, true, "/plain/a.txt"); err != nil { + t.Fatal(err) + } + if len(remote.writes) != 0 { + t.Errorf("writes = %+v, want none once the destination holds the content", remote.writes) + } + if filerClient.updates != 1 { + t.Errorf("updates = %d, want the entry stamped", filerClient.updates) + } + }) + + t.Run("no object on either side: the event fails", func(t *testing.T) { + remote := &recordingRemote{objects: map[string][]byte{}} + filerClient := newFilerClient(uncached) + if err := processCreateEvent(filerClient, filerClient, "", remote, mountedDir, mountLoc, renameIn, true, "/plain/a.txt"); err == nil { + t.Fatal("want the event to fail and hold the offset") + } + }) + + t.Run("a pending marker under the mount names the source", func(t *testing.T) { + marked := proto.Clone(uncached).(*filer_pb.Entry) + marked.Extended = map[string][]byte{filer.ExtRemoteRenameSourceKey: []byte(mountedDir + "/lock/old.txt")} + remote := &recordingRemote{objects: map[string][]byte{"/mnt/lock/old.txt": []byte("data")}} + filerClient := newFilerClient(marked) + if err := processCreateEvent(filerClient, filerClient, "", remote, mountedDir, mountLoc, renameIn, true, "/plain/a.txt"); err != nil { + t.Fatal(err) + } + if len(remote.writes) != 1 || !proto.Equal(remote.writes[0], dest) { + t.Fatalf("writes = %+v, want the marked source copied to %s", remote.writes, remote_storage.FormatLocation(dest)) + } + }) + + t.Run("rewritten since: the superseding event uploads", func(t *testing.T) { + rewritten := &filer_pb.Entry{ + Name: "a.txt", + Attributes: &filer_pb.FuseAttributes{Mtime: 1786096700, FileSize: 4}, + Chunks: []*filer_pb.FileChunk{{FileId: "2,02222222", Size: 4}}, + RemoteEntry: stamp, + } + remote := &recordingRemote{} + filerClient := newFilerClient(rewritten) + if err := processCreateEvent(filerClient, filerClient, "", remote, mountedDir, mountLoc, renameIn, true, "/plain/a.txt"); err != nil { + t.Fatal(err) + } + if len(remote.writes) != 0 { + t.Errorf("writes = %+v, want the rewrite's own event to upload", remote.writes) + } + }) +} + func TestIsSystemPath(t *testing.T) { tests := []struct { dir string @@ -2086,4 +2439,3 @@ func TestRemoteSyncSkipsSystemFolder(t *testing.T) { t.Fatalf("expected 0 remote deletes for system path, got %d deletes", len(remote.deletes)) } } - diff --git a/weed/command/filer_sync.go b/weed/command/filer_sync.go index 529472139e..554f5e3f99 100644 --- a/weed/command/filer_sync.go +++ b/weed/command/filer_sync.go @@ -7,6 +7,7 @@ import ( "os" "regexp" "strings" + "sync" "sync/atomic" "time" @@ -222,7 +223,7 @@ func runFilerSynchronize(cmd *Command, args []string) bool { if state == nil || state.processor == nil { return } - offsetTsNs := state.processor.processedTsWatermark.Load() + offsetTsNs := resumeTsNsFor(state.processor) if offsetTsNs == 0 { return } @@ -475,7 +476,7 @@ func doSubscribeFilerMetaChanges(clientId int32, clientEpoch int32, sourceGrpcDi lastProgressedTsNs = offsetTsNs // collect synchronous offset statsCollect.FilerSyncOffsetGauge.WithLabelValues(sourceFiler.String(), targetFiler.String(), clientName, sourcePath).Set(float64(offsetTsNs)) - return setOffset(targetGrpcDialOption, targetFiler, signaturePrefix, sourceFilerSignature, offsetTsNs) + return setOffset(targetGrpcDialOption, targetFiler, signaturePrefix, sourceFilerSignature, resumeTsNsFor(processor)) }) prefix := sourcePath @@ -495,7 +496,7 @@ func doSubscribeFilerMetaChanges(clientId int32, clientEpoch int32, sourceGrpcDi StopTsNs: 0, EventErrorType: pb.RetryForeverOnError, GetResumeTsNs: func() int64 { - return processor.processedTsWatermark.Load() + return resumeTsNsFor(processor) }, Resubscribe: processor.ResubscribeCh(), // While the source has only read activity it emits no metadata events, so @@ -576,16 +577,77 @@ func getOffsetWithFallback(grpcDialOption grpc.DialOption, filer pb.ServerAddres return lastOffsetTsNs, err } -func setOffset(grpcDialOption grpc.DialOption, filer pb.ServerAddress, signaturePrefix string, signature int32, offsetTsNs int64) error { - return pb.WithFilerClient(false, signature, filer, grpcDialOption, func(client filer_pb.SeaweedFilerClient) error { +// A checkpoint write can outlive its caller's wait and land beside the next +// subscription attempt's writes; issued concurrently they may be applied out +// of order, moving saved progress backward or forward past events still owed +// a replay. Writes to the same persisted key are serialized and each is +// ordered by issuance: a write issued before the last one to land is stale +// and dropped. Deliberate regressions — resume floors, operator seeds — are +// issued after the writes they supersede, so they still apply. State is keyed +// by target filer so unrelated streams — or the two directions of a +// bidirectional sync — never block or shadow each other. +var offsetCheckpointWrites = struct { + sync.Mutex + byKey map[string]*offsetCheckpoint +}{byKey: map[string]*offsetCheckpoint{}} - syncKey := []byte(signaturePrefix + "____") - util.Uint32toBytes(syncKey[len(signaturePrefix):len(signaturePrefix)+4], uint32(signature)) +type offsetCheckpoint struct { + sync.Mutex + lastSeq int64 +} + +// resumeTsNsFor floors the processed watermark by the earliest failure the +// processor saw: a redelivery can clear the pin, but events dropped while it +// was stopped only replay if the resume position stays behind it. It also +// stays one tick below the newest timestamp seen — a watermark on the +// frontier can have finished every job it admitted while more events at the +// same timestamp still sit unread in the log, and an exclusive resume there +// would skip them. +func resumeTsNsFor(p *MetadataProcessor) int64 { + ts := p.processedTsWatermark.Load() + if newestTsNs := p.NewestSeenTsNs(); newestTsNs > 0 && ts >= newestTsNs { + ts = newestTsNs - 1 + } + if failedTsNs := p.EarliestFailedTsNs(); failedTsNs > 0 && failedTsNs-1 < ts { + ts = failedTsNs - 1 + } + return ts +} + +// offsetWriteSeq numbers checkpoint writes in issuance order so a write that +// lands late can be told apart from the fresher write it would shadow. +var offsetWriteSeq atomic.Int64 + +func setOffset(grpcDialOption grpc.DialOption, filer pb.ServerAddress, signaturePrefix string, signature int32, offsetTsNs int64) error { + syncKey := []byte(signaturePrefix + "____") + util.Uint32toBytes(syncKey[len(signaturePrefix):len(signaturePrefix)+4], uint32(signature)) + key := string(filer) + "\x00" + string(syncKey) + + offsetCheckpointWrites.Lock() + checkpoint := offsetCheckpointWrites.byKey[key] + if checkpoint == nil { + checkpoint = &offsetCheckpoint{} + offsetCheckpointWrites.byKey[key] = checkpoint + } + offsetCheckpointWrites.Unlock() + + seq := offsetWriteSeq.Add(1) + checkpoint.Lock() + defer checkpoint.Unlock() + if seq < checkpoint.lastSeq { + return nil + } + checkpoint.lastSeq = seq + + return pb.WithFilerClient(false, signature, filer, grpcDialOption, func(client filer_pb.SeaweedFilerClient) error { valueBuf := make([]byte, 8) util.Uint64toBytes(valueBuf, uint64(offsetTsNs)) - resp, err := client.KvPut(context.Background(), &filer_pb.KvPutRequest{ + // a stalled KvPut must not hold a caller's drain or persistence loop + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + resp, err := client.KvPut(ctx, &filer_pb.KvPutRequest{ Key: syncKey, Value: valueBuf, }) @@ -600,7 +662,6 @@ func setOffset(grpcDialOption grpc.DialOption, filer pb.ServerAddress, signature return nil }) - } func genProcessFunction(sourcePath string, targetPath string, excludePaths []string, reExcludeFileName *regexp.Regexp, excludeFileNames []*wildcard.WildcardMatcher, excludePathPatterns []*wildcard.WildcardMatcher, dataSink sink.ReplicationSink, doDeleteFiles bool, debug bool) func(resp *filer_pb.SubscribeMetadataResponse) error { diff --git a/weed/command/filer_sync_jobs.go b/weed/command/filer_sync_jobs.go index 69554fd9a4..a791158b40 100644 --- a/weed/command/filer_sync_jobs.go +++ b/weed/command/filer_sync_jobs.go @@ -137,6 +137,19 @@ type MetadataProcessor struct { failedSticky bool oldestFailedTsNs int64 + // earliestFailedTsNs is the smallest timestamp this processor ever failed + // on, latched for its lifetime: a redelivery can clear failedTs, but by + // then the processor is stopped and has already dropped unrecorded + // events — possibly sharing the failure's timestamp — so a resume + // position must still sit behind it. + earliestFailedTsNs int64 + + // newestSeenTsNs is the largest TsNs the stream has offered, job or + // marker. A watermark equal to it is not a safe resume point: more + // events at the same timestamp can still sit unread in the log, so the + // persisted position has to stay one tick below it. + newestSeenTsNs int64 + // resubscribeCh closes once a failure has stopped the processor and all // in-flight jobs have drained, asking the metadata follower to drop the // stream so the caller's reconnect replays the pinned events in order — @@ -199,6 +212,21 @@ func (t *MetadataProcessor) OldestFailedTsNs() int64 { return t.oldestFailedTsNs } +// EarliestFailedTsNs returns the smallest timestamp this processor has ever +// failed on, including failures later cleared by a redelivery. +func (t *MetadataProcessor) EarliestFailedTsNs() int64 { + t.activeJobsLock.Lock() + defer t.activeJobsLock.Unlock() + return t.earliestFailedTsNs +} + +// NewestSeenTsNs returns the largest TsNs the stream has offered so far. +func (t *MetadataProcessor) NewestSeenTsNs() int64 { + t.activeJobsLock.Lock() + defer t.activeJobsLock.Unlock() + return t.newestSeenTsNs +} + // ResubscribeCh closes once a job failure has stopped the processor and its // in-flight jobs have drained, signaling the metadata follower to drop the // stream so a reconnect replays what the watermark still covers. @@ -206,6 +234,18 @@ func (t *MetadataProcessor) ResubscribeCh() <-chan struct{} { return t.resubscribeCh } +// Drain blocks until every admitted job has finished. Callers whose stream +// just ended use it before reconnecting: a replayed event must not race a +// still-running job for the same path, or the older write could finish last +// and clobber the newer one. +func (t *MetadataProcessor) Drain() { + t.activeJobsLock.Lock() + defer t.activeJobsLock.Unlock() + for t.activeJobCount > 0 { + t.activeJobsCond.Wait() + } +} + // pathAncestors returns all proper ancestor directories of p. // For "/a/b/c", returns ["/a/b", "/a", "/"]. func pathAncestors(p util.FullPath) []util.FullPath { @@ -339,6 +379,9 @@ func (t *MetadataProcessor) AddSyncJob(resp *filer_pb.SubscribeMetadataResponse) // to it so idle stretches still advance the resume point. t.activeJobsLock.Lock() defer t.activeJobsLock.Unlock() + if resp.TsNs > t.newestSeenTsNs { + t.newestSeenTsNs = resp.TsNs + } if resp.TsNs > t.filteredTsNs { t.filteredTsNs = resp.TsNs } @@ -354,6 +397,10 @@ func (t *MetadataProcessor) AddSyncJob(resp *filer_pb.SubscribeMetadataResponse) t.activeJobsLock.Lock() defer t.activeJobsLock.Unlock() + if resp.TsNs > t.newestSeenTsNs { + t.newestSeenTsNs = resp.TsNs + } + p, newPath, kind := extractJobInfo(resp) eventKey := failedEventKey{tsNs: resp.TsNs, path: p, newPath: newPath, kind: kind} @@ -425,6 +472,9 @@ func (t *MetadataProcessor) AddSyncJob(resp *filer_pb.SubscribeMetadataResponse) // whole mechanism exists to remove. t.stopped = true t.activeJobsCond.Broadcast() + if t.earliestFailedTsNs == 0 || resp.TsNs < t.earliestFailedTsNs { + t.earliestFailedTsNs = resp.TsNs + } if t.failedSticky { if resp.TsNs < t.oldestFailedTsNs { t.oldestFailedTsNs = resp.TsNs diff --git a/weed/command/filer_sync_jobs_test.go b/weed/command/filer_sync_jobs_test.go index 968858638a..20f5ab04da 100644 --- a/weed/command/filer_sync_jobs_test.go +++ b/weed/command/filer_sync_jobs_test.go @@ -4,6 +4,7 @@ import ( "container/heap" "errors" "fmt" + "sync/atomic" "testing" "time" @@ -553,6 +554,61 @@ func TestSamePathBarrierSerialization(t *testing.T) { }) } +// TestConcurrentJobsUpToLimit proves independent-path jobs run in parallel up +// to the concurrency limit: every admitted job is in flight before any +// finishes (a serial loop could only have one running), and admissions past +// the limit wait for a slot rather than running ahead of it. +func TestConcurrentJobsUpToLimit(t *testing.T) { + const limit = 3 + started := make(chan struct{}, limit) + release := make(chan struct{}) + var inFlight, maxInFlight atomic.Int32 + + p := NewMetadataProcessor(func(resp *filer_pb.SubscribeMetadataResponse) error { + cur := inFlight.Add(1) + for { + if old := maxInFlight.Load(); cur <= old || maxInFlight.CompareAndSwap(old, cur) { + break + } + } + started <- struct{}{} + <-release + inFlight.Add(-1) + return nil + }, limit, 0) + + admitDone := make(chan struct{}) + go func() { + defer close(admitDone) + for i := 0; i < limit+1; i++ { + p.AddSyncJob(makeResp("/d", fmt.Sprintf("f%d", i), false, int64(i+1), true)) + } + }() + + for i := 0; i < limit; i++ { + select { + case <-started: + case <-time.After(10 * time.Second): + t.Fatalf("only %d of %d jobs entered flight", i, limit) + } + } + // The extra admission must still be blocked on a full pool. + select { + case <-started: + t.Fatal("a job started with no free slot") + case <-admitDone: + t.Fatal("submission returned past the concurrency limit") + case <-time.After(200 * time.Millisecond): + } + + close(release) + <-admitDone + p.Drain() + if got := maxInFlight.Load(); got != limit { + t.Fatalf("peak in-flight = %d, want %d", got, limit) + } +} + // benchResult prevents the compiler from optimizing away the conflict check. var benchResult bool @@ -1073,3 +1129,107 @@ func TestResubscribeWaitsForSameTsSibling(t *testing.T) { t.Fatal("resubscribe channel never closed after the same-ts jobs drained") } } + +// TestResumeStaysBehindClearedPin verifies the resume floor outlives the +// ledger pin: a redelivery can clear failedTs, but events dropped while the +// processor was stopped — including ones sharing the failure's timestamp — +// still have to replay, so the resume position stays one tick below the +// earliest failure. +func TestResumeStaysBehindClearedPin(t *testing.T) { + fail := true + release := make(chan struct{}) + p := NewMetadataProcessor(func(resp *filer_pb.SubscribeMetadataResponse) error { + name := resp.EventNotification.NewEntry.GetName() + if name == "slow.txt" { + <-release + } + if name == "bad.txt" && fail { + return errors.New("AccessDenied: Access Denied") + } + return nil + }, 10, 0) + + // the slow job holds the drain — and the resubscribe — open so the + // redelivery still lands on this processor + p.AddSyncJob(makeResp("/dir", "slow.txt", false, 300, true)) + p.AddSyncJob(makeResp("/dir", "bad.txt", false, 200, true)) + deadline := time.Now().Add(10 * time.Second) + for p.OldestFailedTsNs() != 200 && time.Now().Before(deadline) { + time.Sleep(time.Millisecond) + } + + // a same-ts sibling arriving after the stop is dropped + p.AddSyncJob(makeResp("/dir", "sib.txt", false, 200, true)) + p.activeJobsLock.Lock() + dropped := p.activeJobTs[200] == 0 + p.activeJobsLock.Unlock() + if !dropped { + t.Fatal("sibling admitted after the failure stopped the processor") + } + + // the redelivery runs and clears the pin + fail = false + p.AddSyncJob(makeResp("/dir", "bad.txt", false, 200, true)) + deadline = time.Now().Add(10 * time.Second) + for p.OldestFailedTsNs() != 0 && time.Now().Before(deadline) { + time.Sleep(time.Millisecond) + } + if got := p.OldestFailedTsNs(); got != 0 { + t.Fatalf("oldest failed = %d after the redelivery, want 0", got) + } + + if got := p.EarliestFailedTsNs(); got != 200 { + t.Fatalf("earliest failed = %d, want the floor held at 200", got) + } + if got := resumeTsNsFor(p); got != 199 { + t.Fatalf("resume = %d, want 199 so the dropped sibling replays", got) + } + + close(release) + waitForJobsToDrain(t, p) +} + +// TestResumeStaysBelowNewestSeen covers the frontier case: when every admitted +// job has finished, the watermark sits on the newest timestamp seen — but the +// log can still hold same-timestamp events the stream has not delivered yet. +// An exclusive resume there would skip them, so the position stays one tick +// below until a newer timestamp proves the block is closed. +func TestResumeStaysBelowNewestSeen(t *testing.T) { + p := NewMetadataProcessor(func(resp *filer_pb.SubscribeMetadataResponse) error { + return nil + }, 10, 0) + + // the event at 100 finishes before a same-timestamp sibling could even be + // read; the watermark reaches the frontier but cannot persist there + p.AddSyncJob(makeResp("/dir", "a.txt", false, 100, true)) + waitForJobsToDrain(t, p) + if got := resumeTsNsFor(p); got != 99 { + t.Fatalf("resume = %d at the frontier, want 99 so an unread same-ts event replays", got) + } + + // once a newer timestamp is seen the 100-block is closed: a slow job at + // 200 in flight pins nothing below it, so 100 is now a safe position + release := make(chan struct{}) + p2 := NewMetadataProcessor(func(resp *filer_pb.SubscribeMetadataResponse) error { + if resp.TsNs == 200 { + <-release + } + return nil + }, 10, 0) + p2.AddSyncJob(makeResp("/dir", "c.txt", false, 100, true)) + p2.AddSyncJob(makeResp("/dir", "b.txt", false, 200, true)) + deadline := time.Now().Add(10 * time.Second) + for p2.processedTsWatermark.Load() != 100 && time.Now().Before(deadline) { + time.Sleep(time.Millisecond) + } + if got := resumeTsNsFor(p2); got != 100 { + t.Fatalf("resume = %d below the frontier, want 100", got) + } + + // finishing the 200 job lands the watermark back on the frontier + close(release) + waitForJobsToDrain(t, p2) + if got := resumeTsNsFor(p2); got != 199 { + t.Fatalf("resume = %d at the frontier, want 199", got) + } +} diff --git a/weed/command/filer_sync_offset_test.go b/weed/command/filer_sync_offset_test.go index 835bb70fa0..93caf23e5c 100644 --- a/weed/command/filer_sync_offset_test.go +++ b/weed/command/filer_sync_offset_test.go @@ -1,6 +1,10 @@ package command -import "testing" +import ( + "testing" + + "github.com/seaweedfs/seaweedfs/weed/util" +) // The offset key is a persistence contract: a changed form orphans saved // checkpoints, so the historical forms must survive verbatim and the @@ -74,3 +78,50 @@ func TestSyncOffset_HistoricalFallbackThenIndependence(t *testing.T) { t.Fatalf("B read = (%d, %v), want (111, nil)", got, err) } } + +// TestSyncOffset_StaleWriteDropped: a checkpoint write issued before a write +// that already landed under the same key is stale and must not apply, while +// a deliberately regressed value issued later — a resume floor or an +// operator seed — still does. +func TestSyncOffset_StaleWriteDropped(t *testing.T) { + filerAddr, dial := startKvFiler(t) + prefix := getSignaturePrefixByPath("/data", "/backup-stale") + sourceSig := int32(77) + + if err := setOffset(dial, filerAddr, prefix, sourceSig, 300); err != nil { + t.Fatalf("seed offset: %v", err) + } + + // Pin the checkpoint past the next write's issuance seq, simulating a + // write that was issued before a newer one already landed. + syncKey := []byte(prefix + "____") + util.Uint32toBytes(syncKey[len(prefix):len(prefix)+4], uint32(sourceSig)) + mapKey := string(filerAddr) + "\x00" + string(syncKey) + offsetCheckpointWrites.Lock() + checkpoint := offsetCheckpointWrites.byKey[mapKey] + offsetCheckpointWrites.Unlock() + if checkpoint == nil { + t.Fatal("no checkpoint state for the seeded key") + } + checkpoint.Lock() + checkpoint.lastSeq = offsetWriteSeq.Load() + 1000 + checkpoint.Unlock() + + if err := setOffset(dial, filerAddr, prefix, sourceSig, 100); err != nil { + t.Fatalf("stale write: %v", err) + } + if got, err := getOffset(dial, filerAddr, prefix, sourceSig); err != nil || got != 300 { + t.Fatalf("after stale write = (%d, %v), want (300, nil)", got, err) + } + + // a later-issued regression is deliberate and must apply + checkpoint.Lock() + checkpoint.lastSeq = 0 + checkpoint.Unlock() + if err := setOffset(dial, filerAddr, prefix, sourceSig, 199); err != nil { + t.Fatalf("floored write: %v", err) + } + if got, err := getOffset(dial, filerAddr, prefix, sourceSig); err != nil || got != 199 { + t.Fatalf("after floored write = (%d, %v), want (199, nil)", got, err) + } +} diff --git a/weed/command/master.go b/weed/command/master.go index 81ffa6db94..64b9883bfa 100644 --- a/weed/command/master.go +++ b/weed/command/master.go @@ -94,7 +94,7 @@ func init() { m.volumeSizeLimitMB = cmdMaster.Flag.Uint("volumeSizeLimitMB", util.DefaultVolumeSizeLimitMB, "Master stops directing writes to oversized volumes.") m.fileSizeLimitMB = cmdMaster.Flag.Int("fileSizeLimitMB", 256, "limit the file size accepted by /submit, should match the volume servers' -fileSizeLimitMB (-volume.fileSizeLimitMB under weed server or weed mini, which set this for you)") m.volumePreallocate = cmdMaster.Flag.Bool("volumePreallocate", false, "Preallocate disk space for volumes.") - m.maxParallelVacuumPerServer = cmdMaster.Flag.Int("maxParallelVacuumPerServer", 1, "maximum number of volumes to vacuum in parallel per volume server") + m.maxParallelVacuumPerServer = cmdMaster.Flag.Int("maxParallelVacuumPerServer", 1, "maximum number of volumes to vacuum in parallel per volume server; 0 for no limit (deprecated, use master.vacuum.max_parallel_per_server in master.toml)") m.vacuumIntervalSeconds = cmdMaster.Flag.Int("vacuumIntervalSeconds", 840, "seconds between automatic vacuum sweeps") m.vacuumDeleteEmptyAfterSeconds = cmdMaster.Flag.Int("vacuumDeleteEmptyAfterSeconds", 0, "automatic sweep deletes volume copies that stay empty this many seconds; 0 disables") // m.pulseSeconds = cmdMaster.Flag.Int("pulseSeconds", 5, "number of seconds between heartbeats") @@ -488,5 +488,6 @@ func (m *MasterOptions) toMasterOption(whiteList []string) *weed_server.MasterOp MetricsIntervalSec: *m.metricsIntervalSec, TelemetryUrl: *m.telemetryUrl, TelemetryEnabled: *m.telemetryEnabled, + SingleMaster: isSingleMasterMode(*m.peers), } } diff --git a/weed/command/mini.go b/weed/command/mini.go index 9fd8d0963f..05a2c1f752 100644 --- a/weed/command/mini.go +++ b/weed/command/mini.go @@ -445,7 +445,7 @@ func initMiniMasterFlags() { miniMasterOptions.peers = cmdMini.Flag.String("master.peers", "", "all master nodes in comma separated ip:masterPort list (default: none for single master)") miniMasterOptions.volumeSizeLimitMB = cmdMini.Flag.Uint("master.volumeSizeLimitMB", defaultMiniVolumeSizeMB, "Master stops directing writes to oversized volumes (default: 128MB for mini)") miniMasterOptions.volumePreallocate = cmdMini.Flag.Bool("master.volumePreallocate", false, "Preallocate disk space for volumes.") - miniMasterOptions.maxParallelVacuumPerServer = cmdMini.Flag.Int("master.maxParallelVacuumPerServer", 1, "maximum number of volumes to vacuum in parallel on one volume server") + miniMasterOptions.maxParallelVacuumPerServer = cmdMini.Flag.Int("master.maxParallelVacuumPerServer", 1, "maximum number of volumes to vacuum in parallel on one volume server; 0 for no limit (deprecated, use master.vacuum.max_parallel_per_server in master.toml)") miniMasterOptions.vacuumIntervalSeconds = cmdMini.Flag.Int("master.vacuumIntervalSeconds", 840, "seconds between automatic vacuum sweeps") miniMasterOptions.vacuumDeleteEmptyAfterSeconds = cmdMini.Flag.Int("master.vacuumDeleteEmptyAfterSeconds", 0, "automatic sweep deletes volume copies that stay empty this many seconds; 0 disables") miniMasterOptions.defaultReplication = cmdMini.Flag.String("master.defaultReplication", "", "Default replication type if not specified.") @@ -516,6 +516,9 @@ func initMiniVolumeFlags() { miniOptions.v.hasSlowRead = cmdMini.Flag.Bool("volume.hasSlowRead", true, "if true, prevents slow reads from blocking other requests") miniOptions.v.readBufferSizeMB = cmdMini.Flag.Int("volume.readBufferSizeMB", 4, "read buffer size in MB") miniOptions.v.allowUntrustedRemoteEndpoints = cmdMini.Flag.Bool("volume.allowUntrustedRemoteEndpoints", false, "if true, FetchAndWriteNeedle accepts arbitrary remote S3 endpoints including loopback / link-local hosts. Default rejects internal / metadata endpoints.") + miniOptions.v.fsyncMinBatchBytes = cmdMini.Flag.Int64("volume.fsyncMinBatchBytes", 1*1024*1024, "minimum bytes an fsync batch gathers before a drained queue flushes without waiting") + miniOptions.v.fsyncMinBatchRequests = cmdMini.Flag.Int("volume.fsyncMinBatchRequests", 128, "minimum requests an fsync batch gathers before a drained queue flushes without waiting") + miniOptions.v.fsyncBatchWait = cmdMini.Flag.Duration("volume.fsyncBatchWait", 500*time.Microsecond, "how long a below-minimum fsync batch waits for more requests once the queue drains; 0 flushes immediately") miniOptions.v.preStopSeconds = cmdMini.Flag.Int("volume.preStopSeconds", 1, "number of seconds between stop send heartbeats and stop volume server (default: 1 for mini)") miniOptions.v.setDiskIOProbeDefaults() } diff --git a/weed/command/s3.go b/weed/command/s3.go index 958afb8eab..e2d377c64e 100644 --- a/weed/command/s3.go +++ b/weed/command/s3.go @@ -6,7 +6,6 @@ import ( "crypto/x509" "fmt" "io/ioutil" - "net" "net/http" "os" "runtime" @@ -429,23 +428,19 @@ func (s3opt *S3Options) startS3Server() bool { if runtime.GOOS != "windows" { localSocket := *s3opt.localSocket - if localSocket == "" { + explicit := localSocket != "" + if !explicit { localSocket = fmt.Sprintf("/tmp/seaweedfs-s3-%d.sock", *s3opt.port) } - if err := os.Remove(localSocket); err != nil && !os.IsNotExist(err) { - glog.Fatalf("Failed to remove %s, error: %s", localSocket, err.Error()) + if s3SocketListener := pb.ListenOnLocalSocket(localSocket); s3SocketListener != nil { + socketServer := newHttpServer(router, nil) + httpServers = append(httpServers, socketServer) + go func() { + if err := socketServer.Serve(s3SocketListener); err != nil && err != http.ErrServerClosed { + glog.Fatalf("Failed to start S3 http server: %v", err) + } + }() } - s3SocketListener, err := net.Listen("unix", localSocket) - if err != nil { - glog.Fatalf("Failed to listen on %s: %v", localSocket, err) - } - socketServer := newHttpServer(router, nil) - httpServers = append(httpServers, socketServer) - go func() { - if err := socketServer.Serve(s3SocketListener); err != nil && err != http.ErrServerClosed { - glog.Fatalf("Failed to start S3 http server: %v", err) - } - }() } listenAddress := fmt.Sprintf("%s:%d", *s3opt.bindIp, *s3opt.port) diff --git a/weed/command/scaffold/admin.toml b/weed/command/scaffold/admin.toml index e1af67d7a7..99709abc03 100644 --- a/weed/command/scaffold/admin.toml +++ b/weed/command/scaffold/admin.toml @@ -66,6 +66,9 @@ # replica_placement = "" # fail planning when the placement constraints cannot be met instead of relaxing them # strict_placement = false +# volume-slot deadband for shard placement: nodes whose free volume slots differ by at +# most this much tie and fewer live shards wins; applied times data shards (0 = strict) +# free_slots_deadband = 0 # max retry attempts for a failed erasure coding job # retry_limit = 1 # seconds to wait between retry attempts diff --git a/weed/command/scaffold/master.toml b/weed/command/scaffold/master.toml index b7a344e231..dfa8472030 100644 --- a/weed/command/scaffold/master.toml +++ b/weed/command/scaffold/master.toml @@ -4,6 +4,13 @@ # /etc/seaweedfs/master.toml # this file is read by master +[master] +# cap on writable volumes per volume layout (collection + replica placement + +# ttl + disk type) that write assignments spread over; fullest picked first. +# 0 = no limit +max_writable_volumes_per_layout = 0 + + [master.maintenance] # periodically run these scripts are the same as running them from 'weed shell' # Scripts are skipped while an admin server is connected. @@ -11,6 +18,15 @@ scripts = """{{DEFAULT_MAINTENANCE_SCRIPTS}}""" sleep_minutes = 17 # sleep minutes between each script execution +# vacuum reclaims deleted space by compacting volumes +[master.vacuum] +# concurrent volume compactions per volume server; overrides the deprecated +# -maxParallelVacuumPerServer flag (default 1) when set; 0 = no limit +# max_parallel_per_server = 1 +# concurrent volume copies compacting within one rack; 0 = no limit +max_parallel_per_rack = 0 + + [master.sequencer] type = "raft" # Choose [raft|snowflake] type for storing the file id sequence # when sequencer.type = snowflake, the snowflake id must be different from other masters diff --git a/weed/command/scaffold/notification.toml b/weed/command/scaffold/notification.toml index ebbd240f9f..7d327984e4 100644 --- a/weed/command/scaffold/notification.toml +++ b/weed/command/scaffold/notification.toml @@ -81,3 +81,16 @@ workers = 5 # optional: concurrent worker buffer_size = 10000 # optional: event buffer size (default: 10000, range: 100-1000000) # event_types = ["create", "update", "delete", "rename"] # optional: filter by event types (default: all) # path_prefixes = ["/important", "/data"] # optional: filter by path prefixes (default: all) +# +# Additional endpoints can be declared as named subsections, each with its own +# endpoint, filters, retry and worker settings, e.g.: +# [notification.webhook.file_ingress] +# enabled = true +# endpoint = "https://ingress.example.com/hook" +# event_types = ["create"] +# path_prefixes = ["/buckets/uploads/data"] +# +# [notification.webhook.metadata_extractor] +# enabled = true +# endpoint = "https://metadata.example.com/hook" +# path_prefixes = ["/buckets/repository"] diff --git a/weed/command/server.go b/weed/command/server.go index a9b27bc064..a1d17a5fab 100644 --- a/weed/command/server.go +++ b/weed/command/server.go @@ -97,7 +97,7 @@ func init() { masterOptions.peers = cmdServer.Flag.String("master.peers", "", "all master nodes in comma separated ip:masterPort list") masterOptions.volumeSizeLimitMB = cmdServer.Flag.Uint("master.volumeSizeLimitMB", util.DefaultVolumeSizeLimitMB, "Master stops directing writes to oversized volumes.") masterOptions.volumePreallocate = cmdServer.Flag.Bool("master.volumePreallocate", false, "Preallocate disk space for volumes.") - masterOptions.maxParallelVacuumPerServer = cmdServer.Flag.Int("master.maxParallelVacuumPerServer", 1, "maximum number of volumes to vacuum in parallel on one volume server") + masterOptions.maxParallelVacuumPerServer = cmdServer.Flag.Int("master.maxParallelVacuumPerServer", 1, "maximum number of volumes to vacuum in parallel on one volume server; 0 for no limit (deprecated, use master.vacuum.max_parallel_per_server in master.toml)") masterOptions.vacuumIntervalSeconds = cmdServer.Flag.Int("master.vacuumIntervalSeconds", 840, "seconds between automatic vacuum sweeps") masterOptions.vacuumDeleteEmptyAfterSeconds = cmdServer.Flag.Int("master.vacuumDeleteEmptyAfterSeconds", 0, "automatic sweep deletes volume copies that stay empty this many seconds; 0 disables") masterOptions.defaultReplication = cmdServer.Flag.String("master.defaultReplication", "", "Default replication type if not specified.") @@ -164,6 +164,9 @@ func init() { serverOptions.v.hasSlowRead = cmdServer.Flag.Bool("volume.hasSlowRead", true, " if true, this prevents slow reads from blocking other requests, but large file read P99 latency will increase.") serverOptions.v.readBufferSizeMB = cmdServer.Flag.Int("volume.readBufferSizeMB", 4, " larger values can optimize query performance but will increase some memory usage,Use with hasSlowRead normally") serverOptions.v.allowUntrustedRemoteEndpoints = cmdServer.Flag.Bool("volume.allowUntrustedRemoteEndpoints", false, "if true, FetchAndWriteNeedle accepts arbitrary remote S3 endpoints including loopback / link-local hosts. Default rejects internal / metadata endpoints.") + serverOptions.v.fsyncMinBatchBytes = cmdServer.Flag.Int64("volume.fsyncMinBatchBytes", 1*1024*1024, "minimum bytes an fsync batch gathers before a drained queue flushes without waiting") + serverOptions.v.fsyncMinBatchRequests = cmdServer.Flag.Int("volume.fsyncMinBatchRequests", 128, "minimum requests an fsync batch gathers before a drained queue flushes without waiting") + serverOptions.v.fsyncBatchWait = cmdServer.Flag.Duration("volume.fsyncBatchWait", 500*time.Microsecond, "how long a below-minimum fsync batch waits for more requests once the queue drains; 0 flushes immediately") serverOptions.v.setDiskIOProbeDefaults() s3Options.port = cmdServer.Flag.Int("s3.port", 8333, "s3 server http listen port") @@ -422,7 +425,6 @@ func runServer(cmd *Command, args []string) bool { if *isStartingSftp { go func() { time.Sleep(2 * time.Second) - sftpOptions.localSocket = filerOptions.localSocket sftpOptions.startSftpServer() }() } diff --git a/weed/command/sftp.go b/weed/command/sftp.go index ed4a28b79c..eb482d018b 100644 --- a/weed/command/sftp.go +++ b/weed/command/sftp.go @@ -3,8 +3,6 @@ package command import ( "context" "fmt" - "net" - "os" "runtime" "time" @@ -180,22 +178,18 @@ func (sftpOpt *SftpOptions) startSftpServer() bool { // Set up Unix socket if on non-Windows platforms if runtime.GOOS != "windows" { localSocket := *sftpOpt.localSocket - if localSocket == "" { + explicit := localSocket != "" + if !explicit { localSocket = fmt.Sprintf("/tmp/seaweedfs-sftp-%d.sock", *sftpOpt.port) } - if err := os.Remove(localSocket); err != nil && !os.IsNotExist(err) { - glog.Fatalf("Failed to remove %s, error: %s", localSocket, err.Error()) + if sftpSocketListener := pb.ListenOnLocalSocket(localSocket); sftpSocketListener != nil { + go func() { + // start on local unix socket + if err := service.Serve(sftpSocketListener); err != nil { + glog.Fatalf("Failed to serve SFTP on socket %s: %v", localSocket, err) + } + }() } - go func() { - // start on local unix socket - sftpSocketListener, err := net.Listen("unix", localSocket) - if err != nil { - glog.Fatalf("Failed to listen on %s: %v", localSocket, err) - } - if err := service.Serve(sftpSocketListener); err != nil { - glog.Fatalf("Failed to serve SFTP on socket %s: %v", localSocket, err) - } - }() } // Start the SFTP service on TCP diff --git a/weed/command/volume.go b/weed/command/volume.go index 4646d9130a..95b76409c1 100644 --- a/weed/command/volume.go +++ b/weed/command/volume.go @@ -76,6 +76,9 @@ type VolumeServerOptions struct { readBufferSizeMB *int ldbTimeout *int64 allowUntrustedRemoteEndpoints *bool + fsyncMinBatchBytes *int64 + fsyncMinBatchRequests *int + fsyncBatchWait *time.Duration debug *bool debugPort *int diskIOProbe *bool @@ -147,6 +150,22 @@ func (v *VolumeServerOptions) setDiskIOProbeDefaults() { } } +// syncBatchOption resolves the fsync batch window, defaulting each field so +// commands or embedders that leave the option pointers unset still start. +func (v *VolumeServerOptions) syncBatchOption() storage.SyncBatchOption { + option := storage.DefaultSyncBatchOption + if v.fsyncMinBatchBytes != nil { + option.MinBytes = *v.fsyncMinBatchBytes + } + if v.fsyncMinBatchRequests != nil { + option.MinRequests = *v.fsyncMinBatchRequests + } + if v.fsyncBatchWait != nil { + option.Wait = *v.fsyncBatchWait + } + return option +} + func (v *VolumeServerOptions) applyDiskIOProbeConfig() { v.setDiskIOProbeDefaults() @@ -230,6 +249,9 @@ func init() { v.hasSlowRead = cmdVolume.Flag.Bool("hasSlowRead", true, " if true, this prevents slow reads from blocking other requests, but large file read P99 latency will increase.") v.readBufferSizeMB = cmdVolume.Flag.Int("readBufferSizeMB", 4, " larger values can optimize query performance but will increase some memory usage,Use with hasSlowRead normally.") v.allowUntrustedRemoteEndpoints = cmdVolume.Flag.Bool("volume.allowUntrustedRemoteEndpoints", false, "if true, FetchAndWriteNeedle accepts arbitrary remote S3 endpoints including loopback / link-local hosts. Default rejects internal / metadata endpoints.") + v.fsyncMinBatchBytes = cmdVolume.Flag.Int64("fsyncMinBatchBytes", 1*1024*1024, "minimum bytes an fsync batch gathers before a drained queue flushes without waiting") + v.fsyncMinBatchRequests = cmdVolume.Flag.Int("fsyncMinBatchRequests", 128, "minimum requests an fsync batch gathers before a drained queue flushes without waiting") + v.fsyncBatchWait = cmdVolume.Flag.Duration("fsyncBatchWait", 500*time.Microsecond, "how long a below-minimum fsync batch waits for more requests once the queue drains; 0 flushes immediately") v.debug = cmdVolume.Flag.Bool("debug", false, "serves runtime profiling data via pprof on the port specified by -debug.port") v.debugPort = cmdVolume.Flag.Int("debug.port", 6060, "http port for debugging") v.setDiskIOProbeDefaults() @@ -460,6 +482,7 @@ func (v VolumeServerOptions) startVolumeServer(volumeFolders, maxVolumeCounts, v *v.ldbTimeout, *v.allowUntrustedRemoteEndpoints, diskProbeConfig, + v.syncBatchOption(), ) // starting grpc server grpcS := v.startGrpcService(volumeServer) @@ -573,15 +596,54 @@ func (v VolumeServerOptions) isSeparatedPublicPort() bool { return *v.publicPort != *v.port } +// multiHttpdownServer gracefully stops an http server serving on both the +// bound ip and the localhost listeners as if it were a single server. +type multiHttpdownServer []httpdown.Server + +// Wait returns the first member's non-nil error as soon as it happens, so a +// failing localhost listener is reported even while the bound-ip listener +// keeps serving, and nil once every member has finished cleanly. +func (m multiHttpdownServer) Wait() error { + errs := make(chan error, len(m)) + for _, s := range m { + go func(s httpdown.Server) { errs <- s.Wait() }(s) + } + for range m { + if err := <-errs; err != nil { + return err + } + } + return nil +} + +// Stop stops all members concurrently so the localhost listener closes as +// soon as shutdown begins instead of after the bound-ip listener drains. +func (m multiHttpdownServer) Stop() error { + errs := make(chan error, len(m)) + for _, s := range m { + go func(s httpdown.Server) { errs <- s.Stop() }(s) + } + var firstErr error + for range m { + if err := <-errs; err != nil && firstErr == nil { + firstErr = err + } + } + return firstErr +} + func (v VolumeServerOptions) startGrpcService(vs volume_server_pb.VolumeServerServer) *grpc.Server { grpcPort := *v.portGrpc - grpcL, err := util.NewListener(util.JoinHostPort(*v.bindIp, grpcPort), 0) + grpcL, grpcLocalL, err := util.NewIpAndLocalListeners(*v.bindIp, grpcPort, 0) if err != nil { glog.Fatalf("failed to listen on grpc port %d: %v", grpcPort, err) } grpcS := pb.NewGrpcServer(security.LoadServerTLS(util.GetViper(), "grpc.volume")) volume_server_pb.RegisterVolumeServerServer(grpcS, vs) reflection.Register(grpcS) + if grpcLocalL != nil { + go grpcS.Serve(grpcLocalL) + } go func() { if err := grpcS.Serve(grpcL); err != nil { glog.Fatalf("start gRPC service failed, %s", err) @@ -594,13 +656,16 @@ func (v VolumeServerOptions) startGrpcService(vs volume_server_pb.VolumeServerSe func (v VolumeServerOptions) startPublicHttpService(handler http.Handler) httpdown.Server { publicListeningAddress := util.JoinHostPort(*v.bindIp, *v.publicPort) glog.V(0).Infoln("Start Seaweed volume server", version.Version(), "public at", publicListeningAddress) - publicListener, e := util.NewListener(publicListeningAddress, time.Duration(*v.idleConnectionTimeout)*time.Second) + publicListener, publicLocalListener, e := util.NewIpAndLocalListeners(*v.bindIp, *v.publicPort, time.Duration(*v.idleConnectionTimeout)*time.Second) if e != nil { glog.Fatalf("Volume server listener error:%v", e) } pubHttp := httpdown.HTTP{StopTimeout: 5 * time.Minute, KillTimeout: 5 * time.Minute} - publicHttpDown := pubHttp.Serve(&http.Server{Handler: handler}, publicListener) + publicHttpDown := multiHttpdownServer{pubHttp.Serve(&http.Server{Handler: handler}, publicListener)} + if publicLocalListener != nil { + publicHttpDown = append(publicHttpDown, pubHttp.Serve(&http.Server{Handler: handler}, publicLocalListener)) + } go func() { if err := publicHttpDown.Wait(); err != nil { glog.Errorf("public http down wait failed, %v", err) @@ -627,7 +692,7 @@ func (v VolumeServerOptions) startClusterHttpService(handler http.Handler) (http listeningAddress := util.JoinHostPort(*v.bindIp, *v.port) glog.V(0).Infof("Start Seaweed volume server %s at %s", version.Version(), listeningAddress) - listener, e := util.NewListener(listeningAddress, time.Duration(*v.idleConnectionTimeout)*time.Second) + listener, localListener, e := util.NewIpAndLocalListeners(*v.bindIp, *v.port, time.Duration(*v.idleConnectionTimeout)*time.Second) if e != nil { glog.Fatalf("Volume server listener error:%v", e) } @@ -659,11 +724,15 @@ func (v VolumeServerOptions) startClusterHttpService(handler http.Handler) (http httpS.TLSConfig.GetCertificate = getCert } - clusterHttpServer := httpDown.Serve(httpS, listener) + clusterHttpServers := multiHttpdownServer{httpDown.Serve(httpS, listener)} + if localListener != nil { + localHttpS := &http.Server{Handler: handler, TLSConfig: httpS.TLSConfig} + clusterHttpServers = append(clusterHttpServers, httpDown.Serve(localHttpS, localListener)) + } go func() { - if e := clusterHttpServer.Wait(); e != nil { + if e := clusterHttpServers.Wait(); e != nil { glog.Fatalf("Volume server fail to serve: %v", e) } }() - return clusterHttpServer, closeCert + return clusterHttpServers, closeCert } diff --git a/weed/ec/ec_encode.go b/weed/ec/ec_encode.go index 765810af01..8fde7babc0 100644 --- a/weed/ec/ec_encode.go +++ b/weed/ec/ec_encode.go @@ -121,7 +121,7 @@ func ProcessEcEncodeBatch(env *Env, writer io.Writer, volumeIds []needle.VolumeI if err := EcBalance(env, balanceCollections, "", rp, diskType, maxParallelization, 0, applyBalancing, skippedNodes, nil, volumeIds); err != nil { return fmt.Errorf("re-balance ec shards for collection(s) %v: %w", balanceCollections, err) } - if err := verifyEcShardsBeforeDelete(env, volumeIds, diskType, applyBalancing); err != nil { + if err := verifyEcShardsBeforeDelete(env, volumeIds, diskType, applyBalancing, rp); err != nil { return fmt.Errorf("verify EC shards before deleting originals: %w", err) } // Past verify the EC copy is recoverable; a delete failure below must not @@ -727,6 +727,54 @@ func ecShardsClumpedOnOneNode(topoInfo *master_pb.TopologyInfo, vid needle.Volum return "", false } +// verifyEcShardAttempts and verifyEcShardRetryInterval bound how long +// verifyEcShardsBeforeDelete waits for post-rebalance shard locations to +// propagate through volume-server heartbeats before giving up. They are +// package vars so tests can shrink the wait. +var verifyEcShardAttempts = 10 +var verifyEcShardRetryInterval = 2 * time.Second + +// ecShardRackByNode maps every volume server's address to its "dc:rack" +// failure-domain key, matching how the balance planner keys racks. +func ecShardRackByNode(topoInfo *master_pb.TopologyInfo) map[pb.ServerAddress]string { + rackOf := make(map[pb.ServerAddress]string) + EachDataNode(topoInfo, func(dc DataCenterId, rack RackId, dn *master_pb.DataNodeInfo) { + rackOf[pb.NewServerAddressFromDataNode(dn)] = fmt.Sprintf("%s:%s", dc, rack) + }) + return rackOf +} + +// ecShardPlacementViolations reports where the newest-generation placement of +// vid breaches the caps the encode asked for: the replica placement's node +// digit (SameRackCount) bounds the shards of the volume one node may hold and +// its rack digit (DiffRackCount) the shards one rack may hold. A zero digit +// leaves that level unbounded; a nil placement binds nothing. byNode counts +// only the newest encode generation -- a stale orphan's shards do not weaken +// the fresh set. +func ecShardPlacementViolations(vid needle.VolumeId, byNode map[pb.ServerAddress]erasure_coding.ShardBits, rackOf map[pb.ServerAddress]string, rp *super_block.ReplicaPlacement) []string { + if rp == nil || (rp.SameRackCount <= 0 && rp.DiffRackCount <= 0) { + return nil + } + rackCount := make(map[string]int) + var violations []string + for addr, bits := range byNode { + count := bits.Count() + if rp.SameRackCount > 0 && count > rp.SameRackCount { + violations = append(violations, fmt.Sprintf("volume %d: %d shards on node %s, over the requested %d per node", vid, count, addr, rp.SameRackCount)) + } + rackCount[rackOf[addr]] += count + } + if rp.DiffRackCount > 0 { + for rack, count := range rackCount { + if count > rp.DiffRackCount { + violations = append(violations, fmt.Sprintf("volume %d: %d shards on rack %s, over the requested %d per rack", vid, count, rack, rp.DiffRackCount)) + } + } + } + sort.Strings(violations) + return violations +} + // ecShardSummaryByNode says where a volume's shards are, one entry per node, // sorted so the message is stable. It names the ids and not just the count: a // set holding shards 0-9 and one holding 4-13 are both "10 shards", and which @@ -789,7 +837,7 @@ func ecShardSummaryByNode(byNode map[pb.ServerAddress]erasure_coding.ShardBits) return summary } -func verifyEcShardsBeforeDelete(env *Env, volumeIds []needle.VolumeId, diskType types.DiskType, expectSpread bool) error { +func verifyEcShardsBeforeDelete(env *Env, volumeIds []needle.VolumeId, diskType types.DiskType, expectSpread bool, rp *super_block.ReplicaPlacement) error { // Shard relocations from the preceding EC balance reach the master via // volume-server heartbeats, so freshly distributed shards may not all be // visible in the master topology immediately. Poll a few times before @@ -802,12 +850,16 @@ func verifyEcShardsBeforeDelete(env *Env, volumeIds []needle.VolumeId, diskType // rebalance ran in apply mode), a volume whose shards all still sit on one // node while another node has free slots also aborts the deletion: losing // that node would lose the volume, so the original is the safer copy. - const maxAttempts = 10 - const retryInterval = 2 * time.Second + // The replica-placement caps the encode asked for are requirements, not + // preferences: a volume still over the node or rack limit aborts the + // deletion the same way. + maxAttempts := verifyEcShardAttempts + retryInterval := verifyEcShardRetryInterval var lastErr error var lastDegraded []string var lastClumped []string + var lastPlacement []string for attempt := 0; attempt < maxAttempts; attempt++ { topoInfo, _, err := env.FetchTopology(0) if err != nil { @@ -817,6 +869,11 @@ func verifyEcShardsBeforeDelete(env *Env, volumeIds []needle.VolumeId, diskType lastErr = nil lastDegraded = lastDegraded[:0] lastClumped = lastClumped[:0] + lastPlacement = lastPlacement[:0] + var rackOf map[pb.ServerAddress]string + if rp != nil && (rp.SameRackCount > 0 || rp.DiffRackCount > 0) { + rackOf = ecShardRackByNode(topoInfo) + } for _, vid := range volumeIds { // Count the shards wherever they landed, as waitForEcShardsToRegister // above already does. generateEcShards writes them beside the source @@ -852,6 +909,10 @@ func verifyEcShardsBeforeDelete(env *Env, volumeIds []needle.VolumeId, diskType continue } } + if violations := ecShardPlacementViolations(vid, byNode, rackOf, rp); len(violations) > 0 { + lastPlacement = append(lastPlacement, violations...) + continue + } if degraded { lastDegraded = append(lastDegraded, fmt.Sprintf("volume %d: %d/%d shards", vid, union.Count(), totalShards)) continue @@ -861,12 +922,12 @@ func verifyEcShardsBeforeDelete(env *Env, volumeIds []needle.VolumeId, diskType vid, union.Count(), totalShards, len(byNode)) } - if lastErr == nil && len(lastDegraded) == 0 && len(lastClumped) == 0 { + if lastErr == nil && len(lastDegraded) == 0 && len(lastClumped) == 0 && len(lastPlacement) == 0 { return nil } if attempt < maxAttempts-1 { - glog.V(0).Infof("EC shard verification incomplete (attempt %d/%d), waiting for shard locations to propagate: %v %v %v", - attempt+1, maxAttempts, lastErr, lastDegraded, lastClumped) + glog.V(0).Infof("EC shard verification incomplete (attempt %d/%d), waiting for shard locations to propagate: %v %v %v %v", + attempt+1, maxAttempts, lastErr, lastDegraded, lastClumped, lastPlacement) time.Sleep(retryInterval) } } @@ -878,6 +939,9 @@ func verifyEcShardsBeforeDelete(env *Env, volumeIds []needle.VolumeId, diskType if len(lastClumped) > 0 { return fmt.Errorf("EC shards still sit on a single node after rebalance even though other nodes have free slots (%v); keeping the original volumes. Run ec.balance -apply, verify the spread, then delete the originals or re-run ec.encode", lastClumped) } + if len(lastPlacement) > 0 { + return fmt.Errorf("EC shard placement does not meet the requested replica placement limits (%v); keeping the original volumes. Loosen -shardReplicaPlacement or add capacity, then re-run ec.encode", lastPlacement) + } glog.Warningf("EC shard set incomplete but recoverable after %d attempts, proceeding with source deletion (rebuild missing shards with ec.rebuild): %v", maxAttempts, lastDegraded) return nil diff --git a/weed/ec/ec_encode_test.go b/weed/ec/ec_encode_test.go index feb6b6030d..2cb0790642 100644 --- a/weed/ec/ec_encode_test.go +++ b/weed/ec/ec_encode_test.go @@ -1,6 +1,7 @@ package ec import ( + "fmt" "testing" "time" @@ -8,6 +9,7 @@ import ( "github.com/seaweedfs/seaweedfs/weed/pb/master_pb" "github.com/seaweedfs/seaweedfs/weed/storage/erasure_coding" "github.com/seaweedfs/seaweedfs/weed/storage/needle" + "github.com/seaweedfs/seaweedfs/weed/storage/super_block" "github.com/seaweedfs/seaweedfs/weed/storage/types" "github.com/seaweedfs/seaweedfs/weed/util/wildcard" "github.com/stretchr/testify/assert" @@ -529,3 +531,125 @@ func TestCollectSourceFreeVolumeCountsRefundsClearedShards(t *testing.T) { assert.Equal(t, 1, counts["3-127.0.0.1:8110"], "no refund on a node the sweep skipped as unreachable") } + +func TestEcShardPlacementViolations(t *testing.T) { + node := func(id string, shardBits uint32) *master_pb.DataNodeInfo { + return &master_pb.DataNodeInfo{ + Id: id, + DiskInfos: map[string]*master_pb.DiskInfo{ + "": { + MaxVolumeCount: 10, + EcShardInfos: []*master_pb.VolumeEcShardInformationMessage{ + {Id: 1, EcIndexBits: shardBits, DiskId: 0}, + }, + }, + }, + } + } + topo := &master_pb.TopologyInfo{ + DataCenterInfos: []*master_pb.DataCenterInfo{{ + Id: "dc1", + RackInfos: []*master_pb.RackInfo{ + {Id: "rack1", DataNodeInfos: []*master_pb.DataNodeInfo{ + node("n1:8080", 0b0011), // shards 0,1 + node("n2:8080", 0b0100), // shard 2 + }}, + {Id: "rack2", DataNodeInfos: []*master_pb.DataNodeInfo{ + node("n3:8080", 0b1000), // shard 3 + }}, + }, + }}, + } + vid := needle.VolumeId(1) + violationsOf := func(topo *master_pb.TopologyInfo, rp *super_block.ReplicaPlacement) []string { + return ecShardPlacementViolations(vid, CollectEcShardBitsByNode(topo, vid), ecShardRackByNode(topo), rp) + } + + // No caps requested: nothing to violate. + assert.Empty(t, violationsOf(topo, nil)) + assert.Empty(t, violationsOf(topo, &super_block.ReplicaPlacement{})) + + // Node cap: n1 holds 2 shards, over the requested 1 per node. + violations := violationsOf(topo, &super_block.ReplicaPlacement{SameRackCount: 1}) + assert.Len(t, violations, 1) + assert.Contains(t, violations[0], "per node") + + // Rack cap: rack1 holds 3 shards, over the requested 2 per rack; the + // same-named rack in another dc is a different failure domain. + topo2 := &master_pb.TopologyInfo{ + DataCenterInfos: []*master_pb.DataCenterInfo{ + {Id: "dc1", RackInfos: []*master_pb.RackInfo{ + {Id: "rack1", DataNodeInfos: []*master_pb.DataNodeInfo{ + node("n1:8080", 0b0111), // shards 0,1,2 + }}, + }}, + {Id: "dc2", RackInfos: []*master_pb.RackInfo{ + {Id: "rack1", DataNodeInfos: []*master_pb.DataNodeInfo{ + node("n9:8080", 0b1000), // shard 3 + }}, + }}, + }, + } + violations = violationsOf(topo2, &super_block.ReplicaPlacement{DiffRackCount: 2}) + assert.Len(t, violations, 1) + assert.Contains(t, violations[0], "dc1:rack1") + + // Caps that fit the layout: no violations. + assert.Empty(t, violationsOf(topo, &super_block.ReplicaPlacement{SameRackCount: 2, DiffRackCount: 3})) +} + +func TestVerifyEcShardsBeforeDeletePlacement(t *testing.T) { + oldAttempts, oldInterval := verifyEcShardAttempts, verifyEcShardRetryInterval + verifyEcShardAttempts, verifyEcShardRetryInterval = 2, time.Millisecond + t.Cleanup(func() { + verifyEcShardAttempts, verifyEcShardRetryInterval = oldAttempts, oldInterval + }) + + vid := needle.VolumeId(1) + topologyWith := func(nodeShards map[string]uint32) *master_pb.TopologyInfo { + var nodes []*master_pb.DataNodeInfo + for id, bits := range nodeShards { + si := erasure_coding.NewShardsInfo() + for sid := 0; sid < erasure_coding.TotalShardsCount; sid++ { + if bits&(1< 0 { a.VolumeGrowthCount = b.VolumeGrowthCount } @@ -561,9 +573,6 @@ func mergePathConf(a, b *filer_pb.FilerConf_PathConf) { a.Rack = util.Nvl(b.Rack, a.Rack) a.DataNode = util.Nvl(b.DataNode, a.DataNode) a.DisableChunkDeletion = b.DisableChunkDeletion || a.DisableChunkDeletion - // worm merges on presence, so a nested rule can turn it off. readOnly, fsync and - // disableChunkDeletion stay OR'ed on purpose: a nested rule must not be able to - // lift a lock the bucket set. if b.Worm != nil { // copy the value: a is often a scratch conf while b is a live trie entry a.Worm = proto.Bool(*b.Worm) diff --git a/weed/filer/filer_conf_test.go b/weed/filer/filer_conf_test.go index 4c218fc3ce..400c5470a0 100644 --- a/weed/filer/filer_conf_test.go +++ b/weed/filer/filer_conf_test.go @@ -102,6 +102,54 @@ func TestWormLegacyFalseIsNotAnOverride(t *testing.T) { assert.False(t, fc.MatchStorageRule("/buckets/b/sub/x").GetWorm()) } +func TestFsyncInheritance(t *testing.T) { + fc := NewFilerConf() + fc.doLoadConf(&filer_pb.FilerConf{ + Version: FilerConfVersion, + Locations: []*filer_pb.FilerConf_PathConf{ + {LocationPrefix: "/buckets/b/", Fsync: proto.Bool(true), Ttl: "7d"}, + {LocationPrefix: "/buckets/b/quiet/", Collection: "quiet"}, + {LocationPrefix: "/buckets/b/scratch/", Fsync: proto.Bool(false)}, + {LocationPrefix: "/buckets/b/scratch/keep/", Fsync: proto.Bool(true)}, + }, + }) + + // a rule that says nothing about fsync keeps inheriting it, along with the ttl + rule := fc.MatchStorageRule("/buckets/b/quiet/x") + assert.True(t, rule.GetFsync()) + assert.Equal(t, "7d", rule.Ttl) + + // an explicit false turns it off, and a deeper rule turns it back on + assert.False(t, fc.MatchStorageRule("/buckets/b/scratch/x").GetFsync()) + assert.True(t, fc.MatchStorageRule("/buckets/b/scratch/keep/x").GetFsync()) + + // paths with no rule at all are unaffected + assert.False(t, fc.MatchStorageRule("/buckets/other/x").GetFsync()) +} + +// TestFsyncLegacyFalseIsNotAnOverride pins that the explicit "fsync": false +// every pre-version-2 configuration carries does not read as a per-path +// opt-out. +func TestFsyncLegacyFalseIsNotAnOverride(t *testing.T) { + const conf = `{ + "locations": [ + {"locationPrefix": "/buckets/b/", "fsync": true}, + {"locationPrefix": "/buckets/b/sub/", "collection": "sub", "fsync": false} + ] + }` + + for _, version := range []string{"", `"version": 0,`, `"version": 1,`} { + fc := NewFilerConf() + assert.NoError(t, fc.LoadFromBytes([]byte(`{`+version+conf[1:]))) + assert.True(t, fc.MatchStorageRule("/buckets/b/sub/x").GetFsync(), version) + } + + // the same file at the current version means what it says + fc := NewFilerConf() + assert.NoError(t, fc.LoadFromBytes([]byte(`{"version": 2,`+conf[1:]))) + assert.False(t, fc.MatchStorageRule("/buckets/b/sub/x").GetFsync()) +} + // TestWormSurvivesRoundTrip guards the write side: an explicit false has to be // stamped along with a version that says to honor it. func TestWormSurvivesRoundTrip(t *testing.T) { @@ -119,6 +167,23 @@ func TestWormSurvivesRoundTrip(t *testing.T) { assert.True(t, reloaded.MatchStorageRule("/buckets/b/other/x").GetWorm()) } +// TestFsyncSurvivesRoundTrip guards the write side: an explicit false has to be +// stamped along with a version that says to honor it. +func TestFsyncSurvivesRoundTrip(t *testing.T) { + fc := NewFilerConf() + fc.SetLocationConf(&filer_pb.FilerConf_PathConf{LocationPrefix: "/buckets/b/", Fsync: proto.Bool(true)}) + fc.SetLocationConf(&filer_pb.FilerConf_PathConf{LocationPrefix: "/buckets/b/sub/", Fsync: proto.Bool(false)}) + fc.SetLocationConf(&filer_pb.FilerConf_PathConf{LocationPrefix: "/buckets/b/other/", Ttl: "7d"}) + + var buf bytes.Buffer + assert.NoError(t, fc.ToText(&buf)) + + reloaded := NewFilerConf() + assert.NoError(t, reloaded.LoadFromBytes(buf.Bytes())) + assert.False(t, reloaded.MatchStorageRule("/buckets/b/sub/x").GetFsync()) + assert.True(t, reloaded.MatchStorageRule("/buckets/b/other/x").GetFsync()) +} + // TestClonePathConf verifies that ClonePathConf copies all exported fields. // Uses reflection to automatically detect new fields added to the protobuf, // ensuring the test fails if ClonePathConf is not updated for new fields. @@ -130,7 +195,7 @@ func TestClonePathConf(t *testing.T) { Replication: "001", Ttl: "7d", DiskType: "ssd", - Fsync: true, + Fsync: proto.Bool(true), VolumeGrowthCount: 5, ReadOnly: true, MaxFileNameLength: 255, @@ -190,9 +255,11 @@ func TestClonePathConf(t *testing.T) { // Verify mutation of clone doesn't affect source clone.Collection = "modified" clone.ReadOnly = false + *clone.Fsync = false *clone.Worm = false assert.Equal(t, "test_collection", src.Collection, "Modifying clone should not affect source Collection") assert.Equal(t, true, src.ReadOnly, "Modifying clone should not affect source ReadOnly") + assert.Equal(t, true, src.GetFsync(), "Modifying clone should not affect source Fsync") assert.Equal(t, true, src.GetWorm(), "Modifying clone should not affect source Worm") } diff --git a/weed/filer/meta_aggregator.go b/weed/filer/meta_aggregator.go index c5d6521c89..310874a2e2 100644 --- a/weed/filer/meta_aggregator.go +++ b/weed/filer/meta_aggregator.go @@ -788,7 +788,22 @@ func (f *Filer) applyTraversedEntry(ctx context.Context, entry *Entry, peerTsNs return fmt.Errorf("check local state for %s: %w", entry.FullPath, err) } if localTsNs >= peerTsNs { - return nil + if localTsNs != 0 { + return nil + } + // Zero timestamps carry no order: keep the path only when an entry + // exists locally — a missing one takes the peer's copy, or an + // epoch-dated entry from a peer reporting no commit time is dropped. + if f.Store == nil { + return nil + } + local, findErr := f.Store.FindEntry(ctx, entry.FullPath) + if findErr != nil && !errors.Is(findErr, filer_pb.ErrNotFound) { + return fmt.Errorf("check local state for %s: %w", entry.FullPath, findErr) + } + if local != nil { + return nil + } } if insertErr := f.Store.InsertEntry(ctx, entry); insertErr != nil { // Entry already exists (root dir, or partial previous bootstrap); diff --git a/weed/filer/meta_aggregator_replay_test.go b/weed/filer/meta_aggregator_replay_test.go index 3e700a47ec..efbc9968f2 100644 --- a/weed/filer/meta_aggregator_replay_test.go +++ b/weed/filer/meta_aggregator_replay_test.go @@ -575,6 +575,43 @@ func TestBootstrapUpdatesExistingByCommitTime(t *testing.T) { } } +// A missing local entry has no commit time to order against: the peer copy +// must apply even when both sides are timestamp-less — an epoch-dated entry +// from a peer that predates EntryTsNs is still bootstrapped, while an +// existing zero-mtime local entry keeps its place. +func TestBootstrapCopiesMissingEntryWithoutCommitTime(t *testing.T) { + store := newStubFilerStore() + f := newReplayTestFiler(store) + missing := util.NewFullPath("/dir", "missing") + existing := util.NewFullPath("/dir", "existing") + + store.entries[string(existing)] = &Entry{ + FullPath: existing, + Attr: Attr{Mtime: time.Unix(0, 0)}, + Content: []byte("zero-mtime local entry"), + } + for _, path := range []util.FullPath{missing, existing} { + entry := &Entry{ + FullPath: path, + Attr: Attr{Mtime: time.Unix(0, 0)}, + Content: []byte("epoch-dated peer entry"), + } + if err := f.applyTraversedEntry(context.Background(), entry, 0); err != nil { + t.Fatalf("applyTraversedEntry %s: %v", path, err) + } + } + got, err := store.FindEntry(context.Background(), missing) + if err != nil { + t.Fatalf("missing local entry must take the peer copy: %v", err) + } + if string(got.Content) != "epoch-dated peer entry" { + t.Fatalf("content = %q, want peer entry applied", got.Content) + } + if got, _ := store.FindEntry(context.Background(), existing); string(got.Content) != "zero-mtime local entry" { + t.Fatalf("content = %q, existing local entry must be kept", got.Content) + } +} + // A delete whose commit record failed to persist still blocks a stale // replayed create: the pending record stands in until the retry lands it. func TestPendingDeleteRecordBlocksStaleReplay(t *testing.T) { diff --git a/weed/filer/stream.go b/weed/filer/stream.go index a34a6c548f..260c6f5c24 100644 --- a/weed/filer/stream.go +++ b/weed/filer/stream.go @@ -415,11 +415,7 @@ func doNewChunkStreamReader(ctx context.Context, lookupFileIdFn wdclient.LookupF func NewChunkStreamReaderFromFiler(ctx context.Context, masterClient *wdclient.MasterClient, chunks []*filer_pb.FileChunk) *ChunkStreamReader { - lookupFileIdFn := func(ctx context.Context, fileId string) (targetUrl []string, err error) { - return masterClient.LookupFileId(ctx, fileId) - } - - return doNewChunkStreamReader(ctx, lookupFileIdFn, chunks) + return doNewChunkStreamReader(ctx, masterClient.GetLookupFileIdFunction(), chunks) } // NewChunkStreamReaderFromLookup creates a ChunkStreamReader from a lookup function. diff --git a/weed/filer/stream_from_filer_lookup_test.go b/weed/filer/stream_from_filer_lookup_test.go new file mode 100644 index 0000000000..51d604a4a6 --- /dev/null +++ b/weed/filer/stream_from_filer_lookup_test.go @@ -0,0 +1,101 @@ +package filer + +import ( + "context" + "fmt" + "io" + "net" + "net/http" + "net/http/httptest" + "strings" + "testing" + "time" + + "google.golang.org/grpc" + "google.golang.org/grpc/credentials/insecure" + + "github.com/seaweedfs/seaweedfs/weed/cluster" + "github.com/seaweedfs/seaweedfs/weed/pb" + "github.com/seaweedfs/seaweedfs/weed/pb/filer_pb" + "github.com/seaweedfs/seaweedfs/weed/pb/master_pb" + "github.com/seaweedfs/seaweedfs/weed/wdclient" +) + +// lookupOnlyMaster announces no volumes on KeepConnected, so the client's +// cache stays empty, but answers LookupVolume with the one volume server. +type lookupOnlyMaster struct { + master_pb.UnimplementedSeaweedServer + volumeUrl string +} + +func (m *lookupOnlyMaster) KeepConnected(stream master_pb.Seaweed_KeepConnectedServer) error { + if _, err := stream.Recv(); err != nil { + return err + } + if err := stream.Send(&master_pb.KeepConnectedResponse{}); err != nil { + return err + } + <-stream.Context().Done() + return nil +} + +func (m *lookupOnlyMaster) LookupVolume(ctx context.Context, req *master_pb.LookupVolumeRequest) (*master_pb.LookupVolumeResponse, error) { + resp := &master_pb.LookupVolumeResponse{} + for _, id := range req.VolumeOrFileIds { + resp.VolumeIdLocations = append(resp.VolumeIdLocations, &master_pb.LookupVolumeResponse_VolumeIdLocation{ + VolumeOrFileId: id, + Locations: []*master_pb.Location{{Url: m.volumeUrl, PublicUrl: m.volumeUrl}}, + }) + } + return resp, nil +} + +// A filer's cached volume map can lose a volume that still exists: a failed +// read invalidates every cached location of it, and the master only announces +// changes. Readers built from the filer's MasterClient must then ask the master +// instead of failing until the process restarts. +func TestChunkStreamReaderFromFilerAsksMasterOnCacheMiss(t *testing.T) { + content := []byte("metadata log chunk") + volume := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + _, _ = w.Write(content) + })) + t.Cleanup(volume.Close) + + master := &lookupOnlyMaster{volumeUrl: strings.TrimPrefix(volume.URL, "http://")} + lis, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatalf("listen: %v", err) + } + grpcServer := grpc.NewServer() + master_pb.RegisterSeaweedServer(grpcServer, master) + go func() { _ = grpcServer.Serve(lis) }() + t.Cleanup(grpcServer.Stop) + _, port, err := net.SplitHostPort(lis.Addr().String()) + if err != nil { + t.Fatalf("split listener address: %v", err) + } + + mc := wdclient.NewMasterClient( + grpc.WithTransportCredentials(insecure.NewCredentials()), + "", cluster.FilerType, pb.ServerAddress("localhost:0"), "", "", + *pb.NewServiceDiscoveryFromMap(map[string]pb.ServerAddress{"m": pb.ServerAddress(fmt.Sprintf("127.0.0.1:0.%s", port))}), + ) + connecting, stopConnecting := context.WithCancel(context.Background()) + t.Cleanup(stopConnecting) + go mc.KeepConnectedToMaster(connecting) + waiting, cancelWaiting := context.WithTimeout(context.Background(), 20*time.Second) + defer cancelWaiting() + mc.WaitUntilConnected(waiting) + if waiting.Err() != nil { + t.Fatal("the master client never connected to the fake master") + } + + chunks := []*filer_pb.FileChunk{{FileId: "1,0123456789", Offset: 0, Size: uint64(len(content))}} + got, err := io.ReadAll(NewChunkStreamReaderFromFiler(context.Background(), mc, chunks)) + if err != nil { + t.Fatalf("read with volume 1 missing from the cache: %v", err) + } + if string(got) != string(content) { + t.Fatalf("read %q, want %q", got, content) + } +} diff --git a/weed/iam/oidc/jti_cache.go b/weed/iam/oidc/jti_cache.go new file mode 100644 index 0000000000..77dab0d044 --- /dev/null +++ b/weed/iam/oidc/jti_cache.go @@ -0,0 +1,410 @@ +package oidc + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "fmt" + "strings" + "sync" + "sync/atomic" + "time" + + "github.com/seaweedfs/seaweedfs/weed/glog" + "github.com/seaweedfs/seaweedfs/weed/iam/providers" +) + +const ( + // clockSkewTolerance matches the typical JWT library clock skew allowance (RFC 7519) + clockSkewTolerance = 5 * time.Minute + // defaultJTICacheSize is the default maximum number of JTI entries to cache + defaultJTICacheSize = 10000 + // defaultCleanupInterval is how often to scan for expired JTI entries + defaultCleanupInterval = 5 * time.Minute + // eagerCleanupMinInterval rate limits opportunistic sweeps near capacity + eagerCleanupMinInterval = 5 * time.Second + // tokenFingerprintPrefix marks replay-cache keys derived from the token's + // own fingerprint, used when the token carries no usable jti claim + tokenFingerprintPrefix = "fp:" +) + +// jtiEntry represents a cached JTI with metadata +type jtiEntry struct { + subject string + expiresAt time.Time + // uncounted marks an entry merged in past the capacity bound: it keeps + // rejecting replays but does not charge the count, so merged excess can + // neither reject new exchanges nor be double-counted when swept. + uncounted bool +} + +// jtiReplayCache wraps the consumed-token store that backs JTI replay +// protection. Each consumed token identifier is held until the token's exp +// plus clock skew, and a second presentation of the same identifier is +// rejected with ErrProviderTokenReplayed. +// +// The cache is bounded by maxSize and uses automatic TTL-based expiration: +// a background worker sweeps expired entries, an eager sweep runs when the +// cache nears capacity, and the consume path reclaims expired slots before +// rejecting a fresh token for capacity. +// +// Thread Safety: +// All methods are safe for concurrent use. The consume path uses atomic +// LoadOrStore to prevent TOCTOU races. +type jtiReplayCache struct { + name string // owning provider name, for log messages + + enabled atomic.Bool + maxSize atomic.Int64 + lastSweep atomic.Int64 // unix nano of the last sweep; rate limits eager cleanups + + // storeMu guards the store/count pair: replay-state handoffs (adoptFrom, + // mergeFrom) repoint them under the write lock while the consume path + // holds a read lock, so a consume can never land in a store the handoff + // already drained and abandoned. + // It is always acquired before sweepMu, never while holding it. + storeMu sync.RWMutex + store *sync.Map // map[string]*jtiEntry - atomic storage for consumed token keys + count *atomic.Int64 // entries in store; shared with the store + + // mu guards the lifecycle of the cleanup worker: cleanupCancel and + // cleanupDone. The worker itself only touches store/count, so holding + // mu while waiting for it is safe. + mu sync.Mutex + cleanupCancel context.CancelFunc + cleanupDone chan struct{} // Closed when cleanup goroutine exits + + // sweepMu serializes cache sweeps so at most one scan runs at a time. + sweepMu sync.Mutex +} + +func newJTIReplayCache(name string) *jtiReplayCache { + return &jtiReplayCache{ + name: name, + store: &sync.Map{}, + count: &atomic.Int64{}, + } +} + +// configure applies the replay-protection settings: a previous worker is +// always stopped, whether the new configuration enables protection or +// disables it. The consumed-token store survives re-configuration: resetting +// it would let already-seen tokens be replayed against the refreshed provider. +func (c *jtiReplayCache) configure(enabled bool, maxSize int64) { + c.mu.Lock() + c.stopWorkerLocked() + if enabled { + if maxSize <= 0 { + maxSize = defaultJTICacheSize + } + // the bound must land before the flag: a consume that sees enabled + // with a zero bound would reject the first token it registers + c.maxSize.Store(maxSize) + ctx, cancel := context.WithCancel(context.Background()) + done := make(chan struct{}) + c.cleanupCancel = cancel + c.cleanupDone = done + c.enabled.Store(true) + c.mu.Unlock() + + go c.cleanupLoop(ctx, done, defaultCleanupInterval) + glog.V(2).Infof("OIDC provider %q: JTI replay protection enabled (max size: %d)", c.name, maxSize) + } else { + c.enabled.Store(false) + c.mu.Unlock() + glog.Warningf("OIDC provider %q: JTI replay protection is DISABLED", c.name) + } +} + +// stopWorkerLocked cancels the current cleanup goroutine and waits briefly +// for it to exit. Caller must hold mu. +func (c *jtiReplayCache) stopWorkerLocked() { + if c.cleanupCancel == nil { + return + } + c.cleanupCancel() + select { + case <-c.cleanupDone: + glog.V(3).Infof("OIDC provider %q: JTI cleanup goroutine exited", c.name) + case <-time.After(2 * time.Second): + glog.Warningf("OIDC provider %q: Timeout waiting for JTI cleanup goroutine", c.name) + } + c.cleanupCancel = nil + c.cleanupDone = nil +} + +// consume marks a token as used and rejects a second presentation of it. +// jti is the token's jti claim (may be empty); tokenExpiresAt is its exp. +// A no-op while protection is disabled. +func (c *jtiReplayCache) consume(token, jti, subject string, tokenExpiresAt time.Time) error { + if !c.enabled.Load() { + return nil + } + + key := jti + if key == "" { + // Providers that issue no jti cannot be tracked by claim, so the + // signed input keys the entry: a repeated presentation of the same + // claims is a replay even when the signature bytes differ (e.g. + // ECDSA s/N-s malleability). + signingInput := token + if i := strings.LastIndexByte(token, '.'); i > 0 { + signingInput = token[:i] + } + sum := sha256.Sum256([]byte(signingInput)) + key = tokenFingerprintPrefix + hex.EncodeToString(sum[:]) + } + + if tokenExpiresAt.IsZero() { + // a cache slot is freed only by expiry, so a token without exp + // would hold it forever and eventually exhaust the bound + return fmt.Errorf("%w: missing exp claim", providers.ErrProviderMissingClaims) + } + expiresAt := tokenExpiresAt.Add(clockSkewTolerance) + if !time.Now().Before(expiresAt) { + return fmt.Errorf("%w: token expired", providers.ErrProviderTokenExpired) + } + + entry := &jtiEntry{ + subject: subject, + expiresAt: expiresAt, + } + + // The read lock pins the store/count pair for the whole test-and-set + // so a replay-state handoff cannot repoint them mid-check and strand + // this consume in an abandoned store. + c.storeMu.RLock() + store, count := c.store, c.count + + // Atomic test-and-set detects replays before any capacity work runs, + // so a replayed token cannot trigger cleanup scans. + if _, loaded := store.LoadOrStore(key, entry); loaded { + c.storeMu.RUnlock() + glog.Warningf("OIDC provider %q: Token replay detected (jti: %s)", + c.name, jtiLogKey(key)) + return fmt.Errorf("%w: jti %s has already been used", + providers.ErrProviderTokenReplayed, jtiLogKey(key)) + } + + // The count is incremented as part of the insert, so concurrent + // validations cannot all slip under the limit: whoever pushes the + // count over evicts their own entry and is rejected. + newCount := count.Add(1) + maxSize := c.maxSize.Load() + if newCount > maxSize { + // Over the bound: expired entries may still occupy slots, so + // reclaim them with a synchronized sweep before rejecting. + c.sweepStore(store, count) + if count.Load() > maxSize { + if _, loaded := store.LoadAndDelete(key); loaded { + count.Add(-1) + } + c.storeMu.RUnlock() + glog.Errorf("OIDC provider %q: JTI cache FULL (%d entries) - rejecting token. "+ + "This may indicate a DoS attack or insufficient cache size. Consider increasing JTICacheMaxSize.", + c.name, maxSize) + return fmt.Errorf("JTI cache capacity exceeded") + } + } else if newCount >= maxSize*9/10 { + glog.Warningf("OIDC provider %q: JTI cache at %d%% capacity (%d/%d), triggering eager cleanup", + c.name, (newCount*100)/maxSize, newCount, maxSize) + go c.performEagerCleanup() + } + c.storeMu.RUnlock() + + glog.V(4).Infof("OIDC provider %q: Registered JTI %s (expires at %v, cache size: %d)", + c.name, jtiLogKey(key), expiresAt, newCount) + return nil +} + +// evictExpired removes entries whose expiry has passed and returns how +// many it deleted. Caller must hold sweepMu. +func (c *jtiReplayCache) evictExpired(now time.Time, store *sync.Map, count *atomic.Int64) int { + removed := 0 + store.Range(func(key, value interface{}) bool { + expiresAt := value.(*jtiEntry).expiresAt + if !expiresAt.IsZero() && now.After(expiresAt) { + // LoadAndDelete reports whether this call actually removed the + // entry, so concurrent deletes cannot decrement the count twice. + if v, loaded := store.LoadAndDelete(key); loaded { + if !v.(*jtiEntry).uncounted { + count.Add(-1) + } + removed++ + } + } + return true + }) + return removed +} + +// state returns the cache's current store/count pair. +func (c *jtiReplayCache) state() (*sync.Map, *atomic.Int64) { + c.storeMu.RLock() + defer c.storeMu.RUnlock() + return c.store, c.count +} + +// sweepStore runs one serialized expiry scan over the given store. It only +// takes sweepMu, so callers already holding storeMu may use it. +func (c *jtiReplayCache) sweepStore(store *sync.Map, count *atomic.Int64) { + c.sweepMu.Lock() + defer c.sweepMu.Unlock() + c.lastSweep.Store(time.Now().UnixNano()) + if removed := c.evictExpired(time.Now(), store, count); removed > 0 { + glog.V(4).Infof("OIDC provider %q: Cleaned up %d expired JTI entries (remaining: %d)", + c.name, removed, count.Load()) + } +} + +// sweepExpired runs a scan that is guaranteed to complete before the +// caller proceeds; the periodic worker uses it so the scan always covers +// the store currently in effect, not one captured earlier. +func (c *jtiReplayCache) sweepExpired() { + store, count := c.state() + c.sweepStore(store, count) +} + +// performEagerCleanup performs an opportunistic cleanup of expired JTI +// entries when the cache approaches capacity. At most one sweep runs at a +// time and no more often than eagerCleanupMinInterval, so a burst of +// requests near the limit cannot each turn into a full-cache scan. +func (c *jtiReplayCache) performEagerCleanup() { + if !c.enabled.Load() { + return + } + if time.Since(time.Unix(0, c.lastSweep.Load())) < eagerCleanupMinInterval { + return + } + store, count := c.state() + if !c.sweepMu.TryLock() { + return + } + defer c.sweepMu.Unlock() + c.lastSweep.Store(time.Now().UnixNano()) + + beforeCount := count.Load() + if removed := c.evictExpired(time.Now(), store, count); removed > 0 { + glog.Infof("OIDC provider %q: Eager cleanup removed %d expired JTI entries (%d -> %d)", + c.name, removed, beforeCount, count.Load()) + } else { + glog.Warningf("OIDC provider %q: Eager cleanup found no expired entries to remove. "+ + "Cache may be filled with valid long-lived tokens. Consider increasing JTICacheMaxSize.", + c.name) + } +} + +// cleanupLoop periodically removes expired JTI entries from the cache. +// This prevents memory growth by removing JTIs after their tokens have +// expired. Runs until ctx is cancelled; done is closed on exit. +func (c *jtiReplayCache) cleanupLoop(ctx context.Context, done chan struct{}, interval time.Duration) { + defer close(done) + + ticker := time.NewTicker(interval) + defer ticker.Stop() + + glog.V(3).Infof("OIDC provider %q: JTI cleanup goroutine started (interval: %v)", c.name, interval) + + for { + select { + case <-ctx.Done(): + glog.V(3).Infof("OIDC provider %q: JTI cleanup goroutine stopping", c.name) + return + case <-ticker.C: + c.sweepExpired() + } + } +} + +// stop gracefully stops the cleanup worker. +// Should be called when the provider is being shut down to prevent goroutine +// leaks. +func (c *jtiReplayCache) stop(ctx context.Context) error { + c.mu.Lock() + cancel, done := c.cleanupCancel, c.cleanupDone + c.mu.Unlock() + + if cancel == nil { + return nil + } + cancel() + + select { + case <-done: + glog.V(2).Infof("OIDC provider %q: Cleanup goroutine shutdown complete", c.name) + case <-ctx.Done(): + glog.Warningf("OIDC provider %q: Shutdown timeout waiting for cleanup goroutine: %v", c.name, ctx.Err()) + return ctx.Err() + } + + c.mu.Lock() + if c.cleanupDone == done { + c.cleanupCancel = nil + c.cleanupDone = nil + } + c.mu.Unlock() + return nil +} + +// adoptFrom takes over the consumed-token store of another cache, so tokens +// it already saw stay rejected across the rebuild. The caller must shut down +// the replaced provider; only one cleanup worker may sweep the shared store +// at a time. Call before the new provider serves validators; sweepMu +// excludes its own cleanup worker during the swap. +func (c *jtiReplayCache) adoptFrom(old *jtiReplayCache) { + if old == nil || old == c { + return + } + c.storeMu.Lock() + c.sweepMu.Lock() + c.store, c.count = old.store, old.count + c.sweepMu.Unlock() + c.storeMu.Unlock() +} + +// mergeFrom hands this cache's store to another cache and then folds the +// other's consumed-token entries in, so rejections the other recorded keep +// holding when its records move to a shared provider. Repointing first means +// tokens the other still consumes — in-flight exchanges or ones routed to it +// before the provider map updates — land in the shared store instead of a +// store this merge already drained. Unlike adoptFrom it swaps nothing of its +// own, so it is safe on a cache that is already serving validators. +func (c *jtiReplayCache) mergeFrom(other *jtiReplayCache) { + if other == nil || other == c { + return + } + c.storeMu.RLock() + defer c.storeMu.RUnlock() + store, count := c.store, c.count + + other.storeMu.Lock() + oldStore := other.store + if oldStore == store { + other.storeMu.Unlock() + return + } + other.store, other.count = store, count + other.storeMu.Unlock() + + c.sweepMu.Lock() + defer c.sweepMu.Unlock() + now := time.Now() + oldStore.Range(func(key, value interface{}) bool { + entry := value.(*jtiEntry) + if expiresAt := entry.expiresAt; !expiresAt.IsZero() && now.After(expiresAt) { + return true + } + // A consumed token is never dropped to fit the bound: merged entries + // stay but do not charge the count, so a store filled past the bound + // by the merge cannot reject new exchanges while it drains by expiry. + store.LoadOrStore(key, &jtiEntry{subject: entry.subject, expiresAt: entry.expiresAt, uncounted: true}) + return true + }) +} + +// jtiLogKey redacts a token identifier for logs and error strings: a stable +// fingerprint that still lets operators correlate repeated presentations. +func jtiLogKey(jti string) string { + sum := sha256.Sum256([]byte(jti)) + return hex.EncodeToString(sum[:8]) +} diff --git a/weed/iam/oidc/oidc_provider.go b/weed/iam/oidc/oidc_provider.go index c2ccf4ab8f..6b115041d5 100644 --- a/weed/iam/oidc/oidc_provider.go +++ b/weed/iam/oidc/oidc_provider.go @@ -6,7 +6,6 @@ import ( "crypto/elliptic" "crypto/rsa" "crypto/sha1" - "crypto/sha256" "crypto/tls" "crypto/x509" "encoding/base64" @@ -20,7 +19,6 @@ import ( "path/filepath" "strings" "sync" - "sync/atomic" "time" "github.com/golang-jwt/jwt/v5" @@ -29,30 +27,6 @@ import ( "github.com/seaweedfs/seaweedfs/weed/security" ) -const ( - // clockSkewTolerance matches the typical JWT library clock skew allowance (RFC 7519) - clockSkewTolerance = 5 * time.Minute - // defaultJTICacheSize is the default maximum number of JTI entries to cache - defaultJTICacheSize = 10000 - // defaultCleanupInterval is how often to scan for expired JTI entries - defaultCleanupInterval = 5 * time.Minute - // eagerCleanupMinInterval rate limits opportunistic sweeps near capacity - eagerCleanupMinInterval = 5 * time.Second - // tokenFingerprintPrefix marks replay-cache keys derived from the token's - // own fingerprint, used when the token carries no usable jti claim - tokenFingerprintPrefix = "fp:" -) - -// jtiEntry represents a cached JTI with metadata -type jtiEntry struct { - subject string - expiresAt time.Time - // uncounted marks an entry merged in past the capacity bound: it keeps - // rejecting replays but does not charge jtiCount, so merged excess can - // neither reject new exchanges nor be double-counted when swept. - uncounted bool -} - // OIDCProvider implements OpenID Connect authentication with JTI replay protection. // // Replay Protection: @@ -84,31 +58,24 @@ type OIDCProvider struct { resolvedJWKSUri string discoveryFailed bool - // JTI replay protection fields. jtiStore and jtiCount are allocated once - // in NewOIDCProvider and survive re-initialization, so a config refresh - // cannot reopen already-consumed tokens for replay. - jtiEnabled atomic.Bool - jtiStore *sync.Map // map[string]*jtiEntry - atomic storage for consumed token keys - jtiCount *atomic.Int64 // entries in jtiStore; shared with the store - jtiMaxSize atomic.Int64 - jtiLastSweep atomic.Int64 // unix nano of the last sweep; rate limits eager cleanups + // jti wraps the consumed-token cache behind the replay protection. It is + // allocated by NewOIDCProvider or lazily by replayCache, and survives + // re-initialization, so a config refresh cannot reopen already-consumed + // tokens for replay. jtiOnce makes a zero-value OIDCProvider safe. + jti *jtiReplayCache + jtiOnce sync.Once +} - // jtiStoreMu guards the jtiStore/jtiCount pair: replay-state handoffs - // (AdoptReplayStateFrom, MergeReplayStateFrom) repoint them under the - // write lock while the consume path holds a read lock, so a consume can - // never land in a store the handoff already drained and abandoned. - // It is always acquired before jtiSweepMu, never while holding it. - jtiStoreMu sync.RWMutex - - // jtiMu guards the lifecycle of the cleanup worker: jtiCleanupCancel and - // jtiCleanupDone. The worker itself only touches jtiStore/jtiCount, so - // holding jtiMu while waiting for it is safe. - jtiMu sync.Mutex - jtiCleanupCancel context.CancelFunc - jtiCleanupDone chan struct{} // Closed when cleanup goroutine exits - - // jtiSweepMu serializes cache sweeps so at most one scan runs at a time. - jtiSweepMu sync.Mutex +// replayCache returns the consumed-token cache, allocating it for providers +// built without NewOIDCProvider so the delegated operations never panic on a +// nil cache. +func (p *OIDCProvider) replayCache() *jtiReplayCache { + p.jtiOnce.Do(func() { + if p.jti == nil { + p.jti = newJTIReplayCache(p.name) + } + }) + return p.jti } // OIDCConfig holds OIDC provider configuration @@ -391,8 +358,7 @@ func NewOIDCProvider(name string) *OIDCProvider { return &OIDCProvider{ name: name, httpClient: &http.Client{Timeout: 30 * time.Second, CheckRedirect: refuseDowngradeRedirect}, - jtiStore: &sync.Map{}, - jtiCount: &atomic.Int64{}, + jti: newJTIReplayCache(name), } } @@ -512,53 +478,12 @@ func (p *OIDCProvider) Initialize(config interface{}) error { if oidcConfig.JTIReplayProtectionEnabled != nil { jtiEnabled = *oidcConfig.JTIReplayProtectionEnabled } - - p.jtiMu.Lock() - p.stopJTICleanupWorkerLocked() - if jtiEnabled { - maxSize := int64(oidcConfig.JTICacheMaxSize) - if maxSize <= 0 { - maxSize = defaultJTICacheSize - } - // the bound must land before the flag: a consume that sees enabled - // with a zero bound would reject the first token it registers - p.jtiMaxSize.Store(maxSize) - ctx, cancel := context.WithCancel(context.Background()) - done := make(chan struct{}) - p.jtiCleanupCancel = cancel - p.jtiCleanupDone = done - p.jtiEnabled.Store(true) - p.jtiMu.Unlock() - - go p.cleanupExpiredJTIs(ctx, done, defaultCleanupInterval) - glog.V(2).Infof("OIDC provider %q: JTI replay protection enabled (max size: %d)", p.name, maxSize) - } else { - p.jtiEnabled.Store(false) - p.jtiMu.Unlock() - glog.Warningf("OIDC provider %q: JTI replay protection is DISABLED", p.name) - } + p.replayCache().configure(jtiEnabled, int64(oidcConfig.JTICacheMaxSize)) // For testing, we'll skip the actual OIDC client initialization return nil } -// stopJTICleanupWorkerLocked cancels the current cleanup goroutine and waits -// briefly for it to exit. Caller must hold jtiMu. -func (p *OIDCProvider) stopJTICleanupWorkerLocked() { - if p.jtiCleanupCancel == nil { - return - } - p.jtiCleanupCancel() - select { - case <-p.jtiCleanupDone: - glog.V(3).Infof("OIDC provider %q: JTI cleanup goroutine exited", p.name) - case <-time.After(2 * time.Second): - glog.Warningf("OIDC provider %q: Timeout waiting for JTI cleanup goroutine", p.name) - } - p.jtiCleanupCancel = nil - p.jtiCleanupDone = nil -} - // validateConfig validates the OIDC configuration func (p *OIDCProvider) validateConfig(config *OIDCConfig) error { if config.Issuer == "" { @@ -935,85 +860,14 @@ func (p *OIDCProvider) validateToken(ctx context.Context, token string, singleUs tokenClaims.Claims[key] = value } - // Extract and validate JTI claim for replay protection. The check runs - // only for single-use callers: a bearer token is meant to be presented on - // every request, so validating it must not consume it. - if singleUse && p.jtiEnabled.Load() { + // Consume the token for replay protection. The check runs only for + // single-use callers: a bearer token is meant to be presented on every + // request, so validating it must not consume it. + if singleUse { jti, _ := claims["jti"].(string) - key := jti - if key == "" { - // Providers that issue no jti cannot be tracked by claim, so the - // signed input keys the entry: a repeated presentation of the same - // claims is a replay even when the signature bytes differ (e.g. - // ECDSA s/N-s malleability). - signingInput := token - if i := strings.LastIndexByte(token, '.'); i > 0 { - signingInput = token[:i] - } - sum := sha256.Sum256([]byte(signingInput)) - key = tokenFingerprintPrefix + hex.EncodeToString(sum[:]) + if err := p.replayCache().consume(token, jti, subject, tokenClaims.ExpiresAt); err != nil { + return nil, err } - - expiresAt := tokenClaims.ExpiresAt - if expiresAt.IsZero() { - // a cache slot is freed only by expiry, so a token without exp - // would hold it forever and eventually exhaust the bound - return nil, fmt.Errorf("%w: missing exp claim", providers.ErrProviderMissingClaims) - } - expiresAt = expiresAt.Add(clockSkewTolerance) - if !time.Now().Before(expiresAt) { - return nil, fmt.Errorf("%w: token expired", providers.ErrProviderTokenExpired) - } - - entry := &jtiEntry{ - subject: subject, - expiresAt: expiresAt, - } - - // The read lock pins the store/count pair for the whole test-and-set - // so a replay-state handoff cannot repoint them mid-check and strand - // this consume in an abandoned store. - p.jtiStoreMu.RLock() - store, count := p.jtiStore, p.jtiCount - - // Atomic test-and-set detects replays before any capacity work runs, - // so a replayed token cannot trigger cleanup scans. - if _, loaded := store.LoadOrStore(key, entry); loaded { - p.jtiStoreMu.RUnlock() - glog.Warningf("OIDC provider %q: Token replay detected (jti: %s)", - p.name, jtiLogKey(key)) - return nil, fmt.Errorf("%w: jti %s has already been used", - providers.ErrProviderTokenReplayed, jtiLogKey(key)) - } - - // The count is incremented as part of the insert, so concurrent - // validations cannot all slip under the limit: whoever pushes the - // count over evicts their own entry and is rejected. - newCount := count.Add(1) - maxSize := p.jtiMaxSize.Load() - if newCount > maxSize { - // Over the bound: expired entries may still occupy slots, so - // reclaim them with a synchronized sweep before rejecting. - p.sweepJTIStore(store, count) - if count.Load() > maxSize { - if _, loaded := store.LoadAndDelete(key); loaded { - count.Add(-1) - } - p.jtiStoreMu.RUnlock() - glog.Errorf("OIDC provider %q: JTI cache FULL (%d entries) - rejecting token. "+ - "This may indicate a DoS attack or insufficient cache size. Consider increasing JTICacheMaxSize.", - p.name, maxSize) - return nil, fmt.Errorf("JTI cache capacity exceeded") - } - } else if newCount >= maxSize*9/10 { - glog.Warningf("OIDC provider %q: JTI cache at %d%% capacity (%d/%d), triggering eager cleanup", - p.name, (newCount*100)/maxSize, newCount, maxSize) - go p.performEagerCleanup() - } - p.jtiStoreMu.RUnlock() - - glog.V(4).Infof("OIDC provider %q: Registered JTI %s (expires at %v, cache size: %d)", - p.name, jtiLogKey(key), expiresAt, newCount) } return tokenClaims, nil @@ -1445,148 +1299,22 @@ func (p *OIDCProvider) mapUserInfoToIdentity(userInfo map[string]interface{}) *p return identity } -// evictExpiredJTIs removes entries whose expiry has passed and returns how -// many it deleted. Caller must hold jtiSweepMu. -func (p *OIDCProvider) evictExpiredJTIs(now time.Time, store *sync.Map, count *atomic.Int64) int { - removed := 0 - store.Range(func(key, value interface{}) bool { - expiresAt := value.(*jtiEntry).expiresAt - if !expiresAt.IsZero() && now.After(expiresAt) { - // LoadAndDelete reports whether this call actually removed the - // entry, so concurrent deletes cannot decrement the count twice. - if v, loaded := store.LoadAndDelete(key); loaded { - if !v.(*jtiEntry).uncounted { - count.Add(-1) - } - removed++ - } - } - return true - }) - return removed -} - -// replayState returns the provider's current store/count pair. -func (p *OIDCProvider) replayState() (*sync.Map, *atomic.Int64) { - p.jtiStoreMu.RLock() - defer p.jtiStoreMu.RUnlock() - return p.jtiStore, p.jtiCount -} - -// sweepJTIStore runs one serialized expiry scan over the given store. It only -// takes jtiSweepMu, so callers already holding jtiStoreMu may use it. -func (p *OIDCProvider) sweepJTIStore(store *sync.Map, count *atomic.Int64) { - p.jtiSweepMu.Lock() - defer p.jtiSweepMu.Unlock() - p.jtiLastSweep.Store(time.Now().UnixNano()) - if removed := p.evictExpiredJTIs(time.Now(), store, count); removed > 0 { - glog.V(4).Infof("OIDC provider %q: Cleaned up %d expired JTI entries (remaining: %d)", - p.name, removed, count.Load()) - } -} - -// sweepExpiredJTIs runs a scan that is guaranteed to complete before the -// caller proceeds; the capacity-rejection path uses it so a fresh token is -// not refused while expired entries still hold slots. -func (p *OIDCProvider) sweepExpiredJTIs() { - store, count := p.replayState() - p.sweepJTIStore(store, count) -} - -// performEagerCleanup performs an opportunistic cleanup of expired JTI -// entries when the cache approaches capacity. At most one sweep runs at a -// time and no more often than eagerCleanupMinInterval, so a burst of -// requests near the limit cannot each turn into a full-cache scan. -func (p *OIDCProvider) performEagerCleanup() { - if !p.jtiEnabled.Load() { - return - } - if time.Since(time.Unix(0, p.jtiLastSweep.Load())) < eagerCleanupMinInterval { - return - } - store, count := p.replayState() - if !p.jtiSweepMu.TryLock() { - return - } - defer p.jtiSweepMu.Unlock() - p.jtiLastSweep.Store(time.Now().UnixNano()) - - beforeCount := count.Load() - if removed := p.evictExpiredJTIs(time.Now(), store, count); removed > 0 { - glog.Infof("OIDC provider %q: Eager cleanup removed %d expired JTI entries (%d -> %d)", - p.name, removed, beforeCount, count.Load()) - } else { - glog.Warningf("OIDC provider %q: Eager cleanup found no expired entries to remove. "+ - "Cache may be filled with valid long-lived tokens. Consider increasing JTICacheMaxSize.", - p.name) - } -} - -// cleanupExpiredJTIs periodically removes expired JTI entries from the cache. -// This prevents memory growth by removing JTIs after their tokens have expired. -// Runs in a background goroutine until ctx is cancelled; done is closed on exit. -func (p *OIDCProvider) cleanupExpiredJTIs(ctx context.Context, done chan struct{}, interval time.Duration) { - defer close(done) - - ticker := time.NewTicker(interval) - defer ticker.Stop() - - glog.V(3).Infof("OIDC provider %q: JTI cleanup goroutine started (interval: %v)", p.name, interval) - - for { - select { - case <-ctx.Done(): - glog.V(3).Infof("OIDC provider %q: JTI cleanup goroutine stopping", p.name) - return - case <-ticker.C: - p.sweepExpiredJTIs() - } - } -} - // Shutdown gracefully stops the JTI cleanup goroutine. // Should be called when the provider is being shut down to prevent goroutine leaks. func (p *OIDCProvider) Shutdown(ctx context.Context) error { - p.jtiMu.Lock() - cancel, done := p.jtiCleanupCancel, p.jtiCleanupDone - p.jtiMu.Unlock() - - if cancel == nil { - return nil - } - cancel() - - select { - case <-done: - glog.V(2).Infof("OIDC provider %q: Cleanup goroutine shutdown complete", p.name) - case <-ctx.Done(): - glog.Warningf("OIDC provider %q: Shutdown timeout waiting for cleanup goroutine: %v", p.name, ctx.Err()) - return ctx.Err() - } - - p.jtiMu.Lock() - if p.jtiCleanupDone == done { - p.jtiCleanupCancel = nil - p.jtiCleanupDone = nil - } - p.jtiMu.Unlock() - return nil + return p.replayCache().stop(ctx) } // AdoptReplayStateFrom takes over the consumed-token store of the provider it // replaces, so tokens it already saw stay rejected across the rebuild. The // caller must shut down the replaced provider; only one cleanup worker may // sweep the shared store at a time. Call before the new provider serves -// validators; jtiSweepMu excludes its own cleanup worker during the swap. +// validators. func (p *OIDCProvider) AdoptReplayStateFrom(old *OIDCProvider) { - if old == nil || old == p { + if old == nil || old == p || old.jti == nil { return } - p.jtiStoreMu.Lock() - p.jtiSweepMu.Lock() - p.jtiStore, p.jtiCount = old.jtiStore, old.jtiCount - p.jtiSweepMu.Unlock() - p.jtiStoreMu.Unlock() + p.replayCache().adoptFrom(old.jti) } // MergeReplayStateFrom hands this provider's store to another provider and @@ -1601,38 +1329,5 @@ func (p *OIDCProvider) MergeReplayStateFrom(other *OIDCProvider) { if other == nil || other == p { return } - p.jtiStoreMu.RLock() - defer p.jtiStoreMu.RUnlock() - store, count := p.jtiStore, p.jtiCount - - other.jtiStoreMu.Lock() - oldStore := other.jtiStore - if oldStore == store { - other.jtiStoreMu.Unlock() - return - } - other.jtiStore, other.jtiCount = store, count - other.jtiStoreMu.Unlock() - - p.jtiSweepMu.Lock() - defer p.jtiSweepMu.Unlock() - now := time.Now() - oldStore.Range(func(key, value interface{}) bool { - entry := value.(*jtiEntry) - if expiresAt := entry.expiresAt; !expiresAt.IsZero() && now.After(expiresAt) { - return true - } - // A consumed token is never dropped to fit the bound: merged entries - // stay but do not charge the count, so a store filled past the bound - // by the merge cannot reject new exchanges while it drains by expiry. - store.LoadOrStore(key, &jtiEntry{subject: entry.subject, expiresAt: entry.expiresAt, uncounted: true}) - return true - }) -} - -// jtiLogKey redacts a token identifier for logs and error strings: a stable -// fingerprint that still lets operators correlate repeated presentations. -func jtiLogKey(jti string) string { - sum := sha256.Sum256([]byte(jti)) - return hex.EncodeToString(sum[:8]) + p.replayCache().mergeFrom(other.replayCache()) } diff --git a/weed/iam/oidc/oidc_provider_test.go b/weed/iam/oidc/oidc_provider_test.go index 3744ec3138..29f7e3e656 100644 --- a/weed/iam/oidc/oidc_provider_test.go +++ b/weed/iam/oidc/oidc_provider_test.go @@ -1067,10 +1067,10 @@ func TestOIDCProviderJTICleanup(t *testing.T) { require.NoError(t, err) // Check cache size is 1 - assert.Equal(t, int64(1), provider.jtiCount.Load()) + assert.Equal(t, int64(1), provider.jti.count.Load()) // Verify entry exists and check its expiration includes clock skew - val, ok := provider.jtiStore.Load("cleanup-test-jti") + val, ok := provider.jti.store.Load("cleanup-test-jti") require.True(t, ok, "JTI entry should exist") entry := val.(*jtiEntry) @@ -1088,21 +1088,21 @@ func TestOIDCProviderJTICleanup(t *testing.T) { // both the entry and the count must actually be cleared cleanupCtx, cleanupCancel := context.WithCancel(context.Background()) cleanupDone := make(chan struct{}) - go provider.cleanupExpiredJTIs(cleanupCtx, cleanupDone, 20*time.Millisecond) + go provider.jti.cleanupLoop(cleanupCtx, cleanupDone, 20*time.Millisecond) defer func() { cleanupCancel() <-cleanupDone }() - provider.jtiStore.Store("expired-cleanup-jti", &jtiEntry{ + provider.jti.store.Store("expired-cleanup-jti", &jtiEntry{ subject: "expired-user", expiresAt: time.Now().Add(-time.Minute), }) - provider.jtiCount.Add(1) + provider.jti.count.Add(1) require.Eventually(t, func() bool { - _, stillThere := provider.jtiStore.Load("expired-cleanup-jti") - return !stillThere && provider.jtiCount.Load() == 1 // only the valid entry remains + _, stillThere := provider.jti.store.Load("expired-cleanup-jti") + return !stillThere && provider.jti.count.Load() == 1 // only the valid entry remains }, 2*time.Second, 10*time.Millisecond, "periodic cleanup should evict the expired entry") } @@ -1124,7 +1124,7 @@ func TestOIDCProviderReinitialization(t *testing.T) { // First initialization require.NoError(t, provider.Initialize(config)) - require.NotNil(t, provider.jtiCleanupCancel, "First cleanup cancel should be set") + require.NotNil(t, provider.jti.cleanupCancel, "First cleanup cancel should be set") // Create and validate a token to ensure JTI protection is working token1 := createTestJWT(t, privateKey, jwt.MapClaims{ @@ -1140,7 +1140,7 @@ func TestOIDCProviderReinitialization(t *testing.T) { // Second initialization should not cause errors require.NoError(t, provider.Initialize(config)) - require.NotNil(t, provider.jtiCleanupCancel, "Second cleanup cancel should be set") + require.NotNil(t, provider.jti.cleanupCancel, "Second cleanup cancel should be set") // The JTI cache survives re-initialization, so the consumed token is // still detected as a replay rather than being usable again @@ -1150,7 +1150,7 @@ func TestOIDCProviderReinitialization(t *testing.T) { // Third initialization to ensure it works multiple times require.NoError(t, provider.Initialize(config)) - require.NotNil(t, provider.jtiCleanupCancel, "Third cleanup cancel should be set") + require.NotNil(t, provider.jti.cleanupCancel, "Third cleanup cancel should be set") // Verify JTI replay protection still works after re-initialization token2 := createTestJWT(t, privateKey, jwt.MapClaims{ @@ -1201,16 +1201,16 @@ func TestOIDCProviderJTIEagerCleanup(t *testing.T) { subject: fmt.Sprintf("expired-user-%d", i), expiresAt: time.Now().Add(-time.Hour), // Already expired } - provider.jtiStore.Store(jti, entry) - provider.jtiCount.Add(1) + provider.jti.store.Store(jti, entry) + provider.jti.count.Add(1) } - assert.Equal(t, int64(5), provider.jtiCount.Load(), "Should have 5 expired entries") + assert.Equal(t, int64(5), provider.jti.count.Load(), "Should have 5 expired entries") // Perform eager cleanup - should remove all expired entries - provider.performEagerCleanup() + provider.jti.performEagerCleanup() - assert.Equal(t, int64(0), provider.jtiCount.Load(), + assert.Equal(t, int64(0), provider.jti.count.Load(), "All expired entries should be removed by eager cleanup") // Now test that adding valid tokens works after cleanup @@ -1228,7 +1228,7 @@ func TestOIDCProviderJTIEagerCleanup(t *testing.T) { } // Should have 9 valid entries now (90% of capacity, triggers eager cleanup warning) - assert.Equal(t, int64(9), provider.jtiCount.Load(), "Should have 9 valid entries") + assert.Equal(t, int64(9), provider.jti.count.Load(), "Should have 9 valid entries") // 10th token should succeed (at capacity but not over) token10 := createTestJWT(t, privateKey, jwt.MapClaims{ @@ -1282,11 +1282,11 @@ func TestOIDCProviderConcurrentCleanupNoNegativeCount(t *testing.T) { subject: fmt.Sprintf("user-%d", i), expiresAt: time.Now().Add(-time.Hour), // Already expired } - provider.jtiStore.Store(jti, entry) - provider.jtiCount.Add(1) + provider.jti.store.Store(jti, entry) + provider.jti.count.Add(1) } - initialCount := provider.jtiCount.Load() + initialCount := provider.jti.count.Load() assert.Equal(t, int64(50), initialCount, "Should have 50 expired entries") // Run multiple concurrent cleanups @@ -1297,14 +1297,14 @@ func TestOIDCProviderConcurrentCleanupNoNegativeCount(t *testing.T) { wg.Add(1) go func() { defer wg.Done() - provider.performEagerCleanup() + provider.jti.performEagerCleanup() }() } wg.Wait() // Counter should be 0 (all entries removed), never negative - finalCount := provider.jtiCount.Load() + finalCount := provider.jti.count.Load() assert.Equal(t, int64(0), finalCount, "Counter should be 0 after concurrent cleanups, got %d", finalCount) assert.GreaterOrEqual(t, finalCount, int64(0), @@ -1312,7 +1312,7 @@ func TestOIDCProviderConcurrentCleanupNoNegativeCount(t *testing.T) { // Verify store is actually empty storeCount := 0 - provider.jtiStore.Range(func(key, value interface{}) bool { + provider.jti.store.Range(func(key, value interface{}) bool { storeCount++ return true }) diff --git a/weed/iam/sts/session_claims.go b/weed/iam/sts/session_claims.go index e7f4ce89c5..0dd0361718 100644 --- a/weed/iam/sts/session_claims.go +++ b/weed/iam/sts/session_claims.go @@ -105,6 +105,13 @@ type STSSessionClaims struct { // a SeaweedFS service minted the session for it (AssumeRoleForPrincipal); // empty for every other session type. SourceAccessKeyId string `json:"src_ak,omitempty"` + + // BoundAccessKeyId is the stored credential the session is bound to: the + // session ends when that key is gone. It is a claim of its own because + // tokens minted before the binding existed carry SourceAccessKeyId as an + // audit field, never a binding — enforcing those would break them. + BoundAccessKeyId string `json:"bind_ak,omitempty"` + BoundAccessKeyFingerprint string `json:"bind_fp,omitempty"` } // NewSTSSessionClaims creates new STS session claims with all required information @@ -163,6 +170,9 @@ func (c *STSSessionClaims) ToSessionInfo(credGen *CredentialGenerator) *SessionI ParentUser: c.ParentUser, // The access key the session was minted for, if recorded SourceAccessKeyId: c.SourceAccessKeyId, + // The stored credential the session is bound to, if any + BoundAccessKeyId: c.BoundAccessKeyId, + BoundAccessKeyFingerprint: c.BoundAccessKeyFingerprint, // Provide the Subject (sub) from registered claims Subject: c.Subject, Credentials: credentials, @@ -263,6 +273,14 @@ func (c *STSSessionClaims) WithSourceAccessKeyId(accessKeyId string) *STSSession return c } +// WithBoundAccessKey binds the session to a stored credential: the key id it +// was vended for and a fingerprint of its secret at mint time. +func (c *STSSessionClaims) WithBoundAccessKey(accessKeyId, fingerprint string) *STSSessionClaims { + c.BoundAccessKeyId = accessKeyId + c.BoundAccessKeyFingerprint = fingerprint + return c +} + // WithParentUser sets the stable per-identity hash for the session. See // ComputeParentUser for the derivation rule. func (c *STSSessionClaims) WithParentUser(parentUser string) *STSSessionClaims { diff --git a/weed/iam/sts/sts_service.go b/weed/iam/sts/sts_service.go index 9083b221e5..99f48dc14d 100644 --- a/weed/iam/sts/sts_service.go +++ b/weed/iam/sts/sts_service.go @@ -320,6 +320,16 @@ type SessionInfo struct { // SourceAccessKeyId is the access key the session was minted for, when the // minting service recorded it (AssumeRoleForPrincipal). SourceAccessKeyId string `json:"sourceAccessKeyId,omitempty"` + + // BoundAccessKeyId is the stored credential the session is bound to, when + // the minting service recorded one (AssumeRoleForPrincipal). A legacy + // SourceAccessKeyId is never treated as a binding. + BoundAccessKeyId string `json:"boundAccessKeyId,omitempty"` + + // BoundAccessKeyFingerprint identifies the credential instance the + // session is bound to, so a key re-created under the same id does not + // match. + BoundAccessKeyFingerprint string `json:"boundAccessKeyFingerprint,omitempty"` } // NewSTSService creates a new STS service @@ -901,6 +911,14 @@ type AssumeRoleForPrincipalRequest struct { // SourceAccessKeyId optionally records the access key the caller // authenticated with, so the session can be tied to that key. SourceAccessKeyId string + + // BoundAccessKeyId binds the session to a stored credential: the session + // ends when that key is gone. Distinct from SourceAccessKeyId, which is + // audit-only on sessions minted before the binding existed. + BoundAccessKeyId string + + // BoundAccessKeyFingerprint identifies the bound credential instance. + BoundAccessKeyFingerprint string } // AssumeRoleForPrincipal issues session credentials for a caller that a @@ -947,7 +965,10 @@ func (s *STSService) AssumeRoleForPrincipal(ctx context.Context, request *Assume // carry no role ID and stay bound by name. return s.issueSession(request.RoleArn, "", request.RoleSessionName, sessionPolicy, request.DurationSeconds, request.ProviderName, request.Principal, - func(c *STSSessionClaims) { c.WithSourceAccessKeyId(request.SourceAccessKeyId) }) + func(c *STSSessionClaims) { + c.WithSourceAccessKeyId(request.SourceAccessKeyId) + c.WithBoundAccessKey(request.BoundAccessKeyId, request.BoundAccessKeyFingerprint) + }) } // ValidateSessionToken validates a session token and returns session information diff --git a/weed/mq/kafka/integration/broker_client.go b/weed/mq/kafka/integration/broker_client.go index 3c3482c86e..428cfdffef 100644 --- a/weed/mq/kafka/integration/broker_client.go +++ b/weed/mq/kafka/integration/broker_client.go @@ -22,9 +22,8 @@ import ( ) const ( - brokerHealthCheckTimeout = 5 * time.Second - brokerHealthCheckAttempts = 3 - brokerHealthCheckBackoff = 250 * time.Millisecond + brokerHealthCheckTimeout = 5 * time.Second + brokerHealthCheckBackoff = 250 * time.Millisecond ) // NewBrokerClientWithFilerAccessor creates a client with a shared filer accessor @@ -102,14 +101,17 @@ func (bc *BrokerClient) Close() error { return bc.conn.Close() } -// HealthCheck verifies the broker connection is working +// HealthCheck verifies the broker connection is working. A freshly announced +// broker may still be acquiring its balancer lock, so keep retrying for the +// same window cluster discovery waits out. func (bc *BrokerClient) HealthCheck() error { if bc.client == nil { return fmt.Errorf("broker client not connected") } + deadline := time.Now().Add(clusterNodeDiscoveryTimeout) var lastErr error - for attempt := 1; attempt <= brokerHealthCheckAttempts; attempt++ { + for attempt := 1; ; attempt++ { ctx, cancel := context.WithTimeout(bc.ctx, brokerHealthCheckTimeout) _, err := bc.client.FindBrokerLeader(ctx, &mq_pb.FindBrokerLeaderRequest{}) cancel() @@ -117,17 +119,18 @@ func (bc *BrokerClient) HealthCheck() error { return nil } lastErr = err - if attempt == brokerHealthCheckAttempts { + if !time.Now().Before(deadline) { break } + backoff := min(time.Duration(attempt)*brokerHealthCheckBackoff, time.Second) select { case <-bc.ctx.Done(): return bc.ctx.Err() - case <-time.After(time.Duration(attempt) * brokerHealthCheckBackoff): + case <-time.After(backoff): } } - return fmt.Errorf("broker health check failed after %d attempts: %v", brokerHealthCheckAttempts, lastErr) + return fmt.Errorf("broker health check failed after %v: %v", clusterNodeDiscoveryTimeout, lastErr) } // GetPartitionRangeInfo gets comprehensive range information from SeaweedMQ broker's native range manager diff --git a/weed/mq/kafka/integration/broker_client_test.go b/weed/mq/kafka/integration/broker_client_test.go new file mode 100644 index 0000000000..efcb8c9153 --- /dev/null +++ b/weed/mq/kafka/integration/broker_client_test.go @@ -0,0 +1,46 @@ +package integration + +import ( + "context" + "errors" + "testing" + "time" + + "google.golang.org/grpc" + + "github.com/seaweedfs/seaweedfs/weed/pb/mq_pb" +) + +type stubMessagingClient struct { + mq_pb.SeaweedMessagingClient + calls int + failFor int + err error +} + +func (s *stubMessagingClient) FindBrokerLeader(ctx context.Context, in *mq_pb.FindBrokerLeaderRequest, opts ...grpc.CallOption) (*mq_pb.FindBrokerLeaderResponse, error) { + s.calls++ + if s.calls <= s.failFor { + return nil, s.err + } + return &mq_pb.FindBrokerLeaderResponse{}, nil +} + +func TestHealthCheckRetriesPastTransientBrokerErrors(t *testing.T) { + stub := &stubMessagingClient{failFor: 2, err: errors.New("lock broker_balancer not found")} + bc := &BrokerClient{ctx: context.Background(), client: stub} + if err := bc.HealthCheck(); err != nil || stub.calls != 3 { + t.Fatalf("err %v after %d calls, want success on the 3rd", err, stub.calls) + } +} + +func TestHealthCheckGivesUpAtDeadline(t *testing.T) { + defer func(orig time.Duration) { clusterNodeDiscoveryTimeout = orig }(clusterNodeDiscoveryTimeout) + clusterNodeDiscoveryTimeout = 300 * time.Millisecond + + stub := &stubMessagingClient{failFor: 1 << 30, err: errors.New("lock broker_balancer not found")} + bc := &BrokerClient{ctx: context.Background(), client: stub} + if err := bc.HealthCheck(); err == nil || stub.calls < 2 { + t.Fatalf("err %v after %d calls, want repeated retries then an error", err, stub.calls) + } +} diff --git a/weed/mq/kafka/integration/seaweedmq_handler_utils.go b/weed/mq/kafka/integration/seaweedmq_handler_utils.go index 33c3590ff4..e17cf4eb69 100644 --- a/weed/mq/kafka/integration/seaweedmq_handler_utils.go +++ b/weed/mq/kafka/integration/seaweedmq_handler_utils.go @@ -52,21 +52,23 @@ func NewSeaweedMQBrokerHandler(masters string, filerGroup string, clientHost str time.Sleep(2 * time.Second) glog.V(1).Infof("Initial connection delay completed") - // Discover brokers from masters using master client + // Discover brokers and filers from masters using master client. A fresh + // node registers only after its announce delay (a filer also waits out its + // empty-membership confirm window), so an empty list right after startup + // is normal: wait rather than fail fast. glog.V(1).Infof("About to call discoverBrokersWithMasterClient...") - brokerAddresses, err := discoverBrokersWithMasterClient(ctx, masterClient, filerGroup) + brokerAddresses, err := waitForClusterNodes(ctx, "brokers", clusterNodeDiscoveryTimeout, func() ([]string, error) { + return discoverBrokersWithMasterClient(ctx, masterClient, filerGroup) + }) if err != nil { glog.Errorf("Broker discovery failed: %v", err) return nil, fmt.Errorf("failed to discover brokers: %v", err) } glog.V(1).Infof("Broker discovery returned: %v", brokerAddresses) - if len(brokerAddresses) == 0 { - return nil, fmt.Errorf("no brokers discovered from masters") - } - - // Discover filers from masters using master client - filerAddresses, err := discoverFilersWithMasterClient(ctx, masterClient, filerGroup) + filerAddresses, err := waitForClusterNodes(ctx, "filers", clusterNodeDiscoveryTimeout, func() ([]pb.ServerAddress, error) { + return discoverFilersWithMasterClient(ctx, masterClient, filerGroup) + }) if err != nil { return nil, fmt.Errorf("failed to discover filers: %v", err) } @@ -143,7 +145,31 @@ func discoverBrokersWithMasterClient(ctx context.Context, masterClient *wdclient return brokers, err } -// discoverFilersWithMasterClient queries masters for available filers using reusable master client +// clusterNodeDiscoveryTimeout bounds how long the gateway waits for the +// masters to list a node; well above a fresh node's announce delay. +var clusterNodeDiscoveryTimeout = 60 * time.Second + +// waitForClusterNodes calls discover until it returns at least one node, an +// error, or timeout passes, retrying once a second while the list is empty. +func waitForClusterNodes[T any](ctx context.Context, nodeType string, timeout time.Duration, discover func() ([]T, error)) ([]T, error) { + deadline := time.Now().Add(timeout) + for { + nodes, err := discover() + if err != nil || len(nodes) > 0 { + return nodes, err + } + if !time.Now().Before(deadline) { + return nil, fmt.Errorf("no %s registered with the masters after %v", nodeType, timeout) + } + glog.V(1).Infof("no %s registered with the masters yet; retrying", nodeType) + select { + case <-ctx.Done(): + return nil, ctx.Err() + case <-time.After(time.Second): + } + } +} + func discoverFilersWithMasterClient(ctx context.Context, masterClient *wdclient.MasterClient, filerGroup string) ([]pb.ServerAddress, error) { var filers []pb.ServerAddress diff --git a/weed/mq/kafka/integration/seaweedmq_handler_utils_wait_test.go b/weed/mq/kafka/integration/seaweedmq_handler_utils_wait_test.go new file mode 100644 index 0000000000..ac3c03115c --- /dev/null +++ b/weed/mq/kafka/integration/seaweedmq_handler_utils_wait_test.go @@ -0,0 +1,38 @@ +package integration + +import ( + "context" + "errors" + "testing" + "time" + + "github.com/seaweedfs/seaweedfs/weed/pb" +) + +// A fresh filer is listed by the masters only after its announce delay, so an +// empty list must be waited out, not handed to the filer client accessor +// (which panics on it). +func TestWaitForFilersWaitsOutEmptyListing(t *testing.T) { + calls := 0 + filers, err := waitForClusterNodes(context.Background(), "filers", 10*time.Second, func() ([]pb.ServerAddress, error) { + calls++ + if calls < 3 { + return nil, nil + } + return []pb.ServerAddress{"filer:8888"}, nil + }) + if err != nil || len(filers) != 1 || calls != 3 { + t.Fatalf("filers %v err %v after %d calls, want 1 filer after 3", filers, err, calls) + } +} + +func TestWaitForFilersGivesUpWithAnError(t *testing.T) { + filers, err := waitForClusterNodes(context.Background(), "filers", 1500*time.Millisecond, func() ([]pb.ServerAddress, error) { return nil, nil }) + if err == nil || len(filers) != 0 { + t.Fatalf("still empty: filers %v err %v, want an error", filers, err) + } + boom := errors.New("master down") + if _, err := waitForClusterNodes(context.Background(), "filers", time.Minute, func() ([]pb.ServerAddress, error) { return nil, boom }); !errors.Is(err, boom) { + t.Fatalf("discovery error: got %v, want it returned at once", err) + } +} diff --git a/weed/notification/configuration.go b/weed/notification/configuration.go index 1c620f2e6a..d271323149 100644 --- a/weed/notification/configuration.go +++ b/weed/notification/configuration.go @@ -26,10 +26,10 @@ func LoadConfiguration(config *util.ViperProxy, prefix string) { return } - validateOneEnabledQueue(config) + validateOneEnabledQueue(config, prefix) for _, queue := range MessageQueues { - if config.GetBool(prefix + queue.GetName() + ".enabled") { + if queueEnabled(config, prefix, queue) { if err := queue.Initialize(config, prefix+queue.GetName()+"."); err != nil { glog.Fatalf("Failed to initialize notification for %s: %+v", queue.GetName(), err) @@ -42,10 +42,26 @@ func LoadConfiguration(config *util.ViperProxy, prefix string) { } -func validateOneEnabledQueue(config *util.ViperProxy) { +// queueEnabled reports whether a queue is switched on — by the flat +// ".enabled" flag or, for queues accepting named sub-instances +// ([notification..]), by any enabled subconfig. +func queueEnabled(config util.Configuration, prefix string, queue MessageQueue) bool { + key := prefix + queue.GetName() + "." + if config.GetBool(key + "enabled") { + return true + } + if q, ok := queue.(interface { + HasEnabledSubconfig(util.Configuration, string) bool + }); ok { + return q.HasEnabledSubconfig(config, key) + } + return false +} + +func validateOneEnabledQueue(config *util.ViperProxy, prefix string) { enabledQueue := "" for _, queue := range MessageQueues { - if config.GetBool(queue.GetName() + ".enabled") { + if queueEnabled(config, prefix, queue) { if enabledQueue == "" { enabledQueue = queue.GetName() } else { diff --git a/weed/notification/webhook/types.go b/weed/notification/webhook/types.go index f177f635a4..1567e832f2 100644 --- a/weed/notification/webhook/types.go +++ b/weed/notification/webhook/types.go @@ -3,7 +3,6 @@ package webhook import ( "fmt" "net/url" - "strconv" "github.com/seaweedfs/seaweedfs/weed/notification" "github.com/seaweedfs/seaweedfs/weed/pb/filer_pb" @@ -30,12 +29,6 @@ func (e eventType) valid() bool { return notification.ValidEventType(string(e)) } -var ( - pubSubHandlerNameTemplate = func(n int) string { - return "webhook_handler_" + strconv.Itoa(n) - } -) - type client interface { sendMessage(message *webhookMessage) error } diff --git a/weed/notification/webhook/webhook_queue.go b/weed/notification/webhook/webhook_queue.go index 2ac9db3ebb..afaa326245 100644 --- a/weed/notification/webhook/webhook_queue.go +++ b/weed/notification/webhook/webhook_queue.go @@ -4,6 +4,9 @@ import ( "context" "errors" "fmt" + "sort" + "strings" + "sync" "time" "github.com/ThreeDotsLabs/watermill" @@ -22,45 +25,176 @@ func init() { notification.MessageQueues = append(notification.MessageQueues, &Queue{}) } +// target is one webhook endpoint with its own filter, http client and +// concurrency budget. The flat [notification.webhook] section is the unnamed +// target; each enabled [notification.webhook.] subsection adds another. +type target struct { + name string + config *config + filter *filter + client client + sem chan struct{} + inbox chan *message.Message + pending chan struct{} +} + +func newTarget(name string, cfg *config) (*target, error) { + t := &target{ + name: name, + config: cfg, + filter: newFilter(cfg), + sem: make(chan struct{}, cfg.nWorkers), + inbox: make(chan *message.Message, cfg.bufferSize), + pending: make(chan struct{}, cfg.bufferSize), + } + hClient, err := newHTTPClient(cfg) + if err != nil { + return nil, fmt.Errorf("failed to create webhook http client: %w", err) + } + t.client = hClient + return t, nil +} + +func (t *target) topic() string { + if t.name == "" { + return pubSubTopicName + } + return pubSubTopicName + "_" + t.name +} + +func (t *target) handlerName() string { + if t.name == "" { + return "webhook_handler" + } + return "webhook_handler_" + t.name +} + type Queue struct { - router *message.Router + routers []*message.Router queueChannel *gochannel.GoChannel - config *config - client client - filter *filter + targets []*target ctx context.Context cancel context.CancelFunc - - // Semaphore for controlling concurrent webhook requests - sem chan struct{} } func (w *Queue) GetName() string { return queueName } +// HasEnabledSubconfig reports whether any [notification.webhook.] +// subsection is enabled, so the queue activates on named targets even when the +// flat [notification.webhook] section is absent. +func (w *Queue) HasEnabledSubconfig(configuration util.Configuration, prefix string) bool { + return len(namedTargetPrefixes(configuration, prefix)) > 0 +} + +// stringMapConfig is implemented by *util.ViperProxy; anything narrower simply +// gets no named-target discovery. +type stringMapConfig interface { + GetStringMap(key string) map[string]interface{} +} + +func namedTargetPrefixes(configuration util.Configuration, prefix string) []string { + mg, ok := configuration.(stringMapConfig) + if !ok { + return nil + } + var prefixes []string + for k, v := range mg.GetStringMap(strings.TrimSuffix(prefix, ".")) { + if _, ok := v.(map[string]interface{}); !ok { + continue + } + p := prefix + k + "." + if !configuration.GetBool(p + "enabled") { + continue + } + prefixes = append(prefixes, p) + } + sort.Strings(prefixes) + return prefixes +} + +func buildTargets(configuration util.Configuration, prefix string) ([]*target, error) { + var prefixes []string + if configuration.GetBool(prefix + "enabled") { + prefixes = append(prefixes, prefix) + } + prefixes = append(prefixes, namedTargetPrefixes(configuration, prefix)...) + + var targets []*target + for _, p := range prefixes { + cfg := newConfigWithDefaults(configuration, p) + if err := cfg.validate(); err != nil { + return nil, err + } + t, err := newTarget(strings.TrimSuffix(strings.TrimPrefix(p, prefix), "."), cfg) + if err != nil { + return nil, err + } + targets = append(targets, t) + } + return targets, nil +} + func (w *Queue) SendMessage(key string, msg proto.Message) error { eventNotification, ok := msg.(*filer_pb.EventNotification) if !ok { return nil } - if w.filter != nil && !w.filter.shouldPublish(key, eventNotification) { + var deferred []routedMessage + for _, t := range w.targets { + if t.filter != nil && !t.filter.shouldPublish(key, eventNotification) { + continue + } + m := newWebhookMessage(key, msg) + if m == nil { + continue + } + wMsg, err := m.toWaterMillMessage() + if err != nil { + return err + } + wMsg.Metadata.Set("target", t.name) + // a free reservation routes the event without waiting on slower + // targets; a saturated target defers to the blocking pass below so + // it applies backpressure instead of dropping or starving the rest + select { + case t.pending <- struct{}{}: + if err := t.enqueue(w.ctx, wMsg); err != nil { + return err + } + default: + deferred = append(deferred, routedMessage{t, wMsg}) + } + } + for _, r := range deferred { + select { + case r.target.pending <- struct{}{}: + case <-w.ctx.Done(): + return w.ctx.Err() + } + if err := r.target.enqueue(w.ctx, r.msg); err != nil { + return err + } + } + return nil +} + +type routedMessage struct { + target *target + msg *message.Message +} + +func (t *target) enqueue(ctx context.Context, wMsg *message.Message) error { + select { + case t.inbox <- wMsg: return nil + case <-ctx.Done(): + t.releasePending() + return ctx.Err() } - - m := newWebhookMessage(key, msg) - if m == nil { - return nil - } - - wMsg, err := m.toWaterMillMessage() - if err != nil { - return err - } - - return w.queueChannel.Publish(pubSubTopicName, wMsg) } func (w *webhookMessage) toWaterMillMessage() (*message.Message, error) { @@ -78,105 +212,158 @@ func (w *webhookMessage) toWaterMillMessage() (*message.Message, error) { } func (w *Queue) Initialize(configuration util.Configuration, prefix string) error { - c := newConfigWithDefaults(configuration, prefix) - - if err := c.validate(); err != nil { + targets, err := buildTargets(configuration, prefix) + if err != nil { return err } + if len(targets) == 0 { + return fmt.Errorf("webhook endpoint is required") + } - return w.initialize(c) + return w.initialize(targets) } -func (w *Queue) initialize(cfg *config) error { +func (w *Queue) initialize(targets []*target) error { w.ctx, w.cancel = context.WithCancel(context.Background()) - w.config = cfg - w.filter = newFilter(cfg) + w.targets = targets - // Initialize semaphore for controlling concurrent webhook requests - w.sem = make(chan struct{}, cfg.nWorkers) - - hClient, err := newHTTPClient(cfg) - if err != nil { - return fmt.Errorf("failed to create webhook http client: %w", err) - } - w.client = hClient - - if err = w.setupWatermillQueue(cfg); err != nil { + if err := w.setupWatermillQueue(targets); err != nil { return fmt.Errorf("failed to setup watermill queue: %w", err) } - if err = w.logDeadLetterMessages(); err != nil { + if err := w.logDeadLetterMessages(); err != nil { return err } return nil } -func (w *Queue) setupWatermillQueue(cfg *config) error { +func (w *Queue) setupWatermillQueue(targets []*target) error { logger := watermill.NewStdLogger(false, false) + bufferSize := 0 + for _, t := range targets { + if t.config.bufferSize > bufferSize { + bufferSize = t.config.bufferSize + } + } pubSubConfig := gochannel.Config{ - OutputChannelBuffer: int64(cfg.bufferSize), + OutputChannelBuffer: int64(bufferSize), Persistent: false, } w.queueChannel = gochannel.NewGoChannel(pubSubConfig, logger) - router, err := message.NewRouter( - message.RouterConfig{ - CloseTimeout: 60 * time.Second, - }, - logger, - ) - if err != nil { - return fmt.Errorf("failed to create router: %w", err) - } - w.router = router + var wg sync.WaitGroup + for _, t := range targets { + // One router per target so each keeps its own retry/backoff settings. + // Handlers subscribe to the target's own topic, so a message is + // delivered to an endpoint exactly once per matching target. + router, err := message.NewRouter( + message.RouterConfig{ + CloseTimeout: 60 * time.Second, + }, + logger, + ) + if err != nil { + return fmt.Errorf("failed to create router: %w", err) + } + w.routers = append(w.routers, router) - retryMiddleware := middleware.Retry{ - MaxRetries: cfg.maxRetries, - InitialInterval: time.Duration(cfg.backoffSeconds) * time.Second, - MaxInterval: time.Duration(cfg.maxBackoffSeconds) * time.Second, - Multiplier: 2.0, - RandomizationFactor: 0.3, - Logger: logger, - }.Middleware + retryMiddleware := middleware.Retry{ + MaxRetries: t.config.maxRetries, + InitialInterval: time.Duration(t.config.backoffSeconds) * time.Second, + MaxInterval: time.Duration(t.config.maxBackoffSeconds) * time.Second, + Multiplier: 2.0, + RandomizationFactor: 0.3, + Logger: logger, + }.Middleware - poisonQueue, err := middleware.PoisonQueue(w.queueChannel, deadLetterTopic) - if err != nil { - return fmt.Errorf("failed to create poison queue: %w", err) - } - - router.AddPlugin(plugin.SignalsHandler) - router.AddMiddleware(retryMiddleware, poisonQueue) - - // Add a single handler to avoid duplicate message delivery. - // With gochannel's default behavior, each handler call creates - // a separate subscription, and all subscriptions receive their own copy of each message. - // Using a single handler ensures each webhook is sent only once. - // Concurrency is controlled via semaphore in handleWebhook based on nWorkers config. - router.AddConsumerHandler( - "webhook_handler", - pubSubTopicName, - w.queueChannel, - w.handleWebhook, - ) - - go func() { - // cancels the queue context so the dead letter logger exists in case context not canceled by the shutdown signal already - defer w.cancel() - - if err := router.Run(w.ctx); err != nil && !errors.Is(err, context.Canceled) { - glog.Errorf("webhook pubsub worker stopped with error: %v", err) + poisonQueue, err := middleware.PoisonQueue(w.queueChannel, deadLetterTopic) + if err != nil { + return fmt.Errorf("failed to create poison queue: %w", err) } - glog.Info("webhook pubsub worker stopped") + router.AddPlugin(plugin.SignalsHandler) + // outermost: the reservation SendMessage took releases once the whole + // chain (retries, dead letter) is done, bounding in-flight deliveries. + // poison wraps retry so the delivery is retried before dead-lettering. + releasePending := func(h message.HandlerFunc) message.HandlerFunc { + return func(msg *message.Message) ([]*message.Message, error) { + defer t.releasePending() + return h(msg) + } + } + router.AddMiddleware(releasePending, poisonQueue, retryMiddleware) + + t := t + router.AddConsumerHandler( + t.handlerName(), + t.topic(), + w.queueChannel, + t.handleWebhook, + ) + + routerStopped := make(chan struct{}) + wg.Add(1) + go func() { + defer wg.Done() + defer close(routerStopped) + if err := router.Run(w.ctx); err != nil && !errors.Is(err, context.Canceled) { + glog.Errorf("webhook pubsub worker stopped with error: %v", err) + } + glog.Info("webhook pubsub worker stopped") + }() + + go w.forwardTargetMessages(t) + + // without a subscriber GoChannel drops the publish as undeliverable + select { + case <-router.Running(): + case <-routerStopped: + return fmt.Errorf("webhook router for target %q stopped before it started", t.name) + case <-w.ctx.Done(): + return w.ctx.Err() + } + } + + // the queue context lives until every target's worker has stopped so the + // dead letter logger keeps running while any of them is still up + go func() { + wg.Wait() + w.cancel() }() return nil } -func (w *Queue) handleWebhook(msg *message.Message) error { +// forwardTargetMessages publishes each queued event to the target's topic in +// order. Publish detaches its delivery, so the pending reservation is what +// keeps a saturated target from queuing without bound. +func (w *Queue) forwardTargetMessages(t *target) { + for { + select { + case msg := <-t.inbox: + if err := w.queueChannel.Publish(t.topic(), msg); err != nil { + glog.Errorf("webhook target %q failed to publish message: %v", t.name, err) + t.releasePending() + } + case <-w.ctx.Done(): + return + } + } +} + +// releasePending frees one delivery reservation. It never blocks: a nacked +// message GoChannel resends reaches the release again after its token is gone. +func (t *target) releasePending() { + select { + case <-t.pending: + default: + } +} + +func (t *target) handleWebhook(msg *message.Message) error { // Acquire semaphore slot (blocks if at capacity) - w.sem <- struct{}{} - defer func() { <-w.sem }() + t.sem <- struct{}{} + defer func() { <-t.sem }() var n filer_pb.EventNotification if err := proto.Unmarshal(msg.Payload, &n); err != nil { @@ -191,8 +378,8 @@ func (w *Queue) handleWebhook(msg *message.Message) error { Notification: &n, } - if err := w.client.sendMessage(webhookMsg); err != nil { - glog.Errorf("failed to send message to webhook %s: %v", webhookMsg.Key, err) + if err := t.client.sendMessage(webhookMsg); err != nil { + glog.Errorf("webhook target %q failed to send message for %s: %v", t.name, webhookMsg.Key, err) return err } @@ -218,10 +405,12 @@ func (w *Queue) logDeadLetterMessages() error { continue } key := "unknown" + targetName := "" if msg.Metadata != nil { if keyValue, exists := msg.Metadata["key"]; exists { key = keyValue } + targetName = msg.Metadata["target"] } payload := "" if msg.Payload != nil { @@ -232,7 +421,7 @@ func (w *Queue) logDeadLetterMessages() error { payload = n.String() } } - glog.Errorf("received dead letter message: %s, key: %s", payload, key) + glog.Errorf("received dead letter message: %s, key: %s, target: %s", payload, key, targetName) case <-w.ctx.Done(): return } diff --git a/weed/notification/webhook/webhook_queue_test.go b/weed/notification/webhook/webhook_queue_test.go index a084ef975b..ba75ecb147 100644 --- a/weed/notification/webhook/webhook_queue_test.go +++ b/weed/notification/webhook/webhook_queue_test.go @@ -4,6 +4,7 @@ import ( "net/http" "net/http/httptest" "strings" + "sync/atomic" "testing" "time" @@ -170,6 +171,29 @@ func TestWebhookMessageSerialization(t *testing.T) { } } +func newQueueForTest(t *testing.T, cfg *config) *Queue { + t.Helper() + tgt, err := newTarget("", cfg) + if err != nil { + t.Fatalf("newTarget() error = %v", err) + } + q := &Queue{} + if err := q.initialize([]*target{tgt}); err != nil { + t.Fatalf("initialize() error = %v", err) + } + return q +} + +func closeQueueForTest(q *Queue) { + if q.cancel != nil { + q.cancel() + } + time.Sleep(100 * time.Millisecond) + for _, r := range q.routers { + r.Close() + } +} + func TestQueueInitialize(t *testing.T) { cfg := &config{ endpoint: "https://example.com/webhook", @@ -182,32 +206,19 @@ func TestQueueInitialize(t *testing.T) { bufferSize: 100, } - q := &Queue{} - err := q.initialize(cfg) - if err != nil { - t.Errorf("Initialize() error = %v", err) - } + q := newQueueForTest(t, cfg) + defer closeQueueForTest(q) - defer func() { - if q.cancel != nil { - q.cancel() - } - time.Sleep(100 * time.Millisecond) - if q.router != nil { - q.router.Close() - } - }() - - if q.router == nil { + if len(q.routers) != 1 { t.Error("Expected router to be initialized") } if q.queueChannel == nil { t.Error("Expected queueChannel to be initialized") } - if q.client == nil { + if q.targets[0].client == nil { t.Error("Expected client to be initialized") } - if q.config == nil { + if q.targets[0].config == nil { t.Error("Expected config to be initialized") } } @@ -225,21 +236,8 @@ func TestQueueSendMessage(t *testing.T) { bufferSize: 10, } - q := &Queue{} - err := q.initialize(cfg) - if err != nil { - t.Fatalf("Failed to initialize queue: %v", err) - } - - defer func() { - if q.cancel != nil { - q.cancel() - } - time.Sleep(100 * time.Millisecond) - if q.router != nil { - q.router.Close() - } - }() + q := newQueueForTest(t, cfg) + defer closeQueueForTest(q) msg := &filer_pb.EventNotification{ NewEntry: &filer_pb.Entry{ @@ -247,8 +245,7 @@ func TestQueueSendMessage(t *testing.T) { }, } - err = q.SendMessage("/test/path", msg) - if err != nil { + if err := q.SendMessage("/test/path", msg); err != nil { t.Errorf("SendMessage() error = %v", err) } } @@ -271,7 +268,7 @@ func TestQueueHandleWebhook(t *testing.T) { } client, _ := newHTTPClient(cfg) - q := &Queue{ + tgt := &target{ client: client, sem: make(chan struct{}, cfg.nWorkers), } @@ -287,8 +284,7 @@ func TestQueueHandleWebhook(t *testing.T) { t.Fatalf("Failed to create watermill message: %v", err) } - err = q.handleWebhook(wmMsg) - if err != nil { + if err := tgt.handleWebhook(wmMsg); err != nil { t.Errorf("handleWebhook() error = %v", err) } } @@ -307,21 +303,8 @@ func TestQueueEndToEnd(t *testing.T) { bufferSize: 10, } - q := &Queue{} - err := q.initialize(cfg) - if err != nil { - t.Fatalf("Failed to initialize queue: %v", err) - } - - defer func() { - if q.cancel != nil { - q.cancel() - } - time.Sleep(100 * time.Millisecond) - if q.router != nil { - q.router.Close() - } - }() + q := newQueueForTest(t, cfg) + defer closeQueueForTest(q) msg := &filer_pb.EventNotification{ NewEntry: &filer_pb.Entry{ @@ -329,8 +312,7 @@ func TestQueueEndToEnd(t *testing.T) { }, } - err = q.SendMessage("/test/path", msg) - if err != nil { + if err := q.SendMessage("/test/path", msg); err != nil { t.Errorf("SendMessage() error = %v", err) } } @@ -347,33 +329,20 @@ func TestQueueRetryMechanism(t *testing.T) { bufferSize: 10, } - q := &Queue{} - err := q.initialize(cfg) - if err != nil { - t.Fatalf("Failed to initialize queue: %v", err) - } - - defer func() { - if q.cancel != nil { - q.cancel() - } - time.Sleep(100 * time.Millisecond) - if q.router != nil { - q.router.Close() - } - }() + q := newQueueForTest(t, cfg) + defer closeQueueForTest(q) // Verify that the queue is properly configured for retries - if q.config.maxRetries != 3 { - t.Errorf("Expected maxRetries=3, got %d", q.config.maxRetries) + if q.targets[0].config.maxRetries != 3 { + t.Errorf("Expected maxRetries=3, got %d", q.targets[0].config.maxRetries) } - if q.config.backoffSeconds != 2 { - t.Errorf("Expected backoffSeconds=2, got %d", q.config.backoffSeconds) + if q.targets[0].config.backoffSeconds != 2 { + t.Errorf("Expected backoffSeconds=2, got %d", q.targets[0].config.backoffSeconds) } - if q.config.maxBackoffSeconds != 10 { - t.Errorf("Expected maxBackoffSeconds=10, got %d", q.config.maxBackoffSeconds) + if q.targets[0].config.maxBackoffSeconds != 10 { + t.Errorf("Expected maxBackoffSeconds=10, got %d", q.targets[0].config.maxBackoffSeconds) } // Test that we can send a message (retry behavior is handled by Watermill middleware) @@ -381,8 +350,7 @@ func TestQueueRetryMechanism(t *testing.T) { NewEntry: &filer_pb.Entry{Name: "test.txt"}, } - err = q.SendMessage("/test/retry", msg) - if err != nil { + if err := q.SendMessage("/test/retry", msg); err != nil { t.Errorf("SendMessage() error = %v", err) } } @@ -419,21 +387,8 @@ func TestQueueNoDuplicateWebhooks(t *testing.T) { bufferSize: 10, } - q := &Queue{} - err := q.initialize(cfg) - if err != nil { - t.Fatalf("Failed to initialize queue: %v", err) - } - - defer func() { - if q.cancel != nil { - q.cancel() - } - time.Sleep(100 * time.Millisecond) - if q.router != nil { - q.router.Close() - } - }() + q := newQueueForTest(t, cfg) + defer closeQueueForTest(q) // Wait for router and subscriber to be fully ready time.Sleep(200 * time.Millisecond) @@ -444,8 +399,7 @@ func TestQueueNoDuplicateWebhooks(t *testing.T) { }, } - err = q.SendMessage("/test/path", msg) - if err != nil { + if err := q.SendMessage("/test/path", msg); err != nil { t.Errorf("SendMessage() error = %v", err) } @@ -607,3 +561,298 @@ func TestQueueSendMessageWithFilter(t *testing.T) { }) } } + +type mapConfig struct { + values map[string]interface{} +} + +func (m *mapConfig) GetString(key string) string { + v, _ := m.values[key].(string) + return v +} + +func (m *mapConfig) GetBool(key string) bool { + v, _ := m.values[key].(bool) + return v +} + +func (m *mapConfig) GetInt(key string) int { + v, _ := m.values[key].(int) + return v +} + +func (m *mapConfig) GetStringSlice(key string) []string { + v, _ := m.values[key].([]string) + return v +} + +func (m *mapConfig) SetDefault(key string, value interface{}) {} + +func (m *mapConfig) GetStringMap(key string) map[string]interface{} { + v, _ := m.values[key].(map[string]interface{}) + return v +} + +func TestNamedTargets(t *testing.T) { + var hitsA, hitsB atomic.Int32 + serverA := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + hitsA.Add(1) + w.WriteHeader(http.StatusOK) + })) + defer serverA.Close() + serverB := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + hitsB.Add(1) + w.WriteHeader(http.StatusOK) + })) + defer serverB.Close() + + cfg := &mapConfig{values: map[string]interface{}{ + "notification.webhook": map[string]interface{}{ + "file_ingress": map[string]interface{}{ + "enabled": true, + "endpoint": serverA.URL, + "path_prefixes": []string{"/data/"}, + }, + "metadata_extractor": map[string]interface{}{ + "enabled": true, + "endpoint": serverB.URL, + "path_prefixes": []string{"/repository/"}, + }, + "disabled_one": map[string]interface{}{ + "enabled": false, + "endpoint": "https://example.com/never", + }, + }, + "notification.webhook.file_ingress.enabled": true, + "notification.webhook.file_ingress.endpoint": serverA.URL, + "notification.webhook.file_ingress.path_prefixes": []string{"/data/"}, + "notification.webhook.metadata_extractor.enabled": true, + "notification.webhook.metadata_extractor.endpoint": serverB.URL, + "notification.webhook.metadata_extractor.path_prefixes": []string{"/repository/"}, + }} + + q := &Queue{} + if !q.HasEnabledSubconfig(cfg, "notification.webhook.") { + t.Fatal("expected named subconfig to be detected as enabled") + } + + targets, err := buildTargets(cfg, "notification.webhook.") + if err != nil { + t.Fatalf("buildTargets() error = %v", err) + } + if len(targets) != 2 { + t.Fatalf("expected 2 targets, got %d", len(targets)) + } + if targets[0].name != "file_ingress" || targets[1].name != "metadata_extractor" { + t.Fatalf("unexpected target names: %s, %s", targets[0].name, targets[1].name) + } + + if err := q.initialize(targets); err != nil { + t.Fatalf("initialize() error = %v", err) + } + defer closeQueueForTest(q) + + time.Sleep(200 * time.Millisecond) + + msg := &filer_pb.EventNotification{ + NewEntry: &filer_pb.Entry{Name: "f.txt"}, + } + for _, key := range []string{"/data/f.txt", "/repository/f.txt", "/elsewhere/f.txt"} { + if err := q.SendMessage(key, msg); err != nil { + t.Errorf("SendMessage(%q) error = %v", key, err) + } + } + + deadline := time.Now().Add(5 * time.Second) + for time.Now().Before(deadline) && (hitsA.Load() == 0 || hitsB.Load() == 0) { + time.Sleep(50 * time.Millisecond) + } + time.Sleep(200 * time.Millisecond) + + if got := hitsA.Load(); got != 1 { + t.Errorf("expected file_ingress to receive 1 event, got %d", got) + } + if got := hitsB.Load(); got != 1 { + t.Errorf("expected metadata_extractor to receive 1 event, got %d", got) + } +} + +func TestQueueBurstKeepsPendingEvents(t *testing.T) { + var hits atomic.Int32 + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + hits.Add(1) + time.Sleep(20 * time.Millisecond) + w.WriteHeader(http.StatusOK) + })) + defer server.Close() + + cfg := &config{ + endpoint: server.URL, + timeoutSeconds: 5, + maxRetries: 0, + backoffSeconds: 1, + maxBackoffSeconds: 1, + nWorkers: 1, + bufferSize: 2, + } + q := newQueueForTest(t, cfg) + defer closeQueueForTest(q) + + const total = 6 + msg := &filer_pb.EventNotification{NewEntry: &filer_pb.Entry{Name: "f.txt"}} + for i := 0; i < total; i++ { + if err := q.SendMessage("/test/path", msg); err != nil { + t.Fatalf("SendMessage() error = %v", err) + } + } + + deadline := time.Now().Add(10 * time.Second) + for time.Now().Before(deadline) && hits.Load() < total { + time.Sleep(50 * time.Millisecond) + } + if got := hits.Load(); got != total { + t.Errorf("expected %d deliveries, got %d", total, got) + } +} + +// A saturated target still applies backpressure to the sender, but it must +// not starve the targets after it: the event reaches a healthy endpoint +// while the slow one is still out of pending slots. +func TestSaturatedTargetDoesNotStarveOthers(t *testing.T) { + var hitsA, hitsB atomic.Int32 + serverA := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + hitsA.Add(1) + w.WriteHeader(http.StatusOK) + })) + defer serverA.Close() + serverB := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + hitsB.Add(1) + w.WriteHeader(http.StatusOK) + })) + defer serverB.Close() + + newTestTarget := func(name, endpoint string) *target { + tgt, err := newTarget(name, &config{ + endpoint: endpoint, + timeoutSeconds: 5, + maxRetries: 0, + backoffSeconds: 1, + maxBackoffSeconds: 1, + nWorkers: 1, + bufferSize: 2, + }) + if err != nil { + t.Fatalf("newTarget() error = %v", err) + } + return tgt + } + slow := newTestTarget("slow", serverA.URL) + fast := newTestTarget("fast", serverB.URL) + + q := &Queue{} + if err := q.initialize([]*target{slow, fast}); err != nil { + t.Fatalf("initialize() error = %v", err) + } + defer closeQueueForTest(q) + + for i := 0; i < cap(slow.pending); i++ { + slow.pending <- struct{}{} + } + + done := make(chan error, 1) + go func() { + done <- q.SendMessage("/test/path", &filer_pb.EventNotification{NewEntry: &filer_pb.Entry{Name: "f.txt"}}) + }() + + deadline := time.Now().Add(5 * time.Second) + for time.Now().Before(deadline) && hitsB.Load() == 0 { + time.Sleep(20 * time.Millisecond) + } + if got := hitsB.Load(); got != 1 { + t.Fatalf("expected fast target to receive the event while slow is saturated, got %d", got) + } + select { + case err := <-done: + t.Fatalf("SendMessage() returned %v while the saturated target had no free slot", err) + default: + } + + <-slow.pending + if err := <-done; err != nil { + t.Fatalf("SendMessage() error = %v", err) + } + + deadline = time.Now().Add(5 * time.Second) + for time.Now().Before(deadline) && hitsA.Load() == 0 { + time.Sleep(20 * time.Millisecond) + } + if got := hitsA.Load(); got != 1 { + t.Errorf("expected the saturated target to deliver once a slot freed, got %d", got) + } +} + +// A disabled flat section keeps no target even when an endpoint is set — +// the scaffold ships exactly that combination, and a named target must not +// reactivate it. +func TestBuildTargetsDisabledFlat(t *testing.T) { + cfg := &mapConfig{values: map[string]interface{}{ + "notification.webhook": map[string]interface{}{ + "enabled": false, + "endpoint": "https://your-server.com/webhook", + "named": map[string]interface{}{ + "enabled": true, + "endpoint": "https://example.com/hook", + }, + }, + "notification.webhook.enabled": false, + "notification.webhook.endpoint": "https://your-server.com/webhook", + "notification.webhook.named.enabled": true, + "notification.webhook.named.endpoint": "https://example.com/hook", + }} + + targets, err := buildTargets(cfg, "notification.webhook.") + if err != nil { + t.Fatalf("buildTargets() error = %v", err) + } + if len(targets) != 1 || targets[0].name != "named" { + t.Fatalf("expected only the named target, got %v", targets) + } +} + +// A failing endpoint must be retried maxRetries times before the event is +// dead-lettered — the poison middleware has to sit outside the retry chain +// or it swallows the first error and retries never run. +func TestQueueRetriesBeforeDeadLetter(t *testing.T) { + var hits atomic.Int32 + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + hits.Add(1) + w.WriteHeader(http.StatusInternalServerError) + })) + defer server.Close() + + cfg := &config{ + endpoint: server.URL, + timeoutSeconds: 5, + maxRetries: 2, + backoffSeconds: 1, + maxBackoffSeconds: 1, + nWorkers: 1, + bufferSize: 10, + } + q := newQueueForTest(t, cfg) + defer closeQueueForTest(q) + + msg := &filer_pb.EventNotification{NewEntry: &filer_pb.Entry{Name: "f.txt"}} + if err := q.SendMessage("/test/path", msg); err != nil { + t.Fatalf("SendMessage() error = %v", err) + } + + want := int32(cfg.maxRetries + 1) + deadline := time.Now().Add(15 * time.Second) + for time.Now().Before(deadline) && hits.Load() < want { + time.Sleep(50 * time.Millisecond) + } + if got := hits.Load(); got != want { + t.Errorf("expected %d delivery attempts (1 + %d retries), got %d", want, cfg.maxRetries, got) + } +} diff --git a/weed/operation/upload_content.go b/weed/operation/upload_content.go index 704f70f49d..081d295e66 100644 --- a/weed/operation/upload_content.go +++ b/weed/operation/upload_content.go @@ -17,6 +17,7 @@ import ( "strconv" "strings" "sync" + "sync/atomic" "time" "github.com/seaweedfs/seaweedfs/weed/util/request_id" @@ -63,6 +64,10 @@ type UploadOption struct { MaxAttempts int // <=0 uses the default GenUploadUrl func(host, fileId string) string // if nil → fallback "http://{host}/{fileId}" FilerJwt security.FilerJwtProvider // credential for proxy chunk URLs; nil → process-wide jwt.filer_signing + + // stallWatchdogDisarmed turns the watchdog off for this upload's later + // attempts once it fired: a re-arm could burn every retry. + stallWatchdogDisarmed bool } type UploadResult struct { @@ -125,6 +130,7 @@ var ( // StatusCode 0 means the request never got a response — a transport failure. type uploadStatusError struct { StatusCode int + stalled bool // the stall watchdog cancelled this attempt's request err error } @@ -163,6 +169,11 @@ type HTTPClient interface { // Uploader type Uploader struct { httpClient HTTPClient + // fastUploadHosts records the wire rate of the last sizeable upload to + // each host, so a later upload stalling far below it can be blamed on + // that one connection rather than the link and get cancelled for a retry + // on a fresh conn. + fastUploadHosts sync.Map // host:port → fastHostMark } func NewUploader() (*Uploader, error) { @@ -328,6 +339,11 @@ func (uploader *Uploader) doUpload(ctx context.Context, reader io.Reader, option } func (uploader *Uploader) retriedUploadData(ctx context.Context, data []byte, option *UploadOption) (uploadResult *UploadResult, err error) { + // A stall disarms the watchdog for this upload's remaining attempts only; + // keep that on a per-call option so a caller reusing UploadOption starts + // the next upload armed. + localOption := *option + option = &localOption maxAttempts := option.MaxAttempts if maxAttempts <= 0 { maxAttempts = 3 @@ -348,6 +364,10 @@ func (uploader *Uploader) retriedUploadData(ctx context.Context, data []byte, op if ctx.Err() != nil { return nil, ctx.Err() } + var statusErr *uploadStatusError + if errors.As(err, &statusErr) && statusErr.stalled { + option.stallWatchdogDisarmed = true + } glog.WarningfCtx(ctx, "uploading %d to %s: %v", i, option.UploadUrl, err) } return @@ -415,14 +435,15 @@ func (uploader *Uploader) doUploadData(ctx context.Context, data []byte, option _, err = w.Write(encryptedData) return }, len(encryptedData), &UploadOption{ - UploadUrl: option.UploadUrl, - Filename: "", - Cipher: false, - IsInputCompressed: false, - MimeType: "", - PairMap: nil, - Jwt: option.Jwt, - BytesBuffer: option.BytesBuffer, + UploadUrl: option.UploadUrl, + Filename: "", + Cipher: false, + IsInputCompressed: false, + MimeType: "", + PairMap: nil, + Jwt: option.Jwt, + BytesBuffer: option.BytesBuffer, + stallWatchdogDisarmed: option.stallWatchdogDisarmed, }) if uploadResult == nil { return @@ -443,15 +464,16 @@ func (uploader *Uploader) doUploadData(ctx context.Context, data []byte, option _, err = w.Write(data) return }, len(data), &UploadOption{ - UploadUrl: option.UploadUrl, - Filename: option.Filename, - Cipher: false, - IsInputCompressed: contentIsGzipped, - MimeType: option.MimeType, - PairMap: option.PairMap, - Jwt: option.Jwt, - Md5: option.Md5, - BytesBuffer: option.BytesBuffer, + UploadUrl: option.UploadUrl, + Filename: option.Filename, + Cipher: false, + IsInputCompressed: contentIsGzipped, + MimeType: option.MimeType, + PairMap: option.PairMap, + Jwt: option.Jwt, + Md5: option.Md5, + BytesBuffer: option.BytesBuffer, + stallWatchdogDisarmed: option.stallWatchdogDisarmed, }) if uploadResult == nil { return @@ -514,7 +536,14 @@ func (uploader *Uploader) upload_content(ctx context.Context, fillBufferFunction } else { reqReader = bytes.NewReader(option.BytesBuffer.Bytes()) } - req, postErr := http.NewRequestWithContext(ctx, http.MethodPost, option.UploadUrl, reqReader) + // bodySize is what the conn must move: originalDataSize counts bytes + // before compression, which would inflate both the recorded rate and the + // stall budget for a pre-compressed body. + bodySize := int64(reqReader.Len()) + reqCtx, cancelUpload := context.WithCancel(ctx) + defer cancelUpload() + + req, postErr := http.NewRequestWithContext(reqCtx, http.MethodPost, option.UploadUrl, reqReader) if postErr != nil { glog.V(1).InfofCtx(ctx, "create upload request %s: %v", option.UploadUrl, postErr) return nil, fmt.Errorf("create upload request %s: %v", option.UploadUrl, postErr) @@ -529,7 +558,46 @@ func (uploader *Uploader) upload_content(ctx context.Context, fillBufferFunction request_id.InjectToRequest(ctx, req) + var uploadDone atomic.Bool + defer uploadDone.Store(true) + // bodySentAt marks when the transport finished draining the request body; + // past it the wait is server-side commit, which the transport's response + // header timeout already bounds — the stall watchdog only blames a conn + // that is still trickling the body. + var curBodySentAt atomic.Pointer[atomic.Int64] + bodySentAt := &atomic.Int64{} + curBodySentAt.Store(bodySentAt) + req.Body = &watchdogBody{ReadCloser: req.Body, sentAt: bodySentAt} + mark, hostWasFast := uploader.fastUploadHosts.Load(req.URL.Host) + hostWasFast = hostWasFast && !option.stallWatchdogDisarmed && time.Since(time.Unix(0, mark.(fastHostMark).at)) < fastUploadHostMemory + stallFired := func() { + if !uploadDone.CompareAndSwap(false, true) { + return + } + // The host was moving bodies at full rate a moment ago, so an + // upload still trickling far below that rate rides a wedged pooled + // conn; cancel drops the conn so the retry dials a fresh one. + // CompareAndDelete leaves a newer mark a concurrent success stored. + uploader.fastUploadHosts.CompareAndDelete(req.URL.Host, mark) + glog.WarningfCtx(ctx, "upload %d bytes to %s stalled on a pooled connection, retrying on a fresh one", originalDataSize, option.UploadUrl) + cancelUpload() + } + var stallTimer *time.Timer + if hostWasFast { + defer func() { + if stallTimer != nil { + stallTimer.Stop() + } + }() + stallTimer = time.AfterFunc(uploadStallTimeout(bodySize, mark.(fastHostMark).bps), func() { + if curBodySentAt.Load().Load() == 0 { + stallFired() + } + }) + } + // print("+") + started := time.Now() resp, post_err := uploader.httpClient.Do(req) defer util_http.CloseResponse(resp) if post_err != nil { @@ -548,7 +616,13 @@ func (uploader *Uploader) upload_content(ctx context.Context, fillBufferFunction // "connection reset" error with a misleading "Body length 0". if req.GetBody != nil { if newBody, gbErr := req.GetBody(); gbErr == nil { - req.Body = newBody + bodySentAt = &atomic.Int64{} + curBodySentAt.Store(bodySentAt) + started = time.Now() + if stallTimer != nil && !uploadDone.Load() { + stallTimer.Reset(uploadStallTimeout(bodySize, mark.(fastHostMark).bps)) + } + req.Body = &watchdogBody{ReadCloser: newBody, sentAt: bodySentAt} resp, post_err = uploader.httpClient.Do(req) defer util_http.CloseResponse(resp) } else { @@ -561,22 +635,40 @@ func (uploader *Uploader) upload_content(ctx context.Context, fillBufferFunction } if post_err != nil { stats.UploadErrorCounter.WithLabelValues("0").Inc() - return nil, &uploadStatusError{StatusCode: 0, err: fmt.Errorf("upload %s %d bytes to %v: %v", option.Filename, originalDataSize, option.UploadUrl, post_err)} + return nil, &uploadStatusError{StatusCode: 0, stalled: reqCtx.Err() != nil && ctx.Err() == nil, err: fmt.Errorf("upload %s %d bytes to %v: %v", option.Filename, originalDataSize, option.UploadUrl, post_err)} } + // Rate is bounded by elapsed-to-response: the server cannot answer before + // the body lands, so this can only understate link speed — a drain-time + // reading mistakes kernel-buffer absorption for wire speed and would mark + // a slow link fast. The mark still only lands on a verified upload: a + // fast rejection says nothing about how the conn drains. + movedIn := time.Since(started) + // print("-") var ret UploadResult etag := getEtag(resp) if resp.StatusCode == http.StatusNoContent { + if bps := uploadObservedBps(bodySize, movedIn); bps > 0 { + uploader.fastUploadHosts.Store(req.URL.Host, fastHostMark{at: time.Now().UnixNano(), bps: bps}) + } ret.ETag = etag ret.ContentMd5 = resp.Header.Get("Content-MD5") return &ret, nil } + if stallTimer != nil { + // The header timeout does not cover the response body: keep the + // watchdog armed for the read so a conn that wedges mid-reply gets + // cancelled for a fresh retry. + respStallTimer := time.AfterFunc(uploadStallMinWait, stallFired) + defer respStallTimer.Stop() + } resp_body, ra_err := io.ReadAll(resp.Body) + uploadDone.Store(true) if ra_err != nil { stats.UploadErrorCounter.WithLabelValues(strconv.Itoa(resp.StatusCode)).Inc() - return nil, &uploadStatusError{StatusCode: resp.StatusCode, err: fmt.Errorf("read response body %v: %w", option.UploadUrl, ra_err)} + return nil, &uploadStatusError{StatusCode: resp.StatusCode, stalled: reqCtx.Err() != nil && ctx.Err() == nil, err: fmt.Errorf("read response body %v: %w", option.UploadUrl, ra_err)} } unmarshal_err := json.Unmarshal(resp_body, &ret) @@ -589,6 +681,9 @@ func (uploader *Uploader) upload_content(ctx context.Context, fillBufferFunction stats.UploadErrorCounter.WithLabelValues(strconv.Itoa(resp.StatusCode)).Inc() return nil, &uploadStatusError{StatusCode: resp.StatusCode, err: fmt.Errorf("unmarshalled error %v: %v", option.UploadUrl, ret.Error)} } + if bps := uploadObservedBps(bodySize, movedIn); bps > 0 { + uploader.fastUploadHosts.Store(req.URL.Host, fastHostMark{at: time.Now().UnixNano(), bps: bps}) + } ret.ETag = etag ret.ContentMd5 = resp.Header.Get("Content-MD5") return &ret, nil @@ -601,3 +696,68 @@ func getEtag(r *http.Response) (etag string) { } return } + +// uploadRateMinBytes is the smallest body whose measured rate can mark a +// host: smaller uploads are dominated by round-trip and commit overhead, not +// wire speed. +const uploadRateMinBytes = int64(256 << 10) + +var ( + // uploadStallSlowFactor is how far below the host's last observed rate an + // upload may run before its conn is presumed wedged: at 8, a host that + // just moved a body at 8MiB/s tolerates down to 1MiB/s. + uploadStallSlowFactor = int64(8) + uploadStallMinWait = 10 * time.Second + fastUploadHostMemory = time.Minute +) + +// fastHostMark remembers the wire rate of a host's last sizeable upload, so a +// later upload running far below it blames the pooled conn rather than the +// link — no fixed rate fits every deployment's links. +type fastHostMark struct { + at int64 // UnixNano + bps int64 // wire bytes/sec measured elapsed-to-response +} + +// uploadStallTimeout is the most an upload may take on a conn whose host last +// proved bps: the time the body needs at an uploadStallSlowFactor of that +// rate, with a floor so a fast conn is not raced on small bodies. +func uploadStallTimeout(size, bps int64) time.Duration { + if bps > 0 { + if d := time.Duration(size/bps) * time.Duration(uploadStallSlowFactor) * time.Second; d > uploadStallMinWait { + return d + } + } + return uploadStallMinWait +} + +// uploadObservedBps returns the wire rate a completed upload proved on its +// host, or 0 when the body was too small to measure. Elapsed-to-response can +// only understate the link's speed, which keeps the mark — and the stall +// budget derived from it — conservative. +func uploadObservedBps(size int64, elapsed time.Duration) int64 { + if size < uploadRateMinBytes || elapsed <= 0 { + return 0 + } + return int64(float64(size) / elapsed.Seconds()) +} + +// watchdogBody records when the transport drains the request body: before that +// a stuck send blames the conn; after it the wait is the server's. +type watchdogBody struct { + io.ReadCloser + sentAt *atomic.Int64 +} + +func (b *watchdogBody) Read(p []byte) (int, error) { + n, err := b.ReadCloser.Read(p) + if err == io.EOF { + b.sentAt.Store(time.Now().UnixNano()) + } + return n, err +} + +func (b *watchdogBody) Close() error { + b.sentAt.Store(time.Now().UnixNano()) + return b.ReadCloser.Close() +} diff --git a/weed/operation/upload_content_test.go b/weed/operation/upload_content_test.go index ebbfb722f4..d80c8cd591 100644 --- a/weed/operation/upload_content_test.go +++ b/weed/operation/upload_content_test.go @@ -8,10 +8,12 @@ import ( "errors" "fmt" "io" + "net" "net/http" "net/http/httptest" "strings" "sync" + "sync/atomic" "testing" "time" @@ -199,6 +201,372 @@ func (c *bodyCapturingHTTPClient) Do(req *http.Request) (*http.Response, error) }, nil } +// stallingHTTPClient parks the first stalls calls without draining the request +// body (a conn that never drains) until the request ctx is cancelled, and +// delays the rest by slowFor after draining. +type stallingHTTPClient struct { + mu sync.Mutex + stalls int + slowFor time.Duration + requestCtxs []context.Context +} + +func (c *stallingHTTPClient) Do(req *http.Request) (*http.Response, error) { + c.mu.Lock() + c.requestCtxs = append(c.requestCtxs, req.Context()) + stall := c.stalls > 0 + if stall { + c.stalls-- + } + c.mu.Unlock() + + if stall { + <-req.Context().Done() + return nil, req.Context().Err() + } + if req.Body != nil { + _, _ = io.Copy(io.Discard, req.Body) + } + if c.slowFor > 0 { + time.Sleep(c.slowFor) + } + return &http.Response{ + StatusCode: http.StatusCreated, + Header: make(http.Header), + Body: io.NopCloser(strings.NewReader(`{"size":1}`)), + }, nil +} + +// markHostFast arms the stall watchdog the way a recent fast upload would: +// the recorded rate is high enough that the adaptive budget lands on +// uploadStallMinWait for test-sized bodies. +func markHostFast(u *Uploader, host string) { + u.fastUploadHosts.Store(host, fastHostMark{at: time.Now().UnixNano(), bps: 1 << 30}) +} + +// A host that just served a fast upload arms the stall watchdog: a request +// parked well past the floor is riding a wedged conn, so it is cancelled and +// the retry runs on a fresh conn without the watchdog. +func TestUploadStallWatchdogRetriesOnFreshConn(t *testing.T) { + defer func(orig time.Duration) { uploadStallMinWait = orig }(uploadStallMinWait) + uploadStallMinWait = 50 * time.Millisecond + + client := &stallingHTTPClient{stalls: 1} + uploader := newUploader(client) + markHostFast(uploader, "test-host:8080") + + result, err := uploader.UploadData(context.Background(), make([]byte, 300*1024), &UploadOption{ + UploadUrl: "http://test-host:8080/1,deadbeef", + MaxAttempts: 2, + }) + if err != nil { + t.Fatalf("upload should have retried on a fresh conn: %v", err) + } + if result.RetryCount != 1 || len(client.requestCtxs) != 2 { + t.Fatalf("got retries=%d calls=%d, want one retry on a fresh conn", result.RetryCount, len(client.requestCtxs)) + } + if err := client.requestCtxs[0].Err(); err == nil { + t.Fatalf("first attempt context was not cancelled") + } +} + +// readStallHTTPClient answers with a response body that never delivers until +// the request ctx is cancelled — the conn that took the request then wedged on +// the read side. +type readStallHTTPClient struct { + mu sync.Mutex + stall bool +} + +func (c *readStallHTTPClient) Do(req *http.Request) (*http.Response, error) { + if req.Body != nil { + _, _ = io.Copy(io.Discard, req.Body) + } + c.mu.Lock() + stall := c.stall + c.stall = false + c.mu.Unlock() + var body io.ReadCloser = io.NopCloser(strings.NewReader(`{"size":1}`)) + if stall { + body = &ctxBlockedBody{ctx: req.Context()} + } + return &http.Response{ + StatusCode: http.StatusCreated, + Header: make(http.Header), + Body: body, + }, nil +} + +type ctxBlockedBody struct{ ctx context.Context } + +func (b *ctxBlockedBody) Read([]byte) (int, error) { + <-b.ctx.Done() + return 0, b.ctx.Err() +} + +func (b *ctxBlockedBody) Close() error { return nil } + +// The watchdog stays armed through the response read: a conn that accepts the +// body then wedges on the reply is as wedged as one that never drains. +func TestUploadStallWatchdogCoversResponseRead(t *testing.T) { + defer func(orig time.Duration) { uploadStallMinWait = orig }(uploadStallMinWait) + uploadStallMinWait = 50 * time.Millisecond + + client := &readStallHTTPClient{stall: true} + uploader := newUploader(client) + markHostFast(uploader, "test-host:8080") + + result, err := uploader.UploadData(context.Background(), make([]byte, 300*1024), &UploadOption{ + UploadUrl: "http://test-host:8080/1,deadbeef", + MaxAttempts: 2, + }) + if err != nil { + t.Fatalf("upload should have retried past the stalled read: %v", err) + } + if result.RetryCount != 1 { + t.Fatalf("got retries=%d, want one retry on a fresh conn", result.RetryCount) + } +} + +// rearmHTTPClient parks the first attempt on a conn that never drains and +// re-marks the host fast as it fails — a concurrent healthy upload can restore +// the mark while this upload is between attempts. The retry drains slowly, so +// a re-armed watchdog would cancel it too. +type rearmHTTPClient struct { + uploader *Uploader + calls int32 + slowFor time.Duration +} + +func (c *rearmHTTPClient) Do(req *http.Request) (*http.Response, error) { + if atomic.AddInt32(&c.calls, 1) == 1 { + <-req.Context().Done() + markHostFast(c.uploader, req.URL.Host) + return nil, req.Context().Err() + } + // Park past the watchdog floor like a real wedged conn: a re-armed + // watchdog must cancel this attempt instead of the sleep masking it. + select { + case <-req.Context().Done(): + return nil, req.Context().Err() + case <-time.After(c.slowFor): + } + if req.Body != nil { + _, _ = io.Copy(io.Discard, req.Body) + } + return &http.Response{ + StatusCode: http.StatusCreated, + Header: make(http.Header), + Body: io.NopCloser(strings.NewReader(`{"size":1}`)), + }, nil +} + +// A re-proven host must not re-arm the watchdog for the retry: the conn it +// cancelled says nothing about the next conn, and re-arming could burn every +// remaining attempt. +func TestUploadStallWatchdogStaysDisarmedOnRetry(t *testing.T) { + defer func(orig time.Duration) { uploadStallMinWait = orig }(uploadStallMinWait) + uploadStallMinWait = 40 * time.Millisecond + + uploader := newUploader(nil) + uploader.httpClient = &rearmHTTPClient{uploader: uploader, slowFor: 60 * time.Millisecond} + markHostFast(uploader, "test-host:8080") + + result, err := uploader.UploadData(context.Background(), make([]byte, 300*1024), &UploadOption{ + UploadUrl: "http://test-host:8080/1,deadbeef", + MaxAttempts: 2, + }) + if err != nil { + t.Fatalf("retry should run unbounded once the watchdog has fired: %v", err) + } + if result.RetryCount != 1 { + t.Fatalf("got retries=%d, want one retry", result.RetryCount) + } +} + +// The disarm is per upload: a caller reusing its UploadOption still gets an +// armed watchdog on the next call — a second stalled conn must cancel too. +func TestUploadStallWatchdogRearmsForNextUpload(t *testing.T) { + defer func(orig time.Duration) { uploadStallMinWait = orig }(uploadStallMinWait) + uploadStallMinWait = 40 * time.Millisecond + + client := &stallingHTTPClient{} + uploader := newUploader(client) + markHostFast(uploader, "test-host:8080") + + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + // IsInputCompressed keeps the body large enough to re-prove the host: + // compressed near-zeros would send too few bytes to re-mark it. + option := &UploadOption{UploadUrl: "http://test-host:8080/1,deadbeef", MaxAttempts: 2, IsInputCompressed: true} + for i := 0; i < 2; i++ { + client.mu.Lock() + client.stalls = 1 + client.mu.Unlock() + result, err := uploader.UploadData(ctx, make([]byte, 300*1024), option) + if err != nil { + t.Fatalf("upload %d should have retried on a fresh conn: %v", i, err) + } + if result.RetryCount != 1 { + t.Fatalf("upload %d got retries=%d, want its own stall caught", i, result.RetryCount) + } + } + if option.stallWatchdogDisarmed { + t.Fatalf("a stall must not disarm the watchdog on the caller's option") + } +} + +// A real transport must drop the conn a cancelled request was riding: the +// retry has to dial again rather than land back on the parked conn. The first +// conn here wedges mid-reply — headers flush, then the body never arrives. +func TestUploadStallWatchdogDialsFreshConn(t *testing.T) { + defer func(orig time.Duration) { uploadStallMinWait = orig }(uploadStallMinWait) + uploadStallMinWait = 50 * time.Millisecond + + var requests, dials int32 + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + _, _ = io.Copy(io.Discard, r.Body) + if atomic.AddInt32(&requests, 1) == 1 { + w.WriteHeader(http.StatusCreated) + w.(http.Flusher).Flush() + <-r.Context().Done() + return + } + w.WriteHeader(http.StatusCreated) + _, _ = w.Write([]byte(`{"size":1}`)) + })) + defer srv.Close() + + tr := &http.Transport{ + DialContext: func(ctx context.Context, network, addr string) (net.Conn, error) { + atomic.AddInt32(&dials, 1) + return new(net.Dialer).DialContext(ctx, network, addr) + }, + } + defer tr.CloseIdleConnections() + uploader := newUploader(&http.Client{Transport: tr}) + + host := strings.TrimPrefix(srv.URL, "http://") + markHostFast(uploader, host) + + result, err := uploader.UploadData(context.Background(), make([]byte, 300*1024), &UploadOption{ + UploadUrl: srv.URL + "/1,deadbeef", + MaxAttempts: 2, + }) + if err != nil { + t.Fatalf("upload should have retried on a fresh conn: %v", err) + } + if result.RetryCount != 1 || atomic.LoadInt32(&dials) != 2 { + t.Fatalf("got retries=%d dials=%d, want one retry on a fresh conn", result.RetryCount, dials) + } +} + +// Once the body drains the wait is the server's commit: a write that takes +// longer than the stall floor to acknowledge must not be cancelled — the conn +// is healthy, the server is just busy. +func TestUploadStallWatchdogAllowsSlowCommit(t *testing.T) { + defer func(orig time.Duration) { uploadStallMinWait = orig }(uploadStallMinWait) + uploadStallMinWait = 30 * time.Millisecond + + client := &stallingHTTPClient{slowFor: 80 * time.Millisecond} + uploader := newUploader(client) + markHostFast(uploader, "test-host:8080") + + result, err := uploader.UploadData(context.Background(), make([]byte, 300*1024), &UploadOption{ + UploadUrl: "http://test-host:8080/1,deadbeef", + MaxAttempts: 1, + }) + if err != nil || result.RetryCount != 0 { + t.Fatalf("a slow commit must not be cancelled: result=%+v err=%v", result, err) + } +} + +// A fast rejection proves nothing about how the conn drains, so it must not +// mark the host fast and arm the watchdog for the next upload. +func TestUploadStallWatchdogIgnoresFailedWrites(t *testing.T) { + uploader := newUploader(&rejectHTTPClient{}) + _, err := uploader.UploadData(context.Background(), make([]byte, 300*1024), &UploadOption{ + UploadUrl: "http://test-host:8080/1,deadbeef", + MaxAttempts: 1, + }) + if err == nil { + t.Fatalf("rejected upload should fail") + } + if _, ok := uploader.fastUploadHosts.Load("test-host:8080"); ok { + t.Fatalf("a failed upload must not mark the host fast") + } +} + +type rejectHTTPClient struct{} + +func (c *rejectHTTPClient) Do(req *http.Request) (*http.Response, error) { + if req.Body != nil { + _, _ = io.Copy(io.Discard, req.Body) + } + return &http.Response{ + StatusCode: http.StatusInternalServerError, + Header: make(http.Header), + Body: io.NopCloser(strings.NewReader(`{"error":"volume is read only"}`)), + }, nil +} + +// Without a recent fast upload the watchdog stays off: a chronically slow link +// is the link, not a conn, and must not get its uploads cancelled. +func TestUploadStallWatchdogKeepsUnprovenHost(t *testing.T) { + defer func(orig time.Duration) { uploadStallMinWait = orig }(uploadStallMinWait) + uploadStallMinWait = 20 * time.Millisecond + + client := &stallingHTTPClient{slowFor: 60 * time.Millisecond} + uploader := newUploader(client) + + result, err := uploader.UploadData(context.Background(), make([]byte, 300*1024), &UploadOption{ + UploadUrl: "http://slow-host:8080/1,deadbeef", + MaxAttempts: 1, + }) + if err != nil { + t.Fatalf("unproven host upload should not be cancelled: %v", err) + } + if result.RetryCount != 0 || len(client.requestCtxs) != 1 { + t.Fatalf("got retries=%d calls=%d, want a single unbounded attempt", result.RetryCount, len(client.requestCtxs)) + } +} + +// Any sizeable completed upload marks its host at the rate it proved — slow +// or fast — because the stall budget scales off that rate. Only a too-small +// body fails to measure. +func TestUploadObservedBps(t *testing.T) { + if bps := uploadObservedBps(4<<20, 4*time.Second); bps != 1<<20 { + t.Fatalf("4MiB in 4s observed %d B/s, want 1MiB/s", bps) + } + if bps := uploadObservedBps(4<<20, 3*time.Hour); bps <= 0 { + t.Fatalf("a slow upload still proves the rate it moved at") + } + if bps := uploadObservedBps(64*1024, time.Millisecond); bps != 0 { + t.Fatalf("a small upload says nothing about host speed") + } + if bps := uploadObservedBps(4<<20, 0); bps != 0 { + t.Fatalf("a zero-duration sample is not a measurement") + } +} + +// The stall budget scales with the rate the host last proved, so the same +// body gets little slack on a fast link and plenty on a slow one — no fixed +// floor fits both. +func TestUploadStallTimeoutScalesWithMeasuredRate(t *testing.T) { + // 8MiB/s observed → a 64MiB body gets 8x the time it needs at that rate. + if d := uploadStallTimeout(64<<20, 8<<20); d != 64*time.Second { + t.Fatalf("stall budget = %v, want 64s", d) + } + // The same body on a 64MiB/s host gets 8s — under the floor, so clamped. + if d := uploadStallTimeout(64<<20, 64<<20); d != uploadStallMinWait { + t.Fatalf("stall budget = %v, want the min wait %v", d, uploadStallMinWait) + } + // A tiny body on a fast host cannot race its own conn. + if d := uploadStallTimeout(1, 1<<30); d != uploadStallMinWait { + t.Fatalf("stall budget = %v, want the min wait %v", d, uploadStallMinWait) + } +} + type fsyncAssignClient struct { filer_pb.SeaweedFilerClient response *filer_pb.AssignVolumeResponse diff --git a/weed/pb/filer.proto b/weed/pb/filer.proto index c693ab93eb..877617b1be 100644 --- a/weed/pb/filer.proto +++ b/weed/pb/filer.proto @@ -786,7 +786,8 @@ message FilerConf { string replication = 3; string ttl = 4; string disk_type = 5; - bool fsync = 6; + // unset inherits from the enclosing path rule, set overrides it + optional bool fsync = 6; uint32 volume_growth_count = 7; bool read_only = 8; string data_center = 9; diff --git a/weed/pb/filer_pb/filer.pb.go b/weed/pb/filer_pb/filer.pb.go index abe1cfef29..b3ab3697e6 100644 --- a/weed/pb/filer_pb/filer.pb.go +++ b/weed/pb/filer_pb/filer.pb.go @@ -1,7 +1,7 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: // protoc-gen-go v1.36.6 -// protoc v3.21.12 +// protoc v7.35.0 // source: filer.proto package filer_pb @@ -7102,20 +7102,21 @@ func (x *LocateBrokerResponse_Resource) GetResourceCount() int32 { } type FilerConf_PathConf struct { - state protoimpl.MessageState `protogen:"open.v1"` - LocationPrefix string `protobuf:"bytes,1,opt,name=location_prefix,json=locationPrefix,proto3" json:"location_prefix,omitempty"` - Collection string `protobuf:"bytes,2,opt,name=collection,proto3" json:"collection,omitempty"` - Replication string `protobuf:"bytes,3,opt,name=replication,proto3" json:"replication,omitempty"` - Ttl string `protobuf:"bytes,4,opt,name=ttl,proto3" json:"ttl,omitempty"` - DiskType string `protobuf:"bytes,5,opt,name=disk_type,json=diskType,proto3" json:"disk_type,omitempty"` - Fsync bool `protobuf:"varint,6,opt,name=fsync,proto3" json:"fsync,omitempty"` - VolumeGrowthCount uint32 `protobuf:"varint,7,opt,name=volume_growth_count,json=volumeGrowthCount,proto3" json:"volume_growth_count,omitempty"` - ReadOnly bool `protobuf:"varint,8,opt,name=read_only,json=readOnly,proto3" json:"read_only,omitempty"` - DataCenter string `protobuf:"bytes,9,opt,name=data_center,json=dataCenter,proto3" json:"data_center,omitempty"` - Rack string `protobuf:"bytes,10,opt,name=rack,proto3" json:"rack,omitempty"` - DataNode string `protobuf:"bytes,11,opt,name=data_node,json=dataNode,proto3" json:"data_node,omitempty"` - MaxFileNameLength uint32 `protobuf:"varint,12,opt,name=max_file_name_length,json=maxFileNameLength,proto3" json:"max_file_name_length,omitempty"` - DisableChunkDeletion bool `protobuf:"varint,13,opt,name=disable_chunk_deletion,json=disableChunkDeletion,proto3" json:"disable_chunk_deletion,omitempty"` + state protoimpl.MessageState `protogen:"open.v1"` + LocationPrefix string `protobuf:"bytes,1,opt,name=location_prefix,json=locationPrefix,proto3" json:"location_prefix,omitempty"` + Collection string `protobuf:"bytes,2,opt,name=collection,proto3" json:"collection,omitempty"` + Replication string `protobuf:"bytes,3,opt,name=replication,proto3" json:"replication,omitempty"` + Ttl string `protobuf:"bytes,4,opt,name=ttl,proto3" json:"ttl,omitempty"` + DiskType string `protobuf:"bytes,5,opt,name=disk_type,json=diskType,proto3" json:"disk_type,omitempty"` + // unset inherits from the enclosing path rule, set overrides it + Fsync *bool `protobuf:"varint,6,opt,name=fsync,proto3,oneof" json:"fsync,omitempty"` + VolumeGrowthCount uint32 `protobuf:"varint,7,opt,name=volume_growth_count,json=volumeGrowthCount,proto3" json:"volume_growth_count,omitempty"` + ReadOnly bool `protobuf:"varint,8,opt,name=read_only,json=readOnly,proto3" json:"read_only,omitempty"` + DataCenter string `protobuf:"bytes,9,opt,name=data_center,json=dataCenter,proto3" json:"data_center,omitempty"` + Rack string `protobuf:"bytes,10,opt,name=rack,proto3" json:"rack,omitempty"` + DataNode string `protobuf:"bytes,11,opt,name=data_node,json=dataNode,proto3" json:"data_node,omitempty"` + MaxFileNameLength uint32 `protobuf:"varint,12,opt,name=max_file_name_length,json=maxFileNameLength,proto3" json:"max_file_name_length,omitempty"` + DisableChunkDeletion bool `protobuf:"varint,13,opt,name=disable_chunk_deletion,json=disableChunkDeletion,proto3" json:"disable_chunk_deletion,omitempty"` // unset inherits from the enclosing path rule, set overrides it Worm *bool `protobuf:"varint,14,opt,name=worm,proto3,oneof" json:"worm,omitempty"` WormGracePeriodSeconds uint64 `protobuf:"varint,15,opt,name=worm_grace_period_seconds,json=wormGracePeriodSeconds,proto3" json:"worm_grace_period_seconds,omitempty"` @@ -7191,8 +7192,8 @@ func (x *FilerConf_PathConf) GetDiskType() string { } func (x *FilerConf_PathConf) GetFsync() bool { - if x != nil { - return x.Fsync + if x != nil && x.Fsync != nil { + return *x.Fsync } return false } @@ -7772,10 +7773,10 @@ const file_filer_proto_rawDesc = "" + "\x03key\x18\x01 \x01(\fR\x03key\x12\x14\n" + "\x05value\x18\x02 \x01(\fR\x05value\"%\n" + "\rKvPutResponse\x12\x14\n" + - "\x05error\x18\x01 \x01(\tR\x05error\"\xe1\x05\n" + + "\x05error\x18\x01 \x01(\tR\x05error\"\xf0\x05\n" + "\tFilerConf\x12\x18\n" + "\aversion\x18\x01 \x01(\x05R\aversion\x12:\n" + - "\tlocations\x18\x02 \x03(\v2\x1c.filer_pb.FilerConf.PathConfR\tlocations\x1a\xfd\x04\n" + + "\tlocations\x18\x02 \x03(\v2\x1c.filer_pb.FilerConf.PathConfR\tlocations\x1a\x8c\x05\n" + "\bPathConf\x12'\n" + "\x0flocation_prefix\x18\x01 \x01(\tR\x0elocationPrefix\x12\x1e\n" + "\n" + @@ -7783,8 +7784,8 @@ const file_filer_proto_rawDesc = "" + "collection\x12 \n" + "\vreplication\x18\x03 \x01(\tR\vreplication\x12\x10\n" + "\x03ttl\x18\x04 \x01(\tR\x03ttl\x12\x1b\n" + - "\tdisk_type\x18\x05 \x01(\tR\bdiskType\x12\x14\n" + - "\x05fsync\x18\x06 \x01(\bR\x05fsync\x12.\n" + + "\tdisk_type\x18\x05 \x01(\tR\bdiskType\x12\x19\n" + + "\x05fsync\x18\x06 \x01(\bH\x00R\x05fsync\x88\x01\x01\x12.\n" + "\x13volume_growth_count\x18\a \x01(\rR\x11volumeGrowthCount\x12\x1b\n" + "\tread_only\x18\b \x01(\bR\breadOnly\x12\x1f\n" + "\vdata_center\x18\t \x01(\tR\n" + @@ -7794,11 +7795,12 @@ const file_filer_proto_rawDesc = "" + "\tdata_node\x18\v \x01(\tR\bdataNode\x12/\n" + "\x14max_file_name_length\x18\f \x01(\rR\x11maxFileNameLength\x124\n" + "\x16disable_chunk_deletion\x18\r \x01(\bR\x14disableChunkDeletion\x12\x17\n" + - "\x04worm\x18\x0e \x01(\bH\x00R\x04worm\x88\x01\x01\x129\n" + + "\x04worm\x18\x0e \x01(\bH\x01R\x04worm\x88\x01\x01\x129\n" + "\x19worm_grace_period_seconds\x18\x0f \x01(\x04R\x16wormGracePeriodSeconds\x12=\n" + "\x1bworm_retention_time_seconds\x18\x10 \x01(\x04R\x18wormRetentionTimeSeconds\x12\x1f\n" + "\vinode_index\x18\x11 \x01(\bR\n" + - "inodeIndexB\a\n" + + "inodeIndexB\b\n" + + "\x06_fsyncB\a\n" + "\x05_worm\"\xba\x01\n" + "&CacheRemoteObjectToLocalClusterRequest\x12\x1c\n" + "\tdirectory\x18\x01 \x01(\tR\tdirectory\x12\x12\n" + diff --git a/weed/pb/filer_pb/filer_client_bfs.go b/weed/pb/filer_pb/filer_client_bfs.go index e4ec78d675..05c2a4387b 100644 --- a/weed/pb/filer_pb/filer_client_bfs.go +++ b/weed/pb/filer_pb/filer_client_bfs.go @@ -115,9 +115,9 @@ func processOneDirectory(ctx context.Context, filerClient FilerClient, parentPat } -func StreamBfs(client SeaweedFilerClient, dir util.FullPath, olderThanTsNs int64, fn func(parentPath util.FullPath, entry *Entry) error) (err error) { +func StreamBfs(ctx context.Context, client SeaweedFilerClient, dir util.FullPath, olderThanTsNs int64, fn func(parentPath util.FullPath, entry *Entry, entryTsNs int64) error) (err error) { glog.V(0).Infof("TraverseBfsMetadata %v if before %v", dir, time.Unix(0, olderThanTsNs)) - ctx, cancel := context.WithCancel(context.Background()) + ctx, cancel := context.WithCancel(ctx) defer cancel() stream, err := client.TraverseBfsMetadata(ctx, &TraverseBfsMetadataRequest{ Directory: string(dir), @@ -133,7 +133,7 @@ func StreamBfs(client SeaweedFilerClient, dir util.FullPath, olderThanTsNs int64 } return fmt.Errorf("traverse bfs metadata: %w", err) } - if err := fn(util.FullPath(resp.Directory), resp.Entry); err != nil { + if err := fn(util.FullPath(resp.Directory), resp.Entry, resp.EntryTsNs); err != nil { return err } } diff --git a/weed/pb/filer_pb/filer_grpc.pb.go b/weed/pb/filer_pb/filer_grpc.pb.go index 8a330ce286..6a2b494a39 100644 --- a/weed/pb/filer_pb/filer_grpc.pb.go +++ b/weed/pb/filer_pb/filer_grpc.pb.go @@ -1,7 +1,7 @@ // Code generated by protoc-gen-go-grpc. DO NOT EDIT. // versions: // - protoc-gen-go-grpc v1.6.2 -// - protoc v3.21.12 +// - protoc v7.35.0 // source: filer.proto package filer_pb diff --git a/weed/pb/filer_pb/filer_vtproto.pb.go b/weed/pb/filer_pb/filer_vtproto.pb.go index cac2d01f9a..44f60f9888 100644 --- a/weed/pb/filer_pb/filer_vtproto.pb.go +++ b/weed/pb/filer_pb/filer_vtproto.pb.go @@ -2659,6 +2659,16 @@ func (m *AtomicRenameEntryRequest) MarshalToSizedBufferVT(dAtA []byte) (int, err i -= len(m.unknownFields) copy(dAtA[i:], m.unknownFields) } + if m.FailIfTargetExists { + i-- + if m.FailIfTargetExists { + dAtA[i] = 1 + } else { + dAtA[i] = 0 + } + i-- + dAtA[i] = 0x30 + } if len(m.Signatures) > 0 { var pksize2 int for _, num := range m.Signatures { @@ -4911,6 +4921,18 @@ func (m *FilerConf_PathConf) MarshalToSizedBufferVT(dAtA []byte) (int, error) { i -= len(m.unknownFields) copy(dAtA[i:], m.unknownFields) } + if m.InodeIndex { + i-- + if m.InodeIndex { + dAtA[i] = 1 + } else { + dAtA[i] = 0 + } + i-- + dAtA[i] = 0x1 + i-- + dAtA[i] = 0x88 + } if m.WormRetentionTimeSeconds != 0 { i = protohelpers.EncodeVarint(dAtA, i, uint64(m.WormRetentionTimeSeconds)) i-- @@ -4984,9 +5006,9 @@ func (m *FilerConf_PathConf) MarshalToSizedBufferVT(dAtA []byte) (int, error) { i-- dAtA[i] = 0x38 } - if m.Fsync { + if m.Fsync != nil { i-- - if m.Fsync { + if *m.Fsync { dAtA[i] = 1 } else { dAtA[i] = 0 @@ -7599,6 +7621,9 @@ func (m *AtomicRenameEntryRequest) SizeVT() (n int) { } n += 1 + protohelpers.SizeOfVarint(uint64(l)) + l } + if m.FailIfTargetExists { + n += 2 + } n += len(m.unknownFields) return n } @@ -8501,7 +8526,7 @@ func (m *FilerConf_PathConf) SizeVT() (n int) { if l > 0 { n += 1 + l + protohelpers.SizeOfVarint(uint64(l)) } - if m.Fsync { + if m.Fsync != nil { n += 2 } if m.VolumeGrowthCount != 0 { @@ -8537,6 +8562,9 @@ func (m *FilerConf_PathConf) SizeVT() (n int) { if m.WormRetentionTimeSeconds != 0 { n += 2 + protohelpers.SizeOfVarint(uint64(m.WormRetentionTimeSeconds)) } + if m.InodeIndex { + n += 3 + } n += len(m.unknownFields) return n } @@ -16531,6 +16559,26 @@ func (m *AtomicRenameEntryRequest) UnmarshalVT(dAtA []byte) error { } else { return fmt.Errorf("proto: wrong wireType = %d for field Signatures", wireType) } + case 6: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field FailIfTargetExists", wireType) + } + var v int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return protohelpers.ErrIntOverflow + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + v |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + m.FailIfTargetExists = bool(v != 0) default: iNdEx = preIndex skippy, err := protohelpers.Skip(dAtA[iNdEx:]) @@ -22263,7 +22311,8 @@ func (m *FilerConf_PathConf) UnmarshalVT(dAtA []byte) error { break } } - m.Fsync = bool(v != 0) + b := bool(v != 0) + m.Fsync = &b case 7: if wireType != 0 { return fmt.Errorf("proto: wrong wireType = %d for field VolumeGrowthCount", wireType) @@ -22497,6 +22546,26 @@ func (m *FilerConf_PathConf) UnmarshalVT(dAtA []byte) error { break } } + case 17: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field InodeIndex", wireType) + } + var v int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return protohelpers.ErrIntOverflow + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + v |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + m.InodeIndex = bool(v != 0) default: iNdEx = preIndex skippy, err := protohelpers.Skip(dAtA[iNdEx:]) diff --git a/weed/pb/grpc_client_server.go b/weed/pb/grpc_client_server.go index 8ba3a16f33..2a119c075e 100644 --- a/weed/pb/grpc_client_server.go +++ b/weed/pb/grpc_client_server.go @@ -140,6 +140,22 @@ func resolveLocalGrpcSocket(address string) string { return localGrpcSockets[port] } +// ListenOnLocalSocket binds a unix socket listener for a local fast path +// next to the TCP listener. A stale socket file is removed first; a path +// that cannot be cleared or bound is skipped, since the socket is optional. +func ListenOnLocalSocket(socketPath string) net.Listener { + if err := os.Remove(socketPath); err != nil && !os.IsNotExist(err) { + glog.Warningf("Failed to remove old socket %s: %v", socketPath, err) + } + lc := net.ListenConfig{Control: setLocalSocketBuffers} + listener, err := lc.Listen(context.Background(), "unix", socketPath) + if err != nil { + glog.Errorf("Failed to listen on Unix socket %s: %v", socketPath, err) + return nil + } + return listener +} + // ServeGrpcOnLocalSocket starts serving a gRPC server on a Unix socket // if one is registered for the given port. func ServeGrpcOnLocalSocket(grpcServer *grpc.Server, grpcPort int) { @@ -147,13 +163,8 @@ func ServeGrpcOnLocalSocket(grpcServer *grpc.Server, grpcPort int) { if socketPath == "" { return } - if err := os.Remove(socketPath); err != nil && !os.IsNotExist(err) { - glog.Warningf("Failed to remove old gRPC socket %s: %v", socketPath, err) - } - lc := net.ListenConfig{Control: setLocalSocketBuffers} - listener, err := lc.Listen(context.Background(), "unix", socketPath) - if err != nil { - glog.Errorf("Failed to listen on gRPC Unix socket %s: %v", socketPath, err) + listener := ListenOnLocalSocket(socketPath) + if listener == nil { return } listener = &localSocketListener{Listener: listener} diff --git a/weed/pb/worker.proto b/weed/pb/worker.proto index fa06886d09..822f89269c 100644 --- a/weed/pb/worker.proto +++ b/weed/pb/worker.proto @@ -381,6 +381,7 @@ message ErasureCodingTaskConfig { repeated string preferred_tags = 5; // Disk tags to prioritize for EC shard placement string replica_placement = 6; // EC shard replica placement (e.g. "020"); empty falls back to master default replication bool strict_placement = 7; // fail planning instead of relaxing placement constraints + int32 free_slots_deadband = 8; // Place free-volume-slot deadband; |ΔfreeSlots| ≤ M×dataShards ranks by liveShards; 0 = strict } // BalanceTaskConfig contains balance-specific configuration diff --git a/weed/pb/worker_pb/worker.pb.go b/weed/pb/worker_pb/worker.pb.go index a516babcce..e2a81e3f5f 100644 --- a/weed/pb/worker_pb/worker.pb.go +++ b/weed/pb/worker_pb/worker.pb.go @@ -2973,16 +2973,17 @@ func (x *VacuumTaskConfig) GetMinIntervalSeconds() int32 { // ErasureCodingTaskConfig contains EC-specific configuration type ErasureCodingTaskConfig struct { - state protoimpl.MessageState `protogen:"open.v1"` - FullnessRatio float64 `protobuf:"fixed64,1,opt,name=fullness_ratio,json=fullnessRatio,proto3" json:"fullness_ratio,omitempty"` // Minimum fullness ratio to trigger EC (0.0-1.0) - QuietForSeconds int32 `protobuf:"varint,2,opt,name=quiet_for_seconds,json=quietForSeconds,proto3" json:"quiet_for_seconds,omitempty"` // Minimum quiet time before EC - MinVolumeSizeMb int32 `protobuf:"varint,3,opt,name=min_volume_size_mb,json=minVolumeSizeMb,proto3" json:"min_volume_size_mb,omitempty"` // Minimum volume size for EC - CollectionFilter string `protobuf:"bytes,4,opt,name=collection_filter,json=collectionFilter,proto3" json:"collection_filter,omitempty"` // Only process volumes from specific collections - PreferredTags []string `protobuf:"bytes,5,rep,name=preferred_tags,json=preferredTags,proto3" json:"preferred_tags,omitempty"` // Disk tags to prioritize for EC shard placement - ReplicaPlacement string `protobuf:"bytes,6,opt,name=replica_placement,json=replicaPlacement,proto3" json:"replica_placement,omitempty"` // EC shard replica placement (e.g. "020"); empty falls back to master default replication - StrictPlacement bool `protobuf:"varint,7,opt,name=strict_placement,json=strictPlacement,proto3" json:"strict_placement,omitempty"` // fail planning instead of relaxing placement constraints - unknownFields protoimpl.UnknownFields - sizeCache protoimpl.SizeCache + state protoimpl.MessageState `protogen:"open.v1"` + FullnessRatio float64 `protobuf:"fixed64,1,opt,name=fullness_ratio,json=fullnessRatio,proto3" json:"fullness_ratio,omitempty"` // Minimum fullness ratio to trigger EC (0.0-1.0) + QuietForSeconds int32 `protobuf:"varint,2,opt,name=quiet_for_seconds,json=quietForSeconds,proto3" json:"quiet_for_seconds,omitempty"` // Minimum quiet time before EC + MinVolumeSizeMb int32 `protobuf:"varint,3,opt,name=min_volume_size_mb,json=minVolumeSizeMb,proto3" json:"min_volume_size_mb,omitempty"` // Minimum volume size for EC + CollectionFilter string `protobuf:"bytes,4,opt,name=collection_filter,json=collectionFilter,proto3" json:"collection_filter,omitempty"` // Only process volumes from specific collections + PreferredTags []string `protobuf:"bytes,5,rep,name=preferred_tags,json=preferredTags,proto3" json:"preferred_tags,omitempty"` // Disk tags to prioritize for EC shard placement + ReplicaPlacement string `protobuf:"bytes,6,opt,name=replica_placement,json=replicaPlacement,proto3" json:"replica_placement,omitempty"` // EC shard replica placement (e.g. "020"); empty falls back to master default replication + StrictPlacement bool `protobuf:"varint,7,opt,name=strict_placement,json=strictPlacement,proto3" json:"strict_placement,omitempty"` // fail planning instead of relaxing placement constraints + FreeSlotsDeadband int32 `protobuf:"varint,8,opt,name=free_slots_deadband,json=freeSlotsDeadband,proto3" json:"free_slots_deadband,omitempty"` // Place free-volume-slot deadband; |ΔfreeSlots| ≤ M×dataShards ranks by liveShards; 0 = strict + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache } func (x *ErasureCodingTaskConfig) Reset() { @@ -3064,6 +3065,13 @@ func (x *ErasureCodingTaskConfig) GetStrictPlacement() bool { return false } +func (x *ErasureCodingTaskConfig) GetFreeSlotsDeadband() int32 { + if x != nil { + return x.FreeSlotsDeadband + } + return 0 +} + // BalanceTaskConfig contains balance-specific configuration type BalanceTaskConfig struct { state protoimpl.MessageState `protogen:"open.v1"` @@ -4293,7 +4301,7 @@ const file_worker_proto_rawDesc = "" + "\x10VacuumTaskConfig\x12+\n" + "\x11garbage_threshold\x18\x01 \x01(\x01R\x10garbageThreshold\x12/\n" + "\x14min_volume_age_hours\x18\x02 \x01(\x05R\x11minVolumeAgeHours\x120\n" + - "\x14min_interval_seconds\x18\x03 \x01(\x05R\x12minIntervalSeconds\"\xc5\x02\n" + + "\x14min_interval_seconds\x18\x03 \x01(\x05R\x12minIntervalSeconds\"\xf5\x02\n" + "\x17ErasureCodingTaskConfig\x12%\n" + "\x0efullness_ratio\x18\x01 \x01(\x01R\rfullnessRatio\x12*\n" + "\x11quiet_for_seconds\x18\x02 \x01(\x05R\x0fquietForSeconds\x12+\n" + @@ -4301,7 +4309,8 @@ const file_worker_proto_rawDesc = "" + "\x11collection_filter\x18\x04 \x01(\tR\x10collectionFilter\x12%\n" + "\x0epreferred_tags\x18\x05 \x03(\tR\rpreferredTags\x12+\n" + "\x11replica_placement\x18\x06 \x01(\tR\x10replicaPlacement\x12)\n" + - "\x10strict_placement\x18\a \x01(\bR\x0fstrictPlacement\"\x9b\x01\n" + + "\x10strict_placement\x18\a \x01(\bR\x0fstrictPlacement\x12.\n" + + "\x13free_slots_deadband\x18\b \x01(\x05R\x11freeSlotsDeadband\"\x9b\x01\n" + "\x11BalanceTaskConfig\x12/\n" + "\x13imbalance_threshold\x18\x01 \x01(\x01R\x12imbalanceThreshold\x12(\n" + "\x10min_server_count\x18\x02 \x01(\x05R\x0eminServerCount\x12+\n" + diff --git a/weed/s3api/auth_credentials.go b/weed/s3api/auth_credentials.go index 46ff18366b..502283a944 100644 --- a/weed/s3api/auth_credentials.go +++ b/weed/s3api/auth_credentials.go @@ -3183,6 +3183,20 @@ func (iam *IdentityAccessManagement) AuthorizeObjectWrite(r *http.Request, ident return iam.authorizeObjectKeyAction(r, identity, http.MethodPut, s3_constants.ACTION_WRITE, bucket, objectKey, "") } +// AuthorizeObjectAclWrite authorizes setting an ACL on one key the request URL +// does not name: a POST Object upload promotes its acl form field and any +// x-amz-grant-* fields onto the upload's ACL headers, which the PUT path only +// applies after an s3:PutObjectAcl grant. +func (iam *IdentityAccessManagement) AuthorizeObjectAclWrite(r *http.Request, identity *Identity, bucket, objectKey string) s3err.ErrorCode { + if !iam.isEnabled() { + return s3err.ErrNone + } + if bucket == "" || objectKey == "" { + return s3err.ErrNone + } + return iam.authorizeObjectKeyAction(r, identity, http.MethodPut, s3_constants.ACTION_WRITE_ACP, bucket, objectKey, "") +} + // authorizeWithIAM authorizes requests using the IAM integration policy engine func (iam *IdentityAccessManagement) authorizeWithIAM(r *http.Request, identity *Identity, action Action, bucket string, object string) s3err.ErrorCode { ctx := r.Context() diff --git a/weed/s3api/auth_signature_v2.go b/weed/s3api/auth_signature_v2.go index 3bda791e64..5148be2218 100644 --- a/weed/s3api/auth_signature_v2.go +++ b/weed/s3api/auth_signature_v2.go @@ -110,6 +110,7 @@ func (iam *IdentityAccessManagement) doesPolicySignatureV2Match(ctx context.Cont return nil, s3err.ErrSignatureDoesNotMatch } s3_constants.RecordAccessKeyId(ctx, cred.AccessKey) + s3_constants.RecordStoredAccessKeyId(ctx, cred.AccessKey, cred.SecretKey) return identity, s3err.ErrNone } @@ -168,6 +169,7 @@ func (iam *IdentityAccessManagement) doesSignV2Match(r *http.Request) (*Identity return nil, s3err.ErrSignatureDoesNotMatch } s3_constants.RecordAccessKeyId(r.Context(), cred.AccessKey) + s3_constants.RecordStoredAccessKeyId(r.Context(), cred.AccessKey, cred.SecretKey) return identity, s3err.ErrNone } @@ -227,6 +229,7 @@ func (iam *IdentityAccessManagement) doesPresignV2SignatureMatch(r *http.Request return nil, s3err.ErrSignatureDoesNotMatch } s3_constants.RecordAccessKeyId(r.Context(), cred.AccessKey) + s3_constants.RecordStoredAccessKeyId(r.Context(), cred.AccessKey, cred.SecretKey) return identity, s3err.ErrNone } diff --git a/weed/s3api/auth_signature_v4.go b/weed/s3api/auth_signature_v4.go index b0ec7fd292..5f96669554 100644 --- a/weed/s3api/auth_signature_v4.go +++ b/weed/s3api/auth_signature_v4.go @@ -412,6 +412,9 @@ func (iam *IdentityAccessManagement) verifyV4Signature(r *http.Request, shouldCh // catalog's vended sessions name it (never a key re-read from the request). // Recorded before the retarget check so that refusal names it too. s3_constants.RecordAccessKeyId(r.Context(), cred.AccessKey) + if found { + s3_constants.RecordStoredAccessKeyId(r.Context(), cred.AccessKey, cred.SecretKey) + } if signedBucket, ok := bucketFromVirtualHost(matchedHost, iam.domain); ok { if routedBucket, _ := s3_constants.GetBucketAndObject(r); routedBucket != "" && routedBucket != signedBucket { glog.V(2).Infof("reject %s %s: signed host %q implies bucket %q but routed to %q", @@ -496,6 +499,19 @@ func (iam *IdentityAccessManagement) validateSTSSessionToken(r *http.Request, se return nil, nil, s3err.ErrExpiredToken } + // A session the table catalog vended for a static access key is bound to + // that key, as a role session is bound to its role's ID: deleting or + // deactivating the key, or deleting or disabling its identity, ends the + // session at its next request instead of at its expiry. Sessions minted + // before the binding existed carry only the audit field and stay unbound. + if sessionInfo.IdentityProvider == icebergVendedCredentialsProvider && sessionInfo.BoundAccessKeyId != "" { + if errCode := iam.vendedSessionKeyLive(sessionInfo.ExternalUserId, sessionInfo.BoundAccessKeyId, sessionInfo.BoundAccessKeyFingerprint); errCode != s3err.ErrNone { + glog.V(2).Infof("vended STS session %s: identity %q or the access key it was vended for is gone, disabled or inactive", + sessionInfo.SessionId, sessionInfo.ExternalUserId) + return nil, nil, errCode + } + } + // Validate required credential fields if sessionInfo.Credentials.SecretAccessKey == "" { glog.Warningf("STS service returned empty SecretAccessKey in credentials") @@ -554,6 +570,38 @@ func (iam *IdentityAccessManagement) validateSTSSessionToken(r *http.Request, se return identity, cred, s3err.ErrNone } +// vendedSessionKeyLive answers ErrNone while owner is a known, enabled +// identity holding accessKey as an active, unexpired credential, and +// ErrInvalidAccessKeyID (as for an unknown key) otherwise. A fingerprint set +// at mint time must still match, so a key re-created under the same id does +// not revive sessions the deleted original had carried. One map lookup under +// the read lock. +func (iam *IdentityAccessManagement) vendedSessionKeyLive(owner, accessKey, fingerprint string) s3err.ErrorCode { + if owner == "" || accessKey == "" { + return s3err.ErrInvalidAccessKeyID + } + fingerprintKey := iam.stsSigningKey() + iam.m.RLock() + defer iam.m.RUnlock() + identity, ok := iam.nameToIdentity[owner] + if !ok || identity == nil || identity.Disabled { + return s3err.ErrInvalidAccessKeyID + } + for _, credential := range identity.Credentials { + if credential == nil || credential.AccessKey != accessKey { + continue + } + if credential.Status == iamAccessKeyStatusInactive || credential.isCredentialExpired() { + return s3err.ErrInvalidAccessKeyID + } + if fingerprint != "" && s3_constants.StoredCredentialFingerprint(fingerprintKey, accessKey, credential.SecretKey) != fingerprint { + return s3err.ErrInvalidAccessKeyID + } + return s3err.ErrNone + } + return s3err.ErrInvalidAccessKeyID +} + // calculateAndVerifySignature contains the core logic for creating the canonical request, // string-to-sign, and comparing the final signature. func calculateAndVerifySignature(secretKey, method, urlPath, queryStr string, extractedSignedHeaders http.Header, authInfo *v4AuthInfo) (string, s3err.ErrorCode) { @@ -874,6 +922,7 @@ func (iam *IdentityAccessManagement) doesPolicySignatureV4Match(ctx context.Cont return nil, s3err.ErrSignatureDoesNotMatch } s3_constants.RecordAccessKeyId(ctx, cred.AccessKey) + s3_constants.RecordStoredAccessKeyId(ctx, cred.AccessKey, cred.SecretKey) return identity, s3err.ErrNone } diff --git a/weed/s3api/filer_multipart.go b/weed/s3api/filer_multipart.go index 6f22429fb6..299c9899cd 100644 --- a/weed/s3api/filer_multipart.go +++ b/weed/s3api/filer_multipart.go @@ -1400,7 +1400,6 @@ func (p *multipartUploadPage) result() *ListMultipartUploadsResult { output := &ListMultipartUploadsResult{ Bucket: input.Bucket, Delimiter: input.Delimiter, - EncodingType: input.EncodingType, KeyMarker: input.KeyMarker, MaxUploads: aws.Int64(int64(p.limit)), Prefix: input.Prefix, @@ -1421,13 +1420,47 @@ func (p *multipartUploadPage) result() *ListMultipartUploadsResult { output.Upload = append(output.Upload, upload) } if p.more { - last := output.Upload[len(output.Upload)-1] + last := candidates[len(candidates)-1] output.IsTruncated = aws.Bool(true) - output.NextKeyMarker, output.NextUploadIdMarker = last.Key, last.UploadId + output.NextKeyMarker, output.NextUploadIdMarker = aws.String(last.key), aws.String(last.entry.Name) + } + if aws.StringValue(input.EncodingType) == s3.EncodingTypeUrl { + // AWS url-encodes these key-named elements when encoding-type=url is + // requested; without it XML 1.0 cannot carry the keys at all and the + // encoder replaces their characters with U+FFFD. + output.EncodingType = input.EncodingType + output.Delimiter = encodeMultipartUploadURLField(output.Delimiter) + output.Prefix = encodeMultipartUploadURLField(output.Prefix) + output.KeyMarker = encodeMultipartUploadURLField(output.KeyMarker) + output.NextKeyMarker = encodeMultipartUploadURLField(output.NextKeyMarker) + for _, upload := range output.Upload { + upload.Key = encodeMultipartUploadURLField(upload.Key) + } + } else { + output.KeyMarker = encodeListContinuationTokenField(output.KeyMarker) + output.NextKeyMarker = encodeListContinuationTokenField(output.NextKeyMarker) } return output } +// urlQueryEscape url-encodes key text per segment: every "/"-separated part is +// query-escaped and the separators are kept, so a client recovers the exact key +// with url.QueryUnescape. +func urlQueryEscape(key string) string { + segments := strings.Split(key, "/") + for i, segment := range segments { + segments[i] = url.QueryEscape(segment) + } + return strings.Join(segments, "/") +} + +func encodeMultipartUploadURLField(value *string) *string { + if value == nil { + return nil + } + return aws.String(urlQueryEscape(*value)) +} + type ListPartsResult struct { XMLName xml.Name `xml:"http://s3.amazonaws.com/doc/2006-03-01/ ListPartsResult"` diff --git a/weed/s3api/filer_multipart_list_encoding_test.go b/weed/s3api/filer_multipart_list_encoding_test.go new file mode 100644 index 0000000000..2449c10585 --- /dev/null +++ b/weed/s3api/filer_multipart_list_encoding_test.go @@ -0,0 +1,76 @@ +package s3api + +import ( + "encoding/xml" + "net/url" + "testing" + + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/service/s3" + "github.com/seaweedfs/seaweedfs/weed/pb/filer_pb" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// With encoding-type=url the upload keys have to be url-encoded, as ListObjects +// does; otherwise XML 1.0 cannot carry a key with a control character, the +// encoder turns it into U+FFFD, and no client can recover the key to abort the +// upload. +func TestSelectMultipartUploadsEncodingTypeURL(t *testing.T) { + entries := []*filer_pb.Entry{ + uploadEntry("1", "m\x01p b+.bin", 10), + uploadEntry("2", "plain/key.txt", 20), + } + input := listUploadsInput("m\x01", "m\x01", "", 1000) + input.EncodingType = aws.String(s3.EncodingTypeUrl) + out := selectMultipartUploads(input, entries) + + require.Len(t, out.Upload, 1) + assert.Equal(t, "m%01p+b%2B.bin", *out.Upload[0].Key) + decoded, err := url.QueryUnescape(*out.Upload[0].Key) + require.NoError(t, err) + assert.Equal(t, "m\x01p b+.bin", decoded) + assert.Equal(t, "m%01", *out.Prefix) + assert.Equal(t, "m%01", *out.KeyMarker) + + body, err := xml.Marshal(out) + require.NoError(t, err) + assert.NotContains(t, string(body), "\uFFFD") + assert.Contains(t, string(body), "m%01p+b%2B.bin") + assert.Contains(t, string(body), "url") +} + +func TestSelectMultipartUploadsEncodingTypeURLNextKeyMarker(t *testing.T) { + entries := []*filer_pb.Entry{ + uploadEntry("1", "a\x1fb", 10), + uploadEntry("2", "c", 20), + } + input := listUploadsInput("", "", "", 1) + input.EncodingType = aws.String(s3.EncodingTypeUrl) + out := selectMultipartUploads(input, entries) + + require.True(t, *out.IsTruncated) + assert.Equal(t, "a%1Fb", *out.NextKeyMarker) + // The upload id is opaque, not a key name. + assert.Equal(t, "1", *out.NextUploadIdMarker) + + body, err := xml.Marshal(out) + require.NoError(t, err) + assert.NotContains(t, string(body), "\uFFFD") +} + +// Without encoding-type=url the response is unchanged, and the keys keep their +// raw form for clients that do not ask for encoding. +func TestSelectMultipartUploadsKeepsKeysRawWithoutEncodingType(t *testing.T) { + entries := []*filer_pb.Entry{ + uploadEntry("1", "m\x01p b+.bin", 10), + uploadEntry("2", "plain/key.txt", 20), + } + out := selectMultipartUploads(listUploadsInput("", "", "", 1000), entries) + + var got []string + for _, upload := range out.Upload { + got = append(got, *upload.Key) + } + assert.Equal(t, []string{"m\x01p b+.bin", "plain/key.txt"}, got) +} diff --git a/weed/s3api/filer_multipart_list_test.go b/weed/s3api/filer_multipart_list_test.go index c00f38acaf..3d2a73789c 100644 --- a/weed/s3api/filer_multipart_list_test.go +++ b/weed/s3api/filer_multipart_list_test.go @@ -4,6 +4,7 @@ import ( "context" "fmt" "math" + "net/url" "slices" "testing" @@ -143,6 +144,68 @@ func TestSelectMultipartUploadsHoldsOnlyThePage(t *testing.T) { } } +// With encoding-type=url the markers and upload keys take the wire form the +// object listings already use: a client that echoes the escaped marker back +// still resumes at the decoded key. +func TestSelectMultipartUploadsUrlEncodesMarkersAndKeys(t *testing.T) { + entries := []*filer_pb.Entry{ + uploadEntry("1", "a\x01b", 10), + uploadEntry("2", "a b", 20), + uploadEntry("3", "b", 30), + } + input := listUploadsInput("", "", "", 1) + input.EncodingType = aws.String(s3.EncodingTypeUrl) + + out := selectMultipartUploads(input, entries) + if *out.Upload[0].Key != "a%01b" || *out.NextUploadIdMarker != "1" { + t.Fatalf("page got key %q upload-id-marker %q", *out.Upload[0].Key, *out.NextUploadIdMarker) + } + marker, err := url.QueryUnescape(*out.NextKeyMarker) + if err != nil || decodeListContinuationToken(marker) != "a\x01b" { + t.Fatalf("next key marker %q does not resume at %q", *out.NextKeyMarker, "a\x01b") + } + + input.KeyMarker = aws.String(decodeListContinuationToken(marker)) + input.UploadIdMarker = out.NextUploadIdMarker + out = selectMultipartUploads(input, entries) + if *out.KeyMarker != "a%01b" || *out.Upload[0].Key != "a+b" || *out.NextKeyMarker != "a+b" { + t.Fatalf("page got marker %q key %q next %q", *out.KeyMarker, *out.Upload[0].Key, *out.NextKeyMarker) + } + + input.KeyMarker = aws.String("a b") + input.UploadIdMarker = out.NextUploadIdMarker + out = selectMultipartUploads(input, entries) + if *out.IsTruncated || *out.Upload[0].Key != "b" { + t.Fatalf("last page got truncated=%v key %q", *out.IsTruncated, *out.Upload[0].Key) + } +} + +// A raw key that already carries the token prefix must survive an +// encoding-type=url round trip: under url mode the wire form is the url escape +// alone, so the request decode only unescapes — running the token decode on +// "base64:..." would misread the raw key as an encoded token. +func TestSelectMultipartUploadsUrlTokenPrefixedKeyRoundTrip(t *testing.T) { + entries := []*filer_pb.Entry{ + uploadEntry("1", "base64:AQ", 10), + uploadEntry("2", "base64:z", 20), + } + input := listUploadsInput("", "", "", 1) + input.EncodingType = aws.String(s3.EncodingTypeUrl) + + out := selectMultipartUploads(input, entries) + resume := decodeEscapedEcho(*out.NextKeyMarker, true, urlQueryEscape) + if resume != "base64:AQ" { + t.Fatalf("next key marker %q resumes at %q, want %q", *out.NextKeyMarker, resume, "base64:AQ") + } + + input.KeyMarker = aws.String(resume) + input.UploadIdMarker = out.NextUploadIdMarker + out = selectMultipartUploads(input, entries) + if *out.IsTruncated || len(out.Upload) != 1 || *out.Upload[0].Key != "base64%3Az" { + t.Fatalf("second page got truncated=%v uploads %v", *out.IsTruncated, out.Upload) + } +} + func TestSelectMultipartUploadsCapsRequestedPageSize(t *testing.T) { // max-uploads is caller-controlled; the page stays bounded at the server cap page := newMultipartUploadPage(listUploadsInput("", "", "", 10*maxUploadsList)) diff --git a/weed/s3api/iceberg/handlers_oauth.go b/weed/s3api/iceberg/handlers_oauth.go index 8e6933c461..54391d04e4 100644 --- a/weed/s3api/iceberg/handlers_oauth.go +++ b/weed/s3api/iceberg/handlers_oauth.go @@ -324,28 +324,28 @@ func mintIcebergToken(identityName, accessKey, secret string, ttlSeconds int) (s // authenticateBearer validates a Bearer token from the Authorization header. // Returns the identity name, identity object, and whether auth succeeded. func (s *Server) authenticateBearer(r *http.Request) (string, interface{}, bool) { - identityName, identity, _, ok := s.authenticateBearerKey(r) + identityName, identity, _, _, ok := s.authenticateBearerKey(r) return identityName, identity, ok } // authenticateBearerKey is authenticateBearer that also returns the access key // the token was minted from (its verified access_key claim), so the catalog // can tell apart the holders of several keys on one identity. -func (s *Server) authenticateBearerKey(r *http.Request) (string, interface{}, string, bool) { +func (s *Server) authenticateBearerKey(r *http.Request) (string, interface{}, string, string, bool) { auth := r.Header.Get("Authorization") if auth == "" { - return "", nil, "", false + return "", nil, "", "", false } if !strings.HasPrefix(strings.ToLower(auth), "bearer ") { - return "", nil, "", false + return "", nil, "", "", false } tokenString := strings.TrimSpace(auth[7:]) if tokenString == "" { - return "", nil, "", false + return "", nil, "", "", false } if s.credentialValidator == nil { - return "", nil, "", false + return "", nil, "", "", false } // Parse the token without verification first to get the access key, @@ -355,18 +355,18 @@ func (s *Server) authenticateBearerKey(r *http.Request) (string, interface{}, st _, _, err := parser.ParseUnverified(tokenString, unverified) if err != nil { glog.V(2).Infof("Iceberg OAuth: failed to parse token: %v", err) - return "", nil, "", false + return "", nil, "", "", false } if unverified.AccessKey == "" { - return "", nil, "", false + return "", nil, "", "", false } // Look up the credential by access key to get the signing key for verification identityName, identity, secretKey, err := s.credentialValidator.GetCredentialByAccessKey(unverified.AccessKey) if err != nil { glog.V(2).Infof("Iceberg OAuth: failed to get credential for access key: %v", err) - return "", nil, "", false + return "", nil, "", "", false } signingKey := deriveSigningKey(unverified.AccessKey, secretKey) @@ -379,12 +379,12 @@ func (s *Server) authenticateBearerKey(r *http.Request) (string, interface{}, st }) if err != nil || !verified.Valid { glog.V(2).Infof("Iceberg OAuth: token verification failed: %v", err) - return "", nil, "", false + return "", nil, "", "", false } // The token's signing key is derived from this access key, so a verified // token was minted from it. - return identityName, identity, unverified.AccessKey, true + return identityName, identity, unverified.AccessKey, secretKey, true } // deriveSigningKey derives a signing key from the access key and secret using HMAC-SHA256. diff --git a/weed/s3api/iceberg/server.go b/weed/s3api/iceberg/server.go index 5ec8de2160..1849f2e6e8 100644 --- a/weed/s3api/iceberg/server.go +++ b/weed/s3api/iceberg/server.go @@ -269,10 +269,11 @@ func (s *Server) Auth(handler http.HandlerFunc) http.HandlerFunc { // The auth scheme is case-insensitive (RFC 7235), matching // authenticateBearer below. if strings.HasPrefix(strings.ToLower(r.Header.Get("Authorization")), "bearer ") { - if identityName, identity, accessKey, ok := s.authenticateBearerKey(r); ok { + if identityName, identity, accessKey, secretKey, ok := s.authenticateBearerKey(r); ok { ctx := r.Context() ctx = s3_constants.SetIdentityNameInContext(ctx, identityName) ctx = s3_constants.SetAccessKeyIdInContext(ctx, accessKey) + ctx = s3_constants.SetStoredAccessKeyIdInContext(ctx, accessKey, secretKey) if identity != nil { ctx = s3_constants.SetIdentityInContext(ctx, identity) } diff --git a/weed/s3api/lance/handlers_oauth.go b/weed/s3api/lance/handlers_oauth.go index 20e791221e..d348a5c200 100644 --- a/weed/s3api/lance/handlers_oauth.go +++ b/weed/s3api/lance/handlers_oauth.go @@ -147,27 +147,27 @@ func mintToken(identityName, accessKey, secret string, ttlSeconds int) (string, // authenticateBearer validates a Bearer token from the Authorization header. // Returns the identity name, identity object, and whether auth succeeded. func (s *Server) authenticateBearer(r *http.Request) (string, interface{}, bool) { - identityName, identity, _, ok := s.authenticateBearerKey(r) + identityName, identity, _, _, ok := s.authenticateBearerKey(r) return identityName, identity, ok } // authenticateBearerKey is authenticateBearer that also returns the access key // the token was minted from (its verified access_key claim). -func (s *Server) authenticateBearerKey(r *http.Request) (string, interface{}, string, bool) { +func (s *Server) authenticateBearerKey(r *http.Request) (string, interface{}, string, string, bool) { auth := r.Header.Get("Authorization") if auth == "" { - return "", nil, "", false + return "", nil, "", "", false } if !strings.HasPrefix(strings.ToLower(auth), "bearer ") { - return "", nil, "", false + return "", nil, "", "", false } tokenString := strings.TrimSpace(auth[7:]) if tokenString == "" { - return "", nil, "", false + return "", nil, "", "", false } if s.credentialValidator == nil { - return "", nil, "", false + return "", nil, "", "", false } // Parse the token without verification first to get the access key, @@ -177,17 +177,17 @@ func (s *Server) authenticateBearerKey(r *http.Request) (string, interface{}, st _, _, err := parser.ParseUnverified(tokenString, unverified) if err != nil { glog.V(2).Infof("Lance OAuth: failed to parse token: %v", err) - return "", nil, "", false + return "", nil, "", "", false } if unverified.AccessKey == "" { - return "", nil, "", false + return "", nil, "", "", false } identityName, identity, secretKey, err := s.credentialValidator.GetCredentialByAccessKey(unverified.AccessKey) if err != nil { glog.V(2).Infof("Lance OAuth: failed to get credential for access key: %v", err) - return "", nil, "", false + return "", nil, "", "", false } signingKey := deriveSigningKey(unverified.AccessKey, secretKey) @@ -200,12 +200,12 @@ func (s *Server) authenticateBearerKey(r *http.Request) (string, interface{}, st }) if err != nil || !verified.Valid { glog.V(2).Infof("Lance OAuth: token verification failed: %v", err) - return "", nil, "", false + return "", nil, "", "", false } // The token's signing key is derived from this access key, so a verified // token was minted from it. - return identityName, identity, unverified.AccessKey, true + return identityName, identity, unverified.AccessKey, secretKey, true } // authenticateApiKey validates an x-api-key header carrying an S3 credential diff --git a/weed/s3api/lance/server.go b/weed/s3api/lance/server.go index d0cb813daa..53effee9e5 100644 --- a/weed/s3api/lance/server.go +++ b/weed/s3api/lance/server.go @@ -177,10 +177,11 @@ func (s *Server) Auth(handler http.HandlerFunc) http.HandlerFunc { // different error that clients cannot act on. // The auth scheme is case-insensitive (RFC 7235). if strings.HasPrefix(strings.ToLower(r.Header.Get("Authorization")), "bearer ") { - if identityName, identity, accessKey, ok := s.authenticateBearerKey(r); ok { + if identityName, identity, accessKey, secretKey, ok := s.authenticateBearerKey(r); ok { ctx := r.Context() ctx = s3_constants.SetIdentityNameInContext(ctx, identityName) ctx = s3_constants.SetAccessKeyIdInContext(ctx, accessKey) + ctx = s3_constants.SetStoredAccessKeyIdInContext(ctx, accessKey, secretKey) if identity != nil { ctx = s3_constants.SetIdentityInContext(ctx, identity) } @@ -195,10 +196,11 @@ func (s *Server) Auth(handler http.HandlerFunc) http.HandlerFunc { if apiKey := r.Header.Get("x-api-key"); apiKey != "" { if identityName, identity, ok := s.authenticateApiKey(apiKey); ok { - accessKey, _, _ := strings.Cut(apiKey, ":") + accessKey, secretKey, _ := strings.Cut(apiKey, ":") ctx := r.Context() ctx = s3_constants.SetIdentityNameInContext(ctx, identityName) ctx = s3_constants.SetAccessKeyIdInContext(ctx, accessKey) + ctx = s3_constants.SetStoredAccessKeyIdInContext(ctx, accessKey, secretKey) if identity != nil { ctx = s3_constants.SetIdentityInContext(ctx, identity) } diff --git a/weed/s3api/policy_engine/README_POLICY_ENGINE.md b/weed/s3api/policy_engine/README_POLICY_ENGINE.md index 97c4344e4a..78e1df5545 100644 --- a/weed/s3api/policy_engine/README_POLICY_ENGINE.md +++ b/weed/s3api/policy_engine/README_POLICY_ENGINE.md @@ -125,6 +125,7 @@ Support for all AWS condition operators: - **Boolean**: `Bool` - **ARN**: `ArnEquals`, `ArnLike` - **Null**: `Null` +- **Set qualifiers**: `ForAllValues:` and `ForAnyValue:` before any of the above, for multi-valued keys such as `s3:RequestObjectTagKeys` ### 4. Condition Keys @@ -357,6 +358,7 @@ The policy engine provides a seamless upgrade path from SeaweedFS's existing sim | `aws:CurrentTime` | Time-based access control | | `s3:ExistingObjectTag/` | Tag-based access control for existing objects | | `s3:RequestObjectTag/` | Tag conditions on the tags a PUT request sets | +| `s3:RequestObjectTagKeys` | The set of tag keys a PUT request sets | | Wildcard Patterns | Support for `*` and `?` in actions and resources | | Principal Matching | `*`, account IDs, and user ARNs | @@ -364,7 +366,6 @@ The policy engine provides a seamless upgrade path from SeaweedFS's existing sim | Feature | GitHub Issue | |---------|--------------| -| `s3:RequestObjectTagKeys` | Check which tag keys are in request | | `s3:x-amz-content-sha256` | Content hash condition | | `s3:x-amz-server-side-encryption` | SSE condition | | `s3:x-amz-storage-class` | Storage class condition | diff --git a/weed/s3api/policy_engine/conditions.go b/weed/s3api/policy_engine/conditions.go index a995c5ddb9..bfb7bf0b94 100644 --- a/weed/s3api/policy_engine/conditions.go +++ b/weed/s3api/policy_engine/conditions.go @@ -669,8 +669,58 @@ func (e *NullEvaluator) Evaluate(conditionValue interface{}, contextValues []str return false } -// GetConditionEvaluator returns the appropriate evaluator for a condition operator +// forAllValuesEvaluator matches when every request value satisfies the base +// operator on its own, and so also when the request carries no value. +type forAllValuesEvaluator struct { + base ConditionEvaluator +} + +func (e *forAllValuesEvaluator) Evaluate(conditionValue interface{}, contextValues []string) bool { + for _, v := range contextValues { + if !e.base.Evaluate(conditionValue, []string{v}) { + return false + } + } + return true +} + +// forAnyValueEvaluator matches when at least one request value satisfies the +// base operator on its own. Checking values one at a time keeps a negated +// base such as StringNotEquals from meaning "none of them". +type forAnyValueEvaluator struct { + base ConditionEvaluator +} + +func (e *forAnyValueEvaluator) Evaluate(conditionValue interface{}, contextValues []string) bool { + for _, v := range contextValues { + if e.base.Evaluate(conditionValue, []string{v}) { + return true + } + } + return false +} + +// GetConditionEvaluator returns the appropriate evaluator for a condition +// operator, including the ForAllValues: and ForAnyValue: set qualifiers. func GetConditionEvaluator(operator string) (ConditionEvaluator, error) { + if base, ok := strings.CutPrefix(operator, "ForAllValues:"); ok { + evaluator, err := baseConditionEvaluator(base) + if err != nil { + return nil, err + } + return &forAllValuesEvaluator{base: evaluator}, nil + } + if base, ok := strings.CutPrefix(operator, "ForAnyValue:"); ok { + evaluator, err := baseConditionEvaluator(base) + if err != nil { + return nil, err + } + return &forAnyValueEvaluator{base: evaluator}, nil + } + return baseConditionEvaluator(operator) +} + +func baseConditionEvaluator(operator string) (ConditionEvaluator, error) { switch operator { case "StringEquals": return &StringEqualsEvaluator{}, nil @@ -732,6 +782,10 @@ const ExistingObjectTagPrefix = "s3:ExistingObjectTag/" // request sets through the X-Amz-Tagging header. const RequestObjectTagPrefix = "s3:RequestObjectTag/" +// RequestObjectTagKeys is the multi-valued condition key listing the keys of +// the tags a request sets through the X-Amz-Tagging header. +const RequestObjectTagKeys = "s3:RequestObjectTagKeys" + // getConditionContextValue resolves the value(s) for a condition key. // For s3:ExistingObjectTag/ conditions, it looks up the tag in objectEntry. // For other condition keys, it looks up the value in contextValues. @@ -780,7 +834,8 @@ func evaluateConditions(conditions PolicyConditions, contextValues map[string][] for key, value := range conditionMap { contextVals := getConditionContextValue(key, contextValues, objectEntry) if original := originalGrants[key]; len(original) != 0 && isGrantConditionKey(key) { - switch operator { + // Extra values can only make ForAllValues fail, so it keeps the canonical list. + switch strings.TrimPrefix(operator, "ForAnyValue:") { case "StringEquals", "StringEqualsIgnoreCase", "StringLike", "ArnEquals", "ArnLike": contextVals = append(append([]string(nil), contextVals...), original...) } diff --git a/weed/s3api/policy_engine/conditions_set_test.go b/weed/s3api/policy_engine/conditions_set_test.go new file mode 100644 index 0000000000..80aac29a0f --- /dev/null +++ b/weed/s3api/policy_engine/conditions_set_test.go @@ -0,0 +1,102 @@ +package policy_engine + +import ( + "testing" +) + +func TestConditionSetOperators(t *testing.T) { + allowed := []string{"department", "project"} + tests := []struct { + name string + operator string + conditionValue interface{} + contextValues []string + expected bool + }{ + {"ForAllValues all match", "ForAllValues:StringEquals", allowed, []string{"department", "project"}, true}, + {"ForAllValues subset matches", "ForAllValues:StringEquals", allowed, []string{"project"}, true}, + {"ForAllValues one outsider fails", "ForAllValues:StringEquals", allowed, []string{"department", "owner"}, false}, + {"ForAllValues empty set matches", "ForAllValues:StringEquals", allowed, []string{}, true}, + {"ForAllValues nil set matches", "ForAllValues:StringEquals", allowed, nil, true}, + {"ForAllValues is case sensitive", "ForAllValues:StringEquals", allowed, []string{"Department"}, false}, + {"ForAllValues StringLike all match", "ForAllValues:StringLike", "proj-*", []string{"proj-a", "proj-b"}, true}, + {"ForAllValues StringLike one outsider fails", "ForAllValues:StringLike", "proj-*", []string{"proj-a", "team"}, false}, + {"ForAllValues StringNotEquals none listed", "ForAllValues:StringNotEquals", allowed, []string{"owner", "team"}, true}, + {"ForAllValues StringNotEquals one listed fails", "ForAllValues:StringNotEquals", allowed, []string{"owner", "project"}, false}, + + {"ForAnyValue one match", "ForAnyValue:StringEquals", allowed, []string{"owner", "project"}, true}, + {"ForAnyValue no match", "ForAnyValue:StringEquals", allowed, []string{"owner", "team"}, false}, + {"ForAnyValue empty set fails", "ForAnyValue:StringEquals", allowed, []string{}, false}, + {"ForAnyValue nil set fails", "ForAnyValue:StringEquals", allowed, nil, false}, + {"ForAnyValue StringLike one match", "ForAnyValue:StringLike", []string{"secret*"}, []string{"owner", "secret-level"}, true}, + {"ForAnyValue StringLike no match", "ForAnyValue:StringLike", []string{"secret*"}, []string{"owner"}, false}, + {"ForAnyValue StringNotEquals one unlisted", "ForAnyValue:StringNotEquals", allowed, []string{"department", "owner"}, true}, + {"ForAnyValue StringNotEquals all listed fails", "ForAnyValue:StringNotEquals", allowed, []string{"department", "project"}, false}, + {"ForAnyValue StringEqualsIgnoreCase", "ForAnyValue:StringEqualsIgnoreCase", allowed, []string{"PROJECT"}, true}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + evaluator, err := GetConditionEvaluator(tt.operator) + if err != nil { + t.Fatalf("Failed to get condition evaluator: %v", err) + } + if got := evaluator.Evaluate(tt.conditionValue, tt.contextValues); got != tt.expected { + t.Errorf("Expected %v, got %v", tt.expected, got) + } + }) + } +} + +func TestGetConditionEvaluatorRejectsUnknownSetOperators(t *testing.T) { + for _, op := range []string{ + "ForAllValues:StringBogus", + "ForAnyValue:", + "ForEachValue:StringEquals", + "ForAllValues:ForAnyValue:StringEquals", + } { + if _, err := GetConditionEvaluator(op); err == nil { + t.Errorf("GetConditionEvaluator accepted %q; expected error", op) + } + } +} + +func TestValidateBucketPolicyAcceptsSetOperators(t *testing.T) { + policy := `{ + "Version": "2012-10-17", + "Statement": [{ + "Effect": "Deny", + "Principal": "*", + "Action": "s3:PutObject", + "Resource": "arn:aws:s3:::test-bucket/*", + "Condition": { + "ForAllValues:StringNotEquals": {"s3:RequestObjectTagKeys": ["department", "project"]}, + "ForAnyValue:StringLike": {"s3:RequestObjectTagKeys": ["secret*"]} + } + }] + }` + engine := NewPolicyEngine() + if err := engine.SetBucketPolicy("test-bucket", policy); err != nil { + t.Fatalf("SetBucketPolicy rejected set operators: %v", err) + } +} + +func TestOriginalGrantsSupplementOnlyForAnyValue(t *testing.T) { + const key = "s3:x-amz-grant-read" + contextValues := map[string][]string{key: {"id=canonical"}} + originalGrants := map[string][]string{key: {"uri=original"}} + + anyValue := PolicyConditions{ + "ForAnyValue:StringEquals": {key: NewStringOrStringSlice("uri=original")}, + } + if !evaluateConditions(anyValue, contextValues, nil, nil, originalGrants) { + t.Error("ForAnyValue should see the original grant values") + } + + allValues := PolicyConditions{ + "ForAllValues:StringEquals": {key: NewStringOrStringSlice("id=canonical")}, + } + if !evaluateConditions(allValues, contextValues, nil, nil, originalGrants) { + t.Error("ForAllValues should be evaluated on the canonical values only") + } +} diff --git a/weed/s3api/policy_engine/engine.go b/weed/s3api/policy_engine/engine.go index 0275f33627..e544b227bd 100644 --- a/weed/s3api/policy_engine/engine.go +++ b/weed/s3api/policy_engine/engine.go @@ -2,9 +2,11 @@ package policy_engine import ( "fmt" + "maps" "net/http" "net/url" "regexp" + "slices" "strings" "sync" "sync/atomic" @@ -441,8 +443,11 @@ func (engine *PolicyEngine) ExtractConditionValuesFromRequest(r *http.Request) m // Note: s3:ExistingObjectTag/ conditions are evaluated using objectEntry // passed to EvaluatePolicy, not extracted from the request. - for key, value := range requestObjectTags(r) { - values[RequestObjectTagPrefix+key] = []string{value} + if tags := requestObjectTags(r); len(tags) > 0 { + for key, value := range tags { + values[RequestObjectTagPrefix+key] = []string{value} + } + values[RequestObjectTagKeys] = slices.Sorted(maps.Keys(tags)) } // S3 bucket-level conditions @@ -514,16 +519,24 @@ func requestObjectTags(r *http.Request) map[string]string { if !storesRequestObjectTags(r) { return nil } - // A malformed header is rejected by the handler with InvalidTag. - parsed, err := url.ParseQuery(tagging) - if err != nil { - return nil - } - // putToFiler rejects a repeated key, but CopyObject stores its last - // value, so that is the one the policy has to see. - tags := make(map[string]string, len(parsed)) - for key, vals := range parsed { - tags[key] = vals[len(vals)-1] + // Parse the same non-empty segments the tag handlers store: url.ParseQuery + // caps at 10,000 parameters, so a header padded with "&" segments would + // hide stored tag keys from every s3:RequestObjectTag* condition. A + // malformed pair is rejected by the handler with InvalidTag. + tags := make(map[string]string) + for _, segment := range strings.Split(tagging, "&") { + if segment == "" { + continue + } + rawKey, rawValue, _ := strings.Cut(segment, "=") + key, errKey := url.QueryUnescape(rawKey) + value, errValue := url.QueryUnescape(rawValue) + if errKey != nil || errValue != nil { + return nil + } + // putToFiler rejects a repeated key, but CopyObject stores its last + // value, so that is the one the policy has to see. + tags[key] = value } return tags } diff --git a/weed/s3api/policy_engine/engine_test.go b/weed/s3api/policy_engine/engine_test.go index 587ffdc115..6f4cccc687 100644 --- a/weed/s3api/policy_engine/engine_test.go +++ b/weed/s3api/policy_engine/engine_test.go @@ -4,6 +4,7 @@ import ( "net/http" "net/http/httptest" "net/url" + "strings" "testing" "github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants" @@ -1051,6 +1052,158 @@ func TestRequestObjectTagConditionDeny(t *testing.T) { } } +func evaluateTaggedRequest(engine *PolicyEngine, method, target string, headers map[string]string, tagging string) PolicyEvaluationResult { + if method == "" { + method = http.MethodPut + } + req := httptest.NewRequest(method, "/test-bucket/test-object"+target, nil) + for k, v := range headers { + req.Header.Set(k, v) + } + if tagging != "" { + req.Header.Set(s3_constants.AmzObjectTagging, tagging) + } + return engine.EvaluatePolicy("test-bucket", &PolicyEvaluationArgs{ + Action: "s3:PutObject", + Resource: "arn:aws:s3:::test-bucket/test-object", + Principal: "*", + Conditions: engine.ExtractConditionValuesFromRequest(req), + }) +} + +func TestRequestObjectTagKeysCondition(t *testing.T) { + engine := NewPolicyEngine() + + policyJSON := `{ + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Principal": "*", + "Action": "s3:PutObject", + "Resource": "arn:aws:s3:::test-bucket/*", + "Condition": { + "ForAllValues:StringEquals": { + "s3:RequestObjectTagKeys": ["department", "project"] + }, + "Null": { + "s3:RequestObjectTagKeys": "false" + } + } + }, + { + "Effect": "Deny", + "Principal": "*", + "Action": "s3:PutObject", + "Resource": "arn:aws:s3:::test-bucket/*", + "Condition": { + "ForAnyValue:StringLike": { + "s3:RequestObjectTagKeys": ["secret*"] + } + } + } + ] + }` + + if err := engine.SetBucketPolicy("test-bucket", policyJSON); err != nil { + t.Fatalf("Failed to set bucket policy: %v", err) + } + + copyHeaders := map[string]string{"X-Amz-Copy-Source": "/src-bucket/src-object"} + replaceHeaders := map[string]string{ + "X-Amz-Copy-Source": "/src-bucket/src-object", + s3_constants.AmzObjectTaggingDirective: "REPLACE", + } + + tests := []struct { + name string + method string + target string + headers map[string]string + tagging string + expected PolicyEvaluationResult + }{ + {"listed key allows", "", "", nil, "department=engineering", PolicyResultAllow}, + {"all listed keys allow", "", "", nil, "department=engineering&project=alpha", PolicyResultAllow}, + {"unlisted key is not allowed", "", "", nil, "department=engineering&owner=bob", PolicyResultIndeterminate}, + {"no tagging header is not allowed", "", "", nil, "", PolicyResultIndeterminate}, + {"repeated key counts once", "", "", nil, "department=a&department=b", PolicyResultAllow}, + {"url-encoded key is decoded", "", "", nil, "depart%6Dent=engineering", PolicyResultAllow}, + {"empty segments add no key", "", "", nil, "&department=engineering&", PolicyResultAllow}, + {"empty key is not allowed", "", "", nil, "department=engineering&=x", PolicyResultIndeterminate}, + {"key case is preserved", "", "", nil, "Department=engineering", PolicyResultIndeterminate}, + {"malformed header is not allowed", "", "", nil, "department=engineering&%zz", PolicyResultIndeterminate}, + {"secret key is denied", "", "", nil, "secret=1", PolicyResultDeny}, + {"secret key behind a padded header is denied", "", "", nil, "secret=1" + strings.Repeat("&", 10001), PolicyResultDeny}, + {"secret key among listed keys is denied", "", "", nil, "department=engineering&secret-level=2", PolicyResultDeny}, + {"url-encoded secret key is denied", "", "", nil, "secr%65t=1", PolicyResultDeny}, + {"secret key on a copy that replaces tags is denied", "", "", replaceHeaders, "secret=1", PolicyResultDeny}, + {"secret key on a multipart create is denied", "POST", "?uploads", nil, "secret=1", PolicyResultDeny}, + {"header is ignored on a copy that keeps source tags", "", "", copyHeaders, "secret=1", PolicyResultIndeterminate}, + {"header is ignored on PutObjectTagging", "", "?tagging", nil, "department=engineering", PolicyResultIndeterminate}, + {"header is ignored on a part upload", "", "?partNumber=1&uploadId=x", nil, "secret=1", PolicyResultIndeterminate}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if result := evaluateTaggedRequest(engine, tt.method, tt.target, tt.headers, tt.tagging); result != tt.expected { + t.Errorf("Expected %v, got %v", tt.expected, result) + } + }) + } +} + +func TestRequestObjectTagKeysRequiredTagDeny(t *testing.T) { + engine := NewPolicyEngine() + + policyJSON := `{ + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Principal": "*", + "Action": "s3:PutObject", + "Resource": "arn:aws:s3:::test-bucket/*" + }, + { + "Effect": "Deny", + "Principal": "*", + "Action": "s3:PutObject", + "Resource": "arn:aws:s3:::test-bucket/*", + "Condition": { + "ForAllValues:StringNotEquals": { + "s3:RequestObjectTagKeys": ["department", "project"] + } + } + } + ] + }` + + if err := engine.SetBucketPolicy("test-bucket", policyJSON); err != nil { + t.Fatalf("Failed to set bucket policy: %v", err) + } + + tests := []struct { + name string + tagging string + expected PolicyEvaluationResult + }{ + {"required key passes", "department=engineering", PolicyResultAllow}, + {"required key with another key passes", "project=alpha&owner=bob", PolicyResultAllow}, + {"only other keys are denied", "owner=bob", PolicyResultDeny}, + {"no tagging header is denied", "", PolicyResultDeny}, + {"malformed header is denied", "department=engineering&%zz", PolicyResultDeny}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if result := evaluateTaggedRequest(engine, "", "", nil, tt.tagging); result != tt.expected { + t.Errorf("Expected %v, got %v", tt.expected, result) + } + }) + } +} + // TestExistingObjectTagConditionMultipleTags tests policies with multiple tag conditions func TestExistingObjectTagConditionMultipleTags(t *testing.T) { engine := NewPolicyEngine() diff --git a/weed/s3api/s3_constants/header.go b/weed/s3api/s3_constants/header.go index cf26ac8f84..a49d40220e 100644 --- a/weed/s3api/s3_constants/header.go +++ b/weed/s3api/s3_constants/header.go @@ -18,6 +18,9 @@ package s3_constants import ( "context" + "crypto/hmac" + "crypto/sha256" + "encoding/hex" "net/http" "strings" "sync/atomic" @@ -355,6 +358,7 @@ const ( contextKeyIgnoredSessionToken contextKey = "s3-ignored-session-token" contextKeyOriginalRequestPath contextKey = "s3-original-request-path" contextKeyAccessKeyId contextKey = "s3-access-key-id" + contextKeyStoredAccessKeyId contextKey = "s3-stored-access-key-id" ) // identityHolder is a mutable container for the authenticated identity name, @@ -365,10 +369,37 @@ const ( // holder installed before authentication is shared across all copies, so the // name written by the inner handler is readable by the outer middleware. type identityHolder struct { - name atomic.Pointer[string] - principalArn atomic.Pointer[string] - identityClaim atomic.Pointer[string] - accessKeyId atomic.Pointer[string] + name atomic.Pointer[string] + principalArn atomic.Pointer[string] + identityClaim atomic.Pointer[string] + accessKeyId atomic.Pointer[string] + storedCredential atomic.Pointer[storedCredential] +} + +// storedCredential is the stored-identity credential a request authenticated +// with: the access key id and its secret, so a credential vendor can bind +// what it mints to this exact credential instance. +type storedCredential struct { + accessKeyId string + secretKey string +} + +// StoredCredentialFingerprint identifies one credential instance. The key id +// alone cannot: an operator can delete a key and re-create it under the same +// id with a different secret. The fingerprint travels inside the vended +// session token, so it is keyed with a server-held key — an unkeyed hash of +// the secret would hand the token holder an offline guessing oracle. +func StoredCredentialFingerprint(fingerprintKey []byte, accessKeyId, secretKey string) string { + if len(fingerprintKey) == 0 { + return "" + } + h := hmac.New(sha256.New, fingerprintKey) + h.Write([]byte("seaweedfs-stored-credential")) + h.Write([]byte{0}) + h.Write([]byte(accessKeyId)) + h.Write([]byte{0}) + h.Write([]byte(secretKey)) + return hex.EncodeToString(h.Sum(nil)[:16]) } // EnsureIdentityHolder attaches a mutable identity holder to the request context @@ -554,6 +585,70 @@ func AccessKeyIdFromContext(ctx context.Context) string { return "" } +// SetStoredAccessKeyIdInContext records the stored-identity credential the +// request authenticated with — set by an authenticator that verified the +// credential against the identity store, so a credential vendor can tell the +// caller apart from an STS session or OIDC user holding a key the store never +// issued. +func SetStoredAccessKeyIdInContext(ctx context.Context, accessKeyId, secretKey string) context.Context { + if accessKeyId == "" { + return ctx + } + sc := storedCredential{accessKeyId, secretKey} + if h, ok := ctx.Value(contextKeyIdentityHolder).(*identityHolder); ok && h != nil { + h.storedCredential.Store(&sc) + } + return context.WithValue(ctx, contextKeyStoredAccessKeyId, sc) +} + +// RecordStoredAccessKeyId stores the stored-credential marker into the +// request's identity holder: a verifier calls it when the credential that +// matched resolves to a key held by a configured identity. A no-op without a +// holder (EnsureIdentityHolder) or without a key. +func RecordStoredAccessKeyId(ctx context.Context, accessKeyId, secretKey string) { + if ctx == nil || accessKeyId == "" { + return + } + sc := storedCredential{accessKeyId, secretKey} + if h, ok := ctx.Value(contextKeyIdentityHolder).(*identityHolder); ok && h != nil { + h.storedCredential.Store(&sc) + } +} + +// StoredAccessKeyIdFromContext retrieves the stored-identity access key id the +// request authenticated with, or "" when it authenticated another way (STS +// session, OIDC bearer, anonymous). +func StoredAccessKeyIdFromContext(ctx context.Context) string { + sc, _ := storedCredentialFromContext(ctx) + return sc.accessKeyId +} + +// StoredCredentialFromContext retrieves the stored-identity access key id and +// secret the request authenticated with, or "", "" when it authenticated +// another way (STS session, OIDC bearer, anonymous). +func StoredCredentialFromContext(ctx context.Context) (string, string) { + sc, ok := storedCredentialFromContext(ctx) + if !ok { + return "", "" + } + return sc.accessKeyId, sc.secretKey +} + +func storedCredentialFromContext(ctx context.Context) (storedCredential, bool) { + if ctx == nil { + return storedCredential{}, false + } + if sc, ok := ctx.Value(contextKeyStoredAccessKeyId).(storedCredential); ok && sc.accessKeyId != "" { + return sc, true + } + if h, ok := ctx.Value(contextKeyIdentityHolder).(*identityHolder); ok && h != nil { + if sc := h.storedCredential.Load(); sc != nil && sc.accessKeyId != "" { + return *sc, true + } + } + return storedCredential{}, false +} + // IgnoreSessionTokenInContext marks the request's session token as tolerated // rather than authenticated: a statically configured access key signed the // request, so the token (which credential vendors like Unity Catalog emit on diff --git a/weed/s3api/s3_constants/header_test.go b/weed/s3api/s3_constants/header_test.go index de780d0d51..2d30f3e859 100644 --- a/weed/s3api/s3_constants/header_test.go +++ b/weed/s3api/s3_constants/header_test.go @@ -313,3 +313,20 @@ func TestGetIdentityNameFromContextWithoutHolder(t *testing.T) { t.Errorf("requester = %q, want admin", got) } } + +func TestStoredCredentialFingerprintIsKeyed(t *testing.T) { + key := []byte("test-signing-key-32-characters!!") + fp := StoredCredentialFingerprint(key, "AKIAEXAMPLE", "secret") + if fp == "" { + t.Fatal("fingerprint is empty with a key") + } + if again := StoredCredentialFingerprint(key, "AKIAEXAMPLE", "secret"); again != fp { + t.Error("fingerprint is not deterministic") + } + if other := StoredCredentialFingerprint([]byte("another-signing-key-padded-out"), "AKIAEXAMPLE", "secret"); other == fp { + t.Error("fingerprint ignores the server key, leaving the secret guessable") + } + if noKey := StoredCredentialFingerprint(nil, "AKIAEXAMPLE", "secret"); noKey != "" { + t.Error("an empty key must yield an empty fingerprint") + } +} diff --git a/weed/s3api/s3_metadata_util.go b/weed/s3api/s3_metadata_util.go index bcd62b0e9d..01e0adc194 100644 --- a/weed/s3api/s3_metadata_util.go +++ b/weed/s3api/s3_metadata_util.go @@ -2,7 +2,6 @@ package s3api import ( "net/http" - "net/url" "strings" "github.com/seaweedfs/seaweedfs/weed/glog" @@ -52,25 +51,16 @@ func ParseS3Metadata(r *http.Request, existing map[string][]byte, isReplace bool // Object tagging if tags := r.Header.Get(s3_constants.AmzObjectTagging); tags != "" { - // Use url.ParseQuery for robust parsing and automatic URL decoding - parsedTags, err := url.ParseQuery(tags) + tagSet, err := parseTagsHeader(tags) if err != nil { - // Return proper S3 error instead of silently dropping tags - glog.Warningf("Invalid S3 tag format in header '%s': %v", tags, err) + glog.Warningf("Invalid S3 tagging header: %v", err) return nil, s3err.ErrInvalidTag } - - // Validate: S3 spec does not allow duplicate tag keys - for key, values := range parsedTags { - if len(values) > 1 { - glog.Warningf("Duplicate tag key '%s' in header '%s'", key, tags) - return nil, s3err.ErrInvalidTag - } - // Tag value can be an empty string but not nil - value := "" - if len(values) > 0 { - value = values[0] - } + if err := ValidateTags(tagSet); err != nil { + glog.Warningf("Invalid tags (%d): %v", len(tagSet), err) + return nil, s3err.ErrInvalidTag + } + for key, value := range tagSet { metadata[s3_constants.AmzObjectTagging+"-"+key] = []byte(value) } } diff --git a/weed/s3api/s3api_bucket_tagging_handlers.go b/weed/s3api/s3api_bucket_tagging_handlers.go index a1b116fd28..9541f38cb1 100644 --- a/weed/s3api/s3api_bucket_tagging_handlers.go +++ b/weed/s3api/s3api_bucket_tagging_handlers.go @@ -68,9 +68,14 @@ func (s3a *S3ApiServer) PutBucketTaggingHandler(w http.ResponseWriter, r *http.R } tags := tagging.ToTags() + // A key may appear only once in a tag set; the map keeps one of them. + if len(tags) != len(tagging.TagSet.Tag) { + glog.Errorf("PutBucketTagging duplicate tag keys %s", r.URL) + s3err.WriteErrorResponse(w, r, s3err.ErrInvalidTag) + return + } - // Validate tags using existing validation - err = ValidateTags(tags) + err = ValidateBucketTags(tags) if err != nil { glog.Errorf("PutBucketTagging ValidateTags error %s: %v", r.URL, err) s3err.WriteErrorResponse(w, r, s3err.ErrInvalidTag) diff --git a/weed/s3api/s3api_bucket_tagging_handlers_test.go b/weed/s3api/s3api_bucket_tagging_handlers_test.go new file mode 100644 index 0000000000..a1cc26b165 --- /dev/null +++ b/weed/s3api/s3api_bucket_tagging_handlers_test.go @@ -0,0 +1,91 @@ +package s3api + +import ( + "fmt" + "net/http" + "net/http/httptest" + "strings" + "testing" + "time" + + "github.com/gorilla/mux" + "github.com/seaweedfs/seaweedfs/weed/pb/s3_pb" + "google.golang.org/protobuf/proto" +) + +// A bucket's tag set holds up to 50 tags, unlike an object's 10, and no key +// twice. +func TestPutBucketTaggingTagCountLimit(t *testing.T) { + const bucket = "tag-limit-bucket" + for _, tc := range []struct { + tags int + repeatKey bool // the last tag repeats the first tag's key + emptyKey bool // the first tag has an empty key + wantCode int + }{ + {tags: 10, wantCode: http.StatusOK}, + {tags: 11, wantCode: http.StatusOK}, + {tags: 50, wantCode: http.StatusOK}, + {tags: 51, wantCode: http.StatusBadRequest}, + {tags: 2, repeatKey: true, wantCode: http.StatusBadRequest}, + {tags: 51, repeatKey: true, wantCode: http.StatusBadRequest}, + {tags: 1, emptyKey: true, wantCode: http.StatusBadRequest}, + } { + name := fmt.Sprintf("%d tags", tc.tags) + if tc.repeatKey { + name += " with a repeated key" + } + if tc.emptyKey { + name += " with an empty key" + } + t.Run(name, func(t *testing.T) { + filer := &fakeTxnFiler{} + s3a := newHeadBucketTestServer(t, filer) + s3a.iam = &IdentityAccessManagement{isAuthEnabled: true} + s3a.bucketConfigCache = NewBucketConfigCache(time.Minute) + s3a.bucketConfigCache.Set(bucket, &BucketConfig{Name: bucket}) + + var body strings.Builder + body.WriteString("") + for i := 0; i < tc.tags; i++ { + key := fmt.Sprintf("key%d", i) + if tc.repeatKey && i == tc.tags-1 { + key = "key0" + } + if tc.emptyKey && i == 0 { + key = "" + } + fmt.Fprintf(&body, "%svalue%d", key, i) + } + body.WriteString("") + + req := httptest.NewRequest(http.MethodPut, "/"+bucket+"?tagging", strings.NewReader(body.String())) + req = mux.SetURLVars(req, map[string]string{"bucket": bucket}) + rr := httptest.NewRecorder() + s3a.PutBucketTaggingHandler(rr, req) + + if rr.Code != tc.wantCode { + t.Fatalf("status = %d, want %d; body: %s", rr.Code, tc.wantCode, rr.Body.String()) + } + if tc.wantCode != http.StatusOK { + if !strings.Contains(rr.Body.String(), "InvalidTag") { + t.Fatalf("body = %s, want InvalidTag", rr.Body.String()) + } + if filer.lastReq != nil { + t.Fatalf("rejected tag set was written to the filer") + } + return + } + if filer.lastReq == nil || len(filer.lastReq.Mutations) != 1 { + t.Fatalf("expected one bucket metadata write, got %+v", filer.lastReq) + } + var stored s3_pb.BucketMetadata + if err := proto.Unmarshal(filer.lastReq.Mutations[0].Content, &stored); err != nil { + t.Fatalf("decode stored bucket metadata: %v", err) + } + if len(stored.Tags) != tc.tags { + t.Fatalf("stored %d tags, want %d", len(stored.Tags), tc.tags) + } + }) + } +} diff --git a/weed/s3api/s3api_embedded_iam.go b/weed/s3api/s3api_embedded_iam.go index 4b837117f4..1c6aa1bec0 100644 --- a/weed/s3api/s3api_embedded_iam.go +++ b/weed/s3api/s3api_embedded_iam.go @@ -1291,9 +1291,9 @@ func (e *EmbeddedIamApi) GetUserPolicy(s3cfg *iam_pb.S3ApiConfiguration, values ctx := context.Background() storedDoc, storeErr := e.credentialManager.GetUserInlinePolicy(ctx, userName, policyName) if storeErr != nil { - glog.Warningf("GetUserPolicy: failed to read stored inline policy %q for user %s: %v; falling back to reconstruction", policyName, userName, storeErr) + return resp, &iamError{Code: iam.ErrCodeServiceFailureException, Error: fmt.Errorf("get inline policy %s for %s: %w", policyName, userName, storeErr)} } - if storeErr == nil && storedDoc != nil { + if storedDoc != nil { policyDocumentJSON, err := json.Marshal(storedDoc) if err != nil { return resp, &iamError{Code: iam.ErrCodeServiceFailureException, Error: err} @@ -1301,6 +1301,14 @@ func (e *EmbeddedIamApi) GetUserPolicy(s3cfg *iam_pb.S3ApiConfiguration, values resp.GetUserPolicyResult.PolicyDocument = string(policyDocumentJSON) return resp, nil } + // The user has stored inline policies, but none by this name + policyNames, err := e.credentialManager.ListUserInlinePolicies(ctx, userName) + if err != nil { + return resp, &iamError{Code: iam.ErrCodeServiceFailureException, Error: fmt.Errorf("list inline policies for %s: %w", userName, err)} + } + if len(policyNames) > 0 { + return resp, &iamError{Code: iam.ErrCodeNoSuchEntityException, Error: fmt.Errorf("the user policy with name %s cannot be found", policyName)} + } } // Fallback: reconstruct from ident.Actions (lossy - fine-grained actions diff --git a/weed/s3api/s3api_embedded_iam_test.go b/weed/s3api/s3api_embedded_iam_test.go index 2e7f9206af..4269958de9 100644 --- a/weed/s3api/s3api_embedded_iam_test.go +++ b/weed/s3api/s3api_embedded_iam_test.go @@ -1979,6 +1979,56 @@ func TestEmbeddedIamGetUserPolicyFallback(t *testing.T) { assert.ElementsMatch(t, []string{"arn:aws:s3:::b-le*", "arn:aws:s3:::b-le*/*"}, stmt.Resource.Strings()) } +// TestEmbeddedIamGetUserPolicyMissingName checks that a policy name the user does not have +// is NoSuchEntity, not a policy rebuilt from the user's other inline policies. +func TestEmbeddedIamGetUserPolicyMissingName(t *testing.T) { + api := NewEmbeddedIamApiForTest() + s3cfg := &iam_pb.S3ApiConfiguration{ + Identities: []*iam_pb.Identity{{Name: "steward"}}, + } + api.mockConfig = s3cfg + + _, iamErr := api.PutUserPolicy(s3cfg, url.Values{ + "UserName": []string{"steward"}, + "PolicyName": []string{"read"}, + "PolicyDocument": []string{`{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Action":"s3:GetObject","Resource":"arn:aws:s3:::b/*"}]}`}, + }) + require.Nil(t, iamErr) + + _, iamErr = api.GetUserPolicy(s3cfg, url.Values{ + "UserName": []string{"steward"}, + "PolicyName": []string{"write"}, + }) + require.NotNil(t, iamErr) + assert.Equal(t, iam.ErrCodeNoSuchEntityException, iamErr.Code) +} + +// TestEmbeddedIamGetUserPolicyStoreError checks that a store failure surfaces as +// ServiceFailure instead of rebuilding a policy from the identity's actions. +func TestEmbeddedIamGetUserPolicyStoreError(t *testing.T) { + api := NewEmbeddedIamApiForTest() + s3cfg := &iam_pb.S3ApiConfiguration{ + Identities: []*iam_pb.Identity{{Name: "steward"}}, + } + api.mockConfig = s3cfg + + _, iamErr := api.PutUserPolicy(s3cfg, url.Values{ + "UserName": []string{"steward"}, + "PolicyName": []string{"read"}, + "PolicyDocument": []string{`{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Action":"s3:GetObject","Resource":"arn:aws:s3:::b/*"}]}`}, + }) + require.Nil(t, iamErr) + + api.credentialManager = &credential.CredentialManager{Store: &memory.MemoryStore{}} + + _, iamErr = api.GetUserPolicy(s3cfg, url.Values{ + "UserName": []string{"steward"}, + "PolicyName": []string{"read"}, + }) + require.NotNil(t, iamErr) + assert.Equal(t, iam.ErrCodeServiceFailureException, iamErr.Code) +} + // TestEmbeddedIamGetUserPolicyUserNotFound tests GetUserPolicy with non-existent user func TestEmbeddedIamGetUserPolicyUserNotFound(t *testing.T) { api := NewEmbeddedIamApiForTest() diff --git a/weed/s3api/s3api_iceberg_credentials.go b/weed/s3api/s3api_iceberg_credentials.go index 50af7cdb99..967a5a79df 100644 --- a/weed/s3api/s3api_iceberg_credentials.go +++ b/weed/s3api/s3api_iceberg_credentials.go @@ -40,8 +40,11 @@ type VendedCredentials struct { // // The access key the caller authenticated with (from ctx) is recorded in the // session: a short hash of it ends the session name, so the audit log's -// requester_arn tells apart the holders of several keys on one identity, and -// the key id itself is carried in the session token. +// requester_arn tells apart the holders of several keys on one identity. When +// the caller is a stored identity holding that key, the key id is also carried +// in the session token, binding the session to it: the session ends when the +// key is deleted or deactivated, or its identity removed or disabled +// (validateSTSSessionToken). func (s3a *S3ApiServer) VendTableCredentials(ctx context.Context, principal, bucket, prefix string, readOnly bool) (*VendedCredentials, error) { if s3a.icebergCredentialRole == "" { return nil, nil @@ -72,6 +75,19 @@ func (s3a *S3ApiServer) VendTableCredentials(ctx context.Context, principal, buc Policy: &policy, SourceAccessKeyId: accessKey, } + // Bind the session to the caller's key when the caller authenticated with a + // stored identity's key — a marker its verifier records, so a caller that + // signed with an STS session key or OIDC token stays unbound even when its + // principal names a stored identity, whose session a binding would refuse + // on every request. The marker survives the key being revoked between + // authentication and this vend: the session then dies on its first request + // instead of living unbound until it expires. The binding travels under + // its own claim — SourceAccessKeyId is audit-only for compatibility with + // tokens minted before the binding existed. + if storedKey, storedSecret := s3_constants.StoredCredentialFromContext(ctx); storedKey != "" { + request.BoundAccessKeyId = storedKey + request.BoundAccessKeyFingerprint = s3_constants.StoredCredentialFingerprint(s3a.iam.stsSigningKey(), storedKey, storedSecret) + } if s3a.icebergCredentialDuration > 0 { duration := s3a.icebergCredentialDuration request.DurationSeconds = &duration @@ -94,10 +110,17 @@ func (s3a *S3ApiServer) VendTableCredentials(ctx context.Context, principal, buc } func (s3a *S3ApiServer) stsService() *sts.STSService { - if s3a.iam == nil || s3a.iam.iamIntegration == nil { + if s3a.iam == nil { return nil } - provider, ok := s3a.iam.iamIntegration.(IAMManagerProvider) + return s3a.iam.stsService() +} + +func (iam *IdentityAccessManagement) stsService() *sts.STSService { + if iam.iamIntegration == nil { + return nil + } + provider, ok := iam.iamIntegration.(IAMManagerProvider) if !ok { return nil } @@ -108,6 +131,13 @@ func (s3a *S3ApiServer) stsService() *sts.STSService { return manager.GetSTSService() } +func (iam *IdentityAccessManagement) stsSigningKey() []byte { + if s := iam.stsService(); s != nil && s.Config != nil { + return s.Config.SigningKey + } + return nil +} + // tablePrefixSessionPolicy narrows a session to one table's files plus the // bucket listing that clients need to resolve them. A read-only session drops // the write verbs: the catalog vends no more authority than the caller holds. diff --git a/weed/s3api/s3api_iceberg_session_name_test.go b/weed/s3api/s3api_iceberg_session_name_test.go index 9a615bef8d..2fd679d24d 100644 --- a/weed/s3api/s3api_iceberg_session_name_test.go +++ b/weed/s3api/s3api_iceberg_session_name_test.go @@ -76,12 +76,14 @@ func TestVendTableCredentialsRecordsTheCallersKey(t *testing.T) { s3a := &S3ApiServer{iam: iam} s3a.SetIcebergCredentialRole("arn:aws:iam::role/IcebergTableAccess", 3600) - for _, key := range []string{sharedKeyA, sharedKeyB, ""} { + for _, k := range []struct{ id, secret string }{{sharedKeyA, sharedSecretA}, {sharedKeyB, sharedSecretB}, {"", ""}} { + key := k.id ctx := context.Background() if key != "" { // As the catalog's auth does: the holder is installed, the verifier records into it. r := s3_constants.EnsureIdentityHolder(httptest.NewRequest(http.MethodGet, "/", nil)) s3_constants.RecordAccessKeyId(r.Context(), key) + s3_constants.RecordStoredAccessKeyId(r.Context(), k.id, k.secret) ctx = r.Context() } creds, err := s3a.VendTableCredentials(ctx, "shared-user", "warehouse", "ns/t", false) diff --git a/weed/s3api/s3api_iceberg_vended_session_test.go b/weed/s3api/s3api_iceberg_vended_session_test.go new file mode 100644 index 0000000000..9b9398cf0f --- /dev/null +++ b/weed/s3api/s3api_iceberg_vended_session_test.go @@ -0,0 +1,234 @@ +package s3api + +import ( + "context" + "fmt" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/seaweedfs/seaweedfs/weed/iam/integration" + "github.com/seaweedfs/seaweedfs/weed/iam/policy" + "github.com/seaweedfs/seaweedfs/weed/iam/sts" + "github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants" + "github.com/seaweedfs/seaweedfs/weed/s3api/s3err" +) + +// vendingServer is an S3ApiServer whose catalog role trusts the vendor, with +// the identity of newSharedIdentityIAM (one identity, two keys). +func vendingServer(t *testing.T) *S3ApiServer { + t.Helper() + mgr := newTestIAMManager(t) + require.NoError(t, mgr.CreateRole(context.Background(), "", "IcebergTableAccess", &integration.RoleDefinition{ + RoleName: "IcebergTableAccess", + TrustPolicy: &policy.PolicyDocument{ + Version: "2012-10-17", + Statement: []policy.Statement{{ + Effect: "Allow", + Principal: map[string]interface{}{"Federated": icebergVendedCredentialsProvider}, + Action: []string{"sts:AssumeRoleWithCredentials"}, + }}, + }, + })) + iam := newSharedIdentityIAM() + iam.SetIAMIntegration(NewS3IAMIntegration(mgr, "")) + s3a := &S3ApiServer{iam: iam} + s3a.SetIcebergCredentialRole("arn:aws:iam::role/IcebergTableAccess", 3600) + return s3a +} + +// vend mints table credentials for principal as if its request had been +// signed with accessKey ("" for a request that recorded none); a non-empty +// secret marks the credential as a stored identity's, as a verifier would. +func vend(t *testing.T, s3a *S3ApiServer, principal, accessKey, secret string) *VendedCredentials { + t.Helper() + ctx := context.Background() + if accessKey != "" { + ctx = s3_constants.SetAccessKeyIdInContext(ctx, accessKey) + } + if secret != "" { + ctx = s3_constants.SetStoredAccessKeyIdInContext(ctx, accessKey, secret) + } + creds, err := s3a.VendTableCredentials(ctx, principal, "warehouse", "ns/t", false) + require.NoError(t, err) + require.NotNil(t, creds) + return creds +} + +// use validates a vended session the way a data-path request carrying it is +// validated. +func use(t *testing.T, s3a *S3ApiServer, creds *VendedCredentials) s3err.ErrorCode { + t.Helper() + r, err := newTestRequest("GET", "https://example.com/warehouse/ns/t/data/f.parquet", 0, nil) + require.NoError(t, err) + _, _, errCode := s3a.iam.validateSTSSessionToken(r, creds.SessionToken, creds.AccessKeyID) + return errCode +} + +func sessionInfo(t *testing.T, s3a *S3ApiServer, creds *VendedCredentials) *sts.SessionInfo { + t.Helper() + info, err := s3a.stsService().ValidateSessionToken(context.Background(), creds.SessionToken) + require.NoError(t, err) + return info +} + +func TestVendedSessionEndsWithItsAccessKey(t *testing.T) { + s3a := vendingServer(t) + sessionA := vend(t, s3a, "shared-user", sharedKeyA, sharedSecretA) + sessionB := vend(t, s3a, "shared-user", sharedKeyB, sharedSecretB) + require.Equal(t, s3err.ErrNone, use(t, s3a, sessionA), "control: the session works while its key does") + require.Equal(t, s3err.ErrNone, use(t, s3a, sessionB)) + + identity := s3a.iam.nameToIdentity["shared-user"] + identity.Credentials[0].Status = iamAccessKeyStatusInactive + assert.Equal(t, s3err.ErrInvalidAccessKeyID, use(t, s3a, sessionA), "key A deactivated") + assert.Equal(t, s3err.ErrNone, use(t, s3a, sessionB), "the session vended through key B keeps working") + + identity.Credentials[0].Status = iamAccessKeyStatusActive + assert.Equal(t, s3err.ErrNone, use(t, s3a, sessionA), "key A reactivated") + + identity.Credentials[0].Expiration = time.Now().Add(-time.Minute).Unix() + assert.Equal(t, s3err.ErrInvalidAccessKeyID, use(t, s3a, sessionA), "key A expired") + identity.Credentials[0].Expiration = 0 + + identity.Credentials = identity.Credentials[1:] + assert.Equal(t, s3err.ErrInvalidAccessKeyID, use(t, s3a, sessionA), "key A deleted") + assert.Equal(t, s3err.ErrNone, use(t, s3a, sessionB), "the session vended through key B keeps working") +} + +func TestVendedSessionEndsWithItsIdentity(t *testing.T) { + s3a := vendingServer(t) + session := vend(t, s3a, "shared-user", sharedKeyA, sharedSecretA) + require.Equal(t, s3err.ErrNone, use(t, s3a, session)) + + s3a.iam.nameToIdentity["shared-user"].Disabled = true + assert.Equal(t, s3err.ErrInvalidAccessKeyID, use(t, s3a, session), "identity disabled") + s3a.iam.nameToIdentity["shared-user"].Disabled = false + assert.Equal(t, s3err.ErrNone, use(t, s3a, session), "identity enabled again") + + delete(s3a.iam.nameToIdentity, "shared-user") + assert.Equal(t, s3err.ErrInvalidAccessKeyID, use(t, s3a, session), "identity deleted") +} + +// A caller that is not a stored identity (a caller signing with another STS +// session's key, an OIDC user, a request that recorded no key) still gets +// credentials, unbound as before: binding them would refuse every request +// made with them. +func TestVendedSessionIsBoundOnlyForAStoredIdentitysKey(t *testing.T) { + s3a := vendingServer(t) + control := sessionInfo(t, s3a, vend(t, s3a, "shared-user", sharedKeyA, sharedSecretA)) + assert.Equal(t, sharedKeyA, control.SourceAccessKeyId, "control") + assert.Equal(t, sharedKeyA, control.BoundAccessKeyId, "control") + + for _, tc := range []struct{ principal, key, why string }{ + {"47ad4828c45b3f337bc3146081ba8f0f", "ASIATEMPORARYKEY0001", "an STS session's subject and temporary key"}, + {"shared-user", "ASIATEMPORARYKEY0001", "an STS session whose subject names a stored identity"}, + {"alice@example.com", "", "an OIDC user"}, + {"shared-user", "", "no recorded key"}, + } { + session := vend(t, s3a, tc.principal, tc.key, "") + info := sessionInfo(t, s3a, session) + assert.Equal(t, tc.key, info.SourceAccessKeyId, tc.why) + assert.Empty(t, info.BoundAccessKeyId, tc.why) + assert.Equal(t, s3err.ErrNone, use(t, s3a, session), tc.why) + } +} + +// Tokens minted before the binding existed record the caller's key under the +// audit claim only. Validating them as bound would end live table credentials +// at upgrade — a chained STS caller's key can never resolve to a stored +// credential. They stay valid until expiry, as they always did. +func TestLegacyVendedSessionStaysUnbound(t *testing.T) { + iam := newSharedIdentityIAM() + iam.iamIntegration = &MockIAMIntegration{ + validateSessionFunc: func(ctx context.Context, token string) (*sts.SessionInfo, error) { + return &sts.SessionInfo{ + SessionId: "sid", + AssumedRoleUser: "arn:aws:sts::assumed-role/Reader/session", + Principal: "arn:aws:sts::assumed-role/Reader/session", + IdentityProvider: icebergVendedCredentialsProvider, + ExternalUserId: "someone-not-in-the-identity-table", + SourceAccessKeyId: "ASIATEMPORARYKEY0001", + Credentials: &sts.Credentials{AccessKeyId: "ASIALEGACY", SecretAccessKey: "secret"}, + ExpiresAt: time.Now().Add(time.Hour), + }, nil + }, + } + r, err := newTestRequest("GET", "https://example.com/bucket/f.parquet", 0, nil) + require.NoError(t, err) + _, _, errCode := iam.validateSTSSessionToken(r, "token", "ASIALEGACY") + assert.Equal(t, s3err.ErrNone, errCode) +} + +// A key re-created under the same id with a different secret does not revive +// sessions bound to the deleted original: the fingerprint recorded at mint +// time no longer matches. +func TestRecreatedKeyDoesNotReviveVendedSession(t *testing.T) { + s3a := vendingServer(t) + session := vend(t, s3a, "shared-user", sharedKeyA, sharedSecretA) + require.Equal(t, s3err.ErrNone, use(t, s3a, session)) + + identity := s3a.iam.nameToIdentity["shared-user"] + identity.Credentials[0] = &Credential{AccessKey: sharedKeyA, SecretKey: "recreated-secret"} + assert.Equal(t, s3err.ErrInvalidAccessKeyID, use(t, s3a, session), "same id, different secret") + + identity.Credentials[0] = &Credential{AccessKey: sharedKeyA, SecretKey: sharedSecretA} + assert.Equal(t, s3err.ErrNone, use(t, s3a, session), "the identical credential is indistinguishable") +} + +// A session for a stored identity stays bound even when the key no longer +// resolves at vend time — revoked between the request's authentication and +// the vend, or held by a different identity now. The session dies on its +// first request instead of living unbound until it expires. +func TestVendedSessionStaysBoundWhenTheKeyIsGoneAtVend(t *testing.T) { + s3a := vendingServer(t) + + identity := s3a.iam.nameToIdentity["shared-user"] + identity.Credentials = identity.Credentials[1:] // sharedKeyA revoked mid-request + session := vend(t, s3a, "shared-user", sharedKeyA, sharedSecretA) + assert.Equal(t, sharedKeyA, sessionInfo(t, s3a, session).SourceAccessKeyId, "still bound") + assert.Equal(t, s3err.ErrInvalidAccessKeyID, use(t, s3a, session), "the dead key's session is refused") +} + +// Only sessions the catalog vended are bound; any other STS session (web +// identity, AssumeRole) is validated as before. +func TestOtherSessionsAreNotBoundToAKey(t *testing.T) { + iam := newSharedIdentityIAM() + iam.iamIntegration = &MockIAMIntegration{ + validateSessionFunc: func(ctx context.Context, token string) (*sts.SessionInfo, error) { + return &sts.SessionInfo{ + SessionId: "sid", + AssumedRoleUser: "arn:aws:sts::assumed-role/Reader/session", + Principal: "arn:aws:sts::assumed-role/Reader/session", + IdentityProvider: "oidc-provider", + ExternalUserId: "someone-not-in-the-identity-table", + SourceAccessKeyId: "AKIAGONE", + Credentials: &sts.Credentials{AccessKeyId: "ASIAWEBID", SecretAccessKey: "secret"}, + ExpiresAt: time.Now().Add(time.Hour), + }, nil + }, + } + r, err := newTestRequest("GET", "https://example.com/bucket/f.parquet", 0, nil) + require.NoError(t, err) + _, _, errCode := iam.validateSTSSessionToken(r, "token", "ASIAWEBID") + assert.Equal(t, s3err.ErrNone, errCode) +} + +// The check on the request path is one read-locked map lookup and a scan of +// one identity's keys. +func BenchmarkVendedSessionKeyLive(b *testing.B) { + iam := newSharedIdentityIAM() + for i := 0; i < 1000; i++ { + name := fmt.Sprintf("user-%04d", i) + iam.nameToIdentity[name] = &Identity{Name: name, Credentials: []*Credential{{AccessKey: name}}} + } + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + if iam.vendedSessionKeyLive("shared-user", sharedKeyB, "") != s3err.ErrNone { + b.Fatal("refused") + } + } +} diff --git a/weed/s3api/s3api_object_handlers_copy.go b/weed/s3api/s3api_object_handlers_copy.go index 2473ed77cc..555dcaf696 100644 --- a/weed/s3api/s3api_object_handlers_copy.go +++ b/weed/s3api/s3api_object_handlers_copy.go @@ -70,6 +70,14 @@ func isValidDirective(value string) bool { return value == "" || value == DirectiveCopy || value == DirectiveReplace } +// copyChangesStorageClass reports whether a copy request asks for a storage +// class other than the source object's. That makes copying an object onto +// itself a change AWS accepts, even with the COPY directive. +func (s3a *S3ApiServer) copyChangesStorageClass(h http.Header, src *filer_pb.Entry) bool { + sc := h.Get(s3_constants.AmzStorageClass) + return sc != "" && sc != s3a.getStorageClassFromExtended(src.Extended) +} + // hasPrefixFold reports whether s starts with prefix, ignoring case. func hasPrefixFold(s, prefix string) bool { return len(s) >= len(prefix) && strings.EqualFold(s[:len(prefix)], prefix) @@ -196,6 +204,11 @@ func (s3a *S3ApiServer) CopyObjectHandler(w http.ResponseWriter, r *http.Request return } + if sc := r.Header.Get(s3_constants.AmzStorageClass); sc != "" && !validateStorageClass(sc) { + s3err.WriteErrorResponse(w, r, s3err.ErrInvalidStorageClass) + return + } + replaceMeta, replaceTagging := replaceDirective(r.Header) // If source object is empty or bucket is empty, reply back invalid copy source. @@ -266,9 +279,10 @@ func (s3a *S3ApiServer) CopyObjectHandler(w http.ResponseWriter, r *http.Request } sameDestination := srcBucket == dstBucket && srcObject == dstObject + changesStorageClass := s3a.copyChangesStorageClass(r.Header, entry) // A self-copy into a versioned bucket writes a new version instead of overwriting in // place, so it is not the no-op AWS rejects. It is how an earlier version is restored. - if sameDestination && !(replaceMeta || replaceTagging) && srcVersioningState != s3_constants.VersioningEnabled { + if sameDestination && !(replaceMeta || replaceTagging || changesStorageClass) && srcVersioningState != s3_constants.VersioningEnabled { s3err.WriteErrorResponse(w, r, s3err.ErrInvalidCopyDest) return } @@ -300,7 +314,16 @@ func (s3a *S3ApiServer) CopyObjectHandler(w http.ResponseWriter, r *http.Request replacesSource := copyReplacesSourceEntry(sameDestination, dstVersioningState, srcVersionId) - if replacesSource && (replaceMeta || replaceTagging) && s3a.canUseMetadataOnlySelfCopy(entry, r, dstBucket, dstObject) { + // A self-copy that only changes the storage class updates the object in + // place and has to keep its encryption as stored. The direct strategy + // compares only KMS key IDs, so SSE-KMS requests stay rejected too. + if replacesSource && changesStorageClass && !(replaceMeta || replaceTagging) && + (IsSSEKMSRequest(r) || !s3a.canUseMetadataOnlySelfCopy(entry, r, dstBucket, dstObject)) { + s3err.WriteErrorResponse(w, r, s3err.ErrInvalidCopyDest) + return + } + + if replacesSource && (replaceMeta || replaceTagging || changesStorageClass) && s3a.canUseMetadataOnlySelfCopy(entry, r, dstBucket, dstObject) { var dstVersionId string var etag string // Bind the entire self-copy update to the snapshot read on each attempt. diff --git a/weed/s3api/s3api_object_handlers_copy_storage_class_test.go b/weed/s3api/s3api_object_handlers_copy_storage_class_test.go new file mode 100644 index 0000000000..e176fe5daa --- /dev/null +++ b/weed/s3api/s3api_object_handlers_copy_storage_class_test.go @@ -0,0 +1,111 @@ +package s3api + +import ( + "encoding/base64" + "net/http" + "net/http/httptest" + "strings" + "testing" + "time" + + "github.com/gorilla/mux" + "github.com/seaweedfs/seaweedfs/weed/pb/filer_pb" + "github.com/seaweedfs/seaweedfs/weed/pb/s3_pb" + "github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants" + "google.golang.org/protobuf/proto" +) + +// AWS changes an object's storage class by copying the object onto itself +// with the COPY directive and a new x-amz-storage-class. Only a self-copy that +// changes nothing is rejected, and a bucket's default encryption is no change. +// A self-copy that would also change the encryption stays rejected. +func TestCopyObjectToItselfChangingStorageClass(t *testing.T) { + const bucket, object = "self-copy", "object" + kmsObject := map[string]string{ + s3_constants.AmzServerSideEncryption: s3_constants.SSEAlgorithmKMS, + s3_constants.AmzServerSideEncryptionAwsKmsKeyId: "key-a", + s3_constants.AmzServerSideEncryptionContext: base64.StdEncoding.EncodeToString([]byte(`{"env":"old"}`)), + } + for _, tt := range []struct { + name string + stored map[string]string // extra object metadata + storageClass string + headers map[string]string // extra request headers + defaultEncryption bool + wantCode int + wantErr string + }{ + {name: "no change", wantCode: http.StatusBadRequest, wantErr: "InvalidRequest"}, + {name: "no change under default encryption", defaultEncryption: true, wantCode: http.StatusBadRequest, wantErr: "InvalidRequest"}, + {name: "invalid storage class", storageClass: "NOT_A_CLASS", wantCode: http.StatusBadRequest, wantErr: "InvalidStorageClass"}, + {name: "new storage class", storageClass: "STANDARD_IA", wantCode: http.StatusOK}, + {name: "current storage class", stored: map[string]string{s3_constants.AmzStorageClass: "STANDARD_IA"}, + storageClass: "STANDARD_IA", wantCode: http.StatusBadRequest, wantErr: "InvalidRequest"}, + {name: "STANDARD without a stored class", storageClass: "STANDARD", wantCode: http.StatusBadRequest, wantErr: "InvalidRequest"}, + {name: "encryption only", headers: map[string]string{s3_constants.AmzServerSideEncryption: "AES256"}, + wantCode: http.StatusBadRequest, wantErr: "InvalidRequest"}, + {name: "new storage class and SSE-KMS context", stored: kmsObject, storageClass: "STANDARD_IA", headers: map[string]string{ + s3_constants.AmzServerSideEncryption: s3_constants.SSEAlgorithmKMS, + s3_constants.AmzServerSideEncryptionAwsKmsKeyId: "key-a", + s3_constants.AmzServerSideEncryptionContext: base64.StdEncoding.EncodeToString([]byte(`{"env":"new"}`)), + }, wantCode: http.StatusBadRequest, wantErr: "InvalidRequest"}, + } { + t.Run(tt.name, func(t *testing.T) { + filer := ©ACLFiler{ambiguousPutFiler: &ambiguousPutFiler{apply: true, entries: map[string]*filer_pb.Entry{}}} + s3a := newPutTestServer(t, startFakeFiler(t, filer)) + s3a.iam = &IdentityAccessManagement{} + s3a.bucketConfigCache = NewBucketConfigCache(time.Minute) + config := &BucketConfig{Name: bucket, Ownership: s3_constants.OwnershipObjectWriter} + if tt.defaultEncryption { + config.Encryption = &s3_pb.EncryptionConfiguration{SseAlgorithm: s3_constants.SSEAlgorithmAES256} + } + s3a.bucketConfigCache.Set(bucket, config) + objectPath := "/buckets/" + bucket + "/" + object + original := &filer_pb.Entry{Name: object, Content: []byte("data"), + Attributes: &filer_pb.FuseAttributes{FileMode: defaultFileMode, FileSize: 4, Mime: "text/plain"}, + Extended: map[string][]byte{"X-Amz-Meta-Owner": []byte("kept")}} + for k, v := range tt.stored { + original.Extended[k] = []byte(v) + } + filer.entries[objectPath] = proto.Clone(original).(*filer_pb.Entry) + + req := httptest.NewRequest(http.MethodPut, "/"+bucket+"/"+object, nil) + req = mux.SetURLVars(req, map[string]string{"bucket": bucket, "object": object}) + req.Header.Set("X-Amz-Copy-Source", "/"+bucket+"/"+object) + req.Header.Set(s3_constants.AmzUserMetaDirective, DirectiveCopy) + if tt.storageClass != "" { + req.Header.Set(s3_constants.AmzStorageClass, tt.storageClass) + } + for k, v := range tt.headers { + req.Header.Set(k, v) + } + rr := httptest.NewRecorder() + s3a.CopyObjectHandler(rr, req) + + if rr.Code != tt.wantCode { + t.Fatalf("status = %d, want %d; body: %s", rr.Code, tt.wantCode, rr.Body.String()) + } + filer.mu.Lock() + defer filer.mu.Unlock() + stored := filer.entries[objectPath] + if tt.wantCode != http.StatusOK { + if !strings.Contains(rr.Body.String(), ""+tt.wantErr+"") { + t.Fatalf("body = %s, want %s", rr.Body.String(), tt.wantErr) + } + if !proto.Equal(stored, original) { + t.Fatalf("rejected copy changed the object: %v", stored) + } + return + } + if got := string(stored.Extended[s3_constants.AmzStorageClass]); got != tt.storageClass { + t.Fatalf("storage class = %q, want %q", got, tt.storageClass) + } + if got := string(stored.Extended["X-Amz-Meta-Owner"]); got != "kept" { + t.Fatalf("user metadata = %q, want it kept under the COPY directive", got) + } + if got := string(stored.Content); got != "data" { + t.Fatalf("content = %q, want %q", got, "data") + } + }) + } +} diff --git a/weed/s3api/s3api_object_handlers_list.go b/weed/s3api/s3api_object_handlers_list.go index 6b049e0e70..76614932ce 100644 --- a/weed/s3api/s3api_object_handlers_list.go +++ b/weed/s3api/s3api_object_handlers_list.go @@ -2,6 +2,7 @@ package s3api import ( "context" + "encoding/base64" "encoding/xml" "errors" "fmt" @@ -11,7 +12,9 @@ import ( "sort" "strconv" "strings" + "unicode/utf8" + "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/service/s3" "github.com/seaweedfs/seaweedfs/weed/glog" "github.com/seaweedfs/seaweedfs/weed/pb/filer_pb" @@ -70,8 +73,8 @@ func toListBucketResultV1(in ListBucketResult) listBucketResultV1 { Metadata: in.Metadata, Name: in.Name, Prefix: in.Prefix, - Marker: in.Marker, - NextMarker: in.NextMarker, + Marker: encodeMarkerEcho(in.Marker, in.EncodingType == s3.EncodingTypeUrl), + NextMarker: encodeMarkerEcho(in.NextMarker, in.EncodingType == s3.EncodingTypeUrl), MaxKeys: in.MaxKeys, Delimiter: in.Delimiter, IsTruncated: in.IsTruncated, @@ -104,9 +107,10 @@ func (s3a *S3ApiServer) ListObjectsV2Handler(w http.ResponseWriter, r *http.Requ return } - marker := continuationToken.string + startAfterKey := decodeMarkerEcho(startAfter, encodingTypeUrl) + marker := decodeListContinuationToken(continuationToken.string) if !continuationToken.set { - marker = startAfter + marker = startAfterKey } // Adjust marker if it ends with delimiter to skip all entries with that prefix @@ -145,27 +149,139 @@ func (s3a *S3ApiServer) ListObjectsV2Handler(w http.ResponseWriter, r *http.Requ IsTruncated: response.IsTruncated, KeyCount: len(response.Contents) + len(response.CommonPrefixes), MaxKeys: uint16(response.MaxKeys), - NextContinuationToken: response.NextMarker, + NextContinuationToken: encodeListContinuationToken(response.NextMarker), Prefix: response.Prefix, StartAfter: startAfter, } if encodingTypeUrl { responseV2.EncodingType = s3.EncodingTypeUrl + responseV2.StartAfter = urlPathEscape(startAfterKey) } glog.V(3).Infof("ListObjectsV2Handler response: %+v", responseV2) writeSuccessResponseXML(w, r, responseV2) } +// continuationTokenEncodingPrefix tags a NextContinuationToken whose payload is the +// base64url encoding of the raw key. XML 1.0 cannot carry control characters, and the +// XML encoder replaces them with U+FFFD, so a token holding such a key would resume +// past a mangled name and skip every key sorting between the two. Keys that are valid +// XML text still travel as the bare key, so the token is unchanged for them. +const continuationTokenEncodingPrefix = "base64:" + +func encodeListContinuationToken(key string) string { + if key == "" { + return "" + } + if isXMLText(key) && !strings.HasPrefix(key, continuationTokenEncodingPrefix) { + return key + } + return continuationTokenEncodingPrefix + base64.RawURLEncoding.EncodeToString([]byte(key)) +} + +func encodeListContinuationTokenField(value *string) *string { + if value == nil { + return nil + } + return aws.String(encodeListContinuationToken(*value)) +} + +// encodeMarkerEcho renders a decoded request marker for the response echo: +// the token encoding keeps an XML-unsafe key legible, and urlPathEscape applies +// the encoding-type=url form the keys in the same response already carry. +func encodeMarkerEcho(marker string, encodingTypeUrl bool) string { + marker = encodeListContinuationToken(marker) + if encodingTypeUrl { + marker = urlPathEscape(marker) + } + return marker +} + +// decodeMarkerEcho reverses the wire encoding on a marker the client echoes +// back: under encoding-type=url the emitted echo was urlPathEscape'd. Only a +// value that is the canonical escaped form of its unescape could have come +// from that encoder — "a%2541" round-trips to itself, "a%41" does not — so +// anything else stays the raw key the client sent. +func decodeMarkerEcho(marker string, encodingTypeUrl bool) string { + return decodeEscapedEcho(marker, encodingTypeUrl, urlPathEscape) +} + +func decodeEscapedEcho(marker string, encodingTypeUrl bool, escape func(string) string) string { + if !encodingTypeUrl { + return marker + } + if unescaped, err := url.QueryUnescape(marker); err == nil && escape(unescaped) == marker { + return unescaped + } + return marker +} + +func decodeListContinuationToken(token string) string { + key, ok := strings.CutPrefix(token, continuationTokenEncodingPrefix) + if !ok { + return token + } + decoded, err := base64.RawURLEncoding.DecodeString(key) + if err != nil { + // Not a token this server wrote; resume at the token as the raw key. + return token + } + decodedKey := string(decoded) + // encodeListContinuationToken only writes a token for a key that cannot + // travel bare, so a decoded key that could have stayed bare — valid XML + // text without the prefix — means the input was a legacy raw-key token, + // not something to decode. + if isXMLText(decodedKey) && !strings.HasPrefix(decodedKey, continuationTokenEncodingPrefix) { + return token + } + // A decoded key that could not travel bare makes the token ambiguous + // ("base64:AQ" reads as both the encoded form of "\x01" and a literal + // legacy key). The encoded reading wins: preferring the raw one would + // resume past real entries for every token this server writes, while a + // legacy token aliasing the encoded form only re-lists keys. + return decodedKey +} + +// isXMLText reports whether s survives an XML 1.0 round trip unchanged, i.e. the +// encoder does not replace any of its characters with U+FFFD. +func isXMLText(s string) bool { + for i := 0; i < len(s); { + r, size := utf8.DecodeRuneInString(s[i:]) + if r == utf8.RuneError && size == 1 { + return false + } + if !isXMLChar(r) { + return false + } + i += size + } + return true +} + +// isXMLChar follows the XML 1.0 Char production the encoding/xml encoder enforces. +func isXMLChar(r rune) bool { + switch { + case r == 0x9 || r == 0xA || r == 0xD: + return true + case r >= 0x20 && r <= 0xD7FF: + return true + case r >= 0xE000 && r <= 0xFFFD: + return true + case r >= 0x10000 && r <= 0x10FFFF: + return true + } + return false +} + func (s3a *S3ApiServer) ListObjectsV1Handler(w http.ResponseWriter, r *http.Request) { // https://docs.aws.amazon.com/AmazonS3/latest/API/API_ListObjects.html // collect parameters bucket, _ := s3_constants.GetBucketAndObject(r) - originalPrefix, marker, delimiter, encodingTypeUrl, maxKeys, allowUnordered, errCode := getListObjectsV1Args(r.URL.Query()) + originalPrefix, listMarker, delimiter, encodingTypeUrl, maxKeys, allowUnordered, errCode := getListObjectsV1Args(r.URL.Query()) - glog.V(2).Infof("ListObjectsV1Handler bucket=%s prefix=%s marker=%s delimiter=%s maxKeys=%d", bucket, originalPrefix, marker, delimiter, maxKeys) + glog.V(2).Infof("ListObjectsV1Handler bucket=%s prefix=%s marker=%s delimiter=%s maxKeys=%d", bucket, originalPrefix, listMarker, delimiter, maxKeys) if errCode != s3err.ErrNone { s3err.WriteErrorResponse(w, r, errCode) @@ -181,8 +297,8 @@ func (s3a *S3ApiServer) ListObjectsV1Handler(w http.ResponseWriter, r *http.Requ } // Adjust marker if it ends with delimiter to skip all entries with that prefix - requestMarker := marker - marker = adjustMarkerForDelimiter(marker, originalPrefix, delimiter) + requestMarker := decodeListContinuationToken(decodeMarkerEcho(listMarker, encodingTypeUrl)) + marker := adjustMarkerForDelimiter(requestMarker, originalPrefix, delimiter) response, err := s3a.listFilerEntries(r.Context(), listObjectsRequest{ bucket: bucket, diff --git a/weed/s3api/s3api_object_handlers_list_token_test.go b/weed/s3api/s3api_object_handlers_list_token_test.go new file mode 100644 index 0000000000..75ffbdcf9f --- /dev/null +++ b/weed/s3api/s3api_object_handlers_list_token_test.go @@ -0,0 +1,328 @@ +package s3api + +import ( + "encoding/xml" + "net/http" + "net/http/httptest" + "net/url" + "strconv" + "testing" + "time" + + "github.com/gorilla/mux" + "github.com/seaweedfs/seaweedfs/weed/pb/filer_pb" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +type listV2TestPage struct { + XMLName xml.Name `xml:"ListBucketResult"` + IsTruncated bool `xml:"IsTruncated"` + NextContinuationToken string `xml:"NextContinuationToken"` + StartAfter string `xml:"StartAfter"` + Contents []struct { + Key string `xml:"Key"` + } `xml:"Contents"` +} + +func listTokenTestFile(name string) *filer_pb.Entry { + return &filer_pb.Entry{Name: name, Attributes: &filer_pb.FuseAttributes{}} +} + +func newListTokenTestServer(t *testing.T, keys []string) *S3ApiServer { + t.Helper() + entries := make([]*filer_pb.Entry, 0, len(keys)) + for _, key := range keys { + entries = append(entries, listTokenTestFile(key)) + } + filer := &fakeBucketDeleteFiler{entriesByDir: map[string][]*filer_pb.Entry{ + "/buckets/test": entries, + }} + s3a := newFailoverTestServer(t, startFakeFiler(t, filer)) + s3a.option.BucketsPath = "/buckets" + s3a.bucketConfigCache = NewBucketConfigCache(time.Minute) + s3a.bucketConfigCache.Set("test", &BucketConfig{Name: "test"}) + return s3a +} + +func listV2Page(t *testing.T, s3a *S3ApiServer, query url.Values) listV2TestPage { + t.Helper() + r := httptest.NewRequest(http.MethodGet, "/test?"+query.Encode(), nil) + r = mux.SetURLVars(r, map[string]string{"bucket": "test"}) + w := httptest.NewRecorder() + s3a.ListObjectsV2Handler(w, r) + require.Equal(t, http.StatusOK, w.Code, w.Body.String()) + var page listV2TestPage + require.NoError(t, xml.Unmarshal(w.Body.Bytes(), &page), w.Body.String()) + return page +} + +// A page that ends on a key XML 1.0 cannot carry must still resume right after +// that key. The token used to be the raw key, which the XML encoder turned into +// U+FFFD, so the next page resumed past the mangled name and skipped every key +// sorting between it and the real one. +func TestListObjectsV2ContinuationTokenWithXMLUnsafeKeys(t *testing.T) { + keys := []string{"a\x01b", "a\x02b", "a\x1fb", "a b", "b", "c"} + + for _, encodingTypeURL := range []bool{false, true} { + t.Run("encoding-type=url:"+strconv.FormatBool(encodingTypeURL), func(t *testing.T) { + s3a := newListTokenTestServer(t, keys) + + var got []string + token := "" + for page := 0; ; page++ { + require.Less(t, page, len(keys), "listing did not terminate") + query := url.Values{"list-type": {"2"}, "max-keys": {"2"}} + if encodingTypeURL { + query.Set("encoding-type", "url") + } + if token != "" { + query.Set("continuation-token", token) + } + result := listV2Page(t, s3a, query) + for _, content := range result.Contents { + key := content.Key + if encodingTypeURL { + decoded, err := url.QueryUnescape(key) + require.NoError(t, err) + key = decoded + } + got = append(got, key) + } + if !result.IsTruncated { + break + } + require.NotEmpty(t, result.NextContinuationToken) + token = result.NextContinuationToken + } + + if encodingTypeURL { + assert.Equal(t, keys, got) + return + } + // Without encoding-type=url the keys themselves still lose control + // characters to the XML encoder, but no key may be skipped. + require.Len(t, got, len(keys)) + assert.Equal(t, "c", got[len(got)-1]) + }) + } +} + +// The StartAfter echo is the wire encoding of the decoded key, so a client +// that feeds the echo back lands on the same resume point instead of +// drifting to the literal escape text. +func TestListObjectsV2StartAfterEchoStable(t *testing.T) { + s3a := newListTokenTestServer(t, []string{"a b", "c"}) + for _, sent := range []string{"a%20b", "a b"} { + result := listV2Page(t, s3a, url.Values{ + "list-type": {"2"}, "encoding-type": {"url"}, "start-after": {sent}, + }) + assert.Equal(t, "a%20b", result.StartAfter, "start-after %q", sent) + require.Len(t, result.Contents, 1) + assert.Equal(t, "c", result.Contents[0].Key) + } +} + +// A token for a key that XML carries unchanged stays the bare key. +func TestListObjectsV2ContinuationTokenUnchangedForNormalKeys(t *testing.T) { + keys := []string{"a\x01b", "a\x02b", "a\x1fb", "a b", "b", "c"} + s3a := newListTokenTestServer(t, keys) + + result := listV2Page(t, s3a, url.Values{"list-type": {"2"}, "max-keys": {"5"}}) + require.True(t, result.IsTruncated) + assert.Equal(t, "b", result.NextContinuationToken) +} + +func TestListObjectsV2ContinuationTokenRoundTrip(t *testing.T) { + for _, key := range []string{"a\x01b", "a\x1fb", "a\xffb", "base64:literal", "dir/file", "a b", "b"} { + token := encodeListContinuationToken(key) + assert.Equal(t, key, decodeListContinuationToken(token), "token %q", token) + } + for _, key := range []string{"b", "dir/file", "a b"} { + assert.Equal(t, key, encodeListContinuationToken(key)) + } + // Tokens written before the encoding existed are still bare keys. + assert.Equal(t, "dir/file", decodeListContinuationToken("dir/file")) +} + +func TestDecodeListContinuationTokenKeepsForeignTokensRaw(t *testing.T) { + // A prefix-looking token that does not decode is a key, not a token. + assert.Equal(t, "base64:%%%", decodeListContinuationToken("base64:%%%")) + // An empty payload is never something encodeListContinuationToken wrote. + assert.Equal(t, "base64:", decodeListContinuationToken("base64:")) + // A legacy bare token is a raw key even when its suffix is valid base64: + // the decoded key could have traveled bare, so the input never was an + // encoded token. + for _, legacy := range []string{"base64:Yw", "base64:YQ", "base64:cGFy", "base64:a2V5LXdpdGg"} { + assert.Equal(t, legacy, decodeListContinuationToken(legacy), "token %q", legacy) + } +} + +// Some tokens are ambiguous: "base64:AQ" is both a possible legacy bare key +// and the encoded form of "\x01". The encoded reading wins because a decoded +// key that cannot travel bare is a shape only encodeListContinuationToken +// writes — and a wrong raw reading skips entries, while a wrong encoded +// reading only re-lists them. +func TestDecodeListContinuationTokenResolvesCollisionEncoded(t *testing.T) { + assert.Equal(t, "\x01", decodeListContinuationToken("base64:AQ")) + assert.Equal(t, "\xff", decodeListContinuationToken("base64:_w")) + assert.Equal(t, "base64:AQ", decodeListContinuationToken("base64:YmFzZTY0OkFR")) +} + +type listV1TestPage struct { + XMLName xml.Name `xml:"ListBucketResult"` + IsTruncated bool `xml:"IsTruncated"` + Marker string `xml:"Marker"` + NextMarker string `xml:"NextMarker"` + Contents []struct { + Key string `xml:"Key"` + } `xml:"Contents"` +} + +func listV1Page(t *testing.T, s3a *S3ApiServer, query url.Values) listV1TestPage { + t.Helper() + r := httptest.NewRequest(http.MethodGet, "/test?"+query.Encode(), nil) + r = mux.SetURLVars(r, map[string]string{"bucket": "test"}) + w := httptest.NewRecorder() + s3a.ListObjectsV1Handler(w, r) + require.Equal(t, http.StatusOK, w.Code, w.Body.String()) + var page listV1TestPage + require.NoError(t, xml.Unmarshal(w.Body.Bytes(), &page), w.Body.String()) + return page +} + +// The V1 marker carries the same resume position as the V2 token, so it gets +// the same treatment: a page ending on an XML-unsafe key must still resume +// right after it. +func TestListObjectsV1MarkerWithXMLUnsafeKeys(t *testing.T) { + keys := []string{"a\x01b", "a\x02b", "a\x1fb", "a b", "b", "c"} + s3a := newListTokenTestServer(t, keys) + + var got []string + marker := "" + for page := 0; ; page++ { + require.Less(t, page, len(keys), "listing did not terminate") + query := url.Values{"max-keys": {"2"}, "encoding-type": {"url"}} + if marker != "" { + query.Set("marker", marker) + } + result := listV1Page(t, s3a, query) + for _, content := range result.Contents { + key, err := url.QueryUnescape(content.Key) + require.NoError(t, err) + got = append(got, key) + } + if !result.IsTruncated { + break + } + require.NotEmpty(t, result.NextMarker) + marker = result.NextMarker + } + require.Equal(t, keys, got) +} + +// A url-encoded marker is only a wire echo when re-escaping its unescaped +// form reproduces it: a real echo decodes, a raw key merely containing '%' +// or a malformed escape passes through untouched. +func TestDecodeMarkerEchoKeepsNonCanonicalRawKey(t *testing.T) { + assert.Equal(t, "a b", decodeMarkerEcho("a%20b", true)) + assert.Equal(t, "a%41", decodeMarkerEcho("a%2541", true)) + assert.Equal(t, "a%41", decodeMarkerEcho("a%41", true)) + assert.Equal(t, "a%2G", decodeMarkerEcho("a%2G", true)) + assert.Equal(t, "a%20b", decodeMarkerEcho("a%20b", false)) +} + +// Under encoding-type=url AWS encodes the StartAfter echo the same as the +// listed keys; a client resuming from the emitted form lands on the raw key. +func TestListObjectsV2UrlStartAfterEchoAndResume(t *testing.T) { + s3a := newListTokenTestServer(t, []string{"a b", "a b0", "a c", "b"}) + + result := listV2Page(t, s3a, url.Values{"list-type": {"2"}, "encoding-type": {"url"}, "start-after": {"a b"}}) + assert.Equal(t, urlPathEscape("a b"), result.StartAfter) + + var got []string + for _, content := range result.Contents { + key, err := url.QueryUnescape(content.Key) + require.NoError(t, err) + got = append(got, key) + } + assert.Equal(t, []string{"a b0", "a c", "b"}, got) + + // Echoing the emitted form back resumes at the same raw key. + result = listV2Page(t, s3a, url.Values{"list-type": {"2"}, "encoding-type": {"url"}, "start-after": {result.StartAfter}}) + got = nil + for _, content := range result.Contents { + key, err := url.QueryUnescape(content.Key) + require.NoError(t, err) + got = append(got, key) + } + assert.Equal(t, []string{"a b0", "a c", "b"}, got) +} + +// A url-encoded marker that is not the canonical urlPathEscape form of any key +// is the client's raw key, not an encoded echo: "a%41" must resume after that +// literal key instead of decoding to "aA" and skipping the keys between them. +func TestListObjectsV1UrlMarkerKeepsNonCanonicalRawKey(t *testing.T) { + s3a := newListTokenTestServer(t, []string{"a%41", "a%42", "aA", "b"}) + + result := listV1Page(t, s3a, url.Values{"encoding-type": {"url"}, "marker": {"a%41"}}) + var got []string + for _, content := range result.Contents { + key, err := url.QueryUnescape(content.Key) + require.NoError(t, err) + got = append(got, key) + } + assert.Equal(t, []string{"a%42", "aA", "b"}, got) +} + +// The echoed request marker carries the client's token, not the decoded key: +// echoing the raw key would mangle an XML-unsafe name to U+FFFD, and under +// encoding-type=url the echo takes the same escaped form as the listed keys. +func TestListObjectsV1MarkerEchoStaysXMLSafe(t *testing.T) { + s3a := newListTokenTestServer(t, []string{"a\x01b", "b", "c"}) + + token := encodeListContinuationToken("a\x01b") + result := listV1Page(t, s3a, url.Values{"max-keys": {"2"}, "marker": {token}}) + assert.Equal(t, token, result.Marker) + + result = listV1Page(t, s3a, url.Values{"max-keys": {"2"}, "marker": {"a b"}, "encoding-type": {"url"}}) + assert.Equal(t, urlPathEscape("a b"), result.Marker) +} + +// A raw-key token resuming at an XML-unsafe prefix must not lose keys when a +// sibling directory shares the key's leading bytes: the start window begins at +// the marker's first path segment and descends only into blocks that can still +// hold keys past it. +func TestListObjectsV2ResumeXMLUnsafePrefixWithSiblingDir(t *testing.T) { + filer := &fakeBucketDeleteFiler{entriesByDir: map[string][]*filer_pb.Entry{ + "/buckets/test/x": { + {Name: "a\x01", IsDirectory: true, Attributes: &filer_pb.FuseAttributes{}}, + listTokenTestFile("a\x01b"), + {Name: "a\x01b", IsDirectory: true, Attributes: &filer_pb.FuseAttributes{}}, + listTokenTestFile("a\x01b0"), + }, + "/buckets/test/x/a\x01": { + listTokenTestFile("b"), + }, + "/buckets/test/x/a\x01b": { + listTokenTestFile("0"), + }, + }} + s3a := newFailoverTestServer(t, startFakeFiler(t, filer)) + s3a.option.BucketsPath = "/buckets" + s3a.bucketConfigCache = NewBucketConfigCache(time.Minute) + s3a.bucketConfigCache.Set("test", &BucketConfig{Name: "test"}) + + result := listV2Page(t, s3a, url.Values{ + "list-type": {"2"}, + "encoding-type": {"url"}, + "prefix": {"x/a\x01b"}, + "max-keys": {"1"}, + "continuation-token": {"x/a\x01b"}, + }) + require.Len(t, result.Contents, 1) + got, err := url.QueryUnescape(result.Contents[0].Key) + require.NoError(t, err) + assert.Equal(t, "x/a\x01b/0", got) + assert.True(t, result.IsTruncated) +} diff --git a/weed/s3api/s3api_object_handlers_multipart.go b/weed/s3api/s3api_object_handlers_multipart.go index 9a98da6627..ddb349bdeb 100644 --- a/weed/s3api/s3api_object_handlers_multipart.go +++ b/weed/s3api/s3api_object_handlers_multipart.go @@ -260,11 +260,17 @@ func (s3a *S3ApiServer) ListMultipartUploadsHandler(w http.ResponseWriter, r *ht return } - prefix, keyMarker, uploadIDMarker, delimiter, maxUploads, encodingType := getBucketMultipartResources(r.URL.Query()) + prefix, multipartKeyMarker, uploadIDMarker, delimiter, maxUploads, encodingType := getBucketMultipartResources(r.URL.Query()) if maxUploads < 0 { s3err.WriteErrorResponse(w, r, s3err.ErrInvalidMaxUploads) return } + // The decode mirrors the response encoding: under encoding-type=url the + // markers go out url-escaped, otherwise they carry the token form. + keyMarker := decodeListContinuationToken(multipartKeyMarker) + if encodingType == s3.EncodingTypeUrl { + keyMarker = decodeEscapedEcho(multipartKeyMarker, true, urlQueryEscape) + } if keyMarker != "" { // Marker not common with prefix is not implemented. if !strings.HasPrefix(keyMarker, prefix) { @@ -290,8 +296,6 @@ func (s3a *S3ApiServer) ListMultipartUploadsHandler(w http.ResponseWriter, r *ht return } - // TODO handle encodingType - writeSuccessResponseXML(w, r, response) } diff --git a/weed/s3api/s3api_object_handlers_postpolicy.go b/weed/s3api/s3api_object_handlers_postpolicy.go index 4fb604abbf..1e8705ba11 100644 --- a/weed/s3api/s3api_object_handlers_postpolicy.go +++ b/weed/s3api/s3api_object_handlers_postpolicy.go @@ -144,6 +144,14 @@ func (s3a *S3ApiServer) PostPolicyBucketHandler(w http.ResponseWriter, r *http.R s3err.WriteErrorResponse(w, r, refineDenialForBucketName(r, identity, errCode)) return } + // An acl or x-amz-grant-* form field rides onto the upload as ACL headers, + // which the PUT path only applies under s3:PutObjectAcl. + if postRequestsAcl(r) { + if errCode := s3a.iam.AuthorizeObjectAclWrite(r, identity, bucket, object); errCode != s3err.ErrNone { + s3err.WriteErrorResponse(w, r, refineDenialForBucketName(r, identity, errCode)) + return + } + } versioningState, err := s3a.getVersioningState(bucket) if err != nil { @@ -275,6 +283,17 @@ func applyPostPolicyFormHeaders(r *http.Request, formValues http.Header) { } } +// postRequestsAcl reports whether the request carries ACL headers, whether +// promoted from the POST form fields or supplied on the request itself. +func postRequestsAcl(r *http.Request) bool { + for _, header := range []string{s3_constants.AmzCannedAcl, s3_constants.AmzAclFullControl, s3_constants.AmzAclRead, s3_constants.AmzAclReadAcp, s3_constants.AmzAclWrite, s3_constants.AmzAclWriteAcp} { + if r.Header.Get(header) != "" { + return true + } + } + return false +} + // Extract form fields and file data from a HTTP POST Policy func extractPostPolicyFormValues(form *multipart.Form) (filePart io.ReadCloser, fileName, fileContentType string, fileSize int64, formValues http.Header, err error) { // / HTML Form values diff --git a/weed/s3api/s3api_object_handlers_postpolicy_authz_test.go b/weed/s3api/s3api_object_handlers_postpolicy_authz_test.go index 1dc6365071..553ff2f0a5 100644 --- a/weed/s3api/s3api_object_handlers_postpolicy_authz_test.go +++ b/weed/s3api/s3api_object_handlers_postpolicy_authz_test.go @@ -3,10 +3,12 @@ package s3api import ( "bytes" "encoding/base64" + "encoding/json" "fmt" "mime/multipart" "net/http" "net/http/httptest" + "sort" "sync" "testing" "time" @@ -26,6 +28,10 @@ const ( postAuthzService = "s3" postAuthzBucket = "testbucket" postAuthzPrincipal = "arn:aws:iam::000000000000:user/tester" + + postAclAccessKey = "AKIAACLTESTTEST" + postAclSecretKey = "secret-key-for-acl-tests" + postAclPrincipal = "arn:aws:iam::000000000000:user/aclwriter" ) // newPostAuthzServer builds an S3ApiServer whose non-admin "tester" identity @@ -45,6 +51,11 @@ func newPostAuthzServer(t *testing.T) *S3ApiServer { Account: &iam_pb.Account{Id: "000000000000", DisplayName: "tester"}, Credentials: []*iam_pb.Credential{{AccessKey: postAuthzAccessKey, SecretKey: postAuthzSecretKey}}, Actions: []string{"Read:" + postAuthzBucket, "Write:" + postAuthzBucket, "List:" + postAuthzBucket}, + }, { + Name: "aclwriter", + Account: &iam_pb.Account{Id: "000000000000", DisplayName: "aclwriter"}, + Credentials: []*iam_pb.Credential{{AccessKey: postAclAccessKey, SecretKey: postAclSecretKey}}, + Actions: []string{"Read:" + postAuthzBucket, "Write:" + postAuthzBucket, "WriteAcp:" + postAuthzBucket}, }}, }) require.NoError(t, err) @@ -58,8 +69,14 @@ func newPostAuthzServer(t *testing.T) *S3ApiServer { "Principal": {"AWS": "%s"}, "Action": ["s3:PutObject"], "Resource": ["arn:aws:s3:::%s/denied/*"] + }, { + "Sid": "DenyAclWriterAclPrefix", + "Effect": "Deny", + "Principal": {"AWS": "%s"}, + "Action": ["s3:PutObjectAcl"], + "Resource": ["arn:aws:s3:::%s/acl-denied/*"] }] - }`, postAuthzPrincipal, postAuthzBucket)) + }`, postAuthzPrincipal, postAuthzBucket, postAclPrincipal, postAuthzBucket)) require.NoError(t, err) iam.policyEngine = policyEngine @@ -79,26 +96,42 @@ func newPostAuthzServer(t *testing.T) *S3ApiServer { // policy conditions are satisfied by the form, so the handler passes signature // and CheckPostPolicy verification and reaches the write-authorization stage. func newSignedPostRequest(t *testing.T, key string) *http.Request { + return newSignedPostRequestFor(t, key, postAuthzAccessKey, postAuthzSecretKey, nil) +} + +// extraFields are added to both the form and the policy conditions so +// CheckPostPolicy accepts them. +func newSignedPostRequestFor(t *testing.T, key, accessKey, secretKey string, extraFields map[string]string) *http.Request { t.Helper() now := time.Now().UTC() amzDate := now.Format(iso8601Format) yyyymmddStr := now.Format(yyyymmdd) - credential := fmt.Sprintf("%s/%s/%s/%s/aws4_request", postAuthzAccessKey, yyyymmddStr, postAuthzRegion, postAuthzService) + credential := fmt.Sprintf("%s/%s/%s/%s/aws4_request", accessKey, yyyymmddStr, postAuthzRegion, postAuthzService) expiration := now.Add(1 * time.Hour).Format("2006-01-02T15:04:05.000Z") - policyJSON := fmt.Sprintf( - `{"expiration":"%s","conditions":[`+ - `["eq","$bucket","%s"],`+ - `["eq","$key","%s"],`+ - `["eq","$x-amz-credential","%s"],`+ - `["eq","$x-amz-algorithm","AWS4-HMAC-SHA256"],`+ - `["eq","$x-amz-date","%s"]`+ - `]}`, - expiration, postAuthzBucket, key, credential, amzDate, - ) - encodedPolicy := base64.StdEncoding.EncodeToString([]byte(policyJSON)) - signature := getSignature(getSigningKey(postAuthzSecretKey, yyyymmddStr, postAuthzRegion, postAuthzService), encodedPolicy) + extraKeys := make([]string, 0, len(extraFields)) + for k := range extraFields { + extraKeys = append(extraKeys, k) + } + sort.Strings(extraKeys) + conditions := []any{ + []any{"eq", "$bucket", postAuthzBucket}, + []any{"eq", "$key", key}, + []any{"eq", "$x-amz-credential", credential}, + []any{"eq", "$x-amz-algorithm", "AWS4-HMAC-SHA256"}, + []any{"eq", "$x-amz-date", amzDate}, + } + for _, k := range extraKeys { + conditions = append(conditions, []any{"eq", "$" + k, extraFields[k]}) + } + policyJSON, err := json.Marshal(map[string]any{ + "expiration": expiration, + "conditions": conditions, + }) + require.NoError(t, err) + encodedPolicy := base64.StdEncoding.EncodeToString(policyJSON) + signature := getSignature(getSigningKey(secretKey, yyyymmddStr, postAuthzRegion, postAuthzService), encodedPolicy) var buf bytes.Buffer writer := multipart.NewWriter(&buf) @@ -107,6 +140,9 @@ func newSignedPostRequest(t *testing.T, key string) *http.Request { require.NoError(t, writer.WriteField("x-amz-credential", credential)) require.NoError(t, writer.WriteField("x-amz-algorithm", "AWS4-HMAC-SHA256")) require.NoError(t, writer.WriteField("x-amz-date", amzDate)) + for _, k := range extraKeys { + require.NoError(t, writer.WriteField(k, extraFields[k])) + } require.NoError(t, writer.WriteField("policy", encodedPolicy)) require.NoError(t, writer.WriteField("x-amz-signature", signature)) filePart, err := writer.CreateFormFile("file", "payload.txt") @@ -170,3 +206,71 @@ func TestAuthorizeObjectWrite(t *testing.T) { s3a.iam.AuthorizeObjectWrite(req, admin, postAuthzBucket, "denied/secret.txt"), "an admin is not confined by the bucket policy") } + +// TestPostPolicyBucketHandlerGatesAclFields drives signed POST uploads carrying +// an acl form field and asserts the handler rejects them the way the PUT path +// does: a writer without the WriteAcp action is refused, and so is a writer the +// bucket policy denies s3:PutObjectAcl on the target key. +func TestPostPolicyBucketHandlerGatesAclFields(t *testing.T) { + s3a := newPostAuthzServer(t) + + rec := httptest.NewRecorder() + s3a.PostPolicyBucketHandler(rec, newSignedPostRequestFor(t, "allowed/report.txt", + postAuthzAccessKey, postAuthzSecretKey, map[string]string{"acl": "public-read"})) + assert.Equal(t, http.StatusForbidden, rec.Code, + "POST with an acl field needs WriteAcp like PUT, body: %s", rec.Body.String()) + + rec = httptest.NewRecorder() + s3a.PostPolicyBucketHandler(rec, newSignedPostRequestFor(t, "acl-denied/secret.txt", + postAclAccessKey, postAclSecretKey, map[string]string{"acl": "public-read"})) + assert.Equal(t, http.StatusForbidden, rec.Code, + "POST with an acl field must honor the bucket-policy PutObjectAcl deny, body: %s", rec.Body.String()) + + for _, field := range []string{"x-amz-grant-read", "x-amz-grant-write", "x-amz-grant-read-acp", "x-amz-grant-write-acp", "x-amz-grant-full-control"} { + rec = httptest.NewRecorder() + s3a.PostPolicyBucketHandler(rec, newSignedPostRequestFor(t, "allowed/report.txt", + postAuthzAccessKey, postAuthzSecretKey, + map[string]string{field: `id="0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef"`})) + assert.Equal(t, http.StatusForbidden, rec.Code, + "POST with an %s field needs WriteAcp like PUT, body: %s", field, rec.Body.String()) + } +} + +func TestPostRequestsAcl(t *testing.T) { + for _, header := range []string{ + s3_constants.AmzCannedAcl, s3_constants.AmzAclFullControl, s3_constants.AmzAclRead, + s3_constants.AmzAclReadAcp, s3_constants.AmzAclWrite, s3_constants.AmzAclWriteAcp, + } { + req := httptest.NewRequest(http.MethodPost, "/"+postAuthzBucket, nil) + req.Header.Set(header, "x") + assert.True(t, postRequestsAcl(req), "%s must trip the ACL gate", header) + } + assert.False(t, postRequestsAcl(httptest.NewRequest(http.MethodPost, "/"+postAuthzBucket, nil))) +} + +// TestAuthorizeObjectAclWrite covers the acl authorization decision at the +// boundary, for the same reason as TestAuthorizeObjectWrite. +func TestAuthorizeObjectAclWrite(t *testing.T) { + s3a := newPostAuthzServer(t) + tester, _, found := s3a.iam.lookupByAccessKey(postAuthzAccessKey) + require.True(t, found) + aclwriter, _, found := s3a.iam.lookupByAccessKey(postAclAccessKey) + require.True(t, found) + + req := httptest.NewRequest(http.MethodPost, "/"+postAuthzBucket, nil) + + assert.Equal(t, s3err.ErrAccessDenied, + s3a.iam.AuthorizeObjectAclWrite(req, tester, postAuthzBucket, "allowed/report.txt"), + "a writer without the WriteAcp action cannot attach an ACL") + assert.Equal(t, s3err.ErrNone, + s3a.iam.AuthorizeObjectAclWrite(req, aclwriter, postAuthzBucket, "allowed/report.txt"), + "the WriteAcp holder clears keys the policy does not deny") + assert.Equal(t, s3err.ErrAccessDenied, + s3a.iam.AuthorizeObjectAclWrite(req, aclwriter, postAuthzBucket, "acl-denied/secret.txt"), + "a bucket-policy PutObjectAcl deny must block the ACL-bearing upload") + + admin := &Identity{Name: "admin", Actions: []Action{s3_constants.ACTION_ADMIN}} + assert.Equal(t, s3err.ErrNone, + s3a.iam.AuthorizeObjectAclWrite(req, admin, postAuthzBucket, "acl-denied/secret.txt"), + "an admin is not confined by the bucket policy") +} diff --git a/weed/s3api/s3api_object_handlers_put.go b/weed/s3api/s3api_object_handlers_put.go index b38a205f16..aa6682712a 100644 --- a/weed/s3api/s3api_object_handlers_put.go +++ b/weed/s3api/s3api_object_handlers_put.go @@ -527,6 +527,22 @@ func (s3a *S3ApiServer) putToFiler(r *http.Request, filePath string, dataReader if !s3_constants.IsValidBucketName(bucket) || (object != "" && !s3_constants.IsValidObjectKey(object)) { return "", s3err.ErrInvalidRequest, SSEResponseMetadata{} } + + // Reject invalid tags before any chunks are uploaded so a failed request + // leaves nothing behind on the volume servers. + var tagSet map[string]string + if tagging := r.Header.Get(s3_constants.AmzObjectTagging); tagging != "" { + parsed, err := parseTagsHeader(tagging) + if err != nil { + glog.Warningf("putToFiler: invalid tagging header for %s: %v", filePath, err) + return "", s3err.ErrInvalidTag, SSEResponseMetadata{} + } + if err := ValidateTags(parsed); err != nil { + glog.Warningf("putToFiler: invalid tags (%d) for %s: %v", len(parsed), filePath, err) + return "", s3err.ErrInvalidTag, SSEResponseMetadata{} + } + tagSet = parsed + } // NEW OPTIMIZATION: Write directly to volume servers, bypassing filer proxy // This eliminates the filer proxy overhead for PUT operations // Note: filePath is now passed directly instead of URL (no parsing needed) @@ -938,26 +954,12 @@ func (s3a *S3ApiServer) putToFiler(r *http.Request, filePath string, dataReader entry.Extended[s3_constants.AmzStorageClass] = []byte(sc) } - // Parse and store object tags from X-Amz-Tagging header - // Fix for GitHub issue #7589: Tags sent during object upload were not being stored - if tagging := r.Header.Get(s3_constants.AmzObjectTagging); tagging != "" { - parsedTags, err := url.ParseQuery(tagging) - if err != nil { - glog.Warningf("putToFiler: Invalid S3 tag format in header '%s': %v", tagging, err) - return "", s3err.ErrInvalidTag, SSEResponseMetadata{} - } - for key, values := range parsedTags { - if len(values) > 1 { - glog.Warningf("putToFiler: Duplicate tag key '%s' in header", key) - return "", s3err.ErrInvalidTag, SSEResponseMetadata{} - } - value := "" - if len(values) > 0 { - value = values[0] - } - entry.Extended[s3_constants.AmzObjectTagging+"-"+key] = []byte(value) - } - glog.V(3).Infof("putToFiler: stored %d tags from X-Amz-Tagging header", len(parsedTags)) + // Store the tags validated before the chunk upload + for key, value := range tagSet { + entry.Extended[s3_constants.AmzObjectTagging+"-"+key] = []byte(value) + } + if len(tagSet) > 0 { + glog.V(3).Infof("putToFiler: stored %d tags from X-Amz-Tagging header", len(tagSet)) } storeSSEMetadata(entry, sseResult) diff --git a/weed/s3api/s3api_object_handlers_tagging.go b/weed/s3api/s3api_object_handlers_tagging.go index 466d8bcfd4..c55b38c09c 100644 --- a/weed/s3api/s3api_object_handlers_tagging.go +++ b/weed/s3api/s3api_object_handlers_tagging.go @@ -139,6 +139,12 @@ func (s3a *S3ApiServer) PutObjectTaggingHandler(w http.ResponseWriter, r *http.R return } tags := tagging.ToTags() + // A key may appear only once in a tag set; the map keeps one of them. + if len(tags) != len(tagging.TagSet.Tag) { + glog.Errorf("PutObjectTaggingHandler duplicate tag keys %s", r.URL) + s3err.WriteErrorResponse(w, r, s3err.ErrInvalidTag) + return + } err = ValidateTags(tags) if err != nil { glog.Errorf("PutObjectTaggingHandler ValidateTags error %s: %v", r.URL, err) diff --git a/weed/s3api/s3api_object_handlers_tagging_test.go b/weed/s3api/s3api_object_handlers_tagging_test.go new file mode 100644 index 0000000000..05dec5a833 --- /dev/null +++ b/weed/s3api/s3api_object_handlers_tagging_test.go @@ -0,0 +1,29 @@ +package s3api + +import ( + "net/http" + "net/http/httptest" + "strings" + "testing" + + "github.com/gorilla/mux" +) + +func TestPutObjectTaggingRejectsDuplicateKeys(t *testing.T) { + s3a := newHeadBucketTestServer(t, &fakeTxnFiler{}) + body := "" + + "a1" + + "a2" + + "" + req := httptest.NewRequest(http.MethodPut, "/b/o?tagging", strings.NewReader(body)) + req = mux.SetURLVars(req, map[string]string{"bucket": "b", "object": "o"}) + rr := httptest.NewRecorder() + s3a.PutObjectTaggingHandler(rr, req) + + if rr.Code != http.StatusBadRequest { + t.Fatalf("status = %d, want %d; body: %s", rr.Code, http.StatusBadRequest, rr.Body.String()) + } + if !strings.Contains(rr.Body.String(), "InvalidTag") { + t.Fatalf("body = %s, want InvalidTag", rr.Body.String()) + } +} diff --git a/weed/s3api/s3api_object_retention.go b/weed/s3api/s3api_object_retention.go index 2d1e8f4ff2..53f92f3574 100644 --- a/weed/s3api/s3api_object_retention.go +++ b/weed/s3api/s3api_object_retention.go @@ -12,6 +12,7 @@ import ( "github.com/seaweedfs/seaweedfs/weed/pb/filer_pb" "github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants" "github.com/seaweedfs/seaweedfs/weed/s3api/s3err" + "github.com/seaweedfs/seaweedfs/weed/util" ) // ==================================================================== @@ -213,6 +214,49 @@ func (s3a *S3ApiServer) getObjectEntry(bucket, object, versionId string) (*filer return entry, nil } +// getObjectEntryForUpdate resolves the entry a metadata update targets and the +// filer path it is written under. It applies the same version resolution as +// getSpecificObjectVersion - "null" names the plain pre-versioning entry, not a +// .versions file - so the write lands on the entry that was read. +func (s3a *S3ApiServer) getObjectEntryForUpdate(bucket, object, versionId string) (entry *filer_pb.Entry, entryPath string, err error) { + entryPath = object + if versionId != "" { + entry, err = s3a.getSpecificObjectVersion(bucket, object, versionId) + if err != nil { + return nil, "", fmt.Errorf("failed to get version %s for object %s/%s: %w", versionId, bucket, object, ErrVersionNotFound) + } + if versionId != "null" { + entryPath = object + ".versions/" + s3a.getVersionFileName(versionId) + } + return entry, entryPath, nil + } + + versioningEnabled, vErr := s3a.isVersioningEnabled(bucket) + if vErr != nil { + return nil, "", fmt.Errorf("error checking versioning: %w", vErr) + } + if !versioningEnabled { + entry, err = s3a.fetchObjectEntryRequired(bucket, object) + if err != nil { + return nil, "", fmt.Errorf("failed to get object %s/%s: %w", bucket, object, ErrObjectNotFound) + } + return entry, entryPath, nil + } + + entry, err = s3a.getLatestObjectVersion(bucket, object) + if err != nil { + return nil, "", fmt.Errorf("failed to get latest version for object %s/%s: %w", bucket, object, ErrLatestVersionNotFound) + } + if entry.Extended != nil { + if versionIdBytes, exists := entry.Extended[s3_constants.ExtVersionIdKey]; exists { + if storedVersionId := string(versionIdBytes); storedVersionId != "null" && isValidVersionID(storedVersionId) { + entryPath = object + ".versions/" + s3a.getVersionFileName(storedVersionId) + } + } + } + return entry, entryPath, nil +} + // ==================================================================== // RETENTION OPERATIONS // ==================================================================== @@ -254,47 +298,9 @@ func (s3a *S3ApiServer) getObjectRetention(bucket, object, versionId string) (*O // setObjectRetention sets object retention configuration func (s3a *S3ApiServer) setObjectRetention(bucket, object, versionId string, retention *ObjectRetention, bypassGovernance bool) error { - var entry *filer_pb.Entry - var err error - var entryPath string - - if versionId != "" { - entry, err = s3a.getSpecificObjectVersion(bucket, object, versionId) - if err != nil { - return fmt.Errorf("failed to get version %s for object %s/%s: %w", versionId, bucket, object, ErrVersionNotFound) - } - entryPath = object + ".versions/" + s3a.getVersionFileName(versionId) - } else { - // Check if versioning is enabled - versioningEnabled, vErr := s3a.isVersioningEnabled(bucket) - if vErr != nil { - return fmt.Errorf("error checking versioning: %w", vErr) - } - - if versioningEnabled { - entry, err = s3a.getLatestObjectVersion(bucket, object) - if err != nil { - return fmt.Errorf("failed to get latest version for object %s/%s: %w", bucket, object, ErrLatestVersionNotFound) - } - // Extract version ID from entry metadata - entryPath = object // default to regular object path - if entry.Extended != nil { - if versionIdBytes, exists := entry.Extended[s3_constants.ExtVersionIdKey]; exists { - versionId = string(versionIdBytes) - // A stored version id must be a valid path segment before it can - // name a .versions file; otherwise fall back to the regular object. - if versionId != "null" && isValidVersionID(versionId) { - entryPath = object + ".versions/" + s3a.getVersionFileName(versionId) - } - } - } - } else { - entry, err = s3a.fetchObjectEntryRequired(bucket, object) - if err != nil { - return fmt.Errorf("failed to get object %s/%s: %w", bucket, object, ErrObjectNotFound) - } - entryPath = object - } + entry, entryPath, err := s3a.getObjectEntryForUpdate(bucket, object, versionId) + if err != nil { + return err } // Check if object is already under retention @@ -355,18 +361,15 @@ func (s3a *S3ApiServer) setObjectRetention(bucket, object, versionId string, ret entry.WormEnforcedAtTsNs = time.Now().UnixNano() } - // Update the entry + // Update the entry in place. UpdateEntry persists the whole fetched entry, so + // attributes, inline content, and remote references survive the metadata change + // where mkFile's fresh entry would drop them. // NOTE: Potential race condition exists if concurrent calls to PutObjectRetention // and PutObjectLegalHold update the same object simultaneously, as they might - // overwrite each other's Extended map changes. This is mitigated by the fact - // that mkFile operations are typically serialized at the filer level, but - // future implementations might consider using atomic update operations or - // entry-level locking for complete safety. - bucketDir := s3a.bucketDir(bucket) - return s3a.mkFile(bucketDir, entryPath, entry.Chunks, func(updatedEntry *filer_pb.Entry) { - updatedEntry.Extended = entry.Extended - updatedEntry.WormEnforcedAtTsNs = entry.WormEnforcedAtTsNs - }) + // overwrite each other's Extended map changes. + dir, name := util.NewFullPath(s3a.bucketDir(bucket), entryPath).DirAndName() + entry.Name = name + return s3a.updateEntry(dir, entry) } // ==================================================================== @@ -399,47 +402,9 @@ func (s3a *S3ApiServer) getObjectLegalHold(bucket, object, versionId string) (*O // setObjectLegalHold sets object legal hold configuration func (s3a *S3ApiServer) setObjectLegalHold(bucket, object, versionId string, legalHold *ObjectLegalHold) error { - var entry *filer_pb.Entry - var err error - var entryPath string - - if versionId != "" { - entry, err = s3a.getSpecificObjectVersion(bucket, object, versionId) - if err != nil { - return fmt.Errorf("failed to get version %s for object %s/%s: %w", versionId, bucket, object, ErrVersionNotFound) - } - entryPath = object + ".versions/" + s3a.getVersionFileName(versionId) - } else { - // Check if versioning is enabled - versioningEnabled, vErr := s3a.isVersioningEnabled(bucket) - if vErr != nil { - return fmt.Errorf("error checking versioning: %w", vErr) - } - - if versioningEnabled { - entry, err = s3a.getLatestObjectVersion(bucket, object) - if err != nil { - return fmt.Errorf("failed to get latest version for object %s/%s: %w", bucket, object, ErrLatestVersionNotFound) - } - // Extract version ID from entry metadata - entryPath = object // default to regular object path - if entry.Extended != nil { - if versionIdBytes, exists := entry.Extended[s3_constants.ExtVersionIdKey]; exists { - versionId = string(versionIdBytes) - // A stored version id must be a valid path segment before it can - // name a .versions file; otherwise fall back to the regular object. - if versionId != "null" && isValidVersionID(versionId) { - entryPath = object + ".versions/" + s3a.getVersionFileName(versionId) - } - } - } - } else { - entry, err = s3a.fetchObjectEntryRequired(bucket, object) - if err != nil { - return fmt.Errorf("failed to get object %s/%s: %w", bucket, object, ErrObjectNotFound) - } - entryPath = object - } + entry, entryPath, err := s3a.getObjectEntryForUpdate(bucket, object, versionId) + if err != nil { + return err } // Update legal hold metadata @@ -449,17 +414,9 @@ func (s3a *S3ApiServer) setObjectLegalHold(bucket, object, versionId string, leg entry.Extended[s3_constants.ExtLegalHoldKey] = []byte(legalHold.Status) - // Update the entry - // NOTE: Potential race condition exists if concurrent calls to PutObjectRetention - // and PutObjectLegalHold update the same object simultaneously, as they might - // overwrite each other's Extended map changes. This is mitigated by the fact - // that mkFile operations are typically serialized at the filer level, but - // future implementations might consider using atomic update operations or - // entry-level locking for complete safety. - bucketDir := s3a.bucketDir(bucket) - return s3a.mkFile(bucketDir, entryPath, entry.Chunks, func(updatedEntry *filer_pb.Entry) { - updatedEntry.Extended = entry.Extended - }) + dir, name := util.NewFullPath(s3a.bucketDir(bucket), entryPath).DirAndName() + entry.Name = name + return s3a.updateEntry(dir, entry) } // ==================================================================== diff --git a/weed/s3api/s3api_object_retention_test.go b/weed/s3api/s3api_object_retention_test.go index 44089deb33..201d471658 100644 --- a/weed/s3api/s3api_object_retention_test.go +++ b/weed/s3api/s3api_object_retention_test.go @@ -1,15 +1,23 @@ package s3api import ( + "context" "fmt" "io" "net/http" "strings" + "sync" "testing" "time" + "google.golang.org/grpc" + "google.golang.org/grpc/credentials/insecure" + "google.golang.org/protobuf/proto" + + "github.com/seaweedfs/seaweedfs/weed/pb" "github.com/seaweedfs/seaweedfs/weed/pb/filer_pb" "github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants" + "github.com/seaweedfs/seaweedfs/weed/wdclient" ) func TestValidateRetention(t *testing.T) { @@ -857,3 +865,95 @@ func TestStoreObjectLockConfigurationClearsStaleYears(t *testing.T) { func timePtr(t time.Time) *time.Time { return &t } + +// retentionTestFiler serves the two filer calls a lock write needs: the lookup +// that resolves the version and the update that persists the modified entry. +type retentionTestFiler struct { + filer_pb.UnimplementedSeaweedFilerServer + + mu sync.Mutex + entries map[string]*filer_pb.Entry + updates []string +} + +func (f *retentionTestFiler) LookupDirectoryEntry(_ context.Context, req *filer_pb.LookupDirectoryEntryRequest) (*filer_pb.LookupDirectoryEntryResponse, error) { + f.mu.Lock() + defer f.mu.Unlock() + if entry, ok := f.entries[req.Directory+"/"+req.Name]; ok { + return &filer_pb.LookupDirectoryEntryResponse{Entry: proto.Clone(entry).(*filer_pb.Entry)}, nil + } + return &filer_pb.LookupDirectoryEntryResponse{}, nil +} + +func (f *retentionTestFiler) UpdateEntry(_ context.Context, req *filer_pb.UpdateEntryRequest) (*filer_pb.UpdateEntryResponse, error) { + f.mu.Lock() + defer f.mu.Unlock() + f.updates = append(f.updates, req.Directory+"/"+req.Entry.Name) + f.entries[req.Directory+"/"+req.Entry.Name] = req.Entry + return &filer_pb.UpdateEntryResponse{}, nil +} + +func newRetentionTestServer(t *testing.T, f *retentionTestFiler) *S3ApiServer { + t.Helper() + addr := startFakeFiler(t, f) + dialOption := grpc.WithTransportCredentials(insecure.NewCredentials()) + return &S3ApiServer{ + option: &S3ApiServerOption{ + Filers: []pb.ServerAddress{addr}, + GrpcDialOption: dialOption, + BucketsPath: "/buckets", + }, + filerClient: wdclient.NewFilerClient([]pb.ServerAddress{addr}, dialOption, ""), + } +} + +// A "null" version id resolves to the pre-versioning entry under the plain +// object key, the same place getSpecificObjectVersion reads it from. Writing +// the lock under .versions/v_null leaves the real entry unlocked. +func TestNullVersionLockWritesToPlainEntry(t *testing.T) { + f := &retentionTestFiler{entries: map[string]*filer_pb.Entry{ + "/buckets/b/key": { + Name: "key", + Chunks: []*filer_pb.FileChunk{{FileId: "1,0000000000000001"}}, + Content: []byte("inline"), + Attributes: &filer_pb.FuseAttributes{ + Mime: "text/plain", + Md5: []byte("etagmd5"), + }, + Extended: map[string][]byte{ + "ETag": []byte("\"etagmd5\""), + }, + }, + }} + s3a := newRetentionTestServer(t, f) + + retainUntil := time.Now().Add(24 * time.Hour).Truncate(time.Second) + err := s3a.setObjectRetention("b", "key", "null", &ObjectRetention{ + Mode: s3_constants.RetentionModeCompliance, + RetainUntilDate: &retainUntil, + }, false) + if err != nil { + t.Fatalf("setObjectRetention: %v", err) + } + if len(f.updates) != 1 || f.updates[0] != "/buckets/b/key" { + t.Fatalf("retention write landed on %v, want /buckets/b/key", f.updates) + } + updated := f.entries["/buckets/b/key"] + if got := string(updated.Extended[s3_constants.ExtObjectLockModeKey]); got != s3_constants.RetentionModeCompliance { + t.Fatalf("plain entry lock mode = %q, want COMPLIANCE", got) + } + if updated.Attributes.Mime != "text/plain" || len(updated.Content) == 0 || len(updated.Chunks) != 1 || string(updated.Extended["ETag"]) != "\"etagmd5\"" { + t.Fatalf("lock write dropped entry fields: %+v", updated) + } + + f.updates = nil + if err := s3a.setObjectLegalHold("b", "key", "null", &ObjectLegalHold{Status: "ON"}); err != nil { + t.Fatalf("setObjectLegalHold: %v", err) + } + if len(f.updates) != 1 || f.updates[0] != "/buckets/b/key" { + t.Fatalf("legal hold write landed on %v, want /buckets/b/key", f.updates) + } + if got := string(f.entries["/buckets/b/key"].Extended[s3_constants.ExtLegalHoldKey]); got != "ON" { + t.Fatalf("plain entry legal hold = %q, want ON", got) + } +} diff --git a/weed/s3api/s3api_object_versioning.go b/weed/s3api/s3api_object_versioning.go index b4a7a4bed9..0e4cc950f3 100644 --- a/weed/s3api/s3api_object_versioning.go +++ b/weed/s3api/s3api_object_versioning.go @@ -448,12 +448,12 @@ func (s3a *S3ApiServer) splitIntoResult(combinedList []versionListItem, bucket, result := &S3ListObjectVersionsResult{ Name: bucket, Prefix: prefix, - KeyMarker: keyMarker, + KeyMarker: encodeListContinuationToken(keyMarker), VersionIdMarker: versionIdMarker, MaxKeys: maxKeys, Delimiter: delimiter, IsTruncated: isTruncated, - NextKeyMarker: nextKeyMarker, + NextKeyMarker: encodeListContinuationToken(nextKeyMarker), NextVersionIdMarker: nextVersionIdMarker, Entries: make([]VersionListEntry, 0, len(combinedList)), } @@ -1906,7 +1906,7 @@ func (s3a *S3ApiServer) ListObjectVersionsHandler(w http.ResponseWriter, r *http prefix := strings.TrimPrefix(originalPrefix, "/") // Note: prefix is used for filtering relative to bucket root, so no leading slash needed - keyMarker := query.Get("key-marker") + keyMarker := decodeListContinuationToken(query.Get("key-marker")) versionIdMarker := query.Get("version-id-marker") delimiter := query.Get("delimiter") diff --git a/weed/s3api/s3api_server.go b/weed/s3api/s3api_server.go index 86b1025081..dc9cb139a2 100644 --- a/weed/s3api/s3api_server.go +++ b/weed/s3api/s3api_server.go @@ -507,8 +507,12 @@ func NewS3ApiServerWithStore(router *mux.Router, option *S3ApiServerOption, expl } else { s3ApiServer.client = &http.Client{ Transport: &http.Transport{ - DialContext: func(_ context.Context, _, _ string) (net.Conn, error) { - return net.Dial("unix", option.LocalFilerSocket) + DialContext: func(ctx context.Context, _, addr string) (net.Conn, error) { + conn, err := (&net.Dialer{}).DialContext(ctx, "unix", option.LocalFilerSocket) + if err != nil { + return util.OutboundDialContext(ctx, "tcp", addr) + } + return conn, nil }, }, } @@ -943,6 +947,17 @@ func (s3a *S3ApiServer) registerRouter(router *mux.Router) { // Object path pattern with (?s) flag to match newlines in object keys const objectPath = "/{object:(?s).+}" + // CORS preflight for service-level requests, answered from the global + // AllowedOrigins rather than a bucket's CORS configuration + serviceOptionsHandler := func(w http.ResponseWriter, r *http.Request) { + if s3a.handleCORSOriginValidation(w, r) { + writeSuccessResponseEmpty(w, r) + } + } + // A path-style bucket router would take the health probe paths as bucket + // names, so their preflight is registered ahead of each one + const healthProbePath = "/{probe:status|healthz|readyz}" + var routers []*mux.Router if s3a.option.DomainName != "" { domainNames := strings.Split(s3a.option.DomainName, ",") @@ -950,6 +965,7 @@ func (s3a *S3ApiServer) registerRouter(router *mux.Router) { // Register path-style domains for _, domain := range pathStyleDomains { + apiRouter.Host(domain).Methods(http.MethodOptions).Path(healthProbePath).HandlerFunc(serviceOptionsHandler) routers = append(routers, apiRouter.Host(domain).PathPrefix("/{bucket}").Subrouter()) apiRouter.Host(domain). Methods(http.MethodGet). @@ -967,6 +983,7 @@ func (s3a *S3ApiServer) registerRouter(router *mux.Router) { // arrive via an IP, an unlisted hostname, or a reverse proxy that rewrites // the Host header still match bucket routes. Host-specific routers above // take precedence because they were registered first. + apiRouter.Methods(http.MethodOptions).Path(healthProbePath).HandlerFunc(serviceOptionsHandler) routers = append(routers, apiRouter.PathPrefix("/{bucket}").Subrouter()) // Get CORS middleware instance with caching @@ -982,8 +999,9 @@ func (s3a *S3ApiServer) registerRouter(router *mux.Router) { bucket.Use(corsMiddleware.Handler) // Bucket-specific OPTIONS handler for CORS preflight requests - // Use PathPrefix to catch all bucket-level preflight routes including /bucket/object - bucket.PathPrefix("/").Methods(http.MethodOptions).HandlerFunc(corsMiddleware.HandleOptionsRequest) + // No path matcher, so it also catches the path-style bucket root (/bucket), + // which PathPrefix("/") would miss, as well as /bucket/ and /bucket/object + bucket.Methods(http.MethodOptions).HandlerFunc(corsMiddleware.HandleOptionsRequest) // each case should follow the next rule: // - requesting object with query must precede any other methods @@ -1218,9 +1236,7 @@ func (s3a *S3ApiServer) registerRouter(router *mux.Router) { return } - if s3a.handleCORSOriginValidation(w, r) { - writeSuccessResponseEmpty(w, r) - } + serviceOptionsHandler(w, r) }) // STS API endpoint for AssumeRoleWithWebIdentity diff --git a/weed/s3api/s3api_server_routing_test.go b/weed/s3api/s3api_server_routing_test.go index 9459fe588f..9316174924 100644 --- a/weed/s3api/s3api_server_routing_test.go +++ b/weed/s3api/s3api_server_routing_test.go @@ -17,6 +17,7 @@ import ( "github.com/seaweedfs/seaweedfs/weed/iam/policy" "github.com/seaweedfs/seaweedfs/weed/iam/sts" "github.com/seaweedfs/seaweedfs/weed/pb/iam_pb" + "github.com/seaweedfs/seaweedfs/weed/s3api/cors" "github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants" "github.com/seaweedfs/seaweedfs/weed/util" "github.com/stretchr/testify/assert" @@ -442,3 +443,89 @@ func TestRouting_BucketRouteMatchesAnyHost(t *testing.T) { }) } } + +// TestRouting_BucketRootPreflightUsesBucketCORS is a regression test: a +// path-style preflight for the bucket root without a trailing +// slash must be evaluated against the bucket CORS rules like /bucket/, not +// answered from the global AllowedOrigins. Service-level paths keep the +// global answer, and virtual-host routing is unchanged. +func TestRouting_BucketRootPreflightUsesBucketCORS(t *testing.T) { + const ( + bucket = "cors-bucket" + allowedOrigin = "https://app.example.com" + foreignOrigin = "https://untrusted.example" + ) + pathStyleBucketPaths := []string{"/" + bucket, "/" + bucket + "/", "/" + bucket + "?list-type=2", "/" + bucket + "/key"} + + cases := []struct { + name string + domainName string + host string + bucketPaths []string // evaluated against the bucket CORS rules + servicePaths []string // answered from the global AllowedOrigins + }{ + { + name: "path-style", + host: "10.0.0.5:8333", + bucketPaths: pathStyleBucketPaths, + servicePaths: []string{"/", "/status", "/healthz", "/readyz"}, + }, + { + name: "path-style domain", + domainName: "s3.example.com,api.s3.example.com", + host: "api.s3.example.com", + bucketPaths: pathStyleBucketPaths, + servicePaths: []string{"/status", "/healthz", "/readyz"}, + }, + { + name: "virtual-host domain", + domainName: "s3.example.com,api.s3.example.com", + host: bucket + ".s3.example.com", + bucketPaths: []string{"/", "/?list-type=2", "/key", "/status"}, + }, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + s3a := setupRoutingTestServer(t) + s3a.option.DomainName = tc.domainName + s3a.option.AllowedOrigins = []string{"*"} + s3a.bucketConfigCache = NewBucketConfigCache(time.Minute) + s3a.bucketConfigCache.Set(bucket, &BucketConfig{ + Name: bucket, + CORS: &cors.CORSConfiguration{CORSRules: []cors.CORSRule{{ + AllowedOrigins: []string{allowedOrigin}, + AllowedMethods: []string{http.MethodGet, http.MethodPut}, + AllowedHeaders: []string{"*"}, + }}}, + }) + router := mux.NewRouter() + s3a.registerRouter(router) + + preflight := func(path, origin string) *httptest.ResponseRecorder { + req := httptest.NewRequest(http.MethodOptions, "http://"+tc.host+path, nil) + req.Header.Set("Origin", origin) + req.Header.Set("Access-Control-Request-Method", http.MethodPut) + rr := httptest.NewRecorder() + router.ServeHTTP(rr, req) + return rr + } + + for _, path := range tc.bucketPaths { + rr := preflight(path, foreignOrigin) + assert.Equal(t, http.StatusForbidden, rr.Code, "foreign origin on %s", path) + assert.Empty(t, rr.Header().Get("Access-Control-Allow-Origin"), "foreign origin on %s", path) + + rr = preflight(path, allowedOrigin) + assert.Equal(t, http.StatusOK, rr.Code, "allowed origin on %s", path) + assert.Equal(t, allowedOrigin, rr.Header().Get("Access-Control-Allow-Origin"), "allowed origin on %s", path) + } + + for _, path := range tc.servicePaths { + rr := preflight(path, foreignOrigin) + assert.Equal(t, http.StatusOK, rr.Code, "service-level %s", path) + assert.Equal(t, "*", rr.Header().Get("Access-Control-Allow-Origin"), "service-level %s", path) + } + }) + } +} diff --git a/weed/s3api/s3lifecycle/router/router.go b/weed/s3api/s3lifecycle/router/router.go index 369172c9c7..81c084e5c1 100644 --- a/weed/s3api/s3lifecycle/router/router.go +++ b/weed/s3api/s3lifecycle/router/router.go @@ -109,11 +109,19 @@ func Route(ctx context.Context, snap *engine.Snapshot, ev *reader.Event, now tim // EXP_DM can fire on two version-folder events: the marker create // (sole survivor immediately) and a noncurrent hard-delete that // leaves only the marker behind. Both reach routeSoleSurvivorMarker. + // A stamped version-file write is also the latest for its logical key + // at write time — the versioned layout emits no bare-key event the + // regular path could see, so ExpirationDays must schedule here or it + // only ever fires via a cold-start / rule-change recovery walk. if versioned && isVersionFolderPath(ev.Key) { - if !hasActiveEventDrivenAction(snap, keys, s3lifecycle.ActionKindExpiredDeleteMarker) { - return nil + var matches []Match + if isVersionFileWrite(ev.NewEntry) { + matches = routeVersionFileExpiration(snap, ev, keys) } - return routeSoleSurvivorMarker(ctx, snap, ev, keys, lister) + if hasActiveEventDrivenAction(snap, keys, s3lifecycle.ActionKindExpiredDeleteMarker) { + matches = append(matches, routeSoleSurvivorMarker(ctx, snap, ev, keys, lister)...) + } + return matches } if ev.NewEntry == nil { @@ -517,6 +525,71 @@ func emitNoncurrentMatches(snap *engine.Snapshot, ev *reader.Event, keys []s3lif return matches } +// isVersionFileWrite reports whether the event's new entry is a genuine +// version-file write: a non-marker file carrying ExtVersionIdKey. The +// version id stamp is what separates it from a literal user file under a +// *.versions/ parent. +func isVersionFileWrite(entry *filer_pb.Entry) bool { + if entry == nil || entry.IsDirectory || isDeleteMarkerEntry(entry) { + return false + } + return len(entry.Extended[s3_constants.ExtVersionIdKey]) > 0 +} + +// routeVersionFileExpiration schedules current-version expiration from a +// version-file write. The file is the latest at write time, and VersionID +// stays empty so the dispatcher resolves the latest at due time — a +// fresher PUT meanwhile resolves to STALE_IDENTITY there, and that PUT's +// own event carried its own schedule. A write that turns out to be an +// older version's metadata update no-ops the same way. +func routeVersionFileExpiration(snap *engine.Snapshot, ev *reader.Event, keys []s3lifecycle.ActionKey) []Match { + logical, ok := logicalKeyFromVersionPath(ev.Key) + if !ok { + return nil + } + entry := ev.NewEntry + if entry.Attributes == nil { + return nil + } + info := &s3lifecycle.ObjectInfo{ + Key: logical, + ModTime: time.Unix(entry.Attributes.Mtime, int64(entry.Attributes.MtimeNs)), + Size: int64(entry.Attributes.FileSize), + IsLatest: true, + } + if tags := extractTags(entry.Extended); len(tags) > 0 { + info.Tags = tags + } + eventTime := time.Unix(0, ev.TsNs) + identity := buildIdentityFromEntry(entry) + var matches []Match + for _, key := range keys { + action := snap.Action(key) + if action == nil || !action.IsActive() || action.Mode != engine.ModeEventDriven { + continue + } + dueTime := s3lifecycle.ComputeDueAt(action.Rule, key.ActionKind, info) + if dueTime.IsZero() { + continue + } + res := s3lifecycle.EvaluateAction(action.Rule, key.ActionKind, info, dueTime) + if res.Action == s3lifecycle.ActionNone { + continue + } + matches = append(matches, Match{ + Key: key, + Action: action, + Result: res, + EventTs: eventTime, + DueTime: dueTime, + Bucket: ev.Bucket, + ObjectKey: logical, + Identity: identity, + }) + } + return matches +} + // routeSoleSurvivorMarker emits an EXP_DM Match against the LOGICAL key // (so the dispatcher can call deleteSpecificObjectVersion) with the // marker's version_id. Handles two events: a marker create (the new diff --git a/weed/s3api/s3lifecycle/router/router_test.go b/weed/s3api/s3lifecycle/router/router_test.go index ddce9a6744..076b0eefb3 100644 --- a/weed/s3api/s3lifecycle/router/router_test.go +++ b/weed/s3api/s3lifecycle/router/router_test.go @@ -458,6 +458,65 @@ func TestRouteVersionedCurrentEventStaysLatest(t *testing.T) { } } +func TestRouteVersionedVersionFileWriteSchedulesExpiration(t *testing.T) { + // A versioned PUT shows up in the meta-log as the version file plus + // the .versions/ directory update — there is no bare-key event for + // the regular path to see. The version file is the latest at write + // time; the match carries its identity with an empty VersionID so + // dispatch CAS-checks the current latest at due time. + rule := &s3lifecycle.Rule{ID: "r", Status: s3lifecycle.StatusEnabled, ExpirationDays: 1} + snap := compileWithVersioned(rule, activatedPrior(rule)) + + now := time.Now() + old := now.AddDate(0, 0, -2) + ev := eventCreate("bk", "logs/foo"+s3_constants.VersionsFolder+"/v_v1", old.Unix(), 1, old.UnixNano()) + ev.NewEntry.Extended = map[string][]byte{ + s3_constants.ExtVersionIdKey: []byte("v1"), + } + + matches := Route(context.Background(), snap, ev, now, nil) + if len(matches) != 1 { + t.Fatalf("expected 1 match (EXPIRATION_DAYS), got %v", matches) + } + m := matches[0] + if m.Key.ActionKind != s3lifecycle.ActionKindExpirationDays { + t.Fatalf("ActionKind=%v, want ExpirationDays", m.Key.ActionKind) + } + if m.ObjectKey != "logs/foo" { + t.Fatalf("ObjectKey=%q, want the logical key", m.ObjectKey) + } + if m.VersionID != "" { + t.Fatalf("VersionID=%q, want empty so dispatch resolves the current latest", m.VersionID) + } + if m.Identity == nil { + t.Fatal("Identity must fingerprint the version file for dispatch CAS") + } + if want := time.Unix(old.Unix(), 0).Add(s3lifecycle.DaysToDuration(1)); !m.DueTime.Equal(want) { + t.Fatalf("DueTime=%v, want %v", m.DueTime, want) + } +} + +func TestRouteVersionedMarkerCreateDoesNotFireExpiration(t *testing.T) { + // A delete marker is a version file too, but ExpirationDays never + // applies to a marker — only EXP_DM, handled by the sole-survivor + // path. Without the marker gate, the expiration match would pin a + // marker identity and dispatch would stack a second marker on top. + rule := &s3lifecycle.Rule{ID: "r", Status: s3lifecycle.StatusEnabled, ExpirationDays: 1} + snap := compileWithVersioned(rule, activatedPrior(rule)) + + now := time.Now() + old := now.AddDate(0, 0, -2) + ev := markerEvent("bk", "logs/gone", "2026-05-09-abc", old.Unix(), old.UnixNano()) + + lister := &recordingLister{survivors: Survivors{Count: 1}} + if got := Route(context.Background(), snap, ev, now, lister); len(got) != 0 { + t.Fatalf("marker create must not fire EXPIRATION_DAYS, got %v", got) + } + if len(lister.calls) != 0 { + t.Fatalf("lister consulted without an EXP_DM rule: calls=%v", lister.calls) + } +} + func TestRouteNonVersionedBucketIgnoresVersionsSuffix(t *testing.T) { // A non-versioned bucket happens to have an object literally named // "logs/foo.versions/v1" — that's just a regular path. The router @@ -796,13 +855,11 @@ func TestRouteVersionedDeleteMarkerNoExpDMRuleSkipsListing(t *testing.T) { } func TestRouteVersionedAllVersionFolderPathsSkipped(t *testing.T) { - // On a versioned bucket the router skips every event whose parent - // directory name ends with ".versions" — both the version files - // SeaweedFS itself writes (logs/foo.versions/v_v1) and any literal - // key the user happens to put under such a parent — because the - // current-vs-noncurrent classification needs the .versions/ - // directory's latest pointer, which isn't carried by these events. - // Bootstrap covers retention for those entries. + // On a versioned bucket, events inside a *.versions/ parent that are + // not stamped version-file writes stay unrouted — current-vs-noncurrent + // classification needs the container's latest pointer, and the file's + // own ExtVersionIdKey is what marks it as a version write the + // expiration path can schedule from. rule := &s3lifecycle.Rule{ID: "r", Status: s3lifecycle.StatusEnabled, ExpirationDays: 1} snap := compileWithVersioned(rule, activatedPrior(rule)) @@ -815,9 +872,11 @@ func TestRouteVersionedAllVersionFolderPathsSkipped(t *testing.T) { ext map[string][]byte }{ { - name: "tracked version file", + // A marker carries the version id but is a marker, not an + // expirable write — it routes to the sole-survivor path. + name: "delete marker", key: "logs/foo.versions/v_v1", - ext: map[string][]byte{s3_constants.ExtVersionIdKey: []byte("v1")}, + ext: map[string][]byte{s3_constants.ExtVersionIdKey: []byte("v1"), s3_constants.ExtDeleteMarkerKey: []byte("true")}, }, { name: "literal-key collision", diff --git a/weed/s3api/s3tables/utils.go b/weed/s3api/s3tables/utils.go index d83e498830..8c76b189af 100644 --- a/weed/s3api/s3tables/utils.go +++ b/weed/s3api/s3tables/utils.go @@ -9,6 +9,7 @@ import ( "regexp" "strings" "time" + "unicode/utf8" "github.com/seaweedfs/seaweedfs/weed/pb/filer_pb" ) @@ -372,17 +373,25 @@ func arnPartitionForRegion(region string) string { // ValidateTags validates tags for S3 Tables. func ValidateTags(tags map[string]string) error { - if len(tags) > 10 { - return fmt.Errorf("validate tags: %d tags more than 10", len(tags)) + return ValidateTagsWithLimit(tags, 10) +} + +// ValidateTagsWithLimit validates a tag set that may hold up to maxTags tags. +func ValidateTagsWithLimit(tags map[string]string, maxTags int) error { + if len(tags) > maxTags { + return fmt.Errorf("validate tags: %d tags more than %d", len(tags), maxTags) } for k, v := range tags { - if len(k) > 128 { + if len(k) == 0 { + return fmt.Errorf("validate tags: empty tag key") + } + if utf8.RuneCountInString(k) > 128 { return fmt.Errorf("validate tags: tag key longer than 128") } if !tagPattern.MatchString(k) { return fmt.Errorf("validate tags key %s error, incorrect key", k) } - if len(v) > 256 { + if utf8.RuneCountInString(v) > 256 { return fmt.Errorf("validate tags: tag value longer than 256") } if !tagPattern.MatchString(v) { diff --git a/weed/s3api/tags.go b/weed/s3api/tags.go index 7b1654529d..49cde79f59 100644 --- a/weed/s3api/tags.go +++ b/weed/s3api/tags.go @@ -53,11 +53,19 @@ func FromTags(tags map[string]string) (t *Tagging) { func parseTagsHeader(tags string) (map[string]string, error) { parsedTags := make(map[string]string) for _, v := range util.StringSplit(tags, "&") { + // url.ParseQuery skips empty segments too, so a copy stores the + // same tag keys that PutObject and bucket policies see. + if v == "" { + continue + } key, value, hasValue := strings.Cut(v, "=") decodedKey, err := url.QueryUnescape(key) if err != nil { return nil, fmt.Errorf("failed to decode tag key '%s': %w", key, err) } + if _, exists := parsedTags[decodedKey]; exists { + return nil, fmt.Errorf("duplicate tag key '%s'", decodedKey) + } if !hasValue { parsedTags[decodedKey] = "" continue @@ -74,3 +82,11 @@ func parseTagsHeader(tags string) (map[string]string, error) { func ValidateTags(tags map[string]string) error { return s3tables.ValidateTags(tags) } + +// maxBucketTags is how many tags a bucket's tag set can hold; an object's +// tag set holds 10. +const maxBucketTags = 50 + +func ValidateBucketTags(tags map[string]string) error { + return s3tables.ValidateTagsWithLimit(tags, maxBucketTags) +} diff --git a/weed/s3api/tags_test.go b/weed/s3api/tags_test.go index 2002d65f28..f313806ae5 100644 --- a/weed/s3api/tags_test.go +++ b/weed/s3api/tags_test.go @@ -1,6 +1,7 @@ package s3api import ( + "strings" "testing" ) @@ -62,6 +63,18 @@ func TestParseTagsHeader(t *testing.T) { expected: nil, expectError: true, }, + { + name: "duplicate key", + input: "key1=value1&key1=value2", + expected: nil, + expectError: true, + }, + { + name: "encoded duplicate key", + input: "key1=value1&%6bey1=value2", + expected: nil, + expectError: true, + }, { name: "plus signs and equals in values", input: "formula=a%2Bb%3Dc&normal=test", @@ -71,6 +84,15 @@ func TestParseTagsHeader(t *testing.T) { }, expectError: false, }, + { + name: "empty segments add no tag", + input: "&key1=value1&&key2=value2&", + expected: map[string]string{ + "key1": "value1", + "key2": "value2", + }, + expectError: false, + }, { name: "unescaped equals in value", input: "token=part1=part2&normal=test", @@ -111,3 +133,19 @@ func TestParseTagsHeader(t *testing.T) { }) } } + +func TestValidateTagsUnicodeLength(t *testing.T) { + // S3 counts tag length in characters, not bytes + multibyteKey := strings.Repeat("中", 128) + multibyteValue := strings.Repeat("中", 256) + + if err := ValidateTags(map[string]string{multibyteKey: multibyteValue}); err != nil { + t.Errorf("Expected 128-char key and 256-char value to pass, got %v", err) + } + if err := ValidateTags(map[string]string{strings.Repeat("中", 129): "v"}); err == nil { + t.Error("Expected 129-char key to fail") + } + if err := ValidateTags(map[string]string{"k": strings.Repeat("中", 257)}); err == nil { + t.Error("Expected 257-char value to fail") + } +} diff --git a/weed/server/filer_server.go b/weed/server/filer_server.go index 41c7256172..8e09191723 100644 --- a/weed/server/filer_server.go +++ b/weed/server/filer_server.go @@ -176,6 +176,12 @@ type FilerServer struct { // replayed backlogs before the filer joins the master's list. startupPeers []pb.ServerAddress + // peerBootstrapPending marks a fresh store whose startup peer scan came + // back empty: an empty answer during a master restart can just mean the + // existing filers have not reconnected yet, so AnnounceOnceCaughtUp + // re-checks membership before registering. + peerBootstrapPending bool + // entryLockTable serializes mutations to the same entry path on this filer. // CreateEntry takes it today; UpdateEntry and DeleteEntry are intended to take // it too as their callers route a key's writes to this node, making it the @@ -367,9 +373,12 @@ func NewFilerServer(defaultMux, readonlyMux *http.ServeMux, option *FilerOption) startFromTime := time.Now().Add(-filer.LogFlushInterval) if isFresh { glog.V(0).Infof("%s bootstrap from peers %+v", option.Host, existingNodes) - if err := fs.filer.MaybeBootstrapFromOnePeer(option.Host, existingNodes, startFromTime); err != nil { + if err := fs.filer.MaybeBootstrapFromOnePeer(context.Background(), option.Host, existingNodes, startFromTime); err != nil { glog.Fatalf("%s bootstrap from %+v: %v", option.Host, existingNodes, err) } + // An empty scan can be a settling master rather than an empty cluster; + // the announce gate then re-checks membership before registering. + fs.peerBootstrapPending = len(existingNodes) == 0 } v.SetDefault("filer.options.s3.empty_folder_cleanup_delay", "2m") if d, err := time.ParseDuration(v.GetString("filer.options.s3.empty_folder_cleanup_delay")); err == nil { @@ -494,16 +503,26 @@ const announceCatchUpCap = time.Minute // routed traffic is answerable the moment the filer is listed. func (fs *FilerServer) AnnounceOnceCaughtUp(announce chan struct{}, waitStart time.Time) { defer close(announce) - if len(fs.startupPeers) == 0 || fs.filer.MetaAggregator == nil { + ma := fs.filer.MetaAggregator + if ma == nil { return } + // The bootstrap re-check and the catch-up wait share one deadline so a + // late peer cannot stretch the registration hold past the cap. deadline := time.Now().Add(announceCatchUpCap) + peers := fs.startupPeers + if fs.peerBootstrapPending { + peers = append(peers, fs.awaitPeerBootstrap(deadline)...) + } + if len(peers) == 0 { + return + } for { // Take the channel before reading the watermarks so a rise in // between still wakes this wait (see DeliveryWatermarkAdvancedChan). - advanced := fs.filer.MetaAggregator.DeliveryWatermarkAdvancedChan() - if fs.filer.MetaAggregator.PeersCaughtUpTo(fs.startupPeers, waitStart.UnixNano()) { - glog.V(0).Infof("%s caught up with %d startup peers, announcing to masters", fs.option.Host, len(fs.startupPeers)) + advanced := ma.DeliveryWatermarkAdvancedChan() + if ma.PeersCaughtUpTo(peers, waitStart.UnixNano()) { + glog.V(0).Infof("%s caught up with %d startup peers, announcing to masters", fs.option.Host, len(peers)) return } remaining := time.Until(deadline) @@ -518,6 +537,60 @@ func (fs *FilerServer) AnnounceOnceCaughtUp(announce chan struct{}, waitStart ti } } +// emptyMembershipConfirmTimeout bounds the announce hold for a peer to +// disprove an empty startup scan on a fresh store. This runs after the ports +// already serve, so a genuinely empty cluster pays a delayed master listing, +// not a stalled startup. +const emptyMembershipConfirmTimeout = 10 * time.Second + +// awaitPeerBootstrap re-polls membership while a fresh store's empty startup +// scan can still be disproved: masters restarted or re-elected report no +// filers until the existing ones reconnect. Peers found late are backfilled +// from before their streams start, then reported so the caller waits for +// their catch-up like startup peers. The whole wait — including each peer +// metadata copy — is bounded by the shared announce deadline. +func (fs *FilerServer) awaitPeerBootstrap(deadline time.Time) (peers []pb.ServerAddress) { + confirmBy := time.Now().Add(emptyMembershipConfirmTimeout) + for time.Now().Before(deadline) { + ctx, cancel := context.WithDeadline(context.Background(), deadline) + nodes, listErr := fs.filer.ListExistingPeerUpdates(ctx) + cancel() + if listErr != nil { + glog.Warningf("%s cannot re-check existing peers: %v; announcing anyway", fs.option.Host, listErr) + return nil + } + var remotes []*master_pb.ClusterNodeUpdate + for _, node := range nodes { + if !pb.ServerAddress(node.Address).Equals(fs.option.Host) { + remotes = append(remotes, node) + } + } + if len(remotes) == 0 { + if !time.Now().Before(confirmBy) { + return nil + } + time.Sleep(time.Second) + continue + } + bootstrapFrom := time.Now().Add(-filer.LogFlushInterval) + ctx, cancel = context.WithDeadline(context.Background(), deadline) + err := fs.filer.MaybeBootstrapFromOnePeer(ctx, fs.option.Host, remotes, bootstrapFrom) + cancel() + if err != nil { + glog.Warningf("%s bootstrap from late peers %+v: %v; retrying", fs.option.Host, remotes, err) + time.Sleep(2 * time.Second) + continue + } + glog.V(0).Infof("%s bootstrap from late-arriving peers %+v", fs.option.Host, remotes) + for _, node := range remotes { + peers = append(peers, pb.ServerAddress(node.Address)) + fs.filer.MetaAggregator.OnPeerUpdate(node, bootstrapFrom) + } + return peers + } + return nil +} + // masterRemovalWaitBudget bounds how long shutdown waits for masters to // drop this filer before gRPC drains anyway. const masterRemovalWaitBudget = 5 * time.Second diff --git a/weed/server/filer_server_handlers_read.go b/weed/server/filer_server_handlers_read.go index 2fa32fa2ff..cf1e2391e2 100644 --- a/weed/server/filer_server_handlers_read.go +++ b/weed/server/filer_server_handlers_read.go @@ -50,10 +50,10 @@ func checkPreconditions(w http.ResponseWriter, r *http.Request, entry *filer.Ent } w.Header().Set("Last-Modified", entry.Attr.Mtime.UTC().Format(http.TimeFormat)) - ifMatchETagHeader := r.Header.Get("If-Match") + ifMatchValues := r.Header.Values("If-Match") ifUnmodifiedSinceHeader := r.Header.Get("If-Unmodified-Since") - if ifMatchETagHeader != "" { - if util.CanonicalizeETag(etag) != util.CanonicalizeETag(ifMatchETagHeader) { + if len(ifMatchValues) != 0 { + if !etagListMatches(strings.Join(ifMatchValues, ","), etag, false) { w.WriteHeader(http.StatusPreconditionFailed) return true } @@ -66,10 +66,10 @@ func checkPreconditions(w http.ResponseWriter, r *http.Request, entry *filer.Ent } } - ifNoneMatchETagHeader := r.Header.Get("If-None-Match") + ifNoneMatchValues := r.Header.Values("If-None-Match") ifModifiedSinceHeader := r.Header.Get("If-Modified-Since") - if ifNoneMatchETagHeader != "" { - if util.CanonicalizeETag(etag) == util.CanonicalizeETag(ifNoneMatchETagHeader) { + if len(ifNoneMatchValues) != 0 { + if etagListMatches(strings.Join(ifNoneMatchValues, ","), etag, true) { SetEtag(w, etag) w.WriteHeader(http.StatusNotModified) return true @@ -87,6 +87,52 @@ func checkPreconditions(w http.ResponseWriter, r *http.Request, entry *filer.Ent return false } +// etagListMatches reports whether an If-Match or If-None-Match value, "*" or a +// list of entity tags, matches etag (RFC 9110 13.1.1, 13.1.2). If-Match uses the +// strong comparison, so a weak tag never matches; If-None-Match the weak one. +func etagListMatches(header, etag string, weak bool) bool { + if strings.TrimSpace(header) == "*" { + return true + } + for _, tag := range splitETagList(header) { + tag = strings.TrimSpace(tag) + if strings.HasPrefix(tag, "W/") { + if !weak { + continue + } + tag = tag[len("W/"):] + } + // Empty list elements are ignored, so padding like a trailing + // comma cannot match an empty opaque tag. + if tag == "" { + continue + } + if util.CanonicalizeETag(tag) == util.CanonicalizeETag(etag) { + return true + } + } + return false +} + +// splitETagList splits an entity-tag list on the commas between tags; a comma +// inside a quoted opaque tag does not split. +func splitETagList(header string) []string { + var tags []string + start, quoted := 0, false + for i := 0; i < len(header); i++ { + switch header[i] { + case '"': + quoted = !quoted + case ',': + if !quoted { + tags = append(tags, header[start:i]) + start = i + 1 + } + } + } + return append(tags, header[start:]) +} + func (fs *FilerServer) GetOrHeadHandler(w http.ResponseWriter, r *http.Request) { ctx := r.Context() path := r.URL.Path diff --git a/weed/server/filer_server_handlers_read_precondition_test.go b/weed/server/filer_server_handlers_read_precondition_test.go new file mode 100644 index 0000000000..053056a130 --- /dev/null +++ b/weed/server/filer_server_handlers_read_precondition_test.go @@ -0,0 +1,115 @@ +package weed_server + +import ( + "net/http" + "net/http/httptest" + "testing" + "time" + + "github.com/seaweedfs/seaweedfs/weed/filer" + "github.com/seaweedfs/seaweedfs/weed/pb/filer_pb" +) + +func TestCheckPreconditionsETagLists(t *testing.T) { + entry := &filer.Entry{Attr: filer.Attr{Mtime: time.Unix(1700000000, 0), Md5: []byte{0xb1, 0x94}}} + const etag = `"b194"` + + tests := []struct { + header string + values []string + want int + }{ + {"If-None-Match", []string{etag}, http.StatusNotModified}, + {"If-None-Match", []string{`*`}, http.StatusNotModified}, + {"If-None-Match", []string{`W/` + etag}, http.StatusNotModified}, + {"If-None-Match", []string{`"x", ` + etag}, http.StatusNotModified}, + {"If-None-Match", []string{`"x"`, etag}, http.StatusNotModified}, + {"If-None-Match", []string{`"x"`}, http.StatusOK}, + {"If-None-Match", []string{`"b194,old"`}, http.StatusOK}, + {"If-Match", []string{etag}, http.StatusOK}, + {"If-Match", []string{`*`}, http.StatusOK}, + {"If-Match", []string{`"x", ` + etag}, http.StatusOK}, + {"If-Match", []string{`"x"`, etag}, http.StatusOK}, + {"If-Match", []string{`W/` + etag}, http.StatusPreconditionFailed}, + {"If-Match", []string{`"x"`}, http.StatusPreconditionFailed}, + {"If-Match", []string{`"b194,old"`}, http.StatusPreconditionFailed}, + } + for _, tt := range tests { + r := httptest.NewRequest(http.MethodGet, "/d/f.txt", nil) + for _, v := range tt.values { + r.Header.Add(tt.header, v) + } + w := httptest.NewRecorder() + got := http.StatusOK + if checkPreconditions(w, r, entry) { + got = w.Code + } + if got != tt.want { + t.Errorf("%s: %q -> %d, want %d", tt.header, tt.values, got, tt.want) + } + } +} + +func TestCheckPreconditionsPresentButEmptyHeaders(t *testing.T) { + entry := &filer.Entry{Attr: filer.Attr{Mtime: time.Unix(1700000000, 0), Md5: []byte{0xb1, 0x94}}} + before := time.Unix(1700000000-60, 0).UTC().Format(http.TimeFormat) + after := time.Unix(1700000000+60, 0).UTC().Format(http.TimeFormat) + + // An empty If-Match line is still an If-Match: it matches nothing, and + // If-Unmodified-Since stays ignored. + r := httptest.NewRequest(http.MethodGet, "/d/f.txt", nil) + r.Header.Set("If-Match", "") + r.Header.Set("If-Unmodified-Since", after) + if w := httptest.NewRecorder(); !checkPreconditions(w, r, entry) || w.Code != http.StatusPreconditionFailed { + t.Errorf("empty If-Match -> %d, want 412", w.Code) + } + + // An empty If-None-Match line matches nothing and suppresses + // If-Modified-Since, so the request proceeds. + r = httptest.NewRequest(http.MethodGet, "/d/f.txt", nil) + r.Header.Set("If-None-Match", "") + r.Header.Set("If-Modified-Since", after) + if checkPreconditions(httptest.NewRecorder(), r, entry) { + t.Error("empty If-None-Match with matching If-Modified-Since -> 304, want 200") + } + + r = httptest.NewRequest(http.MethodGet, "/d/f.txt", nil) + r.Header.Set("If-Modified-Since", before) + if checkPreconditions(httptest.NewRecorder(), r, entry) { + t.Error("old If-Modified-Since alone -> 304, want 200") + } +} + +func TestCheckPreconditionsIgnoresEmptyETagListElements(t *testing.T) { + entry := &filer.Entry{ + Attr: filer.Attr{Mtime: time.Unix(1700000000, 0)}, + Remote: &filer_pb.RemoteEntry{RemoteSize: 1}, + } + + tests := []struct { + header string + values []string + want int + }{ + {"If-None-Match", []string{`"x",`}, http.StatusOK}, + {"If-None-Match", []string{`"x",,"y"`}, http.StatusOK}, + {"If-None-Match", []string{`""`}, http.StatusNotModified}, + {"If-None-Match", []string{`W/""`}, http.StatusNotModified}, + {"If-Match", []string{`"x",`}, http.StatusPreconditionFailed}, + {"If-Match", []string{`""`}, http.StatusOK}, + } + for _, tt := range tests { + r := httptest.NewRequest(http.MethodGet, "/d/f.txt", nil) + for _, v := range tt.values { + r.Header.Add(tt.header, v) + } + w := httptest.NewRecorder() + got := http.StatusOK + if checkPreconditions(w, r, entry) { + got = w.Code + } + if got != tt.want { + t.Errorf("%s: %q -> %d, want %d", tt.header, tt.values, got, tt.want) + } + } +} diff --git a/weed/server/filer_server_handlers_write.go b/weed/server/filer_server_handlers_write.go index 177093c501..9009a0f045 100644 --- a/weed/server/filer_server_handlers_write.go +++ b/weed/server/filer_server_handlers_write.go @@ -337,7 +337,7 @@ func (fs *FilerServer) detectStorageOption(ctx context.Context, requestURI, qCol DataNode: util.Nvl(dataNode, rule.DataNode, fs.option.DataNode), TtlSeconds: ttlSeconds, DiskType: util.Nvl(diskType, overlayDisk, rule.DiskType), - Fsync: rule.Fsync, + Fsync: rule.GetFsync(), VolumeGrowthCount: rule.VolumeGrowthCount, MaxFileNameLength: maxFileNameLength, // A location that survives from a request parameter, a placement diff --git a/weed/server/master_grpc_server_changed_volumes_test.go b/weed/server/master_grpc_server_changed_volumes_test.go index 5436418fb1..0ed15aa132 100644 --- a/weed/server/master_grpc_server_changed_volumes_test.go +++ b/weed/server/master_grpc_server_changed_volumes_test.go @@ -220,7 +220,7 @@ func TestRegistrationRacingCollectionDeleteDoesNotLeak(t *testing.T) { topo.DeleteCollection("c") // the interleaved registration must refuse the dropped layout - if stale.RegisterVolume(&vi, dn) { + if stale.RegisterVolume(&vi, dn, nil) { t.Fatal("registered into a layout dropped with its collection") } diff --git a/weed/server/master_grpc_server_volume.go b/weed/server/master_grpc_server_volume.go index 53bd2c27f0..de1e54333c 100644 --- a/weed/server/master_grpc_server_volume.go +++ b/weed/server/master_grpc_server_volume.go @@ -399,7 +399,7 @@ func (ms *MasterServer) VacuumVolume(ctx context.Context, req *master_pb.VacuumV resp := &master_pb.VacuumVolumeResponse{} - ms.Topo.Vacuum(ms.grpcDialOption, float64(req.GarbageThreshold), ms.option.MaxParallelVacuumPerServer, req.VolumeId, req.Collection, ms.preallocateSize, false, 0) + ms.Topo.Vacuum(ms.grpcDialOption, float64(req.GarbageThreshold), ms.option.MaxParallelVacuumPerServer, ms.option.MaxParallelVacuumPerRack, req.VolumeId, req.Collection, ms.preallocateSize, false, 0) return resp, nil } diff --git a/weed/server/master_server.go b/weed/server/master_server.go index b36679ebcd..595259c7b1 100644 --- a/weed/server/master_server.go +++ b/weed/server/master_server.go @@ -54,6 +54,8 @@ type MasterOption struct { FileSizeLimitMB int VolumePreallocate bool MaxParallelVacuumPerServer int + MaxParallelVacuumPerRack int + MaxWritableVolumes int VacuumIntervalSeconds int VacuumDeleteEmptyAfterSeconds int // PulseSeconds int @@ -64,6 +66,7 @@ type MasterOption struct { MetricsAddress string MetricsIntervalSec int IsFollower bool + SingleMaster bool TelemetryUrl string TelemetryEnabled bool VolumeGrowthDisabled bool @@ -104,7 +107,6 @@ type MasterServer struct { } func NewMasterServer(r *mux.Router, option *MasterOption, peers map[string]pb.ServerAddress) *MasterServer { - v := util.GetViper() signingKey := v.GetString("jwt.signing.key") v.SetDefault("jwt.signing.expires_after_seconds", 10) @@ -127,6 +129,13 @@ func NewMasterServer(r *mux.Router, option *MasterOption, peers map[string]pb.Se option.VolumeGrowthDisabled = v.GetBool("master.volume_growth.disable") v.SetDefault("master.volume_growth.freeze_on_topology_mismatch", true) v.SetDefault("master.volume_growth.state_loss_recovery_wait", "0s") + v.SetDefault("master.vacuum.max_parallel_per_rack", 0) + option.MaxParallelVacuumPerRack = v.GetInt("master.vacuum.max_parallel_per_rack") + if v.IsSet("master.vacuum.max_parallel_per_server") { + option.MaxParallelVacuumPerServer = v.GetInt("master.vacuum.max_parallel_per_server") + } + v.SetDefault("master.max_writable_volumes_per_layout", option.MaxWritableVolumes) + option.MaxWritableVolumes = v.GetInt("master.max_writable_volumes_per_layout") topology.VolumeGrowStrategy.Copy1Count = v.GetUint32("master.volume_growth.copy_1") topology.VolumeGrowStrategy.Copy2Count = v.GetUint32("master.volume_growth.copy_2") @@ -162,6 +171,7 @@ func NewMasterServer(r *mux.Router, option *MasterOption, peers map[string]pb.Se glog.Fatalf("create sequencer failed.") } ms.Topo = topology.NewTopology("topo", seq, uint64(ms.option.VolumeSizeLimitMB)*1024*1024, 5, replicationAsMin) + ms.Topo.SetMaxWritableVolumes(ms.option.MaxWritableVolumes) ms.Topo.SetGrowthFreezeConfig( v.GetBool("master.volume_growth.freeze_on_topology_mismatch"), v.GetDuration("master.volume_growth.state_loss_recovery_wait")) @@ -228,6 +238,7 @@ func NewMasterServer(r *mux.Router, option *MasterOption, peers map[string]pb.Se ms.grpcDialOption, ms.option.GarbageThreshold, ms.option.MaxParallelVacuumPerServer, + ms.option.MaxParallelVacuumPerRack, topology.VolumeGrowStrategy.Threshold, ms.preallocateSize, time.Duration(ms.option.VacuumIntervalSeconds)*time.Second, @@ -592,18 +603,20 @@ func (ms *MasterServer) OnPeerUpdate(update *master_pb.ClusterNodeUpdate, startF // AdmitRaftPeer pulls a master into the raft quorum, if we are the leader and // do not have it yet. A master that starts with no raft state cannot campaign -// under either implementation, so this is its only way in. -func (ms *MasterServer) AdmitRaftPeer(peerAddress pb.ServerAddress) { - if peerAddress.ToHttpAddress() == ms.option.Master.ToHttpAddress() { - return +// under either implementation, so this is its only way in. The returned +// channel resolves with the admission result, or nil when no admission ran. +func (ms *MasterServer) AdmitRaftPeer(peerAddress pb.ServerAddress) <-chan error { + if ms.option.SingleMaster || peerAddress.ToHttpAddress() == ms.option.Master.ToHttpAddress() { + return nil } peerName, ok := ms.missingRaftPeerName(peerAddress) if !ok { - return + return nil } - if _, alreadyAdmitting := ms.raftPeerAdmissions.LoadOrStore(peerName, struct{}{}); alreadyAdmitting { - return + admission := make(chan error, 1) + if inFlight, alreadyAdmitting := ms.raftPeerAdmissions.LoadOrStore(peerName, admission); alreadyAdmitting { + return inFlight.(chan error) } glog.V(0).Infof("adding new raft server: %s", peerName) // The join commits through raft, which waits on the other peers, so keep it @@ -611,10 +624,15 @@ func (ms *MasterServer) AdmitRaftPeer(peerAddress pb.ServerAddress) { // handler that a joining master is still blocked in. go func() { defer ms.raftPeerAdmissions.Delete(peerName) + defer close(admission) if err := ms.raftAddServer(peerName, peerAddress.ToGrpcAddress(), true); err != nil { glog.Warningf("failed adding raft server %s: %v", peerName, err) + admission <- err + return } + admission <- nil }() + return admission } // missingRaftPeerName returns the raft id to admit peerAddress under, and diff --git a/weed/server/master_server_admit_test.go b/weed/server/master_server_admit_test.go new file mode 100644 index 0000000000..012985b5cf --- /dev/null +++ b/weed/server/master_server_admit_test.go @@ -0,0 +1,119 @@ +package weed_server + +import ( + "testing" + "time" + + hashicorpRaft "github.com/hashicorp/raft" + "github.com/seaweedfs/seaweedfs/weed/pb" + "github.com/seaweedfs/seaweedfs/weed/topology" +) + +func TestSingleMasterNeverAdmitsRaftPeers(t *testing.T) { + ms := &MasterServer{option: &MasterOption{SingleMaster: true, Master: "localhost:9333"}} + + // A foreign master announcing itself must not touch the admission path: + // ms.Topo is deliberately nil so any admission attempt panics the test. + if done := ms.AdmitRaftPeer("10.0.0.1:9333"); done != nil { + t.Fatal("single-master mode admitted a foreign raft peer") + } +} + +func TestClusteredLeaderAdmitsMissingPeer(t *testing.T) { + topo := topology.NewTopology("topo", nil, 128*1024*1024, 0, false) + leader, leaderTransport := newInmemRaftLeader(t, "localhost:9333") + topo.HashicorpRaft = leader + ms := &MasterServer{ + option: &MasterOption{Master: "localhost:9333"}, + Topo: topo, + } + + peer := pb.ServerAddress("10.0.0.1:9333") + name, missing := ms.missingRaftPeerName(peer) + if !missing || name != "10.0.0.1:9333" { + t.Fatalf("clustered leader did not report the missing raft peer: %q %v", name, missing) + } + peerRaft := newInmemRaftFollower(t, peer, leaderTransport) + + done := ms.AdmitRaftPeer(peer) + if done == nil { + t.Fatal("clustered leader skipped admission of a missing raft peer") + } + if err := <-done; err != nil { + t.Fatalf("raft peer admission failed: %v", err) + } + + // The joined node's own configuration is the committed view: it only + // learns the quorum it was admitted into through replication. + for deadline := time.Now().Add(10 * time.Second); ; { + if raftHasVoter(peerRaft, string(peer)) { + return + } + if time.Now().After(deadline) { + t.Fatalf("admitted peer never saw itself in the quorum: %v", + peerRaft.GetConfiguration().Configuration().Servers) + } + time.Sleep(10 * time.Millisecond) + } +} + +func raftHasVoter(r *hashicorpRaft.Raft, id string) bool { + for _, server := range r.GetConfiguration().Configuration().Servers { + if server.Suffrage == hashicorpRaft.Voter && string(server.ID) == id { + return true + } + } + return false +} + +func newInmemRaftFollower(t *testing.T, self pb.ServerAddress, peerTransport *hashicorpRaft.InmemTransport) *hashicorpRaft.Raft { + t.Helper() + conf := hashicorpRaft.DefaultConfig() + conf.LocalID = hashicorpRaft.ServerID(self) + addr, transport := hashicorpRaft.NewInmemTransport(hashicorpRaft.ServerAddress(self.ToGrpcAddress())) + transport.Connect(peerTransport.LocalAddr(), peerTransport) + peerTransport.Connect(addr, transport) + r, err := hashicorpRaft.NewRaft(conf, + &StateMachine{topo: topology.NewTopology("fsm", nil, 128*1024*1024, 0, false)}, + hashicorpRaft.NewInmemStore(), hashicorpRaft.NewInmemStore(), + hashicorpRaft.NewInmemSnapshotStore(), transport) + if err != nil { + t.Fatalf("new raft: %v", err) + } + t.Cleanup(func() { r.Shutdown() }) + return r +} + +func newInmemRaftLeader(t *testing.T, self string) (*hashicorpRaft.Raft, *hashicorpRaft.InmemTransport) { + t.Helper() + conf := hashicorpRaft.DefaultConfig() + conf.LocalID = hashicorpRaft.ServerID(self) + conf.HeartbeatTimeout = 50 * time.Millisecond + conf.ElectionTimeout = 50 * time.Millisecond + conf.LeaderLeaseTimeout = 25 * time.Millisecond + conf.CommitTimeout = 5 * time.Millisecond + addr, transport := hashicorpRaft.NewInmemTransport(hashicorpRaft.ServerAddress(self)) + r, err := hashicorpRaft.NewRaft(conf, + &StateMachine{topo: topology.NewTopology("fsm", nil, 128*1024*1024, 0, false)}, + hashicorpRaft.NewInmemStore(), hashicorpRaft.NewInmemStore(), + hashicorpRaft.NewInmemSnapshotStore(), transport) + if err != nil { + t.Fatalf("new raft: %v", err) + } + t.Cleanup(func() { r.Shutdown() }) + + if err := r.BootstrapCluster(hashicorpRaft.Configuration{ + Servers: []hashicorpRaft.Server{{Suffrage: hashicorpRaft.Voter, ID: conf.LocalID, Address: addr}}, + }).Error(); err != nil { + t.Fatalf("bootstrap cluster: %v", err) + } + for deadline := time.Now().Add(10 * time.Second); ; { + if r.State() == hashicorpRaft.Leader { + return r, transport + } + if time.Now().After(deadline) { + t.Fatal("in-memory raft did not elect a leader") + } + time.Sleep(10 * time.Millisecond) + } +} diff --git a/weed/server/master_server_handlers_admin.go b/weed/server/master_server_handlers_admin.go index 293043d679..e6479ea49a 100644 --- a/weed/server/master_server_handlers_admin.go +++ b/weed/server/master_server_handlers_admin.go @@ -61,7 +61,7 @@ func (ms *MasterServer) volumeVacuumHandler(w http.ResponseWriter, r *http.Reque } } // glog.Infoln("garbageThreshold =", gcThreshold) - ms.Topo.Vacuum(ms.grpcDialOption, gcThreshold, ms.option.MaxParallelVacuumPerServer, 0, "", ms.preallocateSize, false, 0) + ms.Topo.Vacuum(ms.grpcDialOption, gcThreshold, ms.option.MaxParallelVacuumPerServer, ms.option.MaxParallelVacuumPerRack, 0, "", ms.preallocateSize, false, 0) ms.dirStatusHandler(w, r) } diff --git a/weed/server/volume_server.go b/weed/server/volume_server.go index ff1c85d044..93351eca44 100644 --- a/weed/server/volume_server.go +++ b/weed/server/volume_server.go @@ -93,6 +93,7 @@ func NewVolumeServer(adminMux, publicMux *http.ServeMux, ip string, ldbTimeout int64, allowUntrustedRemoteEndpoints bool, diskProbeConfig stats.DiskIOProbeConfig, + syncBatch storage.SyncBatchOption, ) *VolumeServer { v := util.GetViper() @@ -149,6 +150,7 @@ func NewVolumeServer(adminMux, publicMux *http.ServeMux, ip string, vs.store = storage.NewStore(vs.grpcDialOption, ip, port, grpcPort, publicUrl, id, folders, maxCounts, minFreeSpaces, idxFolder, vs.needleMapKind, diskTypes, diskTags, ldbTimeout, diskProbeConfig) vs.store.MasterFn = vs.getCurrentMaster + vs.store.SetSyncBatchOption(syncBatch) vs.guard = security.NewGuard(whiteList, signingKey, expiresAfterSec, readSigningKey, readExpiresAfterSec) handleStaticResources(adminMux) diff --git a/weed/server/volume_server_handlers_read.go b/weed/server/volume_server_handlers_read.go index aab0245f5e..16ab5bf96a 100644 --- a/weed/server/volume_server_handlers_read.go +++ b/weed/server/volume_server_handlers_read.go @@ -149,6 +149,20 @@ func (vs *VolumeServer) proxyReqToTargetServer(w http.ResponseWriter, r *http.Re } } +// notModified checks If-Modified-Since only when there is no If-None-Match +// (RFC 9110 13.2.2). +func notModified(r *http.Request, etag string, lastModified uint64) bool { + if inm := r.Header.Values("If-None-Match"); len(inm) != 0 { + return etagListMatches(strings.Join(inm, ","), etag, true) + } + ims := r.Header.Get("If-Modified-Since") + if ims == "" || lastModified == 0 { + return false + } + t, err := time.Parse(http.TimeFormat, ims) + return err == nil && t.Unix() >= int64(lastModified) +} + func (vs *VolumeServer) GetOrHeadHandler(w http.ResponseWriter, r *http.Request) { n := new(needle.Needle) vid, fid, filename, ext, _ := parseURLPath(r.URL.Path) @@ -232,26 +246,29 @@ func (vs *VolumeServer) GetOrHeadHandler(w http.ResponseWriter, r *http.Request) return } if n.Cookie != cookie { + // A meta-only read ended the request; record its success. + if readOption.IsMetaOnly { + if v := vs.store.GetVolume(volumeId); v != nil { + v.NoteMetaOnlySuccess() + } + } glog.V(0).Infof("request %s with cookie:%x expected:%x from %s agent %s", r.URL.Path, cookie, n.Cookie, r.RemoteAddr, r.UserAgent()) NotFound(w) return } if n.LastModified != 0 { w.Header().Set("Last-Modified", time.Unix(int64(n.LastModified), 0).UTC().Format(http.TimeFormat)) - if r.Header.Get("If-Modified-Since") != "" { - if t, parseError := time.Parse(http.TimeFormat, r.Header.Get("If-Modified-Since")); parseError == nil { - if t.Unix() >= int64(n.LastModified) { - w.WriteHeader(http.StatusNotModified) - return - } + } + SetEtag(w, n.Etag()) + if notModified(r, n.Etag(), n.LastModified) { + if readOption.IsMetaOnly { + if v := vs.store.GetVolume(volumeId); v != nil { + v.NoteMetaOnlySuccess() } } - } - if inm := r.Header.Get("If-None-Match"); inm == "\""+n.Etag()+"\"" { w.WriteHeader(http.StatusNotModified) return } - SetEtag(w, n.Etag()) if n.HasPairs() { pairMap := make(map[string]string) @@ -479,6 +496,10 @@ func (vs *VolumeServer) streamWriteResponseContent(filename string, mimeType str AdjustPassthroughHeaders(w, r, filename) if r.Method == http.MethodHead { + // HEAD ends at the metadata read; record its success. + if v := vs.store.GetVolume(volumeId); v != nil { + v.NoteMetaOnlySuccess() + } w.Header().Set("Content-Length", strconv.FormatInt(totalSize, 10)) return } diff --git a/weed/server/volume_server_handlers_read_test.go b/weed/server/volume_server_handlers_read_test.go index 758ff52ea8..1402a9954d 100644 --- a/weed/server/volume_server_handlers_read_test.go +++ b/weed/server/volume_server_handlers_read_test.go @@ -6,6 +6,7 @@ import ( "net/http/httptest" "strings" "testing" + "time" ) func TestInternalErrorWritesTheReadError(t *testing.T) { @@ -59,3 +60,39 @@ func TestShouldAttemptStreamWrite(t *testing.T) { } } } + +func TestNotModifiedChecksIfNoneMatchFirst(t *testing.T) { + const etag = "353dd8be" + const lastModified = 1700000000 + before := time.Unix(lastModified-60, 0).UTC().Format(http.TimeFormat) + after := time.Unix(lastModified+60, 0).UTC().Format(http.TimeFormat) + + tests := []struct { + name string + headers [][2]string + want bool + }{ + {"matching etag", [][2]string{{"If-None-Match", `"353dd8be"`}}, true}, + {"weak etag", [][2]string{{"If-None-Match", `W/"353dd8be"`}}, true}, + {"etag in a list", [][2]string{{"If-None-Match", `"x", "353dd8be"`}}, true}, + {"star", [][2]string{{"If-None-Match", `*`}}, true}, + {"other etag", [][2]string{{"If-None-Match", `"x"`}}, false}, + {"etag across repeated lines", [][2]string{{"If-None-Match", `"x"`}, {"If-None-Match", `"353dd8be"`}}, true}, + {"comma inside quotes", [][2]string{{"If-None-Match", `"353dd8be,old"`}}, false}, + {"other etag ignores If-Modified-Since", [][2]string{{"If-None-Match", `"x"`}, {"If-Modified-Since", after}}, false}, + {"empty If-None-Match ignores If-Modified-Since", [][2]string{{"If-None-Match", ``}, {"If-Modified-Since", after}}, false}, + {"not modified since", [][2]string{{"If-Modified-Since", after}}, true}, + {"modified since", [][2]string{{"If-Modified-Since", before}}, false}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + r := httptest.NewRequest(http.MethodGet, "/3,01637037d6", nil) + for _, kv := range tt.headers { + r.Header.Add(kv[0], kv[1]) + } + if got := notModified(r, etag, lastModified); got != tt.want { + t.Errorf("notModified = %v, want %v", got, tt.want) + } + }) + } +} diff --git a/weed/shell/command_ec_encode.go b/weed/shell/command_ec_encode.go index 0effe470ab..15a10fe516 100644 --- a/weed/shell/command_ec_encode.go +++ b/weed/shell/command_ec_encode.go @@ -54,7 +54,8 @@ func (c *commandEcEncode) Help() string { per node (the data-center digit is not used for EC). For example -shardReplicaPlacement=021 requests at most 1 shard per node and 2 per rack. Only when the final placement meets these limits does losing one rack cost - at most 2 shards; the command does not guarantee that the limits are met. + at most 2 shards, so the encode fails (and keeps the original volumes) + when the cluster cannot honor them. The -collection parameter is a comma-separated list of collection names, with "*" and "?" wildcards, and regex patterns: diff --git a/weed/shell/command_fs_configure.go b/weed/shell/command_fs_configure.go index 29be9b87ae..165e0b8e65 100644 --- a/weed/shell/command_fs_configure.go +++ b/weed/shell/command_fs_configure.go @@ -67,7 +67,7 @@ func (c *commandFsConfigure) Do(args []string, commandEnv *CommandEnv, writer io replication := fsConfigureCommand.String("replication", "", "assign writes with this replication") ttl := fsConfigureCommand.String("ttl", "", "assign writes with this ttl (e.g., 1m, 1h, 1d, 1w, 1y)") diskType := fsConfigureCommand.String("disk", "", "[hdd|ssd|] hard drive or solid state drive or any tag") - fsync := fsConfigureCommand.Bool("fsync", false, "fsync for the writes") + fsync := fsConfigureCommand.Bool("fsync", false, "fsync for the writes; unset inherits from the parent path") isReadOnly := fsConfigureCommand.Bool("readOnly", false, "disable writes") worm := fsConfigureCommand.Bool("worm", false, "write-once-read-many, written files are readonly; unset inherits from the parent path") wormGracePeriod := fsConfigureCommand.Uint64("wormGracePeriod", 0, "grace period before worm is enforced, in seconds") @@ -83,7 +83,7 @@ func (c *commandFsConfigure) Do(args []string, commandEnv *CommandEnv, writer io return nil } - fc, err := filer.ReadFilerConf(commandEnv.option.FilerAddress, commandEnv.option.GrpcDialOption, commandEnv.MasterClient) + fc, err := filer.ReadFilerConf(commandEnv.FilerAddress(), commandEnv.option.GrpcDialOption, commandEnv.MasterClient) if err != nil { return err } @@ -101,7 +101,6 @@ func (c *commandFsConfigure) Do(args []string, commandEnv *CommandEnv, writer io Collection: *collection, Replication: *replication, Ttl: *ttl, - Fsync: *fsync, MaxFileNameLength: uint32(*maxFileNameLength), DiskType: *diskType, VolumeGrowthCount: uint32(*volumeGrowthCount), @@ -113,10 +112,13 @@ func (c *commandFsConfigure) Do(args []string, commandEnv *CommandEnv, writer io WormRetentionTimeSeconds: *wormRetentionTime, } - // worm is only carried when the flag is passed, so a rule that says nothing - // about it keeps inheriting from the enclosing path + // fsync and worm are only carried when the flag is passed, so a rule that + // says nothing about them keeps inheriting from the enclosing path fsConfigureCommand.Visit(func(f *flag.Flag) { - if f.Name == "worm" { + switch f.Name { + case "fsync": + locConf.Fsync = proto.Bool(*fsync) + case "worm": locConf.Worm = proto.Bool(*worm) } }) @@ -152,17 +154,15 @@ func (c *commandFsConfigure) Do(args []string, commandEnv *CommandEnv, writer io fc.DeleteLocationConf(*locationPrefix) } else { fc.AddLocationConf(locConf) - // AddLocationConf merges these boolean fields with OR, which can never - // turn a flag off; let an explicitly passed false win, e.g. -readOnly=false - // to reopen a bucket that quota enforcement locked. worm does not belong - // here: it merges on presence, so the value set above already wins. + // AddLocationConf merges this boolean field with OR, which can never + // turn it off; let an explicitly passed false win, e.g. -readOnly=false + // to reopen a bucket that quota enforcement locked. fsync and worm do + // not belong here: they merge on presence, so the values set above + // already win. if mergedConf, found := fc.GetLocationConf(*locationPrefix); found { fsConfigureCommand.Visit(func(f *flag.Flag) { - switch f.Name { - case "readOnly": + if f.Name == "readOnly" { mergedConf.ReadOnly = *isReadOnly - case "fsync": - mergedConf.Fsync = *fsync } }) } diff --git a/weed/shell/command_fs_meta_save.go b/weed/shell/command_fs_meta_save.go index b936252f62..c52cc421dc 100644 --- a/weed/shell/command_fs_meta_save.go +++ b/weed/shell/command_fs_meta_save.go @@ -75,7 +75,7 @@ func (c *commandFsMetaSave) Do(args []string, commandEnv *CommandEnv, writer io. if fileName == "" { t := time.Now() fileName = fmt.Sprintf("%s-%4d%02d%02d-%02d%02d%02d.meta.gz", - commandEnv.option.FilerAddress.ToHttpAddress(), t.Year(), t.Month(), t.Day(), t.Hour(), t.Minute(), t.Second()) + commandEnv.FilerAddress().ToHttpAddress(), t.Year(), t.Month(), t.Day(), t.Hour(), t.Minute(), t.Second()) } var dst io.Writer @@ -143,7 +143,7 @@ func (c *commandFsMetaSave) Do(args []string, commandEnv *CommandEnv, writer io. }) if err == nil { - fmt.Fprintf(writer, "meta data for http://%s%s is saved to %s\n", commandEnv.option.FilerAddress.ToHttpAddress(), path, fileName) + fmt.Fprintf(writer, "meta data for http://%s%s is saved to %s\n", commandEnv.FilerAddress().ToHttpAddress(), path, fileName) } return err @@ -323,6 +323,9 @@ func doTraverseBfsAndSaving(filerClient filer_pb.FilerClient, writer io.Writer, firstErr = genErr hasErr.Store(true) }) + close(outputChan) + wg.Wait() + <-saveErrChan return genErr } else { if e.IsDirectory { diff --git a/weed/shell/command_fs_verify.go b/weed/shell/command_fs_verify.go index 32128b4b6f..24007793f0 100644 --- a/weed/shell/command_fs_verify.go +++ b/weed/shell/command_fs_verify.go @@ -254,7 +254,7 @@ func (c *commandFsVerify) verifyProcessMetadata(path string, wg *sync.WaitGroup) StopTsNs: time.Now().UnixNano(), EventErrorType: pb.DontLogError, } - err = pb.FollowMetadata(c.env.option.FilerAddress, c.env.option.GrpcDialOption, metadataFollowOption, processEventFn) + err = pb.FollowMetadata(c.env.FilerAddress(), c.env.option.GrpcDialOption, metadataFollowOption, processEventFn) return fileCount, errCount, prunedCount, err } diff --git a/weed/shell/command_mq_topic_compact.go b/weed/shell/command_mq_topic_compact.go index ab3552022c..8171144b1f 100644 --- a/weed/shell/command_mq_topic_compact.go +++ b/weed/shell/command_mq_topic_compact.go @@ -73,7 +73,7 @@ func (c *commandMqTopicCompact) Do(args []string, commandEnv *CommandEnv, writer // read topic configuration fca := filer_client.NewFilerClientAccessor( - []pb.ServerAddress{commandEnv.option.FilerAddress}, + []pb.ServerAddress{commandEnv.FilerAddress()}, commandEnv.option.GrpcDialOption, ) t := topic.NewTopic(*namespace, *topicName) diff --git a/weed/shell/command_remote_meta_sync.go b/weed/shell/command_remote_meta_sync.go index b1ce83140b..ca9320d3f5 100644 --- a/weed/shell/command_remote_meta_sync.go +++ b/weed/shell/command_remote_meta_sync.go @@ -84,7 +84,7 @@ func (c *commandRemoteMetaSync) Do(args []string, commandEnv *CommandEnv, writer } func detectMountInfo(commandEnv *CommandEnv, writer io.Writer, dir string) (*remote_pb.RemoteStorageMapping, string, *remote_pb.RemoteStorageLocation, *remote_pb.RemoteConf, error) { - return filer.DetectMountInfo(commandEnv.option.GrpcDialOption, commandEnv.option.FilerAddress, dir) + return filer.DetectMountInfo(commandEnv.option.GrpcDialOption, commandEnv.FilerAddress(), dir) } /* diff --git a/weed/shell/command_remote_mount.go b/weed/shell/command_remote_mount.go index 1eeb43bf53..debfa27a63 100644 --- a/weed/shell/command_remote_mount.go +++ b/weed/shell/command_remote_mount.go @@ -85,7 +85,7 @@ func (c *commandRemoteMount) Do(args []string, commandEnv *CommandEnv, writer io } // find configuration for remote storage - remoteConf, err := filer.ReadRemoteStorageConf(commandEnv.option.GrpcDialOption, commandEnv.option.FilerAddress, remote_storage.ParseLocationName(*remote)) + remoteConf, err := filer.ReadRemoteStorageConf(commandEnv.option.GrpcDialOption, commandEnv.FilerAddress(), remote_storage.ParseLocationName(*remote)) if err != nil { return fmt.Errorf("find configuration for %s: %v", *remote, err) } @@ -133,7 +133,7 @@ func (c *commandRemoteMount) Do(args []string, commandEnv *CommandEnv, writer io func listExistingRemoteStorageMounts(commandEnv *CommandEnv, writer io.Writer) (mappings *remote_pb.RemoteStorageMapping, err error) { // read current mapping - mappings, err = filer.ReadMountMappings(commandEnv.option.GrpcDialOption, commandEnv.option.FilerAddress) + mappings, err = filer.ReadMountMappings(commandEnv.option.GrpcDialOption, commandEnv.FilerAddress()) if err != nil { return mappings, err } diff --git a/weed/shell/command_remote_mount_buckets.go b/weed/shell/command_remote_mount_buckets.go index 1fa464141c..c011a15e57 100644 --- a/weed/shell/command_remote_mount_buckets.go +++ b/weed/shell/command_remote_mount_buckets.go @@ -62,7 +62,7 @@ func (c *commandRemoteMountBuckets) Do(args []string, commandEnv *CommandEnv, wr } // find configuration for remote storage - remoteConf, err := filer.ReadRemoteStorageConf(commandEnv.option.GrpcDialOption, commandEnv.option.FilerAddress, *remote) + remoteConf, err := filer.ReadRemoteStorageConf(commandEnv.option.GrpcDialOption, commandEnv.FilerAddress(), *remote) if err != nil { return fmt.Errorf("find configuration for %s: %v", *remote, err) } diff --git a/weed/shell/command_remote_uncache.go b/weed/shell/command_remote_uncache.go index e7281c791f..20383a8f40 100644 --- a/weed/shell/command_remote_uncache.go +++ b/weed/shell/command_remote_uncache.go @@ -57,7 +57,7 @@ func (c *commandRemoteUncache) Do(args []string, commandEnv *CommandEnv, writer return nil } - mappings, listErr := filer.ReadMountMappings(commandEnv.option.GrpcDialOption, commandEnv.option.FilerAddress) + mappings, listErr := filer.ReadMountMappings(commandEnv.option.GrpcDialOption, commandEnv.FilerAddress()) if listErr != nil { return listErr } diff --git a/weed/shell/command_remote_unmount.go b/weed/shell/command_remote_unmount.go index 030dd9021a..fd905d00ee 100644 --- a/weed/shell/command_remote_unmount.go +++ b/weed/shell/command_remote_unmount.go @@ -52,7 +52,7 @@ func (c *commandRemoteUnmount) Do(args []string, commandEnv *CommandEnv, writer return nil } - mappings, listErr := filer.ReadMountMappings(commandEnv.option.GrpcDialOption, commandEnv.option.FilerAddress) + mappings, listErr := filer.ReadMountMappings(commandEnv.option.GrpcDialOption, commandEnv.FilerAddress()) if listErr != nil { return listErr } @@ -79,7 +79,7 @@ func (c *commandRemoteUnmount) Do(args []string, commandEnv *CommandEnv, writer } // reset remote sync offset in case the folder is mounted again - if err = remote_storage.SetSyncOffset(commandEnv.option.GrpcDialOption, commandEnv.option.FilerAddress, *dir, time.Now().UnixNano()); err != nil { + if err = remote_storage.SetSyncOffset(commandEnv.option.GrpcDialOption, commandEnv.FilerAddress(), *dir, time.Now().UnixNano()); err != nil { return fmt.Errorf("reset remote.sync offset for %s: %v", *dir, err) } diff --git a/weed/shell/command_s3_bucket_quota_check.go b/weed/shell/command_s3_bucket_quota_check.go index 7313514ec2..eda0c42842 100644 --- a/weed/shell/command_s3_bucket_quota_check.go +++ b/weed/shell/command_s3_bucket_quota_check.go @@ -60,7 +60,7 @@ func (c *commandS3BucketQuotaEnforce) Do(args []string, commandEnv *CommandEnv, } // read existing filer configuration - fc, err := filer.ReadFilerConf(commandEnv.option.FilerAddress, commandEnv.option.GrpcDialOption, commandEnv.MasterClient) + fc, err := filer.ReadFilerConf(commandEnv.FilerAddress(), commandEnv.option.GrpcDialOption, commandEnv.MasterClient) if err != nil { return err } diff --git a/weed/shell/command_s3_clean_uploads.go b/weed/shell/command_s3_clean_uploads.go index 1da7a02d40..1d7bf986c0 100644 --- a/weed/shell/command_s3_clean_uploads.go +++ b/weed/shell/command_s3_clean_uploads.go @@ -106,7 +106,7 @@ func (c *commandS3CleanUploads) cleanupUploads(commandEnv *CommandEnv, writer io fmt.Fprintf(writer, "skip %s: %v\n", staleUpload.Name, checkErr) continue } - deleteUrl := fmt.Sprintf("http://%s%s/%s?recursive=true&ignoreRecursiveError=true", commandEnv.option.FilerAddress.ToHttpAddress(), uploadsDir, staleUpload.Name) + deleteUrl := fmt.Sprintf("http://%s%s/%s?recursive=true&ignoreRecursiveError=true", commandEnv.FilerAddress().ToHttpAddress(), uploadsDir, staleUpload.Name) if completed { deleteUrl += "&skipChunkDeletion=true" } diff --git a/weed/shell/command_s3_iam_client.go b/weed/shell/command_s3_iam_client.go index eb619100a1..832471d0bf 100644 --- a/weed/shell/command_s3_iam_client.go +++ b/weed/shell/command_s3_iam_client.go @@ -28,7 +28,7 @@ func (ce *CommandEnv) withIamClient(fn func(ctx context.Context, client iam_pb.S ctx, cancel := context.WithTimeout(iamAdminAuthContext(context.Background()), iamRequestTimeout) defer cancel() return fn(ctx, iam_pb.NewSeaweedIdentityAccessManagementClient(conn)) - }, ce.option.FilerAddress.ToGrpcAddress(), false, ce.option.GrpcDialOption) + }, ce.FilerAddress().ToGrpcAddress(), false, ce.option.GrpcDialOption) } func iamAdminAuthContext(ctx context.Context) context.Context { diff --git a/weed/shell/command_volume_fsck.go b/weed/shell/command_volume_fsck.go index 7578c71823..ef6b61bfab 100644 --- a/weed/shell/command_volume_fsck.go +++ b/weed/shell/command_volume_fsck.go @@ -271,19 +271,33 @@ func (c *commandVolumeFsck) Do(args []string, commandEnv *CommandEnv, writer io. if *c.findMissingChunksInFiler { // collect all filer file ids and paths - if err = c.collectFilerFileIdAndPaths(dataNodeVolumeIdToVInfo, *purgeAbsent, collectModifyFromAtNs, collectCutoffFromAtNs); err != nil { - return fmt.Errorf("collectFilerFileIdAndPaths: %w", err) + var queuePurge func(purgeItem) + var finishPurge func() + if *applyPurging || *purgeAbsent { + queuePurge, finishPurge = startPurgeWorkers(*c.purgeConcurrency, c.httpDelete) } - for dataNodeId, volumeIdToVInfo := range dataNodeVolumeIdToVInfo { - // for each volume, check filer file ids - if err = c.findFilerChunksMissingInVolumeServers(volumeIdToVInfo, dataNodeId, *applyPurging || *purgeAbsent); err != nil { - return fmt.Errorf("findFilerChunksMissingInVolumeServers: %w", err) + err = func() error { + if err := c.collectFilerFileIdAndPaths(dataNodeVolumeIdToVInfo, *purgeAbsent, queuePurge, collectModifyFromAtNs, collectCutoffFromAtNs); err != nil { + return fmt.Errorf("collectFilerFileIdAndPaths: %w", err) } + for dataNodeId, volumeIdToVInfo := range dataNodeVolumeIdToVInfo { + // for each volume, check filer file ids + if err := c.findFilerChunksMissingInVolumeServers(volumeIdToVInfo, dataNodeId, queuePurge); err != nil { + return fmt.Errorf("findFilerChunksMissingInVolumeServers: %w", err) + } + } + return nil + }() + if finishPurge != nil { + finishPurge() + } + if err != nil { + return err } c.purgeEmptyDirectories() } else { // collect all filer file ids - if err = c.collectFilerFileIdAndPaths(dataNodeVolumeIdToVInfo, false, 0, 0); err != nil { + if err = c.collectFilerFileIdAndPaths(dataNodeVolumeIdToVInfo, false, nil, 0, 0); err != nil { return fmt.Errorf("failed to collect file ids from filer: %w", err) } // If any entry's manifest could not be resolved, our in-use fid set @@ -306,7 +320,7 @@ func (c *commandVolumeFsck) Do(args []string, commandEnv *CommandEnv, writer io. return nil } -func (c *commandVolumeFsck) collectFilerFileIdAndPaths(dataNodeVolumeIdToVInfo map[string]map[uint32]VInfo, purgeAbsent bool, collectModifyFromAtNs int64, cutoffFromAtNs int64) error { +func (c *commandVolumeFsck) collectFilerFileIdAndPaths(dataNodeVolumeIdToVInfo map[string]map[uint32]VInfo, purgeAbsent bool, queuePurge func(purgeItem), collectModifyFromAtNs int64, cutoffFromAtNs int64) error { if *c.verbose { fmt.Fprintf(c.writer, "checking each file from filer path %s...\n", c.getCollectFilerFilePath()) } @@ -382,12 +396,6 @@ func (c *commandVolumeFsck) collectFilerFileIdAndPaths(dataNodeVolumeIdToVInfo m }, func(outputChan chan interface{}) error { buffer := make([]byte, readbufferSize) - var queuePurge func(purgeItem) - var finishPurge func() - if purgeAbsent { - queuePurge, finishPurge = startPurgeWorkers(*c.purgeConcurrency, c.httpDelete) - defer finishPurge() - } for item := range outputChan { i := item.(*Item) if f, ok := files[i.vid]; ok { @@ -400,6 +408,13 @@ func (c *commandVolumeFsck) collectFilerFileIdAndPaths(dataNodeVolumeIdToVInfo m if _, err := f.Write([]byte(i.path)); err != nil { return err } + util.Uint32toBytes(buffer[:4], uint32(len(i.hardLinkId))) + if _, err := f.Write(buffer[:4]); err != nil { + return err + } + if _, err := f.Write(i.hardLinkId); err != nil { + return err + } } else if *c.findMissingChunksInFiler { // check if the volume matches the filter if len(c.volumeIds) > 0 { @@ -423,10 +438,10 @@ func (c *commandVolumeFsck) collectFilerFileIdAndPaths(dataNodeVolumeIdToVInfo m }) } -func (c *commandVolumeFsck) findFilerChunksMissingInVolumeServers(volumeIdToVInfo map[uint32]VInfo, dataNodeId string, applyPurging bool) error { +func (c *commandVolumeFsck) findFilerChunksMissingInVolumeServers(volumeIdToVInfo map[uint32]VInfo, dataNodeId string, queuePurge func(purgeItem)) error { for volumeId, vinfo := range volumeIdToVInfo { - checkErr := c.oneVolumeFileIdsCheckOneVolume(dataNodeId, volumeId, applyPurging) + checkErr := c.oneVolumeFileIdsCheckOneVolume(dataNodeId, volumeId, queuePurge) if checkErr != nil { return fmt.Errorf("failed to collect file ids from volume %d on %s: %v", volumeId, vinfo.server, checkErr) } @@ -650,7 +665,7 @@ func (lw *lockedWriter) Write(p []byte) (int, error) { return lw.w.Write(p) } -func (c *commandVolumeFsck) readFilerFileIdFile(volumeId uint32, fn func(needleId types.NeedleId, itemPath util.FullPath)) error { +func (c *commandVolumeFsck) readFilerFileIdFile(volumeId uint32, fn func(needleId types.NeedleId, itemPath util.FullPath, hardLinkId []byte)) error { fp, err := os.Open(getFilerFileIdFile(c.tempFolder, volumeId)) if err != nil { return err @@ -685,13 +700,23 @@ func (c *commandVolumeFsck) readFilerFileIdFile(volumeId uint32, fn func(needleI return fmt.Errorf("read fid path size mismatch for volume %d,%x%08x: got %d want %d", volumeId, item.fileKey, item.cookie, n, pathSize) } item.path = util.FullPath(pathBytes) + item.hardLinkId = nil + if _, err := io.ReadFull(br, buffer[:4]); err != nil { + return fmt.Errorf("read fid hard link id size for volume %d,%x%08x: %w", volumeId, item.fileKey, item.cookie, err) + } + if hardLinkIdSize := util.BytesToUint32(buffer[:4]); hardLinkIdSize > 0 { + item.hardLinkId = make([]byte, int(hardLinkIdSize)) + if _, err := io.ReadFull(br, item.hardLinkId); err != nil { + return fmt.Errorf("read fid hard link id for volume %d,%x%08x: %w", volumeId, item.fileKey, item.cookie, err) + } + } needleId := types.NeedleId(item.fileKey) - fn(needleId, item.path) + fn(needleId, item.path, item.hardLinkId) } return nil } -func (c *commandVolumeFsck) oneVolumeFileIdsCheckOneVolume(dataNodeId string, volumeId uint32, applyPurging bool) (err error) { +func (c *commandVolumeFsck) oneVolumeFileIdsCheckOneVolume(dataNodeId string, volumeId uint32, queuePurge func(purgeItem)) (err error) { if *c.verbose { fmt.Fprintf(c.writer, "find missing file chunks in dataNodeId %s volume %d ...\n", dataNodeId, volumeId) } @@ -702,11 +727,11 @@ func (c *commandVolumeFsck) oneVolumeFileIdsCheckOneVolume(dataNodeId string, vo if err = db.LoadFromIdx(getVolumeFileIdFile(c.tempFolder, dataNodeId, volumeId)); err != nil { return } - if err = c.readFilerFileIdFile(volumeId, func(needleId types.NeedleId, itemPath util.FullPath) { + if err = c.readFilerFileIdFile(volumeId, func(needleId types.NeedleId, itemPath util.FullPath, hardLinkId []byte) { if _, found := db.Get(needleId); !found { fmt.Fprintf(c.writer, "%s\n", itemPath) - if applyPurging { - c.httpDelete(itemPath) + if queuePurge != nil { + queuePurge(purgeItem{path: itemPath, hardLinkId: hardLinkId}) } } }); err != nil { @@ -765,7 +790,7 @@ func (c *commandVolumeFsck) httpDelete(path util.FullPath) { req.URL = &url.URL{ Scheme: "http", - Host: c.env.option.FilerAddress.ToHttpAddress(), + Host: c.env.FilerAddress().ToHttpAddress(), Path: string(path), } @@ -889,7 +914,7 @@ func (c *commandVolumeFsck) oneVolumeFileIdsSubtractFilerFileIds(dataNodeId stri return } - if err = c.readFilerFileIdFile(volumeId, func(filerNeedleId types.NeedleId, itemPath util.FullPath) { + if err = c.readFilerFileIdFile(volumeId, func(filerNeedleId types.NeedleId, itemPath util.FullPath, _ []byte) { inUseCount++ if *c.verifyNeedle && !vinfo.isEcVolume { if needleValue, ok := volumeFileIdDb.Get(filerNeedleId); ok && !needleValue.Size.IsDeleted() { diff --git a/weed/shell/command_volume_fsck_test.go b/weed/shell/command_volume_fsck_test.go index 50bdd0f8fa..7804b85224 100644 --- a/weed/shell/command_volume_fsck_test.go +++ b/weed/shell/command_volume_fsck_test.go @@ -1,12 +1,14 @@ package shell import ( + "bytes" "context" "fmt" "io" "net" "net/http" "net/http/httptest" + "os" "sort" "strings" "sync/atomic" @@ -16,6 +18,8 @@ import ( "github.com/seaweedfs/seaweedfs/weed/filer" "github.com/seaweedfs/seaweedfs/weed/pb" "github.com/seaweedfs/seaweedfs/weed/pb/filer_pb" + "github.com/seaweedfs/seaweedfs/weed/storage/needle_map" + "github.com/seaweedfs/seaweedfs/weed/storage/types" "github.com/seaweedfs/seaweedfs/weed/util" "google.golang.org/grpc" "google.golang.org/grpc/credentials/insecure" @@ -310,3 +314,109 @@ func TestVolumeFsckPurgeHardLinksSerialized(t *testing.T) { t.Fatalf("processed %d deletes, want 8", got) } } + +// Orphans found by the per-volume check go through the same purge queue as +// the missing-volume path, and the fid file keeps each entry's hard link id +// so same-id deletes still land on one worker. +func TestVolumeFsckCheckOneVolumeQueuesOrphansWithHardLinkId(t *testing.T) { + tempDir := t.TempDir() + var verbose bool + var buf bytes.Buffer + c := &commandVolumeFsck{tempFolder: tempDir, verbose: &verbose, writer: &buf} + + volumeId := uint32(7) + db := needle_map.NewMemDb() + if err := db.Set(types.NeedleId(11), types.ToOffset(16), types.Size(1)); err != nil { + t.Fatalf("seed volume idx: %v", err) + } + if err := db.SaveToIdx(getVolumeFileIdFile(tempDir, "dn1", volumeId)); err != nil { + t.Fatalf("write volume idx: %v", err) + } + db.Close() + + f, err := os.Create(getFilerFileIdFile(tempDir, volumeId)) + if err != nil { + t.Fatalf("create fid file: %v", err) + } + writeFid := func(fileKey uint64, cookie uint32, p string, hardLinkId []byte) { + buffer := make([]byte, readbufferSize) + util.Uint64toBytes(buffer, fileKey) + util.Uint32toBytes(buffer[8:], cookie) + util.Uint32toBytes(buffer[12:], uint32(len(p))) + if _, err := f.Write(buffer); err != nil { + t.Fatalf("write fid header: %v", err) + } + if _, err := f.Write([]byte(p)); err != nil { + t.Fatalf("write fid path: %v", err) + } + util.Uint32toBytes(buffer[:4], uint32(len(hardLinkId))) + if _, err := f.Write(buffer[:4]); err != nil { + t.Fatalf("write fid hard link id size: %v", err) + } + if _, err := f.Write(hardLinkId); err != nil { + t.Fatalf("write fid hard link id: %v", err) + } + } + writeFid(11, 1, "/keep/a", nil) + writeFid(22, 2, "/orphan/b", []byte{1, 2, 3}) + writeFid(33, 3, "/orphan/c", nil) + if err := f.Close(); err != nil { + t.Fatalf("close fid file: %v", err) + } + + var queued []purgeItem + if err := c.oneVolumeFileIdsCheckOneVolume("dn1", volumeId, func(it purgeItem) { + queued = append(queued, it) + }); err != nil { + t.Fatalf("check volume: %v", err) + } + if len(queued) != 2 { + t.Fatalf("queued %d purges, want 2 (the two orphans): %v", len(queued), queued) + } + if queued[0].path != "/orphan/b" || queued[1].path != "/orphan/c" { + t.Fatalf("queued paths %v, want /orphan/b then /orphan/c", queued) + } + if !bytes.Equal(queued[0].hardLinkId, []byte{1, 2, 3}) { + t.Fatalf("queued hardLinkId %v, want [1 2 3]", queued[0].hardLinkId) + } + if queued[1].hardLinkId != nil { + t.Fatalf("queued hardLinkId %v, want nil", queued[1].hardLinkId) + } +} + +// A .fid file cut short inside the hard link id field is corrupt input, not a +// record without a link id: the reader must fail instead of emitting the entry +// with a nil id. +func TestReadFilerFileIdFileRejectsTruncatedHardLinkId(t *testing.T) { + tempDir := t.TempDir() + var verbose bool + c := &commandVolumeFsck{tempFolder: tempDir, verbose: &verbose, writer: io.Discard} + + f, err := os.Create(getFilerFileIdFile(tempDir, 7)) + if err != nil { + t.Fatalf("create fid file: %v", err) + } + buffer := make([]byte, readbufferSize) + util.Uint64toBytes(buffer, 11) + util.Uint32toBytes(buffer[8:], 1) + util.Uint32toBytes(buffer[12:], uint32(len("/orphan/b"))) + if _, err := f.Write(buffer); err != nil { + t.Fatalf("write fid header: %v", err) + } + if _, err := f.Write([]byte("/orphan/b")); err != nil { + t.Fatalf("write fid path: %v", err) + } + if _, err := f.Write([]byte{0, 0}); err != nil { + t.Fatalf("write partial hard link id size: %v", err) + } + if err := f.Close(); err != nil { + t.Fatalf("close fid file: %v", err) + } + + err = c.readFilerFileIdFile(7, func(types.NeedleId, util.FullPath, []byte) { + t.Fatal("callback fired for a truncated record") + }) + if err == nil { + t.Fatal("expected an error for a truncated hard link id size") + } +} diff --git a/weed/shell/commands.go b/weed/shell/commands.go index 3bbb363b18..43b681d4d7 100644 --- a/weed/shell/commands.go +++ b/weed/shell/commands.go @@ -4,7 +4,10 @@ import ( "context" "fmt" "io" + "math/rand/v2" + "os" "strings" + "sync" "time" "github.com/seaweedfs/seaweedfs/weed/cluster" @@ -43,6 +46,7 @@ type CommandEnv struct { noLock bool forceNoLock bool verbose bool + filerMu sync.Mutex } func NewCommandEnv(options *ShellOptions) *CommandEnv { @@ -147,10 +151,75 @@ var _ = filer_pb.FilerClient(&CommandEnv{}) func (ce *CommandEnv) WithFilerClient(streamingMode bool, fn func(filer_pb.SeaweedFilerClient) error) error { - return pb.WithGrpcFilerClient(streamingMode, 0, ce.option.FilerAddress, ce.option.GrpcDialOption, fn) + return pb.WithGrpcFilerClient(streamingMode, 0, ce.FilerAddress(), ce.option.GrpcDialOption, fn) } +// FilerAddress returns the discovered or configured filer, running discovery +// on first use so commands that never need a filer skip the wait entirely. +// A failed or empty discovery is not cached: each later call retries until a +// filer registers, so a long-lived shell can still find one. +func (ce *CommandEnv) FilerAddress() pb.ServerAddress { + ce.filerMu.Lock() + defer ce.filerMu.Unlock() + ce.discoverFiler() + return ce.option.FilerAddress +} + +func (ce *CommandEnv) discoverFiler() { + if ce.option.FilerAddress != "" || ce.MasterClient == nil || ce.option.FilerGroup == nil { + return + } + connectTimeout := ce.option.ConnectTimeout + if connectTimeout <= 0 { + connectTimeout = 30 * time.Second + } + ctx, cancel := context.WithTimeout(context.Background(), connectTimeout) + defer cancel() + var filers []pb.ServerAddress + var listErr error + for len(filers) == 0 && ctx.Err() == nil { + filers = filers[:0] + listErr = ce.MasterClient.WithClient(ctx, false, func(client master_pb.SeaweedClient) error { + resp, err := client.ListClusterNodes(ctx, &master_pb.ListClusterNodesRequest{ + ClientType: cluster.FilerType, + FilerGroup: *ce.option.FilerGroup, + }) + if err != nil { + return err + } + + for _, clusterNode := range resp.ClusterNodes { + filers = append(filers, pb.ServerAddress(clusterNode.Address)) + } + return nil + }) + if len(filers) == 0 { + if listErr != nil && !util.IsTransientError(listErr) { + break + } + select { + case <-ctx.Done(): + case <-time.After(time.Second): + } + } + } + if len(filers) > 0 { + ce.option.FilerAddress = filers[rand.IntN(len(filers))] + } else if listErr != nil { + fmt.Fprintf(os.Stderr, "cannot list filers from master %s: %v\n", *ce.option.Masters, listErr) + } else { + fmt.Fprintf(os.Stderr, "no filer registered with master %s within %s\n", *ce.option.Masters, connectTimeout) + } + if ce.option.Debug { + if len(filers) > 0 { + fmt.Fprintf(os.Stderr, "master: %s filers: %v\n", *ce.option.Masters, filers) + } else { + fmt.Fprintf(os.Stderr, "master: %s\n", *ce.option.Masters) + } + } +} + func (ce *CommandEnv) AdjustedUrl(location *filer_pb.Location) string { return location.Url } diff --git a/weed/shell/commands_test.go b/weed/shell/commands_test.go new file mode 100644 index 0000000000..8456413b7a --- /dev/null +++ b/weed/shell/commands_test.go @@ -0,0 +1,159 @@ +package shell + +import ( + "context" + "fmt" + "net" + "sync/atomic" + "testing" + "time" + + "github.com/seaweedfs/seaweedfs/weed/pb/master_pb" + "google.golang.org/grpc" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/credentials/insecure" + "google.golang.org/grpc/status" +) + +type fakeDiscoveryServer struct { + master_pb.UnimplementedSeaweedServer + leader string + filers atomic.Int32 + listErr atomic.Value +} + +func (s *fakeDiscoveryServer) KeepConnected(stream master_pb.Seaweed_KeepConnectedServer) error { + if _, err := stream.Recv(); err != nil { + return err + } + if err := stream.Send(&master_pb.KeepConnectedResponse{ + VolumeLocation: &master_pb.VolumeLocation{Leader: s.leader}, + }); err != nil { + return err + } + <-stream.Context().Done() + return nil +} + +func (s *fakeDiscoveryServer) ListClusterNodes(context.Context, *master_pb.ListClusterNodesRequest) (*master_pb.ListClusterNodesResponse, error) { + if err, _ := s.listErr.Load().(error); err != nil { + return nil, err + } + resp := &master_pb.ListClusterNodesResponse{} + for i := int32(0); i < s.filers.Load(); i++ { + resp.ClusterNodes = append(resp.ClusterNodes, &master_pb.ListClusterNodesResponse_ClusterNode{ + Address: fmt.Sprintf("127.0.0.1:88%02d", i), + }) + } + return resp, nil +} + +func startFakeDiscoveryMaster(t *testing.T, srv *fakeDiscoveryServer) string { + t.Helper() + lis, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatalf("listen: %v", err) + } + grpcServer := grpc.NewServer() + master_pb.RegisterSeaweedServer(grpcServer, srv) + go func() { _ = grpcServer.Serve(lis) }() + t.Cleanup(grpcServer.GracefulStop) + + _, port, _ := net.SplitHostPort(lis.Addr().String()) + return fmt.Sprintf("127.0.0.1:0.%s", port) +} + +func discoveryEnv(addr string, timeout time.Duration) (*CommandEnv, context.CancelFunc) { + masters := addr + filerGroup := "" + options := &ShellOptions{ + Masters: &masters, + GrpcDialOption: grpc.WithTransportCredentials(insecure.NewCredentials()), + FilerGroup: &filerGroup, + Directory: "/", + ConnectTimeout: timeout, + } + ce := NewCommandEnv(options) + ctx, cancel := context.WithCancel(context.Background()) + go ce.MasterClient.KeepConnectedToMaster(ctx) + ce.MasterClient.WaitUntilConnected(ctx) + return ce, cancel +} + +func TestFilerDiscoveryRetriesUntilFilerAppears(t *testing.T) { + srv := &fakeDiscoveryServer{} + addr := startFakeDiscoveryMaster(t, srv) + srv.leader = addr + + ce, cancel := discoveryEnv(addr, 10*time.Second) + defer cancel() + + go func() { + time.Sleep(1200 * time.Millisecond) + srv.filers.Store(1) + }() + + got := ce.FilerAddress() + if got != "127.0.0.1:8800" { + t.Fatalf("expected the late-registering filer, got %q", got) + } +} + +func TestFilerDiscoveryReturnsEmptyAtDeadline(t *testing.T) { + srv := &fakeDiscoveryServer{} + addr := startFakeDiscoveryMaster(t, srv) + srv.leader = addr + + ce, cancel := discoveryEnv(addr, 1500*time.Millisecond) + defer cancel() + + start := time.Now() + if got := ce.FilerAddress(); got != "" { + t.Fatalf("expected no filer, got %q", got) + } + if elapsed := time.Since(start); elapsed < time.Second { + t.Fatalf("discovery returned after %v, expected it to poll until the deadline", elapsed) + } +} + +func TestFilerDiscoveryRetriesAfterEmptyResult(t *testing.T) { + srv := &fakeDiscoveryServer{} + addr := startFakeDiscoveryMaster(t, srv) + srv.leader = addr + + ce, cancel := discoveryEnv(addr, 1500*time.Millisecond) + defer cancel() + + if got := ce.FilerAddress(); got != "" { + t.Fatalf("expected no filer, got %q", got) + } + srv.filers.Store(1) + if got := ce.FilerAddress(); got != "127.0.0.1:8800" { + t.Fatalf("expected the late-registered filer on retry, got %q", got) + } +} + +func TestFilerDiscoveryStopsOnNonTransientError(t *testing.T) { + srv := &fakeDiscoveryServer{} + addr := startFakeDiscoveryMaster(t, srv) + srv.leader = addr + srv.listErr.Store(status.Error(codes.PermissionDenied, "denied")) + + ce, cancel := discoveryEnv(addr, 30*time.Second) + defer cancel() + + start := time.Now() + if got := ce.FilerAddress(); got != "" { + t.Fatalf("expected no filer, got %q", got) + } + if elapsed := time.Since(start); elapsed > 10*time.Second { + t.Fatalf("discovery kept polling a non-transient error for %v", elapsed) + } +} + +func TestFilerAddressConfiguredSkipsDiscovery(t *testing.T) { + ce := &CommandEnv{option: &ShellOptions{FilerAddress: "127.0.0.1:8888"}} + if got := ce.FilerAddress(); got != "127.0.0.1:8888" { + t.Fatalf("expected the configured filer, got %q", got) + } +} diff --git a/weed/shell/s3tables_helpers.go b/weed/shell/s3tables_helpers.go index d595a5af74..18c5f90a72 100644 --- a/weed/shell/s3tables_helpers.go +++ b/weed/shell/s3tables_helpers.go @@ -20,7 +20,7 @@ func withFilerClient(commandEnv *CommandEnv, fn func(client filer_pb.SeaweedFile return pb.WithGrpcClient(context.Background(), false, 0, func(conn *grpc.ClientConn) error { client := filer_pb.NewSeaweedFilerClient(conn) return fn(client) - }, commandEnv.option.FilerAddress.ToGrpcAddress(), false, commandEnv.option.GrpcDialOption) + }, commandEnv.FilerAddress().ToGrpcAddress(), false, commandEnv.option.GrpcDialOption) } func executeS3Tables(commandEnv *CommandEnv, operation string, req interface{}, resp interface{}, accountID string) error { diff --git a/weed/shell/shell_liner.go b/weed/shell/shell_liner.go index 6aae477b0f..303f2ab2ad 100644 --- a/weed/shell/shell_liner.go +++ b/weed/shell/shell_liner.go @@ -5,16 +5,12 @@ import ( "context" "fmt" "io" - "math/rand/v2" "os" "path" "slices" "strings" "time" - "github.com/seaweedfs/seaweedfs/weed/cluster" - "github.com/seaweedfs/seaweedfs/weed/pb" - "github.com/seaweedfs/seaweedfs/weed/pb/master_pb" "github.com/seaweedfs/seaweedfs/weed/util" "github.com/seaweedfs/seaweedfs/weed/util/grace" @@ -74,38 +70,6 @@ func RunShell(options ShellOptions) error { return err } - if commandEnv.option.FilerAddress == "" { - var filers []pb.ServerAddress - discoverCtx, cancelDiscover := context.WithTimeout(ctx, connectTimeout) - defer cancelDiscover() - listErr := commandEnv.MasterClient.WithClient(discoverCtx, false, func(client master_pb.SeaweedClient) error { - resp, err := client.ListClusterNodes(discoverCtx, &master_pb.ListClusterNodesRequest{ - ClientType: cluster.FilerType, - FilerGroup: *options.FilerGroup, - }) - if err != nil { - return err - } - - for _, clusterNode := range resp.ClusterNodes { - filers = append(filers, pb.ServerAddress(clusterNode.Address)) - } - return nil - }) - if len(filers) > 0 { - commandEnv.option.FilerAddress = filers[rand.IntN(len(filers))] - } else if listErr != nil { - fmt.Fprintf(os.Stderr, "cannot list filers from master %s: %v\n", *options.Masters, listErr) - } - if options.Debug { - if len(filers) > 0 { - fmt.Fprintf(os.Stderr, "master: %s filers: %v\n", *options.Masters, filers) - } else { - fmt.Fprintf(os.Stderr, "master: %s\n", *options.Masters) - } - } - } - if interactive { for { cmd, err := line.Prompt("> ") diff --git a/weed/storage/disk_location.go b/weed/storage/disk_location.go index ce76412a76..e904bae7dc 100644 --- a/weed/storage/disk_location.go +++ b/weed/storage/disk_location.go @@ -58,6 +58,7 @@ type DiskLocation struct { isDiskSpaceLow atomic.Bool isDiskUnavailable atomic.Bool + syncBatch SyncBatchOption closeCh chan struct{} } @@ -119,6 +120,7 @@ func NewDiskLocation(dir string, maxVolumeCount int32, minFreeSpace util.MinFree MaxVolumeCount: maxVolumeCount, OriginalMaxVolumeCount: maxVolumeCount, MinFreeSpace: minFreeSpace, + syncBatch: DefaultSyncBatchOption, } location.volumes = make(map[needle.VolumeId]*Volume) location.ecVolumes = make(map[needle.VolumeId]*erasure_coding.EcVolume) diff --git a/weed/storage/disk_location_ec_realworld_test.go b/weed/storage/disk_location_ec_realworld_test.go index 42ce29bd78..f54682a47d 100644 --- a/weed/storage/disk_location_ec_realworld_test.go +++ b/weed/storage/disk_location_ec_realworld_test.go @@ -5,9 +5,24 @@ import ( "path/filepath" "testing" + "github.com/seaweedfs/seaweedfs/weed/stats" "github.com/seaweedfs/seaweedfs/weed/storage/erasure_coding" ) +// requireDiskSpaceForEncoding skips the subtest when dir's filesystem cannot +// hold the .dat file plus the shards EC encoding writes for it, so ENOSPC on +// space-constrained runners doesn't surface as a test failure. +func requireDiskSpaceForEncoding(t *testing.T, dir string, datFileSize int64, expectedShardSize int64) { + t.Helper() + needBytes := uint64(datFileSize) + + uint64(expectedShardSize)*erasure_coding.TotalShardsCount + + 16*erasure_coding.ErasureCodingSmallBlockSize + if free := stats.NewDiskStatus(dir).Free; free < needBytes { + t.Skipf("insufficient free space in %s: have %d bytes, need ~%d to encode a %d-byte volume", + dir, free, needBytes, datFileSize) + } +} + // TestCalculateExpectedShardSizeWithRealEncoding validates our shard size calculation // by actually running EC encoding on real files and comparing the results func TestCalculateExpectedShardSizeWithRealEncoding(t *testing.T) { @@ -52,6 +67,10 @@ func TestCalculateExpectedShardSizeWithRealEncoding(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { + // Calculate expected shard size using our function + expectedShardSize := calculateExpectedShardSize(tt.datFileSize, erasure_coding.DataShardsCount) + requireDiskSpaceForEncoding(t, tempDir, tt.datFileSize, expectedShardSize) + // Create a test .dat file with the specified size baseFileName := filepath.Join(tempDir, "test_volume") datFileName := baseFileName + ".dat" @@ -82,9 +101,6 @@ func TestCalculateExpectedShardSizeWithRealEncoding(t *testing.T) { } datFile.Close() - // Calculate expected shard size using our function - expectedShardSize := calculateExpectedShardSize(tt.datFileSize, erasure_coding.DataShardsCount) - // Run actual EC encoding _, err = erasure_coding.WriteEcFiles(baseFileName, erasure_coding.BackgroundECContext()) if err != nil { @@ -146,6 +162,9 @@ func TestCalculateExpectedShardSizeEdgeCases(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { + expectedShardSize := calculateExpectedShardSize(tt.datFileSize, erasure_coding.DataShardsCount) + requireDiskSpaceForEncoding(t, tempDir, tt.datFileSize, expectedShardSize) + baseFileName := filepath.Join(tempDir, tt.name) datFileName := baseFileName + ".dat" @@ -163,9 +182,6 @@ func TestCalculateExpectedShardSizeEdgeCases(t *testing.T) { datFile.Write(data) datFile.Close() - // Calculate expected - expectedShardSize := calculateExpectedShardSize(tt.datFileSize, erasure_coding.DataShardsCount) - // Run actual EC encoding _, err = erasure_coding.WriteEcFiles(baseFileName, erasure_coding.BackgroundECContext()) if err != nil { diff --git a/weed/storage/erasure_coding/ecbalancer/balancer.go b/weed/storage/erasure_coding/ecbalancer/balancer.go index 0c8541fea6..2e8ad0ffa5 100644 --- a/weed/storage/erasure_coding/ecbalancer/balancer.go +++ b/weed/storage/erasure_coding/ecbalancer/balancer.go @@ -1284,7 +1284,8 @@ func volumeShardCount(node *Node, vk volKey) int { } // nodeLiveShardCount is the disk-level EC shard cardinality (master EcShards), -// not the UI "EC volumes" count (len of the volume map). +// not the UI "EC volumes" count (len of the volume map). Place uses it as the +// remainder after even-this-volume and freeSlots (with deadband). func nodeLiveShardCount(n *Node) int { if n == nil { return 0 diff --git a/weed/storage/erasure_coding/ecbalancer/place.go b/weed/storage/erasure_coding/ecbalancer/place.go index c9d8e438ef..6ea7e84a72 100644 --- a/weed/storage/erasure_coding/ecbalancer/place.go +++ b/weed/storage/erasure_coding/ecbalancer/place.go @@ -2,6 +2,7 @@ package ecbalancer import ( "fmt" + "math" "sort" "strings" @@ -28,6 +29,11 @@ type Constraints struct { PreferredTags []string ReplicaPlacement *super_block.ReplicaPlacement Ratio func(collection string) (dataShards, parityShards int) + // FreeSlotsDeadband (M) is free volume slots. After even-this-volume, + // Place ranks that evenness class by maxFree, keeps |free−maxFree| ≤ + // M×dataShards, then lower liveShards. 0 is only maxFree then live + // (old strict freeSlots; liveShards only on an exact free tie). + FreeSlotsDeadband int } // DiskTypePolicy controls how Constraints.DiskType constrains placement. @@ -193,7 +199,7 @@ func (t *Topology) Place(vid uint32, collection string, need []int, c Constraint eligible := func(d *disk) bool { return typeEligible(d) && (len(tt) == 0 || diskHasAnyTag(d, tt)) } - res, err := t.tryPlace(vk, need, dataShards, parityShards, racks, rackKeys, mode, c.ReplicaPlacement, eligible, prefer) + res, err := t.tryPlace(vk, need, dataShards, parityShards, racks, rackKeys, mode, c.ReplicaPlacement, eligible, prefer, volumeDeadbandToShardSlots(c.FreeSlotsDeadband, dataShards)) if err != nil { lastErr = err continue @@ -223,7 +229,7 @@ func tagTiers(preferredTags []string) [][]string { // `eligible`, with `prefer` (may be nil) ranking soft-preferred disks first. It // journals reservations and rolls them all back if any shard cannot be placed, so // a failed tier leaves the snapshot unchanged for the next attempt. -func (t *Topology) tryPlace(vk volKey, need []int, dataShards, parityShards int, racks map[string]*rack, rackKeys []string, mode PlacementMode, rp *super_block.ReplicaPlacement, eligible func(*disk) bool, prefer func(*disk) bool) (*PlaceResult, error) { +func (t *Topology) tryPlace(vk volKey, need []int, dataShards, parityShards int, racks map[string]*rack, rackKeys []string, mode PlacementMode, rp *super_block.ReplicaPlacement, eligible func(*disk) bool, prefer func(*disk) bool, freeSlotsDeadband int) (*PlaceResult, error) { result := &PlaceResult{Destinations: make(map[int]Destination, len(need))} // Per-type shard ids per rack (even caps), total shard count per rack @@ -297,7 +303,7 @@ func (t *Topology) tryPlace(vk volKey, need []int, dataShards, parityShards int, typeTotal = parityShards } for _, rl := range attempts { - node, diskID, spilled, ok := chooseShardDest(vk, sid, isData, dataShards, typeTotal, numEligibleRacks, parityShards, rackCap(rl), racks, rackKeys, rp, eligible, prefer, shardsPerRack[isData], rackShardCount, bearing, rl) + node, diskID, spilled, ok := chooseShardDest(vk, sid, isData, dataShards, typeTotal, numEligibleRacks, parityShards, rackCap(rl), racks, rackKeys, rp, eligible, prefer, shardsPerRack[isData], rackShardCount, bearing, rl, freeSlotsDeadband) if !ok { continue } @@ -350,12 +356,13 @@ func (t *Topology) tryPlace(vk volKey, need []int, dataShards, parityShards int, // chooseShardDest selects a (node, disk) for one shard at the given relaxation // level: pick a rack (total-shards-per-rack cap + even per-type cap + -// ReplicaPlacement caps + two-pass anti-affinity to the opposite type), then the -// least-loaded eligible node, then the best eligible disk. If no node in the -// chosen rack fits (e.g. every node is at SameRackCount), the next-best rack is -// tried. The third return reports whether the disk spilled off the -// soft-preferred type. ok=false when no rack/node/disk fits. -func chooseShardDest(vk volKey, sid int, isData bool, dataShards, typeTotal, numEligibleRacks, maxPerDisk, maxTotalPerRack int, racks map[string]*rack, rackKeys []string, rp *super_block.ReplicaPlacement, eligible func(*disk) bool, prefer func(*disk) bool, shardsPerRackType map[string][]int, rackShardCount map[string]int, bearing map[bool]map[string]bool, rl relaxation) (*Node, uint32, bool, bool) { +// ReplicaPlacement caps + two-pass anti-affinity to the opposite type, then +// freeSlots deadband / liveShards), then the least-loaded eligible node, then +// the best eligible disk. If no node in the chosen rack fits (e.g. every node +// is at SameRackCount), the next-best rack is tried. The third return reports +// whether the disk spilled off the soft-preferred type. ok=false when no +// rack/node/disk fits. +func chooseShardDest(vk volKey, sid int, isData bool, dataShards, typeTotal, numEligibleRacks, maxPerDisk, maxTotalPerRack int, racks map[string]*rack, rackKeys []string, rp *super_block.ReplicaPlacement, eligible func(*disk) bool, prefer func(*disk) bool, shardsPerRackType map[string][]int, rackShardCount map[string]int, bearing map[bool]map[string]bool, rl relaxation, freeSlotsDeadband int) (*Node, uint32, bool, bool) { maxPerRack := numEligibleRacks*typeTotal + 1 // effectively unlimited when caps are relaxed if rl.caps { if maxPerRack = ceilDivide(typeTotal, numEligibleRacks); maxPerRack < 1 { @@ -389,12 +396,11 @@ func chooseShardDest(vk volKey, sid int, isData bool, dataShards, typeTotal, num return !tried[r] && racks[r].freeSlots > 0 && rackShardRoom(racks[r], vk, eligible, maxPerDisk) > 0 } for { - destRack, ok := pickTarget(rackKeys, shardsPerRackType, maxPerRack, anti, hasRoom, withinLimit, - func(r string) int { return racks[r].freeSlots }) + destRack, ok := pickRackEligible(rackKeys, shardsPerRackType, maxPerRack, anti, hasRoom, withinLimit, racks, freeSlotsDeadband) if !ok { return nil, 0, false, false } - if node := pickNodeInRackEligible(racks[destRack], vk, rp, eligible, maxPerDisk); node != nil { + if node := pickNodeInRackEligible(racks[destRack], vk, rp, eligible, maxPerDisk, freeSlotsDeadband); node != nil { if diskID, ok, spilled := pickBestDiskEligible(node, vk, eligible, prefer, sid, dataShards, maxPerDisk); ok { return node, diskID, spilled, true } @@ -482,18 +488,83 @@ func rackShardRoomUnderRP(r *rack, vk volKey, eligible func(*disk) bool, maxPerD return room } +// pickRackEligible is pickTarget for Place racks. Even this-type / cap / +// two-pass anti-affinity stay. Among that evenness class the remainder is +// max rack freeSlots, keep |free−maxFree| ≤ shard-slot deadband, then +// lower liveShards (sum of nodeLiveShardCount). M=0 is only maxFree then +// live (old strict freeSlots). pickTarget itself is unchanged so balancer +// / DC paths stay without M. +func pickRackEligible(candidates []string, shardsPerTarget map[string][]int, maxPerTarget int, antiAffinity map[string]bool, hasFreeSlots, withinLimit func(string) bool, racks map[string]*rack, freeSlotsDeadband int) (string, bool) { + try := func(skipAnti bool) (string, bool) { + var keys []string + var free, live []int + bestCount := maxPerTarget + 1 + for _, c := range candidates { + if skipAnti && antiAffinity[c] { + continue + } + if !hasFreeSlots(c) { + continue + } + if len(shardsPerTarget[c]) >= maxPerTarget { + continue + } + if !withinLimit(c) { + continue + } + cnt := len(shardsPerTarget[c]) + if len(keys) == 0 || cnt < bestCount { + keys, free, live = keys[:0], free[:0], live[:0] + bestCount = cnt + } else if cnt > bestCount { + continue + } + f := 0 + if r := racks[c]; r != nil { + f = r.freeSlots + } + keys = append(keys, c) + free = append(free, f) + live = append(live, rackLiveShardCount(racks[c])) + } + if len(keys) == 0 { + return "", false + } + return keys[pickDeadbandRemainder(free, live, freeSlotsDeadband)], true + } + if len(antiAffinity) > 0 { + if t, ok := try(true); ok { + return t, true + } + } + return try(false) +} + +func rackLiveShardCount(r *rack) int { + if r == nil { + return 0 + } + total := 0 + for _, n := range r.nodes { + total += nodeLiveShardCount(n) + } + return total +} + // pickNodeInRackEligible is pickNodeInRack restricted to nodes with an eligible // disk that can take another shard of the volume (free slot, under maxPerDisk). // FromActiveTopology keeps all disk types/tags in the snapshot, so without this a // node with free volume slots but no eligible disk could be chosen. // -// Even first (this volume, machine then node). Machine freeSlots remainder -// (mFree) was dropped: keep host even (mCount); remainder is disk freeSlots -// then liveShards so thin disks fill first. SameRackCount / 2+2 unchanged. -func pickNodeInRackEligible(r *rack, vk volKey, rp *super_block.ReplicaPlacement, eligible func(*disk) bool, maxPerDisk int) *Node { +// Even first (this volume, machine then node). Among that class: max node +// freeSlots, keep |free−maxFree| ≤ shard-slot deadband, then lower +// liveShards. Machine freeSlots (mFree) stays dropped as on origin/master. +// SameRackCount / 2+2 unchanged. M=0 is only maxFree then live. Callers +// convert UI volume M via volumeDeadbandToShardSlots. +func pickNodeInRackEligible(r *rack, vk volKey, rp *super_block.ReplicaPlacement, eligible func(*disk) bool, maxPerDisk, freeSlotsDeadband int) *Node { machineShards := countShardsByHost(vk, r.nodes) - var best *Node - var bestMCount, bestNCount, bestNFree, bestNScore int + var nodes []*Node + var bestMCount, bestNCount int for _, id := range sortedNodeKeys(r.nodes) { node := r.nodes[id] if nodeShardRoom(node, vk, eligible, maxPerDisk) <= 0 { @@ -504,27 +575,68 @@ func pickNodeInRackEligible(r *rack, vk volKey, rp *super_block.ReplicaPlacement continue } mCount := machineShards[node.host] - nScore := nodeLiveShardCount(node) - better := false - switch { - case best == nil: - better = true - case mCount != bestMCount: - better = mCount < bestMCount - case count != bestNCount: - better = count < bestNCount - case node.freeSlots != bestNFree: - better = node.freeSlots > bestNFree - default: - better = nScore < bestNScore + if len(nodes) == 0 || mCount < bestMCount || (mCount == bestMCount && count < bestNCount) { + nodes = nodes[:0] + bestMCount, bestNCount = mCount, count + } else if mCount != bestMCount || count != bestNCount { + continue } - if better { - best, bestMCount, bestNCount, bestNFree, bestNScore = node, mCount, count, node.freeSlots, nScore + nodes = append(nodes, node) + } + if len(nodes) == 0 { + return nil + } + free := make([]int, len(nodes)) + live := make([]int, len(nodes)) + for i, n := range nodes { + free[i] = n.freeSlots + live[i] = nodeLiveShardCount(n) + } + return nodes[pickDeadbandRemainder(free, live, freeSlotsDeadband)] +} + +// pickDeadbandRemainder ranks same-evenness candidates: maxFree of the +// class, keep |free−maxFree|≤m, then min live. Ties keep the earlier +// index (scan order). M=0 keeps only maxFree then live. Shared by node +// and rack so the band cannot drift. +func pickDeadbandRemainder(free, live []int, m int) int { + maxFree := free[0] + for _, f := range free[1:] { + if f > maxFree { + maxFree = f + } + } + best := -1 + for i, f := range free { + if absIntDiff(f, maxFree) > m { + continue + } + if best < 0 || live[i] < live[best] { + best = i } } return best } +func absIntDiff(a, b int) int { + if a > b { + return a - b + } + return b - a +} + +// volumeDeadbandToShardSlots converts UI volume-slot M to the shard-slot +// deadband Place compares against node.freeSlots. OSS 10+4 uses dataShards=10. +func volumeDeadbandToShardSlots(volumes, dataShards int) int { + if volumes <= 0 || dataShards <= 0 { + return 0 + } + if volumes > math.MaxInt/dataShards { + return math.MaxInt + } + return volumes * dataShards +} + // pickBestDiskEligible chooses the best eligible disk on a node, ranking // soft-preferred disks (prefer != nil && prefer(d)) ahead of others so disk-type // Prefer uses the preferred type when available but spills otherwise. Returns the diff --git a/weed/storage/erasure_coding/ecbalancer/place_test.go b/weed/storage/erasure_coding/ecbalancer/place_test.go index cf8f47f2ee..32eef6edba 100644 --- a/weed/storage/erasure_coding/ecbalancer/place_test.go +++ b/weed/storage/erasure_coding/ecbalancer/place_test.go @@ -2,6 +2,7 @@ package ecbalancer import ( "fmt" + "math" "sort" "testing" @@ -914,3 +915,235 @@ func TestPlace10Plus4ImbalanceTiebreak(t *testing.T) { }) } } + +func TestVolumeDeadbandToShardSlots(t *testing.T) { + if got := volumeDeadbandToShardSlots(0, 10); got != 0 { + t.Fatalf("M=0 → %d, want 0", got) + } + if got := volumeDeadbandToShardSlots(1, 10); got != 10 { + t.Fatalf("M=1 at 10+4 → %d, want 10", got) + } + if got := volumeDeadbandToShardSlots(1, 8); got != 8 { + t.Fatalf("M=1 at 8+4 → %d, want 8", got) + } + if got := volumeDeadbandToShardSlots(50, 10); got != 500 { + t.Fatalf("M=50 at 10+4 → %d, want 500", got) + } + if got := volumeDeadbandToShardSlots(-3, 10); got != 0 { + t.Fatalf("negative M → %d, want 0", got) + } +} + +func TestAbsIntDiff(t *testing.T) { + if got := absIntDiff(105, 100); got != 5 { + t.Fatalf("absIntDiff(105, 100)=%d, want 5", got) + } + if got := absIntDiff(100, 105); got != 5 { + t.Fatalf("absIntDiff(100, 105)=%d, want 5", got) + } + if got := absIntDiff(100, 100); got != 0 { + t.Fatalf("absIntDiff(100, 100)=%d, want 0", got) + } + if got := absIntDiff(math.MaxInt, math.MaxInt-5); got != 5 { + t.Fatalf("absIntDiff(MaxInt, MaxInt-5)=%d, want 5", got) + } + if got := absIntDiff(math.MaxInt-5, math.MaxInt); got != 5 { + t.Fatalf("absIntDiff(MaxInt-5, MaxInt)=%d, want 5", got) + } + if got := absIntDiff(math.MinInt+5, math.MinInt); got != 5 { + t.Fatalf("absIntDiff(MinInt+5, MinInt)=%d, want 5", got) + } + if got := absIntDiff(math.MinInt, math.MinInt+5); got != 5 { + t.Fatalf("absIntDiff(MinInt, MinInt+5)=%d, want 5", got) + } +} + +func placeFirstShardNode(t *testing.T, topo *Topology, m int) string { + t.Helper() + res, err := topo.Place(1, "c1", []int{0}, Constraints{FreeSlotsDeadband: m}, PlaceStrict) + if err != nil { + t.Fatalf("Place M=%d: %v", m, err) + } + d, ok := res.Destinations[0] + if !ok { + t.Fatal("missing shard 0") + } + return d.Node +} + +// TestFreeSlotsDeadbandDefersToLiveShards: two nodes, empty this-volume. +// b has slightly more freeSlots but more live shards. M=0 follows freeSlots +// (b); M=1 volume (×10 shard slots at 10+4) covers the gap and follows +// liveShards (a). Sorted-id would always pick a. +func TestFreeSlotsDeadbandDefersToLiveShards(t *testing.T) { + build := func() *Topology { + topo := NewTopology() + a := topo.AddNode("a:8080", "dc1", "dc1:rack0", 100) + a.AddDisk(0, "", 100, 10) + b := topo.AddNode("b:8081", "dc1", "dc1:rack0", 105) + b.AddDisk(0, "", 105, 50) + return topo + } + if got := placeFirstShardNode(t, build(), 0); got != "b:8081" { + t.Fatalf("M=0: got %s, want b (strict freeSlots)", got) + } + if got := placeFirstShardNode(t, build(), 1); got != "a:8080" { + t.Fatalf("M=1 volume: got %s, want a (liveShards remainder)", got) + } +} + +// TestFreeSlotsDeadbandZeroExactTieUsesLiveShards: equal freeSlots, M=0 +// remainder is liveShards, not the lower sorted id. +func TestFreeSlotsDeadbandZeroExactTieUsesLiveShards(t *testing.T) { + topo := NewTopology() + a := topo.AddNode("a:8080", "dc1", "dc1:rack0", 100) + a.AddDisk(0, "", 100, 50) + b := topo.AddNode("b:8081", "dc1", "dc1:rack0", 100) + b.AddDisk(0, "", 100, 10) + if got := placeFirstShardNode(t, topo, 0); got != "b:8081" { + t.Fatalf("equal freeSlots M=0: got %s, want b (lower liveShards)", got) + } +} + +// TestFreeSlotsDeadbandStillHonorsLargeGap: gap above M still follows freeSlots +// even if that node is fatter on liveShards. +func TestFreeSlotsDeadbandStillHonorsLargeGap(t *testing.T) { + topo := NewTopology() + a := topo.AddNode("a:8080", "dc1", "dc1:rack0", 100) + a.AddDisk(0, "", 100, 10) + b := topo.AddNode("b:8081", "dc1", "dc1:rack0", 200) + b.AddDisk(0, "", 200, 80) + if got := placeFirstShardNode(t, topo, 1); got != "b:8081" { + t.Fatalf("M=1 volume gap=100 shards: got %s, want b (freeSlots still wins)", got) + } +} + +// TestFreeSlotsDeadbandScalesWithRatio: M=1 volume is 8 shard slots at 8+4 +// and 10 at default 10+4. Gap of 9 shards: 8+4 still follows freeSlots; 10+4 +// ties and follows liveShards. +func TestFreeSlotsDeadbandScalesWithRatio(t *testing.T) { + build := func() *Topology { + topo := NewTopology() + a := topo.AddNode("a:8080", "dc1", "dc1:rack0", 100) + a.AddDisk(0, "", 100, 10) + b := topo.AddNode("b:8081", "dc1", "dc1:rack0", 109) + b.AddDisk(0, "", 109, 50) + return topo + } + place := func(topo *Topology, ratio func(string) (int, int)) string { + t.Helper() + res, err := topo.Place(1, "c1", []int{0}, Constraints{FreeSlotsDeadband: 1, Ratio: ratio}, PlaceStrict) + if err != nil { + t.Fatalf("Place: %v", err) + } + d, ok := res.Destinations[0] + if !ok { + t.Fatal("missing shard 0") + } + return d.Node + } + if got := place(build(), func(string) (int, int) { return 8, 4 }); got != "b:8081" { + t.Fatalf("8+4 M=1 gap=9: got %s, want b (deadband 8, freeSlots)", got) + } + if got := place(build(), nil); got != "a:8080" { + t.Fatalf("10+4 M=1 gap=9: got %s, want a (deadband 10, liveShards)", got) + } +} + +func rackExtraTopo(thinFree, thinLive, fatFree, fatLive int) *Topology { + topo := NewTopology() + add := func(rack, id string, free, live int) { + n := topo.AddNode(id, "dc1", rack, free) + n.AddDisk(0, "", free, live) + } + add("dc1:rack0", "thin:8080", thinFree, thinLive) + add("dc1:rack1", "fat1:8080", fatFree, fatLive) + add("dc1:rack2", "fat2:8080", fatFree, fatLive) + add("dc1:rack3", "fat3:8080", fatFree, fatLive) + return topo +} + +func place104PerRack(t *testing.T, topo *Topology, m int) map[string]int { + t.Helper() + res, err := topo.Place(1, "c1", allShards(), Constraints{FreeSlotsDeadband: m}, PlaceStrict) + if err != nil { + t.Fatalf("Place M=%d: %v", m, err) + } + got := map[string]int{} + for _, d := range res.Destinations { + got[d.Rack]++ + } + return got +} + +// TestRackDeadbandDefersToLiveShards: 10+4 extras are 4+4+3+3. Thin rack has +// fewer freeSlots (gap 5) but fewer liveShards. M=1 volume is 10 shard slots, +// so the gap is a tie and extras go to the thinner live rack. +func TestRackDeadbandDefersToLiveShards(t *testing.T) { + got := place104PerRack(t, rackExtraTopo(100, 10, 105, 50), 1) + if got["rack0"] != 4 { + t.Fatalf("M=1 gap=5: thin rack0 got %d, want 4 (liveShards remainder); per-rack %v", got["rack0"], got) + } +} + +// TestRackDeadbandStillHonorsLargeGap: same thin-vs-fat live, but freeSlots +// gap 100 is above M=1 (10 shard slots), so extras still follow freeSlots. +func TestRackDeadbandStillHonorsLargeGap(t *testing.T) { + got := place104PerRack(t, rackExtraTopo(100, 10, 200, 50), 1) + if got["rack0"] != 3 { + t.Fatalf("M=1 gap=100: thin rack0 got %d, want 3 (freeSlots still wins); per-rack %v", got["rack0"], got) + } +} + +// Greptile chain at M=1 volume (10 shard slots): 120/300, 110/200, 100/100. +// Pairwise vs the current winner walks 120→110→100 or 100→110→120 depending +// on scan order. maxFree=120 keeps |Δ|≤10, so 110 wins — not 100 (out of +// band) and not 120 (fatter live). +func TestNodeDeadbandAnchorsOnMaxFree(t *testing.T) { + run := func(t *testing.T, frees, lives [3]int, want string) { + t.Helper() + topo := NewTopology() + ids := [3]string{"n0:8080", "n1:8081", "n2:8082"} + for i, id := range ids { + n := topo.AddNode(id, "dc1", "dc1:rack0", frees[i]) + n.AddDisk(0, "", frees[i], lives[i]) + } + if got := placeFirstShardNode(t, topo, 1); got != want { + t.Fatalf("got %s, want %s (110 in-band, not 100/120)", got, want) + } + } + t.Run("high-to-low", func(t *testing.T) { + run(t, [3]int{120, 110, 100}, [3]int{300, 200, 100}, "n1:8081") + }) + t.Run("low-to-high", func(t *testing.T) { + run(t, [3]int{100, 110, 120}, [3]int{100, 200, 300}, "n1:8081") + }) +} + +func TestRackDeadbandAnchorsOnMaxFree(t *testing.T) { + run := func(t *testing.T, frees, lives [3]int, want string) { + t.Helper() + topo := NewTopology() + for i := 0; i < 3; i++ { + n := topo.AddNode(fmt.Sprintf("n%d:8080", i), "dc1", fmt.Sprintf("dc1:rack%d", i), frees[i]) + n.AddDisk(0, "", frees[i], lives[i]) + } + res, err := topo.Place(1, "c1", []int{0}, Constraints{FreeSlotsDeadband: 1}, PlaceStrict) + if err != nil { + t.Fatalf("Place: %v", err) + } + d, ok := res.Destinations[0] + if !ok { + t.Fatal("missing shard 0") + } + if d.Rack != want { + t.Fatalf("got rack %s, want %s (110 in-band, not 100/120)", d.Rack, want) + } + } + t.Run("high-to-low", func(t *testing.T) { + run(t, [3]int{120, 110, 100}, [3]int{300, 200, 100}, "rack1") + }) + t.Run("low-to-high", func(t *testing.T) { + run(t, [3]int{100, 110, 120}, [3]int{100, 200, 300}, "rack1") + }) +} diff --git a/weed/storage/store.go b/weed/storage/store.go index 0d2b51b50d..d213ee6f9e 100644 --- a/weed/storage/store.go +++ b/weed/storage/store.go @@ -1359,6 +1359,14 @@ func (s *Store) GetVolumeSizeLimit() uint64 { return s.volumeSizeLimit.Load() } +// SetSyncBatchOption applies the durable-write batch window to every disk. +// It runs at startup, before any volume takes a write. +func (s *Store) SetSyncBatchOption(syncBatch SyncBatchOption) { + for _, location := range s.Locations { + location.syncBatch = syncBatch + } +} + func (s *Store) SetPreallocate(x bool) { s.preallocate.Store(x) } diff --git a/weed/storage/volume_idx_repair_test.go b/weed/storage/volume_idx_repair_test.go index 197ebfade4..6b155477c5 100644 --- a/weed/storage/volume_idx_repair_test.go +++ b/weed/storage/volume_idx_repair_test.go @@ -1,6 +1,8 @@ package storage import ( + "bytes" + "errors" "os" "path/filepath" "testing" @@ -219,3 +221,96 @@ func TestRepairIdxHeadTombstones_KeepsDeletedNeedlesDeleted(t *testing.T) { t.Fatalf("needle 1 should have been recovered: %v", err) } } + +// TestStaleIndexEntry_HealsOnWrite covers a damaged index entry that resolves +// to a different needle: reads must miss instead of serving the wrong needle, +// and the next write for the key must re-index it rather than be rejected as +// a cookie conflict. +func TestStaleIndexEntry_HealsOnWrite(t *testing.T) { + dir := t.TempDir() + written := writeTestVolume(t, dir, 5) + + v := mustReload(t, dir) + defer v.Close() + + // point key 9 at needle 2's record with a size that does not match the + // record, as a corrupt index entry would look + const staleKey = 9 + nv, ok := v.nm.Get(types.Uint64ToNeedleId(2)) + if !ok { + t.Fatal("needle 2 missing from index") + } + if err := v.nm.Put(types.Uint64ToNeedleId(staleKey), nv.Offset, nv.Size+1); err != nil { + t.Fatalf("plant stale index entry: %v", err) + } + + if _, err := v.readNeedle(newEmptyNeedle(staleKey), nil, nil); err == nil { + t.Fatal("stale-index read served needle 2's data") + } + + stream := newEmptyNeedle(staleKey) + stream.DataSize = written[2].DataSize + if err := v.readNeedleDataInto(stream, &ReadOption{ReadBufferSize: 1024}, &bytes.Buffer{}, 0, int64(stream.DataSize)); err != ErrorNotFound { + t.Fatalf("stale-index stream: want ErrorNotFound, got %v", err) + } + + n := newRandomNeedle(staleKey) + n.Cookie = 0xABCD + if _, _, _, err := v.writeNeedle2(n, true, false, false); err != nil { + t.Fatalf("write over stale index entry: %v", err) + } + if got := mustRead(t, v, staleKey); !bytes.Equal(got, n.Data) { + t.Fatalf("needle %d after re-index: got %d bytes, want %d", staleKey, len(got), len(n.Data)) + } +} + +// TestStaleIndexEntry_SurvivesFailedWrite checks that a stale index entry is +// only replaced once the new record commits: a durable write rolled back by +// a failed fsync leaves the index exactly as it found it. +func TestStaleIndexEntry_SurvivesFailedWrite(t *testing.T) { + v, counting := newCountingVolume(t) + + existing := fixedNeedle(2, "existing") + if _, _, _, err := v.writeNeedle2(existing, true, true, true); err != nil { + t.Fatalf("write needle 2: %v", err) + } + nv, ok := v.nm.Get(existing.Id) + if !ok { + t.Fatal("needle 2 missing from index") + } + + staleId := types.Uint64ToNeedleId(9) + if err := v.nm.Put(staleId, nv.Offset, nv.Size+1); err != nil { + t.Fatalf("plant stale index entry: %v", err) + } + + counting.syncErr = errors.New("batch fsync failed") + if _, _, _, err := v.writeNeedle2(fixedNeedle(9, "replacement"), true, true, false); err == nil { + t.Fatal("a write whose fsync failed must not be acknowledged") + } + + entry, ok := v.nm.Get(staleId) + if !ok { + t.Fatal("a failed write dropped the index entry it meant to replace") + } + if entry.Offset != nv.Offset || entry.Size != nv.Size+1 { + t.Fatalf("index entry changed under a failed write: %+v, want %+v", entry, nv) + } +} + +// TestStaleIndexEntry_KeepsCookieConflictForSameKey checks that an entry +// resolving to the same needle id still enforces the cookie check: only +// cross-key corruption is dropped. +func TestStaleIndexEntry_KeepsCookieConflictForSameKey(t *testing.T) { + dir := t.TempDir() + writeTestVolume(t, dir, 3) + + v := mustReload(t, dir) + defer v.Close() + + n := newRandomNeedle(2) + n.Cookie = 0xABCD + if _, _, _, err := v.writeNeedle2(n, true, false, false); err == nil { + t.Fatal("write with a foreign cookie over an existing key should be rejected") + } +} diff --git a/weed/storage/volume_read.go b/weed/storage/volume_read.go index f93c1a570e..3112cabb15 100644 --- a/weed/storage/volume_read.go +++ b/weed/storage/volume_read.go @@ -61,10 +61,17 @@ func (v *Volume) readNeedle(n *needle.Needle, readOption *ReadOption, onReadSize err = n.ReadNeedleMeta(v.DataBackend, nv.Offset.ToActualOffset()+int64(MaxPossibleVolumeSize), readSize, v.Version()) } if err != nil { + v.checkReadWriteError(err) return 0, err } if !n.IsCompressed() && !n.IsChunkedManifest() { readOption.IsMetaOnly = true + // A must-meta-only read ends here, so its success may reset the + // streak. For a streamed GET the meta read precedes the payload read, + // which reports the outcome itself. + if readOption.MustMetaOnly { + v.checkReadWriteError(nil) + } } } if readOption == nil || !readOption.IsMetaOnly { @@ -91,6 +98,13 @@ func (v *Volume) readNeedle(n *needle.Needle, readOption *ReadOption, onReadSize return -1, ErrorNotFound } +// NoteMetaOnlySuccess records a request that ended at the metadata read +// (e.g. a HEAD or a cookie mismatch), so its success may reset the streak — +// unlike a meta read that precedes a payload read. +func (v *Volume) NoteMetaOnlySuccess() { + v.checkReadWriteError(nil) +} + // read needle at a specific offset func (v *Volume) readNeedleMetaAt(n *needle.Needle, offset int64, size int32) (err error) { v.dataFileAccessLock.RLock() @@ -176,6 +190,55 @@ func (v *Volume) readNeedleDataInto(n *needle.Needle, readOption *ReadOption, wr } }() + // The header is not part of the streamed body, so verify the indexed + // offset really holds this needle before sending another needle's data. + // A slow read dropped the access lock after the index lookup; take it + // back for the probe so a compaction commit cannot swap the backend + // under it, and resolve the offset the way the streaming loop does. + probeHeader := func() error { + if readOption.VolumeRevision != v.SuperBlock.CompactionRevision { + if v.nm == nil { + return ErrorNotFound + } + refreshed, found := v.nm.Get(n.Id) + if !found || refreshed.Offset.IsZero() { + return ErrorNotFound + } + nv = refreshed + actualOffset = nv.Offset.ToActualOffset() + readOption.VolumeRevision = v.SuperBlock.CompactionRevision + } + header := make([]byte, NeedleHeaderSize) + headerCount, headerErr := v.DataBackend.ReadAt(header, actualOffset) + if headerErr == io.EOF && headerCount == NeedleHeaderSize { + headerErr = nil + } + if headerErr != nil { + if headerErr != io.EOF { + v.checkReadWriteError(headerErr) + } + return fmt.Errorf("ReadNeedleData header: %w", headerErr) + } + probe := new(needle.Needle) + probe.ParseNeedleHeader(header) + if probe.Id != n.Id && probe.Size != nv.Size { + glog.V(0).Infof("volume %d index entry for needle %x resolves to needle %x; reporting not-found", + v.Id, uint64(n.Id), uint64(probe.Id)) + return ErrorNotFound + } + return nil + } + if readOption.HasSlowRead { + v.dataFileAccessLock.RLock() + } + probeErr := probeHeader() + if readOption.HasSlowRead { + v.dataFileAccessLock.RUnlock() + } + if probeErr != nil { + return probeErr + } + // read needle data crc := needle.CRC(0) var pending []byte @@ -255,11 +318,6 @@ func (v *Volume) readNeedleDataInto(n *needle.Needle, readOption *ReadOption, wr break } } - // Whole-needle read completed without a backend error — clear any - // pending EIO streak. If a non-EIO failure happens later (CRC etc.) - // we still return that error to the caller, but the disk itself - // produced clean bytes. - v.checkReadWriteError(nil) if checkCRC && (n.Checksum != crc && uint32(n.Checksum) != crc.Value()) { // the crc.Value() function is to be deprecated. this double checking is for backward compatibility // with seaweed version using crc.Value() instead of uint32(crc), which appears in commit 056c480eb @@ -268,6 +326,9 @@ func (v *Volume) readNeedleDataInto(n *needle.Needle, readOption *ReadOption, wr stats.VolumeServerHandlerCounter.WithLabelValues(stats.ErrorCRC).Inc() return fmt.Errorf("ReadNeedleData checksum %v expected %v for Needle: %v,%v", crc, n.Checksum, v.Id, n) } + // The read succeeded only once the checksum passes; a failed needle + // leaves the io error streak untouched, matching the streamed path. + v.checkReadWriteError(nil) if pending != nil { if _, err = writer.Write(pending); err != nil { return fmt.Errorf("ReadNeedleData write: %w", err) diff --git a/weed/storage/volume_write.go b/weed/storage/volume_write.go index a04fa746cc..f119425fff 100644 --- a/weed/storage/volume_write.go +++ b/weed/storage/volume_write.go @@ -4,7 +4,9 @@ import ( "bytes" "errors" "fmt" + "io" "os" + "time" "github.com/seaweedfs/seaweedfs/weed/glog" "github.com/seaweedfs/seaweedfs/weed/storage/backend" @@ -322,22 +324,45 @@ func (v *Volume) appendWriteRequest(n *needle.Needle, checkCookie bool, fsync bo // check whether existing needle cookie matches nv, ok := v.nm.Get(n.Id) - if ok { + if ok && !nv.Offset.IsZero() { existingNeedle, _, _, existingNeedleReadErr := needle.ReadNeedleHeader(v.DataBackend, v.Version(), nv.Offset.ToActualOffset()) if existingNeedleReadErr != nil { err = fmt.Errorf("reading existing needle: %w", existingNeedleReadErr) return } - if n.Cookie == 0 && !checkCookie { - // this is from batch deletion, and read back again when tailing a remote volume - // which only happens when checkCookie == false and fsync == false - n.Cookie = existingNeedle.Cookie + if existingNeedle.Id != n.Id && existingNeedle.Size != nv.Size && + OffsetSize == 4 && nv.Offset.ToActualOffset() < int64(MaxPossibleVolumeSize) { + // a 4-byte index entry can index past the 32 GiB wrap: the low + // offset may hold another needle while the real record sits at + // the wrapped offset, where reads already retry on a mismatch + wrapped, _, _, wrappedErr := needle.ReadNeedleHeader(v.DataBackend, v.Version(), + nv.Offset.ToActualOffset()+int64(MaxPossibleVolumeSize)) + if wrappedErr == nil { + existingNeedle = wrapped + } else if wrappedErr != io.EOF { + err = fmt.Errorf("reading existing needle at wrapped offset: %w", wrappedErr) + return + } } - if existingNeedle.Cookie != n.Cookie { - glog.V(0).Infof("write cookie mismatch: existing %s, new %s", - needle.NewFileIdFromNeedle(v.Id, existingNeedle), needle.NewFileIdFromNeedle(v.Id, n)) - err = fmt.Errorf("mismatching cookie %x", n.Cookie) - return + if existingNeedle.Id != n.Id && existingNeedle.Size != nv.Size { + // the entry resolves to a record that is neither the indexed key + // nor the indexed size, so it is stale rather than a duplicate — + // skip the cookie check and let the publish step overwrite it + // once the append commits + glog.V(0).Infof("volume %d index entry for %s resolves to %s; overwriting stale index entry", + v.Id, needle.NewFileIdFromNeedle(v.Id, n), needle.NewFileIdFromNeedle(v.Id, existingNeedle)) + } else { + if n.Cookie == 0 && !checkCookie { + // this is from batch deletion, and read back again when tailing a remote volume + // which only happens when checkCookie == false and fsync == false + n.Cookie = existingNeedle.Cookie + } + if existingNeedle.Cookie != n.Cookie { + glog.V(0).Infof("write cookie mismatch: existing %s, new %s", + needle.NewFileIdFromNeedle(v.Id, existingNeedle), needle.NewFileIdFromNeedle(v.Id, n)) + err = fmt.Errorf("mismatching cookie %x", n.Cookie) + return + } } } @@ -637,6 +662,29 @@ func (v *Volume) processBatchRun(run []*needle.AsyncRequest) { } } +const ( + syncBatchMaxBytes = 4 * 1024 * 1024 + syncBatchMaxRequests = 128 +) + +// SyncBatchOption tunes the volume's durable-write worker. Once the queue +// drains with the batch still below MinBytes and MinRequests, the batch stays +// open up to Wait for stragglers instead of paying an fsync per request. +type SyncBatchOption struct { + MinBytes int64 + MinRequests int + Wait time.Duration +} + +var DefaultSyncBatchOption = SyncBatchOption{ + MinBytes: 1 * 1024 * 1024, + MinRequests: syncBatchMaxRequests, + Wait: 500 * time.Microsecond, +} + +// syncBatchAfter arms the batch window; a var so tests can drive it. +var syncBatchAfter = time.After + // startWorker returns the volume's batch-write channel, creating it and its // goroutine on first use, and nil once stopWorker has run. func (v *Volume) startWorker() chan *needle.AsyncRequest { @@ -648,8 +696,12 @@ func (v *Volume) startWorker() chan *needle.AsyncRequest { if v.asyncRequestsChan != nil { return v.asyncRequestsChan } - requests := make(chan *needle.AsyncRequest, 128) + requests := make(chan *needle.AsyncRequest, syncBatchMaxRequests) v.asyncRequestsChan = requests + syncBatch := DefaultSyncBatchOption + if v.location != nil { + syncBatch = v.location.syncBatch + } go func() { chanClosed := false for { @@ -657,15 +709,25 @@ func (v *Volume) startWorker() chan *needle.AsyncRequest { if chanClosed { break } - currentRequests := make([]*needle.AsyncRequest, 0, 128) + currentRequests := make([]*needle.AsyncRequest, 0, syncBatchMaxRequests) currentBytesToWrite := int64(0) + var batchDeadline time.Time + var batchWait <-chan time.Time for { - request, ok := <-requests + var request *needle.AsyncRequest + queueOpen := true + select { + case request, queueOpen = <-requests: + case <-batchWait: + } // volume may be closed - if !ok { + if !queueOpen { chanClosed = true break } + if request == nil { + break + } if MaxPossibleVolumeSize < v.ContentSize()+uint64(currentBytesToWrite+request.ActualSize) { request.Complete(0, 0, false, fmt.Errorf("volume size limit %d exceeded! current size is %d", MaxPossibleVolumeSize, v.ContentSize())) @@ -674,10 +736,27 @@ func (v *Volume) startWorker() chan *needle.AsyncRequest { currentRequests = append(currentRequests, request) currentBytesToWrite += request.ActualSize // submit at most 4M bytes or 128 requests at one time to decrease request delay. - // it also need to break if there is no data in channel to avoid io hang. - if currentBytesToWrite >= 4*1024*1024 || len(currentRequests) >= 128 || len(requests) == 0 { + if currentBytesToWrite >= syncBatchMaxBytes || len(currentRequests) >= syncBatchMaxRequests { break } + // a refilled queue must not outwait the armed deadline + if !batchDeadline.IsZero() && !time.Now().Before(batchDeadline) { + break + } + if len(requests) > 0 { + continue + } + if syncBatch.Wait <= 0 || currentBytesToWrite >= syncBatch.MinBytes || len(currentRequests) >= syncBatch.MinRequests { + // a drained queue flushes right away once it holds enough + break + } + // flushing now would sync a singleton batch; hold the window + // open for stragglers, armed once per batch so the added + // latency stays bounded by Wait + if batchWait == nil { + batchDeadline = time.Now().Add(syncBatch.Wait) + batchWait = syncBatchAfter(syncBatch.Wait) + } } if len(currentRequests) == 0 { continue diff --git a/weed/storage/volume_write_test.go b/weed/storage/volume_write_test.go index d35cdda2b2..2fa479067b 100644 --- a/weed/storage/volume_write_test.go +++ b/weed/storage/volume_write_test.go @@ -6,12 +6,14 @@ import ( "fmt" "math" "os" + "sync/atomic" "testing" "time" "github.com/stretchr/testify/assert" "github.com/seaweedfs/seaweedfs/weed/stats" + "github.com/seaweedfs/seaweedfs/weed/storage/backend" "github.com/seaweedfs/seaweedfs/weed/storage/needle" "github.com/seaweedfs/seaweedfs/weed/storage/super_block" "github.com/seaweedfs/seaweedfs/weed/storage/types" @@ -497,3 +499,85 @@ func TestDoWriteRequestRefusesReadOnlyBeforeAppend(t *testing.T) { t.Fatalf("idx size changed; needle must not index") } } + +type syncCountingBackend struct { + backend.BackendStorageFile + syncs atomic.Int64 +} + +func (b *syncCountingBackend) Sync() error { + b.syncs.Add(1) + return b.BackendStorageFile.Sync() +} + +// A drained queue used to flush whatever it held, turning durable writes into +// one fsync per request at low concurrency. With a batch window configured, a +// straggler that lands inside the window joins the same sync. +func TestSyncBatchDefersFlushForStragglers(t *testing.T) { + dir := t.TempDir() + + v, err := NewVolume(dir, dir, "", 7, NeedleMapInMemory, &super_block.ReplicaPlacement{}, &needle.TTL{}, 0, needle.GetCurrentVersion(), 0, 0) + if err != nil { + t.Fatalf("volume creation: %v", err) + } + defer v.Close() + + counting := &syncCountingBackend{BackendStorageFile: v.DataBackend} + v.DataBackend = counting + writeBytes := int64(needle.GetActualSize(16, v.Version())) + v.location = &DiskLocation{syncBatch: SyncBatchOption{ + MinBytes: writeBytes * 2, + MinRequests: syncBatchMaxRequests, + Wait: time.Minute, + }} + + armed := make(chan chan time.Time, 4) + prevAfter := syncBatchAfter + syncBatchAfter = func(time.Duration) <-chan time.Time { + window := make(chan time.Time, 1) + armed <- window + return window + } + defer func() { syncBatchAfter = prevAfter }() + + done := make(chan error, 3) + writeAsync := func(id uint64) { + n := new(needle.Needle) + n.Data = make([]byte, 16) + n.Checksum = needle.NewCRC(n.Data) + n.Id = types.Uint64ToNeedleId(id) + _, _, _, writeErr := v.writeNeedle2(n, true, true, false) + done <- writeErr + } + windowFor := func(id uint64) chan time.Time { + go writeAsync(id) + window := <-armed + select { + case <-done: + t.Fatalf("write %d flushed while its batch window was still open", id) + default: + } + return window + } + + windowFor(1) + // the straggler fills the batch to MinBytes, closing the window early + go writeAsync(2) + for i := 0; i < 2; i++ { + if writeErr := <-done; writeErr != nil { + t.Fatalf("write needle: %v", writeErr) + } + } + if syncs := counting.syncs.Load(); syncs != 1 { + t.Fatalf("expected both writes to share one .dat sync, got %d", syncs) + } + + window := windowFor(3) + window <- time.Now() + if writeErr := <-done; writeErr != nil { + t.Fatalf("write needle: %v", writeErr) + } + if syncs := counting.syncs.Load(); syncs != 2 { + t.Fatalf("expected a lone write to take its own sync, got %d total", syncs) + } +} diff --git a/weed/topology/collection.go b/weed/topology/collection.go index 9fe355a083..254c951dc7 100644 --- a/weed/topology/collection.go +++ b/weed/topology/collection.go @@ -14,6 +14,7 @@ type Collection struct { Name string volumeSizeLimit uint64 replicationAsMin bool + maxWritableVolumes int storageType2VolumeLayout *util.ConcurrentReadMap } @@ -40,7 +41,9 @@ func (c *Collection) GetOrCreateVolumeLayout(rp *super_block.ReplicaPlacement, t keyString += string(diskType) } vl := c.storageType2VolumeLayout.Get(keyString, func() interface{} { - return NewVolumeLayout(rp, ttl, diskType, c.volumeSizeLimit, c.replicationAsMin) + layout := NewVolumeLayout(rp, ttl, diskType, c.volumeSizeLimit, c.replicationAsMin) + layout.maxWritableVolumes = c.maxWritableVolumes + return layout }) return vl.(*VolumeLayout) } diff --git a/weed/topology/data_node.go b/weed/topology/data_node.go index 731e607091..444adf7646 100644 --- a/weed/topology/data_node.go +++ b/weed/topology/data_node.go @@ -358,10 +358,10 @@ func (dn *DataNode) matchesLocation(option *VolumeGrowOption) bool { func (dn *DataNode) GetTopology() *Topology { p := dn.Parent() - for p.Parent() != nil { + for p != nil && p.Parent() != nil { p = p.Parent() } - t := p.(*Topology) + t, _ := p.(*Topology) return t } diff --git a/weed/topology/duplicate_volume_id_test.go b/weed/topology/duplicate_volume_id_test.go index 42cb6fd97a..5fb315ea88 100644 --- a/weed/topology/duplicate_volume_id_test.go +++ b/weed/topology/duplicate_volume_id_test.go @@ -1,7 +1,12 @@ package topology import ( + "errors" + "sync/atomic" "testing" + "time" + + "github.com/stretchr/testify/require" "github.com/seaweedfs/seaweedfs/weed/pb/master_pb" "github.com/seaweedfs/seaweedfs/weed/storage" @@ -104,8 +109,125 @@ func TestQuarantineReleasedWhenConflictResolves(t *testing.T) { topo.UnRegisterVolumeLayout(v2, dn2) + // Registration flags duplicates asynchronously, so a flag that was still + // in flight when the conflict resolved releases the surviving copy only + // once it settles. Wait for that release instead of reading once. + deadline := time.Now().Add(5 * time.Second) + for !isWritable(t, topo, v1) { + if time.Now().After(deadline) { + t.Fatal("the surviving copy must be released once the conflict is gone") + } + time.Sleep(10 * time.Millisecond) + } +} + +func TestFailedReleasePushRestoresQuarantineMark(t *testing.T) { + topo := NewTopology("topo", nil, 128, 3, false) + rp, _ := super_block.NewReplicaPlacementFromString("000") + + dn1 := NewDataNode("server1") + dn2 := NewDataNode("server2") + + var pushFails atomic.Bool + topo.QuarantineVolumeFn = func(dn *DataNode, vid needle.VolumeId, readOnly bool) error { + if !readOnly && pushFails.Load() { + return errors.New("volume server unreachable") + } + return nil + } + + v1 := storage.VolumeInfo{Id: 7, Collection: "a", ReplicaPlacement: rp, Ttl: needle.EMPTY_TTL} + v2 := storage.VolumeInfo{Id: 7, Collection: "b", ReplicaPlacement: rp, Ttl: needle.EMPTY_TTL} + dn1.UpdateVolumes([]storage.VolumeInfo{v1}) + dn2.UpdateVolumes([]storage.VolumeInfo{v2}) + topo.RegisterVolumeLayout(v1, dn1) + topo.RegisterVolumeLayout(v2, dn2) + // Registrations flag duplicates asynchronously; let them settle so no + // late mark can land after the release assertions below. + topo.duplicateFlagChecks.Wait() + topo.flagDuplicateVolumeId(v2, dn2) + + // the conflict resolves while the surviving copy is unreachable: the + // writable push fails, so the record and the layout read-only mark must + // both come back instead of advertising a writable replica the server + // still holds read-only + pushFails.Store(true) + topo.UnRegisterVolumeLayout(v2, dn2) + + if !topo.isQuarantined(dn1, v1.Id) { + t.Fatal("a failed writable push must keep the quarantine record for the retry") + } + if isWritable(t, topo, v1) { + t.Fatal("the layout must not advertise a replica the server still holds read-only") + } + + pushFails.Store(false) + // An idle replica drops out of the delta reports once its stored flag + // matches the server's, so the release pass retries off the leader pulse + // rather than waiting for the volume to be reported again. + topo.retryPendingQuarantineReleases() + require.Eventually(t, func() bool { + return !topo.isQuarantined(dn1, v1.Id) && isWritable(t, topo, v1) + }, 5*time.Second, time.Millisecond, "the pending release must retry without another volume report") +} + +func TestQuarantineKeptWhenReplicaUnreadable(t *testing.T) { + topo := NewTopology("topo", nil, 128, 3, false) + rp, _ := super_block.NewReplicaPlacementFromString("000") + + dn1 := NewDataNode("server1") + dn2 := NewDataNode("server2") + + v1 := storage.VolumeInfo{Id: 7, Collection: "a", ReplicaPlacement: rp, Ttl: needle.EMPTY_TTL} + v2 := storage.VolumeInfo{Id: 7, Collection: "b", ReplicaPlacement: rp, Ttl: needle.EMPTY_TTL} + dn1.UpdateVolumes([]storage.VolumeInfo{v1}) + dn2.UpdateVolumes([]storage.VolumeInfo{v2}) + topo.RegisterVolumeLayout(v1, dn1) + topo.RegisterVolumeLayout(v2, dn2) + topo.duplicateFlagChecks.Wait() + topo.flagDuplicateVolumeId(v2, dn2) + + // the surviving copy drops out of its node's disk map while the lookup + // index still holds it; its layout key is then unknown, so the release + // pass must keep the mark rather than assume the conflict resolved + dn1.UpdateVolumes(nil) + topo.UnRegisterVolumeLayout(v2, dn2) + + if !topo.isQuarantined(dn1, v1.Id) { + t.Fatal("an unreadable replica must not lose its quarantine record") + } + if isWritable(t, topo, v1) { + t.Fatal("a replica whose conflict cannot be ruled out must stay read-only") + } + + dn1.UpdateVolumes([]storage.VolumeInfo{v1}) + topo.RegisterVolumeLayout(v1, dn1) + topo.duplicateFlagChecks.Wait() if !isWritable(t, topo, v1) { - t.Fatal("the surviving copy must be released once the conflict is gone") + t.Fatal("once readable again and sole, the copy must be released to writable") + } +} + +func TestStaleDuplicateReportDoesNotQuarantineSurvivor(t *testing.T) { + topo := NewTopology("topo", nil, 128, 3, false) + rp, _ := super_block.NewReplicaPlacementFromString("000") + + dn1 := NewDataNode("server1") + dn2 := NewDataNode("server2") + v1 := storage.VolumeInfo{Id: 7, Collection: "a", ReplicaPlacement: rp, Ttl: needle.EMPTY_TTL} + v2 := storage.VolumeInfo{Id: 7, Collection: "b", ReplicaPlacement: rp, Ttl: needle.EMPTY_TTL} + dn1.UpdateVolumes([]storage.VolumeInfo{v1}) + topo.RegisterVolumeLayout(v1, dn1) + + // dn2's copy is gone before its delayed duplicate check runs; the report + // still names dn2 but its layout no longer holds it. + topo.flagDuplicateVolumeId(v2, dn2) + + if topo.isQuarantined(dn1, v1.Id) || topo.isQuarantined(dn2, v2.Id) { + t.Fatal("a report for a copy that is already gone must not quarantine anything") + } + if !isWritable(t, topo, v1) { + t.Fatal("the surviving copy must stay writable") } } @@ -134,6 +256,47 @@ func TestQuarantineSurvivesResync(t *testing.T) { } } +func TestQuarantineSurvivesReplicaRejoin(t *testing.T) { + topo := NewTopology("topo", nil, 128, 3, false) + rp, _ := super_block.NewReplicaPlacementFromString("001") + + dn1 := NewDataNode("server1") + dn1.Ip = "10.0.0.1" + dn1.Port = 8080 + dn2 := NewDataNode("server2") + dn2.Ip = "10.0.0.2" + dn2.Port = 8080 + dn3 := NewDataNode("server3") + dn3.Ip = "10.0.0.3" + dn3.Port = 8080 + + v1 := storage.VolumeInfo{Id: 7, Collection: "a", ReplicaPlacement: rp, Ttl: needle.EMPTY_TTL} + v2 := storage.VolumeInfo{Id: 7, Collection: "b", ReplicaPlacement: rp, Ttl: needle.EMPTY_TTL} + dn1.UpdateVolumes([]storage.VolumeInfo{v1}) + dn2.UpdateVolumes([]storage.VolumeInfo{v1}) + dn3.UpdateVolumes([]storage.VolumeInfo{v2}) + topo.RegisterVolumeLayout(v1, dn1) + topo.RegisterVolumeLayout(v1, dn2) + topo.RegisterVolumeLayout(v2, dn3) + topo.flagDuplicateVolumeId(v2, dn3) + + // a replica dropped from the lookup index re-registers through the + // self-heal path; rebuilding location flags from reported state must + // not lift the quarantine on any copy + vl := topo.FindVolumeLayout(v1.Collection, v1.ReplicaPlacement, v1.Ttl, types.ToDiskType(v1.DiskType)) + vl.SetVolumeUnavailable(dn1, v1.Id) + topo.SyncDataNodeRegistration([]*master_pb.VolumeInformationMessage{{ + Id: 7, Collection: "a", ReplicaPlacement: uint32(rp.Byte()), Version: 3, + }}, dn1) + + if isWritable(t, topo, v1) { + t.Fatal("a re-registering replica must not clear the quarantine on the surviving copies") + } + if isWritable(t, topo, v2) { + t.Fatal("the conflicting copy must stay quarantined too") + } +} + func TestReplicationChangeIsNotDuplicate(t *testing.T) { topo := NewTopology("topo", nil, 128, 3, false) rpOld, _ := super_block.NewReplicaPlacementFromString("000") @@ -185,3 +348,56 @@ func TestDuplicateVolumeIdAcrossTtls(t *testing.T) { t.Fatal("same collection with a different ttl is still a reused volume id") } } + +func TestFailedWritablePushRetriedOnNextReport(t *testing.T) { + topo := NewTopology("topo", nil, 128, 3, false) + rp, _ := super_block.NewReplicaPlacementFromString("000") + + var failWritable atomic.Bool + var writablePushes atomic.Int32 + topo.QuarantineVolumeFn = func(dn *DataNode, vid needle.VolumeId, readOnly bool) error { + if !readOnly { + writablePushes.Add(1) + if failWritable.Load() { + return errors.New("volume server unreachable") + } + } + return nil + } + + dn1 := NewDataNode("server1") + dn2 := NewDataNode("server2") + v1 := storage.VolumeInfo{Id: 7, Collection: "a", ReplicaPlacement: rp, Ttl: needle.EMPTY_TTL} + v2 := storage.VolumeInfo{Id: 7, Collection: "b", ReplicaPlacement: rp, Ttl: needle.EMPTY_TTL} + dn1.UpdateVolumes([]storage.VolumeInfo{v1}) + dn2.UpdateVolumes([]storage.VolumeInfo{v2}) + // Register on the layouts directly: RegisterVolumeLayout would spawn an + // async flagDuplicateVolumeId per call, and a delayed one can re-quarantine + // the survivor at any later point in this test. + topo.GetVolumeLayout(v1.Collection, v1.ReplicaPlacement, v1.Ttl, types.ToDiskType(v1.DiskType)).RegisterVolume(&v1, dn1, nil) + topo.GetVolumeLayout(v2.Collection, v2.ReplicaPlacement, v2.Ttl, types.ToDiskType(v2.DiskType)).RegisterVolume(&v2, dn2, nil) + topo.flagDuplicateVolumeId(v2, dn2) + + // the conflict resolves while the survivor's writable push cannot land + failWritable.Store(true) + topo.UnRegisterVolumeLayout(v2, dn2) + require.Eventually(t, func() bool { + // The record reappears only after the failed writable push, and the + // restore arms it for retry in the same store - so this pair means the + // next report will take the retry branch rather than observing the + // release mid-flight. + return writablePushes.Load() >= 1 && topo.isQuarantined(dn1, v1.Id) + }, 5*time.Second, time.Millisecond, "a failed writable push must re-arm the record") + + reported := v1 + reported.ReadOnly = true + failWritable.Store(false) + topo.enforceQuarantine(dn1, reported) + + require.Eventually(t, func() bool { + // The release clears the record and the layout mark before the remote + // push lands, so the count has to be part of the condition or the poll + // can pass before the retried push runs. + return !topo.isQuarantined(dn1, v1.Id) && isWritable(t, topo, v1) && writablePushes.Load() >= 2 + }, 5*time.Second, time.Millisecond, "the next volume report must retry the release and its writable push") +} diff --git a/weed/topology/maintenance_mode_test.go b/weed/topology/maintenance_mode_test.go index 27c3f580ab..596ece558f 100644 --- a/weed/topology/maintenance_mode_test.go +++ b/weed/topology/maintenance_mode_test.go @@ -231,7 +231,7 @@ func TestMaintenanceModeHoldsThroughVacuumAndMarkWritable(t *testing.T) { vl.DrainAndRemoveFromWritable(1) topo.SetDataNodeMaintenanceMode(dn1, true) for _, dn := range []*DataNode{dn1, dn2} { - if vl.SetVolumeAvailable(dn, 1, false, false) { + if vl.SetVolumeAvailable(dn, 1, false, false, false) { t.Errorf("vacuum commit on %s made the volume writable with a replica in maintenance", dn.Id()) } } diff --git a/weed/topology/node.go b/weed/topology/node.go index 98e212ba07..be642027e7 100644 --- a/weed/topology/node.go +++ b/weed/topology/node.go @@ -145,9 +145,12 @@ type Node interface { } type NodeImpl struct { - diskUsages *DiskUsages - id NodeId - parent Node + diskUsages *DiskUsages + id NodeId + // parent is an atomic pointer because unlinking runs under the rack's + // lock while readers (vacuum sweeps, usage propagation, String) reach + // it with no shared lock. + parent atomic.Pointer[Node] sync.RWMutex // lock children children map[NodeId]Node // maxVolumeId uses atomic ops so UpAdjustMaxVolumeId (called from the @@ -292,8 +295,8 @@ func (n *NodeImpl) IsLocked() (isTryLock bool) { } func (n *NodeImpl) String() string { - if n.parent != nil { - return n.parent.String() + ":" + string(n.id) + if parent := n.Parent(); parent != nil { + return parent.String() + ":" + string(n.id) } return string(n.id) } @@ -390,7 +393,11 @@ func (n *NodeImpl) ReleaseReservedCapacity(reservationId string) { } } func (n *NodeImpl) SetParent(node Node) { - n.parent = node + if node == nil { + n.parent.Store(nil) + return + } + n.parent.Store(&node) } func (n *NodeImpl) Children() (ret []Node) { @@ -403,7 +410,10 @@ func (n *NodeImpl) Children() (ret []Node) { } func (n *NodeImpl) Parent() Node { - return n.parent + if parent := n.parent.Load(); parent != nil { + return *parent + } + return nil } func (n *NodeImpl) GetValue() interface{} { @@ -482,8 +492,8 @@ func (n *NodeImpl) reserveOneVolumeInternal(r int64, option *VolumeGrowOption, u func (n *NodeImpl) UpAdjustDiskUsageDelta(diskType types.DiskType, diskUsage *DiskUsageCounts) { //can be negative existingDisk := n.getOrCreateDisk(diskType) existingDisk.addDiskUsageCounts(diskUsage) - if n.parent != nil { - n.parent.UpAdjustDiskUsageDelta(diskType, diskUsage) + if parent := n.Parent(); parent != nil { + parent.UpAdjustDiskUsageDelta(diskType, diskUsage) } } func (n *NodeImpl) UpAdjustMaxVolumeId(vid needle.VolumeId) { @@ -497,8 +507,8 @@ func (n *NodeImpl) UpAdjustMaxVolumeId(vid needle.VolumeId) { break } } - if n.parent != nil { - n.parent.UpAdjustMaxVolumeId(vid) + if parent := n.Parent(); parent != nil { + parent.UpAdjustMaxVolumeId(vid) } } func (n *NodeImpl) GetMaxVolumeId() needle.VolumeId { diff --git a/weed/topology/topology.go b/weed/topology/topology.go index 3c68d3586b..3ceda5de78 100644 --- a/weed/topology/topology.go +++ b/weed/topology/topology.go @@ -50,6 +50,7 @@ type Topology struct { volumeSizeLimit uint64 replicationAsMin bool + maxWritableVolumes int vacuumDisabledByOperator atomic.Bool // true when operator manually disables vacuum vacuumDisabledByPlugin atomic.Bool // true when disabled by the vacuum plugin monitor adminServerConnectedFunc func() bool // optional callback to check admin server presence @@ -105,6 +106,21 @@ type Topology struct { // vid collided across layouts. Kept separate from the reported flag so // later heartbeats from the same server cannot quietly undo it. quarantinedVolumeIds sync.Map + // quarantineLock serializes each quarantine record update with the layout + // mark it justifies: a release pass and a heartbeat read-only flag update + // each observe record and mark as one state, never a mark whose record + // was already deleted or a cleared mark whose record is not yet visible. + quarantineLock sync.Mutex + // quarantinePushLocks serialize the remote mark/unmark calls per replica + // so a push decided under an older record state cannot land on the + // volume server after a newer one. Keyed by replica so a slow server + // does not stall pushes to the others. + quarantinePushLocks map[quarantinedVolume]*quarantinePushLock + quarantinePushLocksMu sync.Mutex + + // duplicateFlagChecks tracks in-flight flagDuplicateVolumeId passes so + // tests can wait for the async checks a registration started. + duplicateFlagChecks sync.WaitGroup // QuarantineVolumeFn pushes the quarantine onto the volume server itself // (VolumeMarkReadonly/VolumeMarkWritable). nil keeps quarantine at the @@ -120,6 +136,12 @@ type quarantinedVolume struct { type quarantinedVolumeState struct { reported bool enforced bool + retry bool +} + +type quarantinePushLock struct { + sync.Mutex + refs int } func (t *Topology) isQuarantined(dn *DataNode, vid needle.VolumeId) bool { @@ -127,6 +149,102 @@ func (t *Topology) isQuarantined(dn *DataNode, vid needle.VolumeId) bool { return ok } +// pushQuarantine sends the remote mark only while it still matches the +// record: a read-only push requires the pair to exist, a writable push +// requires it to be gone. The bool reports whether the remote call was +// actually made, so callers can tell a skipped stale push from a failed one. +func (t *Topology) pushQuarantine(dn *DataNode, vid needle.VolumeId, readOnly bool) (bool, error) { + unlock := t.lockQuarantinePush(quarantinedVolume{dn.Id(), vid}) + defer unlock() + return t.sendQuarantine(dn, vid, readOnly) +} + +// sendQuarantine runs the record check and remote mark; the caller holds +// the replica's push lock. +func (t *Topology) sendQuarantine(dn *DataNode, vid needle.VolumeId, readOnly bool) (bool, error) { + t.quarantineLock.Lock() + _, exists := t.quarantinedVolumeIds.Load(quarantinedVolume{dn.Id(), vid}) + t.quarantineLock.Unlock() + if exists != readOnly { + return false, nil + } + return true, t.QuarantineVolumeFn(dn, vid, readOnly) +} + +// releaseQuarantineWritable pushes the writable mark and re-arms the record +// on failure, all under the replica's push lock so a queued stale push cannot +// slip between the failure and the restore and undo the re-armed record. +func (t *Topology) releaseQuarantineWritable(dn *DataNode, vid needle.VolumeId) { + unlock := t.lockQuarantinePush(quarantinedVolume{dn.Id(), vid}) + defer unlock() + if _, err := t.sendQuarantine(dn, vid, false); err != nil { + t.restoreQuarantineRecord(dn, vid) + } +} + +// lockQuarantinePush returns the release for q's push lock, creating the lock +// on first use and dropping it once the last holder lets go. +func (t *Topology) lockQuarantinePush(q quarantinedVolume) func() { + t.quarantinePushLocksMu.Lock() + if t.quarantinePushLocks == nil { + t.quarantinePushLocks = make(map[quarantinedVolume]*quarantinePushLock) + } + l, ok := t.quarantinePushLocks[q] + if !ok { + l = &quarantinePushLock{} + t.quarantinePushLocks[q] = l + } + l.refs++ + t.quarantinePushLocksMu.Unlock() + l.Lock() + return func() { + t.quarantinePushLocksMu.Lock() + l.refs-- + if l.refs == 0 { + delete(t.quarantinePushLocks, q) + } + t.quarantinePushLocksMu.Unlock() + l.Unlock() + } +} + +// takeQuarantineRetryLocked disarms a pending release retry and reports +// whether one was armed. The caller holds quarantineLock. +func (t *Topology) takeQuarantineRetryLocked(q quarantinedVolume) bool { + v, ok := t.quarantinedVolumeIds.Load(q) + if !ok { + return false + } + state := v.(quarantinedVolumeState) + if !state.retry { + return false + } + state.retry = false + t.quarantinedVolumeIds.Store(q, state) + return true +} + +// retryPendingQuarantineReleases re-runs the release pass for every record a +// failed writable push re-armed. The retry in enforceQuarantine only fires +// while a heartbeat keeps carrying the volume, and an idle replica drops out +// of the delta reports once its stored flag matches the server's, so pending +// retries also run off the leader pulse. +func (t *Topology) retryPendingQuarantineReleases() { + vids := make(map[needle.VolumeId]struct{}) + t.quarantineLock.Lock() + t.quarantinedVolumeIds.Range(func(key, _ any) bool { + q := key.(quarantinedVolume) + if t.takeQuarantineRetryLocked(q) { + vids[q.vid] = struct{}{} + } + return true + }) + t.quarantineLock.Unlock() + for vid := range vids { + go t.releaseResolvedQuarantine(vid) + } +} + // enforceQuarantine keeps the latest reported read-only flag for a quarantined // copy so a later release restores the server's true state. The stored report // keeps the server's flag: quarantine lives at the layout level, and folding @@ -134,19 +252,55 @@ func (t *Topology) isQuarantined(dn *DataNode, vid needle.VolumeId) bool { // disagree with the server's and force a full volume list on every heartbeat. func (t *Topology) enforceQuarantine(dn *DataNode, vi storage.VolumeInfo) storage.VolumeInfo { q := quarantinedVolume{dn.Id(), vi.Id} - if v, ok := t.quarantinedVolumeIds.Load(q); ok { - state := v.(quarantinedVolumeState) - if !state.enforced { - state.reported = vi.ReadOnly - } - if t.QuarantineVolumeFn != nil && (!state.enforced || !vi.ReadOnly) { - // The mark never landed, or the server lost it (restart, manual - // unquarantine); push it while the conflict stands. - if t.QuarantineVolumeFn(dn, vi.Id, true) == nil { - state.enforced = true - } - } + t.quarantineLock.Lock() + v, ok := t.quarantinedVolumeIds.Load(q) + if !ok { + t.quarantineLock.Unlock() + return vi + } + if t.takeQuarantineRetryLocked(q) { + // A failed writable push re-armed this record; hand the release pass + // one retry per report. Clearing it first keeps a surviving conflict + // on the normal enforce path afterwards. + t.quarantineLock.Unlock() + go t.releaseResolvedQuarantine(vi.Id) + return vi + } + state := v.(quarantinedVolumeState) + if !state.enforced { + state.reported = vi.ReadOnly + } + if t.QuarantineVolumeFn == nil || (state.enforced && vi.ReadOnly) { t.quarantinedVolumeIds.Store(q, state) + t.quarantineLock.Unlock() + return vi + } + t.quarantineLock.Unlock() + + // The mark never landed on the server, or the server lost it (restart, + // manual unquarantine); push it while the conflict stands. The call is + // remote, so it runs outside the lock. + pushed, err := t.pushQuarantine(dn, vi.Id, true) + if pushed && err == nil { + state.enforced = true + } + t.quarantineLock.Lock() + cur, still := t.quarantinedVolumeIds.Load(q) + if still { + latest := cur.(quarantinedVolumeState) + if !latest.enforced { + latest.reported = state.reported + } + latest.enforced = latest.enforced || state.enforced + t.quarantinedVolumeIds.Store(q, latest) + t.quarantineLock.Unlock() + return vi + } + t.quarantineLock.Unlock() + if state.enforced && !state.reported { + // The conflict resolved while the push was in flight; leave the + // server consistent with the released layout. + t.releaseQuarantineWritable(dn, vi.Id) } return vi } @@ -172,6 +326,7 @@ func NewTopology(id string, seq sequence.Sequencer, volumeSizeLimit uint64, puls t.Configuration = &Configuration{} t.dataNodeIndex = make(map[string]*DataNode) + t.quarantinePushLocks = make(map[quarantinedVolume]*quarantinePushLock) return t } @@ -532,10 +687,20 @@ func (t *Topology) PickForWrite(requestedCount uint64, option *VolumeGrowOption, func (t *Topology) GetVolumeLayout(collectionName string, rp *super_block.ReplicaPlacement, ttl *needle.TTL, diskType types.DiskType) *VolumeLayout { return t.collectionMap.Get(collectionName, func() interface{} { - return NewCollection(collectionName, t.volumeSizeLimit, t.replicationAsMin) + c := NewCollection(collectionName, t.volumeSizeLimit, t.replicationAsMin) + c.maxWritableVolumes = t.maxWritableVolumes + return c }).(*Collection).GetOrCreateVolumeLayout(rp, ttl, diskType) } +// SetMaxWritableVolumes caps how many writable volumes in a volume layout +// (collection + replica placement + ttl + disk type) assignments are spread +// over; 0 means every writable volume is a candidate. It only affects +// collections created afterwards, so call it before serving. +func (t *Topology) SetMaxWritableVolumes(n int) { + t.maxWritableVolumes = n +} + // FindVolumeLayout is the read-only counterpart of GetVolumeLayout: it returns // nil when the collection or its layout does not exist, instead of creating // one. Lets callers check a layout's state without polluting the collection @@ -667,18 +832,31 @@ func (t *Topology) RegisterVolumeLayout(v storage.VolumeInfo, dn *DataNode) { diskType := types.ToDiskType(v.DiskType) for { vl := t.GetVolumeLayout(v.Collection, v.ReplicaPlacement, v.Ttl, diskType) - if vl.RegisterVolume(&v, dn) { - if t.isQuarantined(dn, v.Id) { - vl.SetVolumeQuarantined(dn, v.Id) - } + // RegisterVolume reads the quarantine record and marks under + // accessLock; holding quarantineLock keeps that pair serialized with + // releaseResolvedQuarantine's clear-and-delete. + t.quarantineLock.Lock() + registered := vl.RegisterVolume(&v, dn, func(n *DataNode) bool { + return t.isQuarantined(n, v.Id) + }) + t.quarantineLock.Unlock() + if registered { vl.EnsureCorrectWritables(&v) - go t.flagDuplicateVolumeId(v, dn) + t.flagDuplicateVolumeIdAsync(v, dn) return } // Dropped with its collection; the next lookup creates a fresh one. } } +func (t *Topology) flagDuplicateVolumeIdAsync(v storage.VolumeInfo, dn *DataNode) { + t.duplicateFlagChecks.Add(1) + go func() { + defer t.duplicateFlagChecks.Done() + t.flagDuplicateVolumeId(v, dn) + }() +} + // flagDuplicateVolumeId reports and quarantines v.Id when another data node // already serves it under a different layout key. It runs after registration // so a legitimate same-node move (disk type or collection change) settles @@ -686,13 +864,22 @@ func (t *Topology) RegisterVolumeLayout(v storage.VolumeInfo, dn *DataNode) { // quarantined: either could be the impostor, and arrival order must not // decide which copy stays writable. func (t *Topology) flagDuplicateVolumeId(v storage.VolumeInfo, dn *DataNode) { + // Every pass ends with a release: a conflict could resolve between the + // scan below and the marks, and a check finding no hits is also the retry + // a kept record waits for once its replica reports a readable layout. + defer t.releaseResolvedQuarantine(v.Id) hits := t.findDuplicateVolumeLayouts(v, dn) if len(hits) == 0 { return } - // A conflict could resolve between the scan above and the marks below; - // re-run the release so a stale quarantine is cleared instead of kept. - defer t.releaseResolvedQuarantine(v.Id) + // This runs on a goroutine after registration, so the reporter's copy may + // already be unregistered (a finished move drops the losing copy). The + // surviving copies are then legitimate, and flagging them quarantines a + // healthy volume. + own := t.FindVolumeLayout(v.Collection, v.ReplicaPlacement, v.Ttl, types.ToDiskType(v.DiskType)) + if own == nil || !own.HasDataNode(v.Id, dn) { + return + } stats.MasterDuplicateVolumeIds.WithLabelValues(v.Collection, v.Id.String()).Inc() var otherKeys []string for _, h := range hits { @@ -705,19 +892,36 @@ func (t *Topology) flagDuplicateVolumeId(v storage.VolumeInfo, dn *DataNode) { if vi, err := n.GetVolumesById(v.Id); err == nil { reported = vi.ReadOnly } + t.quarantineLock.Lock() t.quarantinedVolumeIds.LoadOrStore(q, quarantinedVolumeState{reported: reported}) if vl != nil { vl.SetVolumeQuarantined(n, v.Id) } - if t.QuarantineVolumeFn != nil && t.QuarantineVolumeFn(n, v.Id, true) == nil { - if val, ok := t.quarantinedVolumeIds.Load(q); ok { - state := val.(quarantinedVolumeState) - state.enforced = true - t.quarantinedVolumeIds.Store(q, state) - } + t.quarantineLock.Unlock() + if t.QuarantineVolumeFn == nil { + return + } + pushed, err := t.pushQuarantine(n, v.Id, true) + if err != nil { + return + } + t.quarantineLock.Lock() + val, ok := t.quarantinedVolumeIds.Load(q) + if ok { + state := val.(quarantinedVolumeState) + state.enforced = state.enforced || pushed + t.quarantinedVolumeIds.Store(q, state) + t.quarantineLock.Unlock() + return + } + t.quarantineLock.Unlock() + if pushed && !reported { + // The conflict resolved while the push was in flight; leave the + // server consistent with the released layout. + t.releaseQuarantineWritable(n, v.Id) } } - quarantine(dn, t.FindVolumeLayout(v.Collection, v.ReplicaPlacement, v.Ttl, types.ToDiskType(v.DiskType))) + quarantine(dn, own) for _, h := range hits { for _, n := range h.nodes { quarantine(n, h.vl) @@ -727,7 +931,14 @@ func (t *Topology) flagDuplicateVolumeId(v storage.VolumeInfo, dn *DataNode) { // releaseResolvedQuarantine lifts read-only marks once the conflicting copy // is gone, so a finished move does not leave the surviving copy quarantined. +// The writable push is a remote call, so the locked pass only applies the +// layout marks and records which replicas still need it; the pushes run +// after the lock is dropped, a skipped push means a newer record re-armed +// the pair so its quarantine stands, and a failed push re-adds the record +// so a later release pass retries it. func (t *Topology) releaseResolvedQuarantine(vid needle.VolumeId) { + var writablePushes []*DataNode + t.quarantineLock.Lock() t.quarantinedVolumeIds.Range(func(key, val any) bool { q := key.(quarantinedVolume) if q.vid != vid { @@ -738,6 +949,12 @@ func (t *Topology) releaseResolvedQuarantine(vid needle.VolumeId) { t.quarantinedVolumeIds.Delete(key) return true } + if vi.ReplicaPlacement == nil { + // The replica is in the lookup index but unreadable on its node, + // so its layout key is unknown and the conflict cannot be ruled + // out; keep the mark for a later pass. + return true + } if hits := t.findDuplicateVolumeLayouts(vi, dn); len(hits) > 0 { return true } @@ -748,16 +965,38 @@ func (t *Topology) releaseResolvedQuarantine(vid needle.VolumeId) { } else { vl.SetVolumeWritable(dn, vid) if state.enforced && t.QuarantineVolumeFn != nil { - if err := t.QuarantineVolumeFn(dn, vid, false); err != nil { - // The server still rejects writes; keep the mark so a - // later release pass retries the writable push. - return true - } + writablePushes = append(writablePushes, dn) } } t.quarantinedVolumeIds.Delete(key) return true }) + t.quarantineLock.Unlock() + for _, dn := range writablePushes { + t.releaseQuarantineWritable(dn, vid) + } +} + +// restoreQuarantineRecord re-adds a released record after a failed writable +// push so the next volume report retries the release. An existing record keeps its +// reported flag; the server is known read-only, so enforced and retry are set. +// A record restored from scratch also gets its layout read-only mark back: +// the release pass cleared it with the record, and leaving it cleared would +// advertise a writable replica the server still holds read-only. +func (t *Topology) restoreQuarantineRecord(dn *DataNode, vid needle.VolumeId) { + t.quarantineLock.Lock() + defer t.quarantineLock.Unlock() + key := quarantinedVolume{dn.Id(), vid} + if cur, ok := t.quarantinedVolumeIds.LoadOrStore(key, quarantinedVolumeState{enforced: true, retry: true}); ok { + state := cur.(quarantinedVolumeState) + state.enforced = true + state.retry = true + t.quarantinedVolumeIds.Store(key, state) + return + } + if _, vl, _ := t.findQuarantinedVolume(dn.Id(), vid); vl != nil { + vl.SetVolumeReadOnly(dn, vid) + } } func (t *Topology) findQuarantinedVolume(dnId NodeId, vid needle.VolumeId) (*DataNode, *VolumeLayout, storage.VolumeInfo) { @@ -797,7 +1036,6 @@ func ttlEqual(a, b *needle.TTL) bool { return a.ToUint32() == b.ToUint32() } - // findDuplicateVolumeLayouts reports every layout that already holds v.Id on // data nodes other than dn. Volume ids are global, so the same id held by two // nodes under two layouts means the allocator reused an id. @@ -973,7 +1211,9 @@ func (t *Topology) SyncDataNodeRegistration(volumes []*master_pb.VolumeInformati if v.ReplicaPlacement == nil { continue } + t.quarantineLock.Lock() t.GetVolumeLayout(v.Collection, v.ReplicaPlacement, v.Ttl, types.ToDiskType(v.DiskType)).SetReplicaReadOnlyFlag(dn, v.Id, v.ReadOnly || t.isQuarantined(dn, v.Id)) + t.quarantineLock.Unlock() } // Update effective sizes for all reported volumes (decay pending estimates). // If decay brings a volume eagerly removed by RecordAssign back under the @@ -992,12 +1232,20 @@ func (t *Topology) SyncDataNodeRegistration(volumes []*master_pb.VolumeInformati // Without this, the volume stays visible in volume.list/admin UI yet // LookupVolume returns "volume id not found". if !vl.HasDataNode(v.Id, dn) { - for !vl.RegisterVolume(&v, dn) { + for { + t.quarantineLock.Lock() + registered := vl.RegisterVolume(&v, dn, func(n *DataNode) bool { + return t.isQuarantined(n, v.Id) + }) + t.quarantineLock.Unlock() + if registered { + break + } // Dropped with its collection; the next lookup creates a fresh // one, which the calls below must use too. vl = t.GetVolumeLayout(v.Collection, v.ReplicaPlacement, v.Ttl, diskType) } - go t.flagDuplicateVolumeId(v, dn) + t.flagDuplicateVolumeIdAsync(v, dn) // Volumes new to the disk map were registered above, so reaching // here means only the lookup index had lost it. Clients were told // it went when the node dropped out, so the repair has to tell them @@ -1098,18 +1346,28 @@ func (t *Topology) ApplyVolumeChanges(changed []*master_pb.VolumeInformationMess // volume only that index had lost is an arrival as far as clients are // concerned: they were told it went when the node dropped out. becameServable := !vl.HasDataNode(vi.Id, dn) - for becameServable && !vl.RegisterVolume(&vi, dn) { + for becameServable { + t.quarantineLock.Lock() + registered := vl.RegisterVolume(&vi, dn, func(n *DataNode) bool { + return t.isQuarantined(n, vi.Id) + }) + t.quarantineLock.Unlock() + if registered { + break + } // Dropped with its collection; the next lookup creates a fresh one. vl = t.GetVolumeLayout(vi.Collection, vi.ReplicaPlacement, vi.Ttl, types.ToDiskType(vi.DiskType)) } if becameServable { - go t.flagDuplicateVolumeId(vi, dn) + t.flagDuplicateVolumeIdAsync(vi, dn) } if isNew || becameServable || tierTransition || isChanged { newVolumes = append(newVolumes, vi) } if isChanged { + t.quarantineLock.Lock() vl.SetReplicaReadOnlyFlag(dn, vi.Id, vi.ReadOnly || t.isQuarantined(dn, vi.Id)) + t.quarantineLock.Unlock() } vl.UpdateOversizedState(&vi, dn) if vl.UpdateVolumeSize(vi.Id, vi.Size, vi.CompactRevision, true) { diff --git a/weed/topology/topology_event_handling.go b/weed/topology/topology_event_handling.go index a0df67e221..1793ff0e93 100644 --- a/weed/topology/topology_event_handling.go +++ b/weed/topology/topology_event_handling.go @@ -13,7 +13,7 @@ import ( "github.com/seaweedfs/seaweedfs/weed/storage" ) -func (t *Topology) StartRefreshWritableVolumes(grpcDialOption grpc.DialOption, garbageThreshold float64, concurrentVacuumLimitPerVolumeServer int, growThreshold float64, preallocate int64, vacuumInterval time.Duration, deleteEmptyAfter time.Duration) { +func (t *Topology) StartRefreshWritableVolumes(grpcDialOption grpc.DialOption, garbageThreshold float64, concurrentVacuumLimitPerVolumeServer int, concurrentVacuumLimitPerRack int, growThreshold float64, preallocate int64, vacuumInterval time.Duration, deleteEmptyAfter time.Duration) { if vacuumInterval <= 0 { vacuumInterval = 14 * time.Minute } @@ -23,6 +23,7 @@ func (t *Topology) StartRefreshWritableVolumes(grpcDialOption grpc.DialOption, g freshThreshHold := time.Now().Unix() - 3*t.pulse //3 times of sleep interval t.CollectDeadNodeAndFullVolumes(freshThreshHold, t.volumeSizeLimit, growThreshold) t.DecayQuietVolumeSizes() + t.retryPendingQuarantineReleases() } time.Sleep(time.Duration(float32(t.pulse*1e3)*(1+rand.Float32())) * time.Millisecond) } @@ -39,7 +40,7 @@ func (t *Topology) StartRefreshWritableVolumes(grpcDialOption grpc.DialOption, g t.EnableVacuumByPlugin() } if !t.IsVacuumDisabled() { - t.Vacuum(grpcDialOption, garbageThreshold, concurrentVacuumLimitPerVolumeServer, 0, "", preallocate, true, deleteEmptyAfter) + t.Vacuum(grpcDialOption, garbageThreshold, concurrentVacuumLimitPerVolumeServer, concurrentVacuumLimitPerRack, 0, "", preallocate, true, deleteEmptyAfter) } } else { stats.MasterReplicaPlacementMismatch.Reset() diff --git a/weed/topology/topology_test.go b/weed/topology/topology_test.go index 3a89ce14f5..e761ede3c5 100644 --- a/weed/topology/topology_test.go +++ b/weed/topology/topology_test.go @@ -851,7 +851,7 @@ func TestSetVolumeAvailableRepairsMissingVolume(t *testing.T) { } // The vacuum commit re-marks the volume available; it must re-register it. - vl.SetVolumeAvailable(dn, vid, false, false) + vl.SetVolumeAvailable(dn, vid, false, false, false) if got := topo.Lookup("", vid); len(got) != 1 { t.Fatalf("after SetVolumeAvailable: lookup %d got %v, want 1 location", vid, got) } diff --git a/weed/topology/topology_vacuum.go b/weed/topology/topology_vacuum.go index 83e1ab6f47..ea6de330f4 100644 --- a/weed/topology/topology_vacuum.go +++ b/weed/topology/topology_vacuum.go @@ -226,7 +226,9 @@ func (t *Topology) batchVacuumVolumeCommit(grpcDialOption grpc.DialOption, vl *V vl.accessLock.Unlock() for _, dn := range vacuumLocationList.list { - vl.SetVolumeAvailable(dn, vid, isReadOnly, isFullCapacity) + t.quarantineLock.Lock() + vl.SetVolumeAvailable(dn, vid, isReadOnly, isFullCapacity, t.isQuarantined(dn, vid)) + t.quarantineLock.Unlock() } } return isCommitSuccess @@ -251,7 +253,7 @@ func (t *Topology) batchVacuumVolumeCleanup(grpcDialOption grpc.DialOption, vl * } } -func (t *Topology) Vacuum(grpcDialOption grpc.DialOption, garbageThreshold float64, maxParallelVacuumPerServer int, volumeId uint32, collection string, preallocate int64, automatic bool, deleteEmptyAfter time.Duration) { +func (t *Topology) Vacuum(grpcDialOption grpc.DialOption, garbageThreshold float64, maxParallelVacuumPerServer int, maxParallelVacuumPerRack int, volumeId uint32, collection string, preallocate int64, automatic bool, deleteEmptyAfter time.Duration) { // if there is vacuum going on, return immediately swapped := atomic.CompareAndSwapInt64(&t.vacuumLockCounter, 0, 1) @@ -285,7 +287,7 @@ func (t *Topology) Vacuum(grpcDialOption grpc.DialOption, garbageThreshold float t.vacuumOneVolumeId(grpcDialOption, volumeLayout, c, garbageThreshold, locationList, vid, preallocate, false) } } else { - t.vacuumOneVolumeLayout(grpcDialOption, volumeLayout, c, garbageThreshold, maxParallelVacuumPerServer, preallocate, automatic, deleteEmptyAfter) + t.vacuumOneVolumeLayout(grpcDialOption, volumeLayout, c, garbageThreshold, maxParallelVacuumPerServer, maxParallelVacuumPerRack, preallocate, automatic, deleteEmptyAfter) } } if automatic && t.IsVacuumDisabled() { @@ -299,7 +301,7 @@ func (t *Topology) Vacuum(grpcDialOption grpc.DialOption, garbageThreshold float } } -func (t *Topology) vacuumOneVolumeLayout(grpcDialOption grpc.DialOption, volumeLayout *VolumeLayout, c *Collection, garbageThreshold float64, maxParallelVacuumPerServer int, preallocate int64, automatic bool, deleteEmptyAfter time.Duration) { +func (t *Topology) vacuumOneVolumeLayout(grpcDialOption grpc.DialOption, volumeLayout *VolumeLayout, c *Collection, garbageThreshold float64, maxParallelVacuumPerServer int, maxParallelVacuumPerRack int, preallocate int64, automatic bool, deleteEmptyAfter time.Duration) { volumeLayout.accessLock.RLock() todoVolumeMap := make(map[needle.VolumeId]*VolumeLocationList) @@ -314,14 +316,27 @@ func (t *Topology) vacuumOneVolumeLayout(grpcDialOption grpc.DialOption, volumeL t.deleteEmptyVolumes(grpcDialOption, volumeLayout, todoVolumeMap, deleteEmptyAfter) } - // limiter for each volume server + // limiter for each volume server, and for each rack when a rack cap is + // set — several volume servers can share one rack's disks and network. + // Rack keys are resolved once up front: a node that unregisters mid-sweep + // loses its parent, and a later GetRack call would panic. limiter := make(map[NodeId]int) + rackLimiter := make(map[Node]int) + dnRack := make(map[*DataNode]Node) var limiterLock sync.Mutex for _, locationList := range todoVolumeMap { for _, dn := range locationList.list { if _, ok := limiter[dn.Id()]; !ok { limiter[dn.Id()] = maxParallelVacuumPerServer } + if maxParallelVacuumPerRack > 0 { + if _, ok := dnRack[dn]; !ok { + dnRack[dn] = vacuumRackKey(dn) + } + if _, ok := rackLimiter[dnRack[dn]]; !ok { + rackLimiter[dnRack[dn]] = maxParallelVacuumPerRack + } + } } } @@ -333,38 +348,59 @@ func (t *Topology) vacuumOneVolumeLayout(grpcDialOption grpc.DialOption, volumeL for len(todoVolumeMap) > 0 { pendingVolumeMap := make(map[needle.VolumeId]*VolumeLocationList) for vid, locationList := range todoVolumeMap { - hasEnoughQuota := true + // copies on the same server or rack reserve together; a volume + // that alone exceeds a cap waits for the whole bucket to be free + // instead of pending forever + serverNeeds := make(map[NodeId]int) + rackNeeds := make(map[Node]int) for _, dn := range locationList.list { - limiterLock.Lock() - quota := limiter[dn.Id()] - limiterLock.Unlock() - if quota <= 0 { + serverNeeds[dn.Id()]++ + if maxParallelVacuumPerRack > 0 { + rackNeeds[dnRack[dn]]++ + } + } + limiterLock.Lock() + hasEnoughQuota := true + for id, need := range serverNeeds { + serverNeeds[id] = min(need, maxParallelVacuumPerServer) + if limiter[id] < serverNeeds[id] { + hasEnoughQuota = false + } + } + for id, need := range rackNeeds { + rackNeeds[id] = min(need, maxParallelVacuumPerRack) + if rackLimiter[id] < rackNeeds[id] { hasEnoughQuota = false - break } } if !hasEnoughQuota { + limiterLock.Unlock() pendingVolumeMap[vid] = locationList continue } // debit the quota - for _, dn := range locationList.list { - limiterLock.Lock() - limiter[dn.Id()]-- - limiterLock.Unlock() + for id, need := range serverNeeds { + limiter[id] -= need } + for id, need := range rackNeeds { + rackLimiter[id] -= need + } + limiterLock.Unlock() wg.Add(1) executor.Execute(func() { defer wg.Done() t.vacuumOneVolumeId(grpcDialOption, volumeLayout, c, garbageThreshold, locationList, vid, preallocate, true) // credit the quota - for _, dn := range locationList.list { - limiterLock.Lock() - limiter[dn.Id()]++ - limiterLock.Unlock() + limiterLock.Lock() + for id, need := range serverNeeds { + limiter[id] += need } + for id, need := range rackNeeds { + rackLimiter[id] += need + } + limiterLock.Unlock() select { case quotaFreed <- struct{}{}: default: @@ -390,6 +426,17 @@ func (t *Topology) vacuumOneVolumeLayout(grpcDialOption grpc.DialOption, volumeL } +// vacuumRackKey identifies a data node's rack by the rack node itself, so +// racks that share a name in different data centers still get separate +// quotas. A node that already lost its parent falls back to itself, +// degrading to the per-server cap. +func vacuumRackKey(dn *DataNode) Node { + if rack := dn.Parent(); rack != nil { + return rack + } + return dn +} + // skipReadOnly is set by the background scan and all-volumes sweep, where a // read-only flag usually means an unhealthy disk. Even then a copy that is // read-only because its disk is low on space stays eligible — compaction is diff --git a/weed/topology/topology_vacuum_test.go b/weed/topology/topology_vacuum_test.go index 82643f33b2..106011d594 100644 --- a/weed/topology/topology_vacuum_test.go +++ b/weed/topology/topology_vacuum_test.go @@ -257,13 +257,27 @@ func TestDeleteEmptyVolumesKeepsVidWhenCopyDeleteFails(t *testing.T) { type fakeVacuumServer struct { volume_server_pb.UnimplementedVolumeServerServer - mu sync.Mutex - checks map[uint32]*volume_server_pb.VacuumVolumeCheckResponse - committed []uint32 - compactErr error - hang string - entered chan string - cancelled chan struct{} + mu sync.Mutex + checks map[uint32]*volume_server_pb.VacuumVolumeCheckResponse + committed []uint32 + compactErr error + hang string + entered chan string + cancelled chan struct{} + compactGate chan struct{} + compactTracker *compactTracker +} + +type compactTracker struct { + mu sync.Mutex + inFlight int + max int +} + +func (t *compactTracker) maxInFlight() int { + t.mu.Lock() + defer t.mu.Unlock() + return t.max } func (f *fakeVacuumServer) stall(ctx context.Context, phase string) error { @@ -299,6 +313,28 @@ func (f *fakeVacuumServer) VacuumVolumeCompact(_ *volume_server_pb.VacuumVolumeC if f.hang == "compact" { return f.stall(stream.Context(), "compact") } + if f.compactGate != nil { + select { + case f.entered <- "compact": + default: + } + f.compactTracker.mu.Lock() + f.compactTracker.inFlight++ + if f.compactTracker.inFlight > f.compactTracker.max { + f.compactTracker.max = f.compactTracker.inFlight + } + f.compactTracker.mu.Unlock() + defer func() { + f.compactTracker.mu.Lock() + f.compactTracker.inFlight-- + f.compactTracker.mu.Unlock() + }() + select { + case <-f.compactGate: + case <-stream.Context().Done(): + return stream.Context().Err() + } + } return f.compactErr } @@ -456,7 +492,7 @@ func TestVacuumStalledVolumeServerReleasesGuard(t *testing.T) { done := make(chan struct{}) go func() { - topo.Vacuum(dialOption, 0.3, 1, 1, "c", 0, false, 0) + topo.Vacuum(dialOption, 0.3, 1, 0, 1, "c", 0, false, 0) close(done) }() @@ -530,3 +566,197 @@ func TestVacuumSkipsTieredReplica(t *testing.T) { t.Fatalf("committed %v, want only the local volume 2", fake.committed) // volume 2 is the positive control } } + +// A rack cap bounds copies compacting in parallel inside one rack even when +// every data node still has per-server quota left: volumes whose only copy +// sits on different data nodes of the same rack serialize. +func TestVacuumRackLimitSerializesVolumes(t *testing.T) { + tracker := &compactTracker{} + newFake := func(vid uint32) *fakeVacuumServer { + return &fakeVacuumServer{ + checks: map[uint32]*volume_server_pb.VacuumVolumeCheckResponse{vid: {GarbageRatio: 0.9}}, + entered: make(chan string, 1), + compactGate: make(chan struct{}), + compactTracker: tracker, + } + } + fake1, fake2 := newFake(1), newFake(2) + port1, dialOption := startFakeVolumeServer(t, fake1) + port2, _ := startFakeVolumeServer(t, fake2) + + topo := NewTopology("weedfs", sequence.NewMemorySequencer(), 32*1024, 5, false) + rack := topo.GetOrCreateDataCenter("dc1").GetOrCreateRack("rack1") + dns := []*DataNode{ + rack.GetOrCreateDataNode("127.0.0.1", 8080, port1, "127.0.0.1", "dn-1", map[string]uint32{"": 10}), + rack.GetOrCreateDataNode("127.0.0.1", 8081, port2, "127.0.0.1", "dn-2", map[string]uint32{"": 10}), + } + for i, dn := range dns { + v := storage.VolumeInfo{ + Id: needle.VolumeId(i + 1), + Size: 1 << 20, + Collection: "c", + ModifiedAtSecond: time.Now().Unix(), + Version: needle.GetCurrentVersion(), + ReplicaPlacement: &super_block.ReplicaPlacement{}, + Ttl: needle.EMPTY_TTL, + } + dn.UpdateVolumes([]storage.VolumeInfo{v}) + topo.RegisterVolumeLayout(v, dn) + } + + done := make(chan struct{}) + go func() { + topo.Vacuum(dialOption, 0.3, 10, 1, 0, "c", 0, false, 0) + close(done) + }() + + deadline := time.After(15 * time.Second) + var winner *fakeVacuumServer + for winner == nil { + select { + case <-fake1.entered: + winner = fake1 + case <-fake2.entered: + winner = fake2 + case <-deadline: + t.Fatal("no volume reached the compact phase") + } + } + + // while the first compact is gated, the rack cap must keep the second + // volume from starting one + time.Sleep(300 * time.Millisecond) + if got := tracker.maxInFlight(); got != 1 { + t.Fatalf("in-flight compacts = %d, want the rack cap to hold it at 1", got) + } + + close(fake1.compactGate) + close(fake2.compactGate) + + select { + case <-done: + case <-time.After(15 * time.Second): + t.Fatal("vacuum did not finish after the gates opened") + } + for _, fake := range []*fakeVacuumServer{fake1, fake2} { + fake.mu.Lock() + n := len(fake.committed) + fake.mu.Unlock() + if n != 1 { + t.Fatalf("committed volumes = %d, want 1", n) + } + } +} + +// Racks that share a name in different data centers are independent: a per +// rack cap must not serialize volumes sitting in dc1/rack1 and dc2/rack1. +func TestVacuumRackLimitScopesRacksByDataCenter(t *testing.T) { + tracker := &compactTracker{} + newFake := func(vid uint32) *fakeVacuumServer { + return &fakeVacuumServer{ + checks: map[uint32]*volume_server_pb.VacuumVolumeCheckResponse{vid: {GarbageRatio: 0.9}}, + entered: make(chan string, 1), + compactGate: make(chan struct{}), + compactTracker: tracker, + } + } + fake1, fake2 := newFake(1), newFake(2) + port1, dialOption := startFakeVolumeServer(t, fake1) + port2, _ := startFakeVolumeServer(t, fake2) + + topo := NewTopology("weedfs", sequence.NewMemorySequencer(), 32*1024, 5, false) + dns := []*DataNode{ + topo.GetOrCreateDataCenter("dc1").GetOrCreateRack("rack1"). + GetOrCreateDataNode("127.0.0.1", 8080, port1, "127.0.0.1", "dn-1", map[string]uint32{"": 10}), + topo.GetOrCreateDataCenter("dc2").GetOrCreateRack("rack1"). + GetOrCreateDataNode("127.0.0.1", 8081, port2, "127.0.0.1", "dn-2", map[string]uint32{"": 10}), + } + for i, dn := range dns { + v := storage.VolumeInfo{ + Id: needle.VolumeId(i + 1), + Size: 1 << 20, + Collection: "c", + ModifiedAtSecond: time.Now().Unix(), + Version: needle.GetCurrentVersion(), + ReplicaPlacement: &super_block.ReplicaPlacement{}, + Ttl: needle.EMPTY_TTL, + } + dn.UpdateVolumes([]storage.VolumeInfo{v}) + topo.RegisterVolumeLayout(v, dn) + } + + done := make(chan struct{}) + go func() { + topo.Vacuum(dialOption, 0.3, 10, 1, 0, "c", 0, false, 0) + close(done) + }() + + // both volumes sit in different racks, so both compacts start while the + // gates stay closed + deadline := time.After(15 * time.Second) + for _, fake := range []*fakeVacuumServer{fake1, fake2} { + select { + case <-fake.entered: + case <-deadline: + t.Fatal("a volume in an independent rack never reached the compact phase") + } + } + + close(fake1.compactGate) + close(fake2.compactGate) + + select { + case <-done: + case <-time.After(15 * time.Second): + t.Fatal("vacuum did not finish after the gates opened") + } +} + +// A data node that unregisters while its volume is still compacting must not +// crash the sweep when the quota is credited back: the rack key was captured +// before the RPCs ran. +func TestVacuumRackLimitSurvivesUnregister(t *testing.T) { + fake := &fakeVacuumServer{ + checks: map[uint32]*volume_server_pb.VacuumVolumeCheckResponse{1: {GarbageRatio: 0.9}}, + entered: make(chan string, 1), + compactGate: make(chan struct{}), + compactTracker: &compactTracker{}, + } + grpcPort, dialOption := startFakeVolumeServer(t, fake) + + topo := NewTopology("weedfs", sequence.NewMemorySequencer(), 32*1024, 5, false) + dn := topo.GetOrCreateDataCenter("dc1").GetOrCreateRack("rack1"). + GetOrCreateDataNode("127.0.0.1", 8080, grpcPort, "127.0.0.1", "dn-1", map[string]uint32{"": 10}) + v := storage.VolumeInfo{ + Id: needle.VolumeId(1), + Size: 1 << 20, + Collection: "c", + ModifiedAtSecond: time.Now().Unix(), + Version: needle.GetCurrentVersion(), + ReplicaPlacement: &super_block.ReplicaPlacement{}, + Ttl: needle.EMPTY_TTL, + } + dn.UpdateVolumes([]storage.VolumeInfo{v}) + topo.RegisterVolumeLayout(v, dn) + + done := make(chan struct{}) + go func() { + topo.Vacuum(dialOption, 0.3, 1, 1, 0, "c", 0, false, 0) + close(done) + }() + + select { + case <-fake.entered: + case <-time.After(15 * time.Second): + t.Fatal("volume never reached the compact phase") + } + + topo.UnRegisterDataNode(dn) + close(fake.compactGate) + + select { + case <-done: + case <-time.After(15 * time.Second): + t.Fatal("vacuum did not finish after the data node unregistered") + } +} diff --git a/weed/topology/volume_growth_test.go b/weed/topology/volume_growth_test.go index 11e3ba0829..da9cf5affd 100644 --- a/weed/topology/volume_growth_test.go +++ b/weed/topology/volume_growth_test.go @@ -116,7 +116,7 @@ func setup(topologyLayout string) *Topology { } if vi.ReplicaPlacement != nil { vl := topo.GetVolumeLayout(vi.Collection, vi.ReplicaPlacement, needle.EMPTY_TTL, types.HardDriveType) - vl.RegisterVolume(&vi, server) + vl.RegisterVolume(&vi, server, nil) vl.setVolumeWritable(vi.Id) } server.AddOrUpdateVolume(vi) diff --git a/weed/topology/volume_layout.go b/weed/topology/volume_layout.go index 4219c8eff3..c74fefccbc 100644 --- a/weed/topology/volume_layout.go +++ b/weed/topology/volume_layout.go @@ -68,8 +68,11 @@ type VolumeLayout struct { deletingVolumes map[needle.VolumeId]struct{} volumeSizeLimit uint64 replicationAsMin bool - accessLock sync.RWMutex - sizeTracking map[needle.VolumeId]*volumeSizeTracking + // maxWritableVolumes caps how many writables PickForWrite spreads over; + // 0 means every writable volume is a candidate. + maxWritableVolumes int + accessLock sync.RWMutex + sizeTracking map[needle.VolumeId]*volumeSizeTracking // dropped: the layout went away with its collection; late registrations must re-resolve. dropped bool } @@ -129,7 +132,11 @@ func (vl *VolumeLayout) initSizeTracking(vid needle.VolumeId, size uint64, compa // RegisterVolume records a volume location. It refuses a layout dropped with // its collection — the caller must re-resolve, or the bits it sets would leak. -func (vl *VolumeLayout) RegisterVolume(v *storage.VolumeInfo, dn *DataNode) bool { +// Each replica's read-only flag is rebuilt from what its node last reported; +// isQuarantined folds the master's force-read-only marks into that rebuild, +// since reported flags deliberately exclude them. nil restores reported state +// verbatim. +func (vl *VolumeLayout) RegisterVolume(v *storage.VolumeInfo, dn *DataNode, isQuarantined func(*DataNode) bool) bool { vl.accessLock.Lock() defer vl.accessLock.Unlock() @@ -137,27 +144,27 @@ func (vl *VolumeLayout) RegisterVolume(v *storage.VolumeInfo, dn *DataNode) bool return false } - moveLookupOwnership(v.Id, vl.getOrCreateLocationList(v.Id).Set(dn), dn) + location := vl.getOrCreateLocationList(v.Id) + moveLookupOwnership(v.Id, location.Set(dn), dn) if !v.ReadOnly { vl.initSizeTracking(v.Id, v.Size, v.CompactRevision) } + if isQuarantined != nil && isQuarantined(dn) { + location.SetReadOnly(dn, true) + vl.removeFromWritable(v.Id) + } // glog.V(4).Infof("volume %d added to %s len %d copy %d", v.Id, dn.Id(), vl.vid2location[v.Id].Length(), v.ReplicaPlacement.GetCopyCount()) - location := vl.vid2location[v.Id] - for _, dn := range location.list { - if vInfo, err := dn.GetVolumesById(v.Id); err == nil { - if vInfo.ReadOnly { - glog.V(1).Infof("vid %d removed from writable", v.Id) - vl.removeFromWritable(v.Id) - location.SetReadOnly(dn, true) - return true - } - location.SetReadOnly(dn, false) - } else { - glog.V(1).Infof("vid %d removed from writable", v.Id) - vl.removeFromWritable(v.Id) - location.SetReadOnly(dn, false) - return true + for _, n := range location.list { + marked := isQuarantined != nil && isQuarantined(n) + vInfo, err := n.GetVolumesById(v.Id) + if err == nil && !vInfo.ReadOnly && !marked { + location.SetReadOnly(n, false) + continue } + glog.V(1).Infof("vid %d removed from writable", v.Id) + vl.removeFromWritable(v.Id) + location.SetReadOnly(n, marked || (err == nil && vInfo.ReadOnly)) + return true } return true } @@ -646,13 +653,40 @@ func (vl *VolumeLayout) PickForWrite(count uint64, option *VolumeGrowOption) (vi return 0, 0, nil, true, fmt.Errorf("%s", NoWritableVolumes) } if option.DataCenter == "" && option.Rack == "" && option.DataNode == "" { - vid, locationList = vl.pickWeightedByRemaining(vl.writables) + writables := vl.writables + if vl.maxWritableVolumes > 0 && len(writables) > vl.maxWritableVolumes { + writables = vl.fullestWritable(writables) + } + vid, locationList = vl.pickWeightedByRemaining(writables) if locationList == nil || len(locationList.list) == 0 { return 0, 0, nil, false, fmt.Errorf("Strangely vid %s is on no machine!", vid.String()) } return vid, count, locationList.Copy(), false, nil } + if vl.maxWritableVolumes > 0 { + // The cap must hold inside the requested location, so collect every + // matching writable before narrowing to the fullest. + var matches []needle.VolumeId + for _, writableVolumeId := range vl.writables { + volumeLocationList := vl.vid2location[writableVolumeId] + for _, dn := range volumeLocationList.list { + if dn.matchesLocation(option) { + matches = append(matches, writableVolumeId) + break + } + } + } + if len(matches) == 0 { + return vid, count, locationList, true, fmt.Errorf("%s in DataCenter:%v Rack:%v DataNode:%v", NoWritableVolumes, option.DataCenter, option.Rack, option.DataNode) + } + if len(matches) > vl.maxWritableVolumes { + matches = vl.fullestWritable(matches) + } + vid, locationList = vl.weightedPick(matches) + return vid, count, locationList.Copy(), false, nil + } + // Scan from a random offset to collect up to pickSampleSize matching // candidates, avoiding a full scan + allocation in the common case. var sample [pickSampleSize]needle.VolumeId @@ -700,6 +734,35 @@ func (vl *VolumeLayout) pickWeightedByRemaining(candidates []needle.VolumeId) (n return vl.weightedPick(sample[:]) } +// fullestWritable narrows candidates to the maxWritableVolumes fullest ones, +// so assignments concentrate on the fullest writable volume instead of +// spreading across many disks at once. The cap is expected to be small, so the +// top-n are kept by insertion instead of sorting the whole candidate list. +func (vl *VolumeLayout) fullestWritable(candidates []needle.VolumeId) []needle.VolumeId { + n := vl.maxWritableVolumes + top := make([]needle.VolumeId, 0, n) + remaining := make([]uint64, 0, n) + for _, vid := range candidates { + r := vl.remainingSize(vid) + i := len(top) + for i > 0 && (remaining[i-1] > r || (remaining[i-1] == r && top[i-1] > vid)) { + i-- + } + if i == n { + continue + } + top = append(top, 0) + remaining = append(remaining, 0) + copy(top[i+1:], top[i:]) + copy(remaining[i+1:], remaining[i:]) + top[i], remaining[i] = vid, r + if len(top) > n { + top, remaining = top[:n], remaining[:n] + } + } + return top +} + func (vl *VolumeLayout) weightedPick(candidates []needle.VolumeId) (needle.VolumeId, *VolumeLocationList) { if len(candidates) == 1 { vid := candidates[0] @@ -1087,7 +1150,7 @@ func (vl *VolumeLayout) SetVolumeUnavailable(dn *DataNode, vid needle.VolumeId) } return false } -func (vl *VolumeLayout) SetVolumeAvailable(dn *DataNode, vid needle.VolumeId, isReadOnly, isFullCapacity bool) (becameWritable bool) { +func (vl *VolumeLayout) SetVolumeAvailable(dn *DataNode, vid needle.VolumeId, isReadOnly, isFullCapacity, quarantined bool) (becameWritable bool) { restoreActiveVolumeCount := false vl.accessLock.Lock() defer func() { @@ -1108,8 +1171,13 @@ func (vl *VolumeLayout) SetVolumeAvailable(dn *DataNode, vid needle.VolumeId, is // A disconnect during a long vacuum can drop the entry while the volume is // still on the node; re-create it instead of dereferencing a nil location, // so the commit also repairs the split. - moveLookupOwnership(vid, vl.getOrCreateLocationList(vid).Set(dn), dn) + location := vl.getOrCreateLocationList(vid) + moveLookupOwnership(vid, location.Set(dn), dn) + if quarantined { + location.SetReadOnly(dn, true) + return false + } if vInfo.ReadOnly || isReadOnly || isFullCapacity { return false } diff --git a/weed/topology/volume_layout_capacity_recovery_test.go b/weed/topology/volume_layout_capacity_recovery_test.go index 8edb9eead3..686c67e4ba 100644 --- a/weed/topology/volume_layout_capacity_recovery_test.go +++ b/weed/topology/volume_layout_capacity_recovery_test.go @@ -98,7 +98,7 @@ func TestSetVolumeAvailableRestoresActiveCountForCapacityFullVolume(t *testing.T dn := vl.vid2location[1].list[0] vl.accessLock.RUnlock() - if !vl.SetVolumeAvailable(dn, 1, false, false) { + if !vl.SetVolumeAvailable(dn, 1, false, false, false) { t.Fatalf("SetVolumeAvailable should report the volume became writable") } if got := topo.diskUsages.usages[types.HardDriveType].activeVolumeCount; got != initialActive { diff --git a/weed/topology/volume_layout_max_writable_test.go b/weed/topology/volume_layout_max_writable_test.go new file mode 100644 index 0000000000..26b718801e --- /dev/null +++ b/weed/topology/volume_layout_max_writable_test.go @@ -0,0 +1,127 @@ +package topology + +import ( + "slices" + "testing" + + "github.com/seaweedfs/seaweedfs/weed/pb/master_pb" + "github.com/seaweedfs/seaweedfs/weed/storage/needle" + "github.com/seaweedfs/seaweedfs/weed/storage/super_block" + "github.com/seaweedfs/seaweedfs/weed/storage/types" +) + +func maxWritableLayout(t *testing.T, n int) (*Topology, *DataNode, *VolumeLayout) { + t.Helper() + topo := NewTopology("mw", nil, 32*1024, 5, false) + topo.SetMaxWritableVolumes(n) + dn := topo.GetOrCreateDataCenter("dc1").GetOrCreateRack("rack1"). + GetOrCreateDataNode("127.0.0.1", 8080, 18080, "", "", map[string]uint32{"": 100}) + rp, _ := super_block.NewReplicaPlacementFromString("000") + return topo, dn, topo.GetVolumeLayout("c", rp, needle.EMPTY_TTL, types.HardDriveType) +} + +// With a cap, assignments concentrate on the fullest writable volumes so at +// most cap volumes are written in parallel. +func TestPickForWriteCapsWritableVolumes(t *testing.T) { + topo, dn, vl := maxWritableLayout(t, 1) + + topo.SyncDataNodeRegistration([]*master_pb.VolumeInformationMessage{ + sizeTrackingVolume(1, 30000, false), + sizeTrackingVolume(2, 1000, false), + sizeTrackingVolume(3, 500, false), + }, dn) + + for i := 0; i < 20; i++ { + vid, _, _, _, err := vl.PickForWrite(1, &VolumeGrowOption{}) + if err != nil { + t.Fatalf("PickForWrite: %v", err) + } + if vid != needle.VolumeId(1) { + t.Fatalf("picked volume %d, want the fullest writable 1", vid) + } + } +} + +// Once the fullest volume leaves the writable list, the next fullest takes over. +func TestPickForWriteCapFallsToNextWritable(t *testing.T) { + topo, dn, vl := maxWritableLayout(t, 1) + + topo.SyncDataNodeRegistration([]*master_pb.VolumeInformationMessage{ + sizeTrackingVolume(1, 30000, false), + sizeTrackingVolume(2, 1000, false), + }, dn) + topo.SyncDataNodeRegistration([]*master_pb.VolumeInformationMessage{ + sizeTrackingVolume(1, 30000, true), + sizeTrackingVolume(2, 1000, false), + }, dn) + + vid, _, _, _, err := vl.PickForWrite(1, &VolumeGrowOption{}) + if err != nil { + t.Fatalf("PickForWrite: %v", err) + } + if vid != needle.VolumeId(2) { + t.Fatalf("picked volume %d, want 2 after 1 went read-only", vid) + } +} + +// A cap of two still concentrates on the two fullest writables; zero disables +// the cap so every writable volume is a candidate. +func TestPickForWriteCapOfTwoAndDisabled(t *testing.T) { + topo, dn, vl := maxWritableLayout(t, 2) + + topo.SyncDataNodeRegistration([]*master_pb.VolumeInformationMessage{ + sizeTrackingVolume(1, 30000, false), + sizeTrackingVolume(2, 20000, false), + sizeTrackingVolume(3, 500, false), + }, dn) + + // the capped candidate set is deterministic even though the pick among + // them is weighted-random + if got := vl.fullestWritable(vl.writables); !slices.Equal(got, []needle.VolumeId{1, 2}) { + t.Fatalf("capped candidates %v, want [1 2]", got) + } + for i := 0; i < 40; i++ { + vid, _, _, _, err := vl.PickForWrite(1, &VolumeGrowOption{}) + if err != nil { + t.Fatalf("PickForWrite: %v", err) + } + if vid != needle.VolumeId(1) && vid != needle.VolumeId(2) { + t.Fatalf("capped pick reached volume %d, want one of 1,2", vid) + } + } + + vl.maxWritableVolumes = 0 + for i := 0; i < 40; i++ { + if _, _, _, _, err := vl.PickForWrite(1, &VolumeGrowOption{}); err != nil { + t.Fatalf("PickForWrite: %v", err) + } + } +} + +// The cap applies inside the requested location: with cap 1, a write naming +// dc2 must still find dc2's writable volume even though the fullest is in dc1. +func TestPickForWriteCapWithinRequestedLocation(t *testing.T) { + topo := NewTopology("mw", nil, 32*1024, 5, false) + topo.SetMaxWritableVolumes(1) + dn1 := topo.GetOrCreateDataCenter("dc1").GetOrCreateRack("rack1"). + GetOrCreateDataNode("127.0.0.1", 8080, 18080, "", "", map[string]uint32{"": 100}) + dn2 := topo.GetOrCreateDataCenter("dc2").GetOrCreateRack("rack2"). + GetOrCreateDataNode("127.0.0.2", 8080, 18080, "", "", map[string]uint32{"": 100}) + rp, _ := super_block.NewReplicaPlacementFromString("000") + vl := topo.GetVolumeLayout("c", rp, needle.EMPTY_TTL, types.HardDriveType) + + topo.SyncDataNodeRegistration([]*master_pb.VolumeInformationMessage{ + sizeTrackingVolume(1, 30000, false), // fullest, in dc1 + }, dn1) + topo.SyncDataNodeRegistration([]*master_pb.VolumeInformationMessage{ + sizeTrackingVolume(2, 1000, false), // in dc2 + }, dn2) + + vid, _, _, _, err := vl.PickForWrite(1, &VolumeGrowOption{DataCenter: "dc2"}) + if err != nil { + t.Fatalf("PickForWrite dc2: %v", err) + } + if vid != needle.VolumeId(2) { + t.Fatalf("picked volume %d, want dc2's volume 2", vid) + } +} diff --git a/weed/topology/volume_layout_pick_test.go b/weed/topology/volume_layout_pick_test.go index 4fff615a65..aa8cc99c1f 100644 --- a/weed/topology/volume_layout_pick_test.go +++ b/weed/topology/volume_layout_pick_test.go @@ -61,7 +61,7 @@ func setupWithLimit(t testing.TB, topologyLayout string, volumeSizeLimit uint64) } if vi.ReplicaPlacement != nil { vl := topo.GetVolumeLayout(vi.Collection, vi.ReplicaPlacement, needle.EMPTY_TTL, types.HardDriveType) - vl.RegisterVolume(&vi, server) + vl.RegisterVolume(&vi, server, nil) vl.setVolumeWritable(vi.Id) } server.AddOrUpdateVolume(vi) diff --git a/weed/util/wildcard/wildcard_matcher.go b/weed/util/wildcard/wildcard_matcher.go index 0d74ffe6c4..5b2b85f6da 100644 --- a/weed/util/wildcard/wildcard_matcher.go +++ b/weed/util/wildcard/wildcard_matcher.go @@ -155,13 +155,15 @@ func matchWildcardString(pattern, str string) bool { for targetIndex < len(str) { switch { - case patternIndex < len(pattern) && (pattern[patternIndex] == '?' || pattern[patternIndex] == str[targetIndex]): - targetIndex++ - patternIndex++ + // '*' must be checked before literal equality so a '*' in the target + // cannot consume the pattern wildcard without a backtrack point case patternIndex < len(pattern) && pattern[patternIndex] == '*': lastStarIndex = patternIndex lastStarMatchIndex = targetIndex patternIndex++ + case patternIndex < len(pattern) && (pattern[patternIndex] == '?' || pattern[patternIndex] == str[targetIndex]): + targetIndex++ + patternIndex++ case lastStarIndex != -1: patternIndex = lastStarIndex + 1 lastStarMatchIndex++ diff --git a/weed/util/wildcard/wildcard_matcher_test.go b/weed/util/wildcard/wildcard_matcher_test.go index 26f5f43a89..d318206a38 100644 --- a/weed/util/wildcard/wildcard_matcher_test.go +++ b/weed/util/wildcard/wildcard_matcher_test.go @@ -27,6 +27,12 @@ func TestMatchesWildcard(t *testing.T) { {"Pattern with regex special chars", "test[abc]", "test[abc]", true}, {"Pattern with dots", "test.txt", "test.txt", true}, {"Pattern with dots and wildcard", "*.txt", "test.txt", true}, + {"Star in target at pattern star offset", "a/*", "a/*b", true}, + {"Star in target equals pattern", "a/*", "a/*", true}, + {"Star in target past pattern star offset", "a/*", "a/x*y", true}, + {"Question mark matches literal star", "a/?c", "a/*c", true}, + {"Star in target is not a wildcard", "ab*", "a*b", false}, + {"ARN pattern with star in key", "arn:aws:s3:::bucket/finance/*", "arn:aws:s3:::bucket/finance/*star.txt", true}, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { diff --git a/weed/worker/tasks/erasure_coding/config.go b/weed/worker/tasks/erasure_coding/config.go index b356a61b08..e1eba8fb03 100644 --- a/weed/worker/tasks/erasure_coding/config.go +++ b/weed/worker/tasks/erasure_coding/config.go @@ -12,13 +12,14 @@ import ( // Config extends BaseConfig with erasure coding specific settings type Config struct { base.BaseConfig - QuietForSeconds int `json:"quiet_for_seconds"` - FullnessRatio float64 `json:"fullness_ratio"` - CollectionFilter string `json:"collection_filter"` - MinSizeMB int `json:"min_size_mb"` - PreferredTags []string `json:"preferred_tags"` - ReplicaPlacement string `json:"replica_placement"` // e.g. "020"; empty falls back to the master default replication - StrictPlacement bool `json:"strict_placement"` // fail planning instead of relaxing placement constraints + QuietForSeconds int `json:"quiet_for_seconds"` + FullnessRatio float64 `json:"fullness_ratio"` + CollectionFilter string `json:"collection_filter"` + MinSizeMB int `json:"min_size_mb"` + PreferredTags []string `json:"preferred_tags"` + ReplicaPlacement string `json:"replica_placement"` // e.g. "020"; empty falls back to the master default replication + StrictPlacement bool `json:"strict_placement"` // fail planning instead of relaxing placement constraints + FreeSlotsDeadband int `json:"free_slots_deadband"` // Place volume-slot M; |ΔfreeSlots| ≤ M×dataShards is a tie, then liveShards; 0 = strict } // NewDefaultConfig creates a new default erasure coding configuration @@ -184,6 +185,22 @@ func GetConfigSpec() base.ConfigSpec { InputType: "checkbox", CSSClasses: "form-check-input", }, + { + Name: "free_slots_deadband", + JSONName: "free_slots_deadband", + Type: config.FieldTypeInt, + DefaultValue: 0, + MinValue: 0, + MaxValue: 100000, + Required: false, + DisplayName: "Free Volume Deadband (M)", + Description: "Treat |Δfree volume slots| ≤ M as a Place tie and rank by liveShards instead", + HelpText: "0 (default) keeps strict freeSlots ranking. A positive M is volume slots; Place applies M × data shards (10+4 → ×10).", + Placeholder: "0", + Unit: config.UnitCount, + InputType: "number", + CSSClasses: "form-control", + }, }, } } @@ -199,13 +216,14 @@ func (c *Config) ToTaskPolicy() *worker_pb.TaskPolicy { CheckIntervalSeconds: int32(c.ScanIntervalSeconds), TaskConfig: &worker_pb.TaskPolicy_ErasureCodingConfig{ ErasureCodingConfig: &worker_pb.ErasureCodingTaskConfig{ - FullnessRatio: float64(c.FullnessRatio), - QuietForSeconds: int32(c.QuietForSeconds), - MinVolumeSizeMb: int32(c.MinSizeMB), - CollectionFilter: c.CollectionFilter, - PreferredTags: preferredTagsCopy, - ReplicaPlacement: c.ReplicaPlacement, - StrictPlacement: c.StrictPlacement, + FullnessRatio: float64(c.FullnessRatio), + QuietForSeconds: int32(c.QuietForSeconds), + MinVolumeSizeMb: int32(c.MinSizeMB), + CollectionFilter: c.CollectionFilter, + PreferredTags: preferredTagsCopy, + ReplicaPlacement: c.ReplicaPlacement, + StrictPlacement: c.StrictPlacement, + FreeSlotsDeadband: int32(c.FreeSlotsDeadband), }, }, } @@ -231,6 +249,7 @@ func (c *Config) FromTaskPolicy(policy *worker_pb.TaskPolicy) error { c.PreferredTags = append([]string(nil), ecConfig.PreferredTags...) c.ReplicaPlacement = ecConfig.ReplicaPlacement c.StrictPlacement = ecConfig.StrictPlacement + c.FreeSlotsDeadband = int(ecConfig.FreeSlotsDeadband) } return nil diff --git a/weed/worker/tasks/erasure_coding/detection.go b/weed/worker/tasks/erasure_coding/detection.go index de48d6c55d..a422964ef2 100644 --- a/weed/worker/tasks/erasure_coding/detection.go +++ b/weed/worker/tasks/erasure_coding/detection.go @@ -519,11 +519,12 @@ func planECDestinations(snap *ecbalancer.Topology, nodeAddresses map[string]stri mode = ecbalancer.PlaceStrict } res, err := snap.Place(metric.VolumeID, metric.Collection, need, ecbalancer.Constraints{ - DiskType: metric.DiskType, - DiskTypePolicy: ecbalancer.DiskTypePrefer, - PreferredTags: ecConfig.PreferredTags, - ReplicaPlacement: rp, - Ratio: func(string) (int, int) { return dataShards, parityShards }, + DiskType: metric.DiskType, + DiskTypePolicy: ecbalancer.DiskTypePrefer, + PreferredTags: ecConfig.PreferredTags, + ReplicaPlacement: rp, + Ratio: func(string) (int, int) { return dataShards, parityShards }, + FreeSlotsDeadband: ecConfig.FreeSlotsDeadband, }, mode) if err != nil { return nil, nil, err diff --git a/weed/worker/tasks/erasure_coding/detection_strict_test.go b/weed/worker/tasks/erasure_coding/detection_strict_test.go index 7b7cddbb61..8916c3d960 100644 --- a/weed/worker/tasks/erasure_coding/detection_strict_test.go +++ b/weed/worker/tasks/erasure_coding/detection_strict_test.go @@ -39,12 +39,16 @@ func TestPlanECDestinationsStrictPlacement(t *testing.T) { func TestStrictPlacementRoundTripsThroughTaskPolicy(t *testing.T) { cfg := NewDefaultConfig() cfg.StrictPlacement = true + cfg.FreeSlotsDeadband = 3 restored := NewDefaultConfig() require.NoError(t, restored.FromTaskPolicy(cfg.ToTaskPolicy())) require.True(t, restored.StrictPlacement, "strict placement must survive the persisted policy round trip") + require.Equal(t, 3, restored.FreeSlotsDeadband, "free_slots_deadband must survive the persisted policy round trip") restored.StrictPlacement = false + restored.FreeSlotsDeadband = 0 require.NoError(t, restored.FromTaskPolicy(restored.ToTaskPolicy())) require.False(t, restored.StrictPlacement) + require.Zero(t, restored.FreeSlotsDeadband) } diff --git a/weed/worker/tasks/erasure_coding/plugin_handler.go b/weed/worker/tasks/erasure_coding/plugin_handler.go index 36ffd5a59e..5071529b6d 100644 --- a/weed/worker/tasks/erasure_coding/plugin_handler.go +++ b/weed/worker/tasks/erasure_coding/plugin_handler.go @@ -98,7 +98,7 @@ func (h *ErasureCodingHandler) Descriptor() *plugin_pb.JobTypeDescriptor { WorkerConfigForm: &plugin_pb.ConfigForm{ FormId: "erasure-coding-worker", Title: "Erasure Coding Worker Config", - Description: "Worker-side detection thresholds.", + Description: "Worker-side detection and shard placement.", Sections: []*plugin_pb.ConfigSection{ { SectionId: "thresholds", @@ -133,6 +133,13 @@ func (h *ErasureCodingHandler) Descriptor() *plugin_pb.JobTypeDescriptor { Required: true, MinValue: &plugin_pb.ConfigValue{Kind: &plugin_pb.ConfigValue_Int64Value{Int64Value: 1}}, }, + }, + }, + { + SectionId: "placement", + Title: "Shard Placement", + Description: "Where EC shards go: disk tags, replica-placement caps, Place ranking.", + Fields: []*plugin_pb.ConfigField{ { Name: "preferred_tags", Label: "Preferred Tags", @@ -149,6 +156,15 @@ func (h *ErasureCodingHandler) Descriptor() *plugin_pb.JobTypeDescriptor { FieldType: plugin_pb.ConfigFieldType_CONFIG_FIELD_TYPE_STRING, Widget: plugin_pb.ConfigWidget_CONFIG_WIDGET_TEXT, }, + { + Name: "free_slots_deadband", + Label: "Free Volume Deadband (M)", + Description: "Nodes whose free volume slots differ by at most M are a tie; then fewer live shards wins. Applied as M × data shards (10+4 → ×10). 0 = always pick more free slots.", + FieldType: plugin_pb.ConfigFieldType_CONFIG_FIELD_TYPE_INT64, + Widget: plugin_pb.ConfigWidget_CONFIG_WIDGET_NUMBER, + MinValue: &plugin_pb.ConfigValue{Kind: &plugin_pb.ConfigValue_Int64Value{Int64Value: 0}}, + MaxValue: &plugin_pb.ConfigValue{Kind: &plugin_pb.ConfigValue_Int64Value{Int64Value: 100000}}, + }, { Name: "strict_placement", Label: "Strict Placement", @@ -178,6 +194,9 @@ func (h *ErasureCodingHandler) Descriptor() *plugin_pb.JobTypeDescriptor { "strict_placement": { Kind: &plugin_pb.ConfigValue_BoolValue{BoolValue: false}, }, + "free_slots_deadband": { + Kind: &plugin_pb.ConfigValue_Int64Value{Int64Value: 0}, + }, }, }, AdminRuntimeDefaults: &plugin_pb.AdminRuntimeDefaults{ @@ -205,6 +224,9 @@ func (h *ErasureCodingHandler) Descriptor() *plugin_pb.JobTypeDescriptor { "preferred_tags": { Kind: &plugin_pb.ConfigValue_StringValue{StringValue: ""}, }, + "free_slots_deadband": { + Kind: &plugin_pb.ConfigValue_Int64Value{Int64Value: 0}, + }, }, } } @@ -640,6 +662,14 @@ func deriveErasureCodingWorkerConfig(values map[string]*plugin_pb.ConfigValue) * taskConfig.StrictPlacement = pluginworker.ReadBoolConfig(values, "strict_placement", taskConfig.StrictPlacement) + deadband := pluginworker.ReadIntConfig(values, "free_slots_deadband", taskConfig.FreeSlotsDeadband) + if deadband < 0 { + deadband = 0 + } else if deadband > 100000 { + deadband = 100000 + } + taskConfig.FreeSlotsDeadband = deadband + return &erasureCodingWorkerConfig{ TaskConfig: taskConfig, } diff --git a/weed/worker/tasks/erasure_coding/plugin_handler_test.go b/weed/worker/tasks/erasure_coding/plugin_handler_test.go index 6febba84b9..71e221ce4d 100644 --- a/weed/worker/tasks/erasure_coding/plugin_handler_test.go +++ b/weed/worker/tasks/erasure_coding/plugin_handler_test.go @@ -128,6 +128,17 @@ func TestDeriveErasureCodingWorkerConfig(t *testing.T) { if cfg.TaskConfig.MinSizeMB != 128 { t.Fatalf("expected min_size_mb 128, got %d", cfg.TaskConfig.MinSizeMB) } + if cfg.TaskConfig.FreeSlotsDeadband != 0 { + t.Fatalf("expected default free_slots_deadband 0, got %d", cfg.TaskConfig.FreeSlotsDeadband) + } + + values["free_slots_deadband"] = &plugin_pb.ConfigValue{ + Kind: &plugin_pb.ConfigValue_Int64Value{Int64Value: 12}, + } + cfg = deriveErasureCodingWorkerConfig(values) + if cfg.TaskConfig.FreeSlotsDeadband != 12 { + t.Fatalf("expected free_slots_deadband 12, got %d", cfg.TaskConfig.FreeSlotsDeadband) + } } func TestBuildErasureCodingProposal(t *testing.T) { @@ -249,6 +260,51 @@ func TestEmitErasureCodingDetectionDecisionTraceNoTasks(t *testing.T) { } } +func TestErasureCodingDescriptorExposesFreeSlotsDeadband(t *testing.T) { + descriptor := NewErasureCodingHandler(nil, "").Descriptor() + if descriptor == nil || descriptor.WorkerConfigForm == nil { + t.Fatalf("expected worker config form in descriptor") + } + if descriptor.DescriptorVersion != 1 { + t.Fatalf("descriptor version = %d, want 1", descriptor.DescriptorVersion) + } + if !workerConfigFormHasField(descriptor.WorkerConfigForm, "free_slots_deadband") { + t.Fatal("worker config form omitted free_slots_deadband") + } + var sectionID, prev string + for _, section := range descriptor.WorkerConfigForm.Sections { + if section == nil { + continue + } + for _, field := range section.Fields { + if field != nil && field.Name == "free_slots_deadband" { + sectionID = section.SectionId + if field.Label != "Free Volume Deadband (M)" { + t.Fatalf("free_slots_deadband label = %q, want Free Volume Deadband (M)", field.Label) + } + } + if field != nil && field.Name == "replica_placement" { + prev = section.SectionId + } + } + } + if sectionID != "placement" { + t.Fatalf("free_slots_deadband section = %q, want placement", sectionID) + } + if prev != "placement" { + t.Fatalf("replica_placement section = %q, want placement next to free_slots_deadband", prev) + } + for _, section := range descriptor.WorkerConfigForm.Sections { + if section != nil && section.SectionId == "thresholds" { + for _, field := range section.Fields { + if field != nil && field.Name == "free_slots_deadband" { + t.Fatal("free_slots_deadband must not sit under Detection Thresholds") + } + } + } + } +} + func TestErasureCodingDescriptorOmitsLocalExecutionFields(t *testing.T) { descriptor := NewErasureCodingHandler(nil, "").Descriptor() if descriptor == nil || descriptor.WorkerConfigForm == nil { diff --git a/weed/worker/tasks/iceberg/exec_test.go b/weed/worker/tasks/iceberg/exec_test.go index 18cb661ca6..172e896a1e 100644 --- a/weed/worker/tasks/iceberg/exec_test.go +++ b/weed/worker/tasks/iceberg/exec_test.go @@ -4435,3 +4435,438 @@ func TestResolveCompactionRewritePlanFallsBackForUnsupportedSortTransform(t *tes t.Fatalf("expected fallback to %q for unsupported transform, got %+v", defaultRewriteStrategy, plan) } } + +// A writer may percent-escape a partition directory name ("part=1" becomes +// "part%3D1") in the file paths it records in manifests, while the S3 layer +// decodes the key before storing it. Orphan cleanup must match the decoded +// spelling or it deletes live data files once they pass the age window. +func TestRemoveOrphansKeepsPercentEscapedReferences(t *testing.T) { + fs, client := startFakeFiler(t) + + now := time.Now().UnixMilli() + setup := tableSetup{ + BucketName: "test-bucket", + Namespace: "ns", + TableName: "tbl", + Snapshots: []table.Snapshot{ + {SnapshotID: 1, TimestampMs: now, ManifestList: "metadata/snap-1.avro"}, + }, + } + meta := populateTable(t, fs, setup) + + metaDir := path.Join(s3tables.TablesPath, setup.BucketName, setup.tablePath(), "metadata") + dataDir := path.Join(s3tables.TablesPath, setup.BucketName, setup.tablePath(), "data") + oldTime := time.Now().Add(-200 * time.Hour).Unix() + + // Rewrite the manifest so its entries carry the escaped spelling the + // writer recorded. + escapedPath := fmt.Sprintf("s3://%s/%s/data/month_collected_at_1%%3D681/f.parquet", setup.BucketName, setup.tablePath()) + dfBuilder, err := iceberg.NewDataFileBuilder( + meta.PartitionSpec(), + iceberg.EntryContentData, + escapedPath, + iceberg.ParquetFile, + map[int]any{}, + nil, nil, + 10, + 4096, + ) + if err != nil { + t.Fatalf("build escaped data file: %v", err) + } + snapID := int64(1) + var manifestBuf bytes.Buffer + if _, err := iceberg.WriteManifest( + setup.fileRef("metadata", "manifest-1.avro"), &manifestBuf, + meta.Version(), meta.PartitionSpec(), meta.CurrentSchema(), snapID, + []iceberg.ManifestEntry{iceberg.NewManifestEntry(iceberg.EntryStatusADDED, &snapID, nil, nil, dfBuilder.Build())}, + ); err != nil { + t.Fatalf("write manifest: %v", err) + } + fs.putEntry(metaDir, "manifest-1.avro", &filer_pb.Entry{ + Name: "manifest-1.avro", + Attributes: &filer_pb.FuseAttributes{Mtime: oldTime, FileSize: uint64(manifestBuf.Len())}, + Content: manifestBuf.Bytes(), + }) + + // The object lives at the decoded key, the same place the S3 PUT landed it. + fs.putEntry(dataDir, "month_collected_at_1=681", &filer_pb.Entry{ + Name: "month_collected_at_1=681", + IsDirectory: true, + Attributes: &filer_pb.FuseAttributes{Mtime: oldTime}, + }) + partDir := path.Join(dataDir, "month_collected_at_1=681") + fs.putEntry(partDir, "f.parquet", &filer_pb.Entry{ + Name: "f.parquet", + Attributes: &filer_pb.FuseAttributes{Mtime: oldTime, FileSize: 4096}, + Content: []byte("fake-parquet-data"), + }) + + handler := NewHandler(nil) + config := Config{OrphanOlderThanHours: 0, MaxCommitRetries: 3} + + result, _, err := handler.removeOrphans(context.Background(), client, setup.BucketName, setup.tablePath(), config) + if err != nil { + t.Fatalf("removeOrphans failed: %v", err) + } + if !strings.Contains(result, "removed 1 orphan") { + t.Errorf("expected only the unreferenced snap-1-data.parquet removed, got %q", result) + } + if fs.getEntry(partDir, "f.parquet") == nil { + t.Error("data file referenced through an escaped manifest path was deleted") + } +} + +// The sibling of the orphan-sweep case on the read side: a manifest reference +// carrying a percent-escaped segment must find the object the S3 layer stored +// at the decoded key. +func TestLookupFileByIcebergPathDecodesEscapedReference(t *testing.T) { + fs, client := startFakeFiler(t) + + dir := path.Join(s3tables.TablesPath, "bkt", "ns/tbl/data/part=1") + fs.putEntry(dir, "f.parquet", &filer_pb.Entry{ + Name: "f.parquet", + Content: []byte("data"), + }) + literalDir := path.Join(s3tables.TablesPath, "bkt", "ns/tbl/data/part%3D2") + fs.putEntry(literalDir, "g.parquet", &filer_pb.Entry{ + Name: "g.parquet", + Content: []byte("data"), + }) + + entry, err := lookupFileByIcebergPath(context.Background(), client, "bkt", "ns/tbl", "s3://bkt/ns/tbl/data/part%3D1/f.parquet") + if err != nil { + t.Fatalf("escaped reference should resolve to the decoded key: %v", err) + } + if entry == nil || entry.Name != "f.parquet" { + t.Fatalf("resolved to wrong entry: %+v", entry) + } + + // A file genuinely stored under an escaped name still resolves verbatim. + entry, err = lookupFileByIcebergPath(context.Background(), client, "bkt", "ns/tbl", "s3://bkt/ns/tbl/data/part%3D2/g.parquet") + if err != nil { + t.Fatalf("literal escaped key should resolve: %v", err) + } + if entry == nil || entry.Name != "g.parquet" { + t.Fatalf("resolved to wrong entry: %+v", entry) + } + + // A real filer reports misses as filer_pb.ErrNotFound, not a gRPC NotFound + // status; the decoded retry must trigger on that shape too. + entry, err = lookupFileByIcebergPath(context.Background(), errNotFoundClient{client}, "bkt", "ns/tbl", "s3://bkt/ns/tbl/data/part%3D1/f.parquet") + if err != nil { + t.Fatalf("escaped reference should resolve under ErrNotFound misses: %v", err) + } + if entry == nil || entry.Name != "f.parquet" { + t.Fatalf("resolved to wrong entry: %+v", entry) + } +} + +// errNotFoundClient converts gRPC NotFound misses into filer_pb.ErrNotFound, +// the shape a real filer produces through filer_pb.LookupEntry. +type errNotFoundClient struct { + filer_pb.SeaweedFilerClient +} + +func (c errNotFoundClient) LookupDirectoryEntry(ctx context.Context, req *filer_pb.LookupDirectoryEntryRequest, opts ...grpc.CallOption) (*filer_pb.LookupDirectoryEntryResponse, error) { + resp, err := c.SeaweedFilerClient.LookupDirectoryEntry(ctx, req, opts...) + if status.Code(err) == codes.NotFound { + return nil, filer_pb.ErrNotFound + } + return resp, err +} + +func TestExpireSnapshotsDeletesEscapedReferenceAtExistingKey(t *testing.T) { + fs, client := startFakeFiler(t) + + now := time.Now() + old := now.Add(-10 * 24 * time.Hour) + setup := tableSetup{ + BucketName: "test-bucket", + Namespace: "ns", + TableName: "tbl", + Age: 10 * 24 * time.Hour, + Snapshots: []table.Snapshot{ + {SnapshotID: 1, TimestampMs: now.UnixMilli(), ManifestList: "metadata/snap-1.avro"}, + {SnapshotID: 2, TimestampMs: now.UnixMilli() + 1, ManifestList: "metadata/snap-2.avro"}, + }, + } + meta := populateTable(t, fs, setup) + + // Snapshot 1 (expired) references a file the S3 layer stored at the decoded + // key; snapshot 2 (kept) does not reference it. + metaDir := path.Join(s3tables.TablesPath, setup.BucketName, setup.tablePath(), "metadata") + dataDir := path.Join(s3tables.TablesPath, setup.BucketName, setup.tablePath(), "data") + + escapedPath := fmt.Sprintf("s3://%s/%s/data/month%%3D1/f.parquet", setup.BucketName, setup.tablePath()) + dfBuilder, err := iceberg.NewDataFileBuilder( + meta.PartitionSpec(), iceberg.EntryContentData, escapedPath, + iceberg.ParquetFile, map[int]any{}, nil, nil, 10, 4096, + ) + if err != nil { + t.Fatalf("build escaped data file: %v", err) + } + snapID := int64(1) + var manifestBuf bytes.Buffer + if _, err := iceberg.WriteManifest( + setup.fileRef("metadata", "manifest-1.avro"), &manifestBuf, + meta.Version(), meta.PartitionSpec(), meta.CurrentSchema(), snapID, + []iceberg.ManifestEntry{iceberg.NewManifestEntry(iceberg.EntryStatusADDED, &snapID, nil, nil, dfBuilder.Build())}, + ); err != nil { + t.Fatalf("write manifest: %v", err) + } + fs.putEntry(metaDir, "manifest-1.avro", &filer_pb.Entry{ + Name: "manifest-1.avro", + Attributes: &filer_pb.FuseAttributes{Mtime: old.Unix(), FileSize: uint64(manifestBuf.Len())}, + Content: manifestBuf.Bytes(), + }) + + partDir := path.Join(dataDir, "month=1") + fs.putEntry(dataDir, "month=1", &filer_pb.Entry{ + Name: "month=1", IsDirectory: true, Attributes: &filer_pb.FuseAttributes{Mtime: old.Unix()}, + }) + fs.putEntry(partDir, "f.parquet", &filer_pb.Entry{ + Name: "f.parquet", Attributes: &filer_pb.FuseAttributes{Mtime: old.Unix(), FileSize: 4096}, + Content: []byte("fake-parquet-data"), + }) + + handler := NewHandler(nil) + config := Config{ + SnapshotRetentionMs: hoursToMs(0), + MaxSnapshotsToKeep: 1, + MaxCommitRetries: 3, + Operations: "expire_snapshots", + } + + result, _, err := handler.expireSnapshots(context.Background(), client, setup.BucketName, setup.tablePath(), config) + if err != nil { + t.Fatalf("expireSnapshots failed: %v", err) + } + t.Logf("expireSnapshots result: %s", result) + + if fs.getEntry(partDir, "f.parquet") != nil { + t.Error("expired file at the decoded key should have been deleted") + } +} + +// A client that escapes its own percent sign on upload leaves an object under +// a literal %3D-containing key while the manifest spells the same reference. +// Expiration must delete the entry that actually exists, not the decoded +// spelling. +func TestExpireSnapshotsDeletesLiteralPercentKey(t *testing.T) { + fs, client := startFakeFiler(t) + + now := time.Now() + old := now.Add(-10 * 24 * time.Hour) + setup := tableSetup{ + BucketName: "test-bucket", + Namespace: "ns", + TableName: "tbl", + Age: 10 * 24 * time.Hour, + Snapshots: []table.Snapshot{ + {SnapshotID: 1, TimestampMs: now.UnixMilli(), ManifestList: "metadata/snap-1.avro"}, + {SnapshotID: 2, TimestampMs: now.UnixMilli() + 1, ManifestList: "metadata/snap-2.avro"}, + }, + } + meta := populateTable(t, fs, setup) + + metaDir := path.Join(s3tables.TablesPath, setup.BucketName, setup.tablePath(), "metadata") + dataDir := path.Join(s3tables.TablesPath, setup.BucketName, setup.tablePath(), "data") + + escapedPath := fmt.Sprintf("s3://%s/%s/data/month%%3D1/f.parquet", setup.BucketName, setup.tablePath()) + dfBuilder, err := iceberg.NewDataFileBuilder( + meta.PartitionSpec(), iceberg.EntryContentData, escapedPath, + iceberg.ParquetFile, map[int]any{}, nil, nil, 10, 4096, + ) + if err != nil { + t.Fatalf("build escaped data file: %v", err) + } + snapID := int64(1) + var manifestBuf bytes.Buffer + if _, err := iceberg.WriteManifest( + setup.fileRef("metadata", "manifest-1.avro"), &manifestBuf, + meta.Version(), meta.PartitionSpec(), meta.CurrentSchema(), snapID, + []iceberg.ManifestEntry{iceberg.NewManifestEntry(iceberg.EntryStatusADDED, &snapID, nil, nil, dfBuilder.Build())}, + ); err != nil { + t.Fatalf("write manifest: %v", err) + } + fs.putEntry(metaDir, "manifest-1.avro", &filer_pb.Entry{ + Name: "manifest-1.avro", + Attributes: &filer_pb.FuseAttributes{Mtime: old.Unix(), FileSize: uint64(manifestBuf.Len())}, + Content: manifestBuf.Bytes(), + }) + + literalDir := path.Join(dataDir, "month%3D1") + fs.putEntry(dataDir, "month%3D1", &filer_pb.Entry{ + Name: "month%3D1", IsDirectory: true, Attributes: &filer_pb.FuseAttributes{Mtime: old.Unix()}, + }) + fs.putEntry(literalDir, "f.parquet", &filer_pb.Entry{ + Name: "f.parquet", Attributes: &filer_pb.FuseAttributes{Mtime: old.Unix(), FileSize: 4096}, + Content: []byte("fake-parquet-data"), + }) + + handler := NewHandler(nil) + config := Config{ + SnapshotRetentionMs: hoursToMs(0), + MaxSnapshotsToKeep: 1, + MaxCommitRetries: 3, + Operations: "expire_snapshots", + } + + result, _, err := handler.expireSnapshots(context.Background(), client, setup.BucketName, setup.tablePath(), config) + if err != nil { + t.Fatalf("expireSnapshots failed: %v", err) + } + t.Logf("expireSnapshots result: %s", result) + + if fs.getEntry(literalDir, "f.parquet") != nil { + t.Error("expired file at the literal percent key should have been deleted") + } +} + +// A client that escapes an already-escaped name writes a manifest reference +// like month%253D1 while the object sits at month=1. The kept set must hold +// every decoding in the chain, or the expiry pass deletes a file a live +// snapshot still references. +func TestExpireSnapshotsKeepsDoublyEscapedKeptReference(t *testing.T) { + fs, client := startFakeFiler(t) + + now := time.Now() + old := now.Add(-10 * 24 * time.Hour) + setup := tableSetup{ + BucketName: "test-bucket", + Namespace: "ns", + TableName: "tbl", + Age: 10 * 24 * time.Hour, + Snapshots: []table.Snapshot{ + {SnapshotID: 1, TimestampMs: now.UnixMilli(), ManifestList: "metadata/snap-1.avro"}, + {SnapshotID: 2, TimestampMs: now.UnixMilli() + 1, ManifestList: "metadata/snap-2.avro"}, + }, + } + meta := populateTable(t, fs, setup) + + metaDir := path.Join(s3tables.TablesPath, setup.BucketName, setup.tablePath(), "metadata") + dataDir := path.Join(s3tables.TablesPath, setup.BucketName, setup.tablePath(), "data") + + escapedPath := fmt.Sprintf("s3://%s/%s/data/month%%253D1/f.parquet", setup.BucketName, setup.tablePath()) + writeRefManifest := func(fileName string, snapID int64) { + dfBuilder, err := iceberg.NewDataFileBuilder( + meta.PartitionSpec(), iceberg.EntryContentData, escapedPath, + iceberg.ParquetFile, map[int]any{}, nil, nil, 10, 4096, + ) + if err != nil { + t.Fatalf("build escaped data file: %v", err) + } + var manifestBuf bytes.Buffer + if _, err := iceberg.WriteManifest( + setup.fileRef("metadata", fileName), &manifestBuf, + meta.Version(), meta.PartitionSpec(), meta.CurrentSchema(), snapID, + []iceberg.ManifestEntry{iceberg.NewManifestEntry(iceberg.EntryStatusADDED, &snapID, nil, nil, dfBuilder.Build())}, + ); err != nil { + t.Fatalf("write manifest: %v", err) + } + fs.putEntry(metaDir, fileName, &filer_pb.Entry{ + Name: fileName, + Attributes: &filer_pb.FuseAttributes{Mtime: old.Unix(), FileSize: uint64(manifestBuf.Len())}, + Content: manifestBuf.Bytes(), + }) + } + writeRefManifest("manifest-1.avro", 1) + writeRefManifest("manifest-2.avro", 2) + + partDir := path.Join(dataDir, "month=1") + fs.putEntry(dataDir, "month=1", &filer_pb.Entry{ + Name: "month=1", IsDirectory: true, Attributes: &filer_pb.FuseAttributes{Mtime: old.Unix()}, + }) + fs.putEntry(partDir, "f.parquet", &filer_pb.Entry{ + Name: "f.parquet", Attributes: &filer_pb.FuseAttributes{Mtime: old.Unix(), FileSize: 4096}, + Content: []byte("fake-parquet-data"), + }) + + handler := NewHandler(nil) + config := Config{ + SnapshotRetentionMs: hoursToMs(0), + MaxSnapshotsToKeep: 1, + MaxCommitRetries: 3, + Operations: "expire_snapshots", + } + + if _, _, err := handler.expireSnapshots(context.Background(), client, setup.BucketName, setup.tablePath(), config); err != nil { + t.Fatalf("expireSnapshots failed: %v", err) + } + + if fs.getEntry(partDir, "f.parquet") == nil { + t.Error("file referenced by the kept snapshot through a doubly-escaped path was deleted") + } +} + +// A kept reference whose decoded spelling contains a parent segment resolves +// to the cleaned path. The kept set must record the cleaned spelling, or the +// expiry pass misses the protection and deletes a file a live snapshot still +// references. +func TestExpireSnapshotsKeepsDecodedParentReference(t *testing.T) { + fs, client := startFakeFiler(t) + + now := time.Now() + old := now.Add(-10 * 24 * time.Hour) + setup := tableSetup{ + BucketName: "test-bucket", + Namespace: "ns", + TableName: "tbl", + Age: 10 * 24 * time.Hour, + Snapshots: []table.Snapshot{ + {SnapshotID: 1, TimestampMs: now.UnixMilli(), ManifestList: "metadata/snap-1.avro"}, + {SnapshotID: 2, TimestampMs: now.UnixMilli() + 1, ManifestList: "metadata/snap-2.avro"}, + }, + } + meta := populateTable(t, fs, setup) + + metaDir := path.Join(s3tables.TablesPath, setup.BucketName, setup.tablePath(), "metadata") + dataDir := path.Join(s3tables.TablesPath, setup.BucketName, setup.tablePath(), "data") + + writeRefManifest := func(fileName string, snapID int64, filePath string) { + dfBuilder, err := iceberg.NewDataFileBuilder( + meta.PartitionSpec(), iceberg.EntryContentData, filePath, + iceberg.ParquetFile, map[int]any{}, nil, nil, 10, 4096, + ) + if err != nil { + t.Fatalf("build data file: %v", err) + } + var manifestBuf bytes.Buffer + if _, err := iceberg.WriteManifest( + setup.fileRef("metadata", fileName), &manifestBuf, + meta.Version(), meta.PartitionSpec(), meta.CurrentSchema(), snapID, + []iceberg.ManifestEntry{iceberg.NewManifestEntry(iceberg.EntryStatusADDED, &snapID, nil, nil, dfBuilder.Build())}, + ); err != nil { + t.Fatalf("write manifest: %v", err) + } + fs.putEntry(metaDir, fileName, &filer_pb.Entry{ + Name: fileName, + Attributes: &filer_pb.FuseAttributes{Mtime: old.Unix(), FileSize: uint64(manifestBuf.Len())}, + Content: manifestBuf.Bytes(), + }) + } + writeRefManifest("manifest-1.avro", 1, fmt.Sprintf("s3://%s/%s/data/f.parquet", setup.BucketName, setup.tablePath())) + writeRefManifest("manifest-2.avro", 2, fmt.Sprintf("s3://%s/%s/data/tmp/%%2e%%2e/f.parquet", setup.BucketName, setup.tablePath())) + + fs.putEntry(dataDir, "f.parquet", &filer_pb.Entry{ + Name: "f.parquet", Attributes: &filer_pb.FuseAttributes{Mtime: old.Unix(), FileSize: 4096}, + Content: []byte("fake-parquet-data"), + }) + + handler := NewHandler(nil) + config := Config{ + SnapshotRetentionMs: hoursToMs(0), + MaxSnapshotsToKeep: 1, + MaxCommitRetries: 3, + Operations: "expire_snapshots", + } + + if _, _, err := handler.expireSnapshots(context.Background(), client, setup.BucketName, setup.tablePath(), config); err != nil { + t.Fatalf("expireSnapshots failed: %v", err) + } + + if fs.getEntry(dataDir, "f.parquet") == nil { + t.Error("file referenced by the kept snapshot through a decoded parent segment was deleted") + } +} diff --git a/weed/worker/tasks/iceberg/filer_io.go b/weed/worker/tasks/iceberg/filer_io.go index 482c8dc80a..9c05e010d2 100644 --- a/weed/worker/tasks/iceberg/filer_io.go +++ b/weed/worker/tasks/iceberg/filer_io.go @@ -9,6 +9,7 @@ import ( "errors" "fmt" "io" + "net/url" "path" "strconv" "strings" @@ -162,26 +163,66 @@ func loadCurrentMetadata(ctx context.Context, client filer_pb.SeaweedFilerClient return state, nil } +func isFilerNotFound(err error) bool { + return errors.Is(err, filer_pb.ErrNotFound) || status.Code(err) == codes.NotFound +} + +// resolveFilerPath returns the filer path that actually holds a referenced +// file: the verbatim spelling first, then the percent-decoded one. Writers +// may record a path with escaped segments while the S3 layer decoded the key +// before storing the object. +func resolveFilerPath(ctx context.Context, client filer_pb.SeaweedFilerClient, bucketName, dataPath, icebergPath string) (string, error) { + _, fullPath, err := lookupResolvedEntry(ctx, client, bucketName, dataPath, icebergPath) + return fullPath, err +} + +// lookupResolvedEntry returns the entry and filer path holding an +// Iceberg-style reference under the verbatim-first rule. +func lookupResolvedEntry(ctx context.Context, client filer_pb.SeaweedFilerClient, bucketName, dataPath, icebergPath string) (*filer_pb.Entry, string, error) { + fullPath, err := icebergFilerPath(bucketName, dataPath, icebergPath) + if err != nil { + return nil, "", err + } + resp, err := filer_pb.LookupEntry(ctx, client, &filer_pb.LookupDirectoryEntryRequest{ + Directory: path.Dir(fullPath), + Name: path.Base(fullPath), + }) + if err == nil { + return resp.Entry, fullPath, nil + } + if !isFilerNotFound(err) { + return nil, "", err + } + if decoded, decErr := url.PathUnescape(icebergPath); decErr == nil && decoded != icebergPath { + return lookupResolvedEntry(ctx, client, bucketName, dataPath, decoded) + } + return nil, "", err +} + +// addFilerPathSpellings records every filer path lookupResolvedEntry can +// return for a reference: each successive decoding of the reference, with the +// rest of the path left verbatim. An extra spelling only keeps a file; a +// missing one can drop a referenced file. +func addFilerPathSpellings(set map[string]struct{}, bucketName, dataPath, icebergPath string) { + for spelling := icebergPath; ; { + filerPath, err := icebergFilerPath(bucketName, dataPath, spelling) + if err != nil { + return + } + set[filerPath] = struct{}{} + decoded, err := url.PathUnescape(spelling) + if err != nil || decoded == spelling { + return + } + spelling = decoded + } +} + // lookupFileByIcebergPath returns the filer entry holding an Iceberg-style // path. dataPath is the bucket-relative directory the table's files live in. func lookupFileByIcebergPath(ctx context.Context, client filer_pb.SeaweedFilerClient, bucketName, dataPath, icebergPath string) (*filer_pb.Entry, error) { - fullPath, err := icebergFilerPath(bucketName, dataPath, icebergPath) - if err != nil { - return nil, err - } - dir, fileName := path.Dir(fullPath), path.Base(fullPath) - - resp, err := filer_pb.LookupEntry(ctx, client, &filer_pb.LookupDirectoryEntryRequest{ - Directory: dir, - Name: fileName, - }) - if err != nil { - return nil, fmt.Errorf("lookup %s/%s: %w", dir, fileName, err) - } - if resp == nil || resp.Entry == nil { - return nil, fmt.Errorf("file not found: %s/%s", dir, fileName) - } - return resp.Entry, nil + entry, _, err := lookupResolvedEntry(ctx, client, bucketName, dataPath, icebergPath) + return entry, err } // loadFileByIcebergPath loads a file from the filer given an Iceberg-style path. diff --git a/weed/worker/tasks/iceberg/operations.go b/weed/worker/tasks/iceberg/operations.go index 2266ad0ae3..8b7bb22f24 100644 --- a/weed/worker/tasks/iceberg/operations.go +++ b/weed/worker/tasks/iceberg/operations.go @@ -9,6 +9,7 @@ import ( "math/rand/v2" "path" "sort" + "strings" "time" "github.com/apache/iceberg-go" @@ -90,9 +91,7 @@ func (h *Handler) expireSnapshots( // Resolve kept file paths for consistent comparison keptFilerPaths := make(map[string]struct{}, len(keptFiles)) for f := range keptFiles { - if filerPath, err := icebergFilerPath(bucketName, dataPath, f); err == nil { - keptFilerPaths[filerPath] = struct{}{} - } + addFilerPathSpellings(keptFilerPaths, bucketName, dataPath, f) } // Use MetadataBuilder to remove snapshots and create new metadata @@ -118,11 +117,18 @@ func (h *Handler) expireSnapshots( } // Delete files exclusively referenced by expired snapshots (best-effort) + tableBase := path.Join(s3tables.TablesPath, bucketName, dataPath) + "/" deletedCount := 0 for filePath := range expiredFiles { - resolved, err := icebergFilerPath(bucketName, dataPath, filePath) + resolved, err := resolveFilerPath(ctx, filerClient, bucketName, dataPath, filePath) if err != nil { - glog.Warningf("iceberg maintenance: cannot resolve expired file %s: %v", filePath, err) + if !isFilerNotFound(err) { + glog.Warningf("iceberg maintenance: cannot resolve expired file %s: %v", filePath, err) + } + continue + } + if !strings.HasPrefix(resolved, tableBase) { + glog.Warningf("iceberg maintenance: refusing to delete %s outside table path", resolved) continue } if _, stillReferenced := keptFilerPaths[resolved]; stillReferenced { @@ -357,9 +363,11 @@ func collectOrphanCandidates( referencedFilerPaths := make(map[string]struct{}, len(referencedFiles)) for ref := range referencedFiles { - if filerPath, err := icebergFilerPath(bucketName, dataPath, ref); err == nil { - referencedFilerPaths[filerPath] = struct{}{} - } + // Writers percent-escape partition directory names (e.g. "a=1" stored + // as "a%3D1" in the manifest path), but the S3 layer decodes the key + // before it lands on the filer. Match every spelling the resolver can + // return, or a live file is counted as an orphan and deleted. + addFilerPathSpellings(referencedFilerPaths, bucketName, dataPath, ref) } tableBasePath := path.Join(s3tables.TablesPath, bucketName, dataPath) diff --git a/weed/worker/tasks/iceberg/row_lineage.go b/weed/worker/tasks/iceberg/row_lineage.go index c751f4a98b..f6aa85c2aa 100644 --- a/weed/worker/tasks/iceberg/row_lineage.go +++ b/weed/worker/tasks/iceberg/row_lineage.go @@ -368,12 +368,16 @@ func copyFileRows(ctx context.Context, reader *parquet.Reader, entry iceberg.Man }) } -// mergedFileFirstRowID is the _row_id of the merged output's first row, or nil when it carries none: the value the -// manifest entry's first_row_id must name, since rewritten rows keep their IDs rather than taking fresh ones. +// mergedFileFirstRowID is the _row_id of the merged output's first row, or nil when the file must be committed +// without a first_row_id: a null _row_id would inherit first_row_id+position, an id another row already holds. +// Committed without one, the nulls take fresh ids from the table's next-row-id and materialized values still +// override inheritance. func mergedFileFirstRowID(data []byte) (*int64, error) { - reader := parquet.NewReader(bytes.NewReader(data)) - defer reader.Close() - schema := reader.Schema() + file, err := parquet.OpenFile(bytes.NewReader(data), int64(len(data))) + if err != nil { + return nil, err + } + schema := file.Schema() if schema == nil { return nil, nil } @@ -381,6 +385,11 @@ func mergedFileFirstRowID(data []byte) (*int64, error) { if !ok { return nil, nil } + if rowIDHasNulls(file, leaf.ColumnIndex) { + return nil, nil + } + reader := parquet.NewReader(file) + defer reader.Close() rows := make([]parquet.Row, 1) n, err := reader.ReadRows(rows) if err != nil && err != io.EOF { @@ -402,6 +411,22 @@ func mergedFileFirstRowID(data []byte) (*int64, error) { return nil, nil } +// rowIDHasNulls reports whether any row group's _row_id column chunk counts a null. A chunk whose statistics +// cannot be read counts as having one: a first_row_id named on a hidden null is the failure this guards. +func rowIDHasNulls(file *parquet.File, columnIndex int) bool { + for _, rg := range file.RowGroups() { + chunks := rg.ColumnChunks() + if columnIndex >= len(chunks) { + return true + } + cc, ok := chunks[columnIndex].(*parquet.FileColumnChunk) + if !ok || cc.NullCount() > 0 { + return true + } + } + return false +} + // lineageCodec is the codec of the first data column leaf that declares one, // ZSTD when none does. func lineageCodec(base []parquet.Field) compress.Codec { diff --git a/weed/worker/tasks/iceberg/row_lineage_first_id_test.go b/weed/worker/tasks/iceberg/row_lineage_first_id_test.go new file mode 100644 index 0000000000..07d9929640 --- /dev/null +++ b/weed/worker/tasks/iceberg/row_lineage_first_id_test.go @@ -0,0 +1,63 @@ +package iceberg + +import ( + "bytes" + "testing" + + "github.com/parquet-go/parquet-go" +) + +type lineageRow struct { + ID int64 `parquet:"id"` + RowID *int64 `parquet:"_row_id,optional"` +} + +func lineageFile(t *testing.T, ids ...*int64) []byte { + t.Helper() + var buf bytes.Buffer + w := parquet.NewGenericWriter[lineageRow](&buf) + rows := make([]lineageRow, len(ids)) + for i, id := range ids { + rows[i] = lineageRow{ID: int64(i), RowID: id} + } + if _, err := w.Write(rows); err != nil { + t.Fatal(err) + } + if err := w.Close(); err != nil { + t.Fatal(err) + } + return buf.Bytes() +} + +func i64(v int64) *int64 { return &v } + +// A merged v3 file names its first row's id as first_row_id only when every +// row carries a materialized id: a null row would inherit first_row_id plus +// its position, an id another row already holds (high review of the 4.49 +// series; 4.49's own code took the first row's id whatever followed). +func TestMergedFileFirstRowIDOnlyWithoutNullIDs(t *testing.T) { + cases := []struct { + name string + ids []*int64 + want *int64 + }{ + {"every row materialized", []*int64{i64(40), i64(41), i64(42)}, i64(40)}, + {"materialized first, a null later", []*int64{i64(40), nil, i64(42)}, nil}, + {"null first", []*int64{nil, i64(41)}, nil}, + {"all null", []*int64{nil, nil}, nil}, + } + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + got, err := mergedFileFirstRowID(lineageFile(t, c.ids...)) + if err != nil { + t.Fatal(err) + } + switch { + case c.want == nil && got != nil: + t.Fatalf("first_row_id %d, want none", *got) + case c.want != nil && (got == nil || *got != *c.want): + t.Fatalf("first_row_id %v, want %d", got, *c.want) + } + }) + } +}